Use soong's finder to find included makefiles in mk2rbc

Having soong generate a list of makefiles for mk2rbc
to look through is much faster than having mk2rbc search
itself. Profiling the readLinesFromeFile() function that
reads the list of makefiles shows it takes on the order
of 200 microseconds, much faster than the ~5 seconds
it takes for mk2rbc to search the tree itself.

This CL also allows include statements that are prefixed
with a variable. The concern with this was that there
would be a lot of load statemnts emitted for generic
include statements, causing the generated code to look
ugly, and converting and loading all those files could
cause performance issues. On the performance issues
front, there's already a check that it doesn't result
in over 150 potentially included files. We can lower
that number if necessary, but it's probably good for now.
On the generated code front, while it's true that it's
ugly, it's better to have working but ugly generated
code than refusing to generate anything working at all.

To ensure the soong finder step isn't slowed down due
to having to find a bunch of new makefiles, I profiled
the combination of newSourceFinder and FindSources in main.go:

Baseline incremental:
338.011634ms
340.853335ms
348.541762ms
333.229644ms
349.124824ms

Baseline clean:
1.003836419s
1.006203912s
996.193648ms
1.031005604s
1.03691152s

Modified incremental:
349.029285ms
349.264496ms
351.774948ms
337.63187ms
359.425306ms

Modified clean:
1.028238704s
1.053103506s
1.032757506s
1.016631201s
1.04288108s

So we can see the times are barely affected by this change.

Fixes: 213508006
Test: go test
Change-Id: Iab18bfb127ba3b7e63f2c01f69064805a8398764

mk2rbc/cmd/mk2rbc.go

@@ -57,6 +57,7 @@ var (
 	cpuProfile            = flag.String("cpu_profile", "", "write cpu profile to file")
 	traceCalls            = flag.Bool("trace_calls", false, "trace function calls")
 	inputVariables        = flag.String("input_variables", "", "starlark file containing product config and global variables")
+	makefileList          = flag.String("makefile_list", "", "path to a list of all makefiles in the source tree, generated by soong's finder. If not provided, mk2rbc will find the makefiles itself (more slowly than if this flag was provided)")
 )
 
 func init() {
@@ -79,7 +80,7 @@ func init() {
 var backupSuffix string
 var tracedVariables []string
 var errorLogger = errorSink{data: make(map[string]datum)}
-var makefileFinder = &LinuxMakefileFinder{}
+var makefileFinder mk2rbc.MakefileFinder
 
 func main() {
 	flag.Usage = func() {
@@ -133,6 +134,16 @@ func main() {
 		pprof.StartCPUProfile(f)
 		defer pprof.StopCPUProfile()
 	}
+
+	if *makefileList != "" {
+		makefileFinder = &FileListMakefileFinder{
+			cachedMakefiles: nil,
+			filePath:        *makefileList,
+		}
+	} else {
+		makefileFinder = &FindCommandMakefileFinder{}
+	}
+
 	// Find out global variables
 	getConfigVariables()
 	getSoongVariables()
@@ -519,17 +530,17 @@ func stringsWithFreq(items []string, topN int) (string, int) {
 	return res, len(sorted)
 }
 
-type LinuxMakefileFinder struct {
+// FindCommandMakefileFinder is an implementation of mk2rbc.MakefileFinder that
+// runs the unix find command to find all the makefiles in the source tree.
+type FindCommandMakefileFinder struct {
 	cachedRoot      string
 	cachedMakefiles []string
 }
 
-func (l *LinuxMakefileFinder) Find(root string) []string {
+func (l *FindCommandMakefileFinder) Find(root string) []string {
 	if l.cachedMakefiles != nil && l.cachedRoot == root {
 		return l.cachedMakefiles
 	}
-	l.cachedRoot = root
-	l.cachedMakefiles = make([]string, 0)
 
 	// Return all *.mk files but not in hidden directories.
 
@@ -548,9 +559,60 @@ func (l *LinuxMakefileFinder) Find(root string) []string {
 		panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
 	}
 	scanner := bufio.NewScanner(stdout)
+	result := make([]string, 0)
 	for scanner.Scan() {
-		l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./"))
+		result = append(result, strings.TrimPrefix(scanner.Text(), "./"))
 	}
 	stdout.Close()
+	err = scanner.Err()
+	if err != nil {
+		panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
+	}
+	l.cachedRoot = root
+	l.cachedMakefiles = result
+	return l.cachedMakefiles
+}
+
+// FileListMakefileFinder is an implementation of mk2rbc.MakefileFinder that
+// reads a file containing the list of makefiles in the android source tree.
+// This file is generated by soong's finder, so that it can be computed while
+// soong is already walking the source tree looking for other files. If the root
+// to find makefiles under is not the root of the android source tree, it will
+// fall back to using FindCommandMakefileFinder.
+type FileListMakefileFinder struct {
+	FindCommandMakefileFinder
+	cachedMakefiles []string
+	filePath        string
+}
+
+func (l *FileListMakefileFinder) Find(root string) []string {
+	root, err1 := filepath.Abs(root)
+	wd, err2 := filepath.Abs(*rootDir)
+	if root != wd || err1 != nil || err2 != nil {
+		return l.FindCommandMakefileFinder.Find(root)
+	}
+	if l.cachedMakefiles != nil {
+		return l.cachedMakefiles
+	}
+
+	file, err := os.Open(l.filePath)
+	if err != nil {
+		panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
+	}
+	defer file.Close()
+
+	result := make([]string, 0)
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) > 0 {
+			result = append(result, line)
+		}
+	}
+
+	if err = scanner.Err(); err != nil {
+		panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
+	}
+	l.cachedMakefiles = result
 	return l.cachedMakefiles
 }

scripts/rbc-run

@@ -9,9 +9,10 @@ declare -r output_root="${OUT_DIR:-out}"
 declare -r runner="${output_root}/soong/rbcrun"
 declare -r converter="${output_root}/soong/mk2rbc"
 declare -r launcher="${output_root}/rbc/launcher.rbc"
+declare -r makefile_list="${output_root}/.module_paths/configuration.list"
 declare -r makefile="$1"
 declare -r input_variables="$2"
 shift 2
-"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" "${makefile}"
+"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" --makefile_list="${makefile_list}" "${makefile}"
 "${runner}" RBC_OUT="make,global" RBC_DEBUG="${RBC_DEBUG:-}" $@ "${launcher}"
 

ui/build/finder.go

@@ -87,8 +87,8 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
 			// Bazel top-level file to mark a directory as a Bazel workspace.
 			"WORKSPACE",
 		},
-		// Bazel Starlark configuration files.
-		IncludeSuffixes: []string{".bzl"},
+		// Bazel Starlark configuration files and all .mk files for product/board configuration.
+		IncludeSuffixes: []string{".bzl", ".mk"},
 	}
 	dumpDir := config.FileListDir()
 	f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
@@ -110,6 +110,19 @@ func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []s
 	return entries.DirNames, matches
 }
 
+func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
+	matches := []string{}
+	for _, foundName := range entries.FileNames {
+		if foundName != "Android.mk" &&
+			foundName != "AndroidProducts.mk" &&
+			foundName != "CleanSpec.mk" &&
+			strings.HasSuffix(foundName, ".mk") {
+			matches = append(matches, foundName)
+		}
+	}
+	return entries.DirNames, matches
+}
+
 // FindSources searches for source files known to <f> and writes them to the filesystem for
 // use later.
 func FindSources(ctx Context, config Config, f *finder.Finder) {
@@ -172,6 +185,13 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
 		ctx.Fatalf("Could not find modules: %v", err)
 	}
 
+	// Recursively look for all product/board config files.
+	configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles)
+	err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list"))
+	if err != nil {
+		ctx.Fatalf("Could not export product/board configuration list: %v", err)
+	}
+
 	if config.Dist() {
 		f.WaitForDbDump()
 		// Dist the files.db plain text database.