Merge "Use soong's finder to find included makefiles in mk2rbc"

mk2rbc/cmd/mk2rbc.go

@@ -57,6 +57,7 @@ var (
 	cpuProfile            = flag.String("cpu_profile", "", "write cpu profile to file")
 	traceCalls            = flag.Bool("trace_calls", false, "trace function calls")
 	inputVariables        = flag.String("input_variables", "", "starlark file containing product config and global variables")
+	makefileList          = flag.String("makefile_list", "", "path to a list of all makefiles in the source tree, generated by soong's finder. If not provided, mk2rbc will find the makefiles itself (more slowly than if this flag was provided)")
 )
 
 func init() {
@@ -79,7 +80,7 @@ func init() {
 var backupSuffix string
 var tracedVariables []string
 var errorLogger = errorSink{data: make(map[string]datum)}
-var makefileFinder = &LinuxMakefileFinder{}
+var makefileFinder mk2rbc.MakefileFinder
 
 func main() {
 	flag.Usage = func() {
@@ -133,6 +134,16 @@ func main() {
 		pprof.StartCPUProfile(f)
 		defer pprof.StopCPUProfile()
 	}
+
+	if *makefileList != "" {
+		makefileFinder = &FileListMakefileFinder{
+			cachedMakefiles: nil,
+			filePath:        *makefileList,
+		}
+	} else {
+		makefileFinder = &FindCommandMakefileFinder{}
+	}
+
 	// Find out global variables
 	getConfigVariables()
 	getSoongVariables()
@@ -519,17 +530,17 @@ func stringsWithFreq(items []string, topN int) (string, int) {
 	return res, len(sorted)
 }
 
-type LinuxMakefileFinder struct {
+// FindCommandMakefileFinder is an implementation of mk2rbc.MakefileFinder that
+// runs the unix find command to find all the makefiles in the source tree.
+type FindCommandMakefileFinder struct {
 	cachedRoot      string
 	cachedMakefiles []string
 }
 
-func (l *LinuxMakefileFinder) Find(root string) []string {
+func (l *FindCommandMakefileFinder) Find(root string) []string {
 	if l.cachedMakefiles != nil && l.cachedRoot == root {
 		return l.cachedMakefiles
 	}
-	l.cachedRoot = root
-	l.cachedMakefiles = make([]string, 0)
 
 	// Return all *.mk files but not in hidden directories.
 
@@ -548,9 +559,60 @@ func (l *LinuxMakefileFinder) Find(root string) []string {
 		panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
 	}
 	scanner := bufio.NewScanner(stdout)
+	result := make([]string, 0)
 	for scanner.Scan() {
-		l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./"))
+		result = append(result, strings.TrimPrefix(scanner.Text(), "./"))
 	}
 	stdout.Close()
+	err = scanner.Err()
+	if err != nil {
+		panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
+	}
+	l.cachedRoot = root
+	l.cachedMakefiles = result
+	return l.cachedMakefiles
+}
+
+// FileListMakefileFinder is an implementation of mk2rbc.MakefileFinder that
+// reads a file containing the list of makefiles in the android source tree.
+// This file is generated by soong's finder, so that it can be computed while
+// soong is already walking the source tree looking for other files. If the root
+// to find makefiles under is not the root of the android source tree, it will
+// fall back to using FindCommandMakefileFinder.
+type FileListMakefileFinder struct {
+	FindCommandMakefileFinder
+	cachedMakefiles []string
+	filePath        string
+}
+
+func (l *FileListMakefileFinder) Find(root string) []string {
+	root, err1 := filepath.Abs(root)
+	wd, err2 := filepath.Abs(*rootDir)
+	if root != wd || err1 != nil || err2 != nil {
+		return l.FindCommandMakefileFinder.Find(root)
+	}
+	if l.cachedMakefiles != nil {
+		return l.cachedMakefiles
+	}
+
+	file, err := os.Open(l.filePath)
+	if err != nil {
+		panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
+	}
+	defer file.Close()
+
+	result := make([]string, 0)
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) > 0 {
+			result = append(result, line)
+		}
+	}
+
+	if err = scanner.Err(); err != nil {
+		panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
+	}
+	l.cachedMakefiles = result
 	return l.cachedMakefiles
 }

scripts/rbc-run

@@ -9,9 +9,10 @@ declare -r output_root="${OUT_DIR:-out}"
 declare -r runner="${output_root}/soong/rbcrun"
 declare -r converter="${output_root}/soong/mk2rbc"
 declare -r launcher="${output_root}/rbc/launcher.rbc"
+declare -r makefile_list="${output_root}/.module_paths/configuration.list"
 declare -r makefile="$1"
 declare -r input_variables="$2"
 shift 2
-"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" "${makefile}"
+"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" --makefile_list="${makefile_list}" "${makefile}"
 "${runner}" RBC_OUT="make,global" RBC_DEBUG="${RBC_DEBUG:-}" $@ "${launcher}"
 

ui/build/finder.go

@@ -87,8 +87,8 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
 			// Bazel top-level file to mark a directory as a Bazel workspace.
 			"WORKSPACE",
 		},
-		// Bazel Starlark configuration files.
+		// Bazel Starlark configuration files and all .mk files for product/board configuration.
-		IncludeSuffixes: []string{".bzl"},
+		IncludeSuffixes: []string{".bzl", ".mk"},
 	}
 	dumpDir := config.FileListDir()
 	f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
@@ -110,6 +110,19 @@ func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []s
 	return entries.DirNames, matches
 }
 
+func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
+	matches := []string{}
+	for _, foundName := range entries.FileNames {
+		if foundName != "Android.mk" &&
+			foundName != "AndroidProducts.mk" &&
+			foundName != "CleanSpec.mk" &&
+			strings.HasSuffix(foundName, ".mk") {
+			matches = append(matches, foundName)
+		}
+	}
+	return entries.DirNames, matches
+}
+
 // FindSources searches for source files known to <f> and writes them to the filesystem for
 // use later.
 func FindSources(ctx Context, config Config, f *finder.Finder) {
@@ -172,6 +185,13 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
 		ctx.Fatalf("Could not find modules: %v", err)
 	}
 
+	// Recursively look for all product/board config files.
+	configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles)
+	err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list"))
+	if err != nil {
+		ctx.Fatalf("Could not export product/board configuration list: %v", err)
+	}
+
 	if config.Dist() {
 		f.WaitForDbDump()
 		// Dist the files.db plain text database.