Merge "Use Finder for getPathsToIgnoredBuildFiles() to try and speed up builds with slow hard drives."

cmd/soong_build/main.go

@@ -17,7 +17,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -364,54 +363,41 @@ func touch(path string) {
 // - won't be overwritten by corresponding bp2build generated files
 //
 // And return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
-func getPathsToIgnoredBuildFiles(topDir string, outDir string, generatedRoot string) ([]string, error) {
+func getPathsToIgnoredBuildFiles(topDir string, generatedRoot string, srcDirBazelFiles []string) []string {
 	paths := make([]string, 0)
 
-	err := filepath.WalkDir(topDir, func(fFullPath string, fDirEntry fs.DirEntry, err error) error {
+	for _, srcDirBazelFileRelativePath := range srcDirBazelFiles {
+		srcDirBazelFileFullPath := shared.JoinPath(topDir, srcDirBazelFileRelativePath)
+		fileInfo, err := os.Stat(srcDirBazelFileFullPath)
 		if err != nil {
-			// Warn about error, but continue trying to walk the directory tree
-			fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fFullPath, err)
-			return nil
+			// Warn about error, but continue trying to check files
+			fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", srcDirBazelFileFullPath, err)
+			continue
 		}
-		if fDirEntry.IsDir() {
+		if fileInfo.IsDir() {
 			// Don't ignore entire directories
-			return nil
+			continue
 		}
-		if !(fDirEntry.Name() == "BUILD" || fDirEntry.Name() == "BUILD.bazel") {
+		if !(fileInfo.Name() == "BUILD" || fileInfo.Name() == "BUILD.bazel") {
 			// Don't ignore this file - it is not a build file
-			return nil
+			continue
 		}
-		f := strings.TrimPrefix(fFullPath, topDir+"/")
-		if strings.HasPrefix(f, ".repo/") {
-			// Don't check for files to ignore in the .repo dir (recursively)
-			return fs.SkipDir
-		}
-		if strings.HasPrefix(f, outDir+"/") {
-			// Don't check for files to ignore in the out dir (recursively)
-			return fs.SkipDir
-		}
-		if strings.HasPrefix(f, generatedRoot) {
-			// Don't check for files to ignore in the bp2build dir (recursively)
-			// NOTE: This is usually under outDir
-			return fs.SkipDir
-		}
-		fDir := filepath.Dir(f)
-		if android.ShouldKeepExistingBuildFileForDir(fDir) {
+		srcDirBazelFileDir := filepath.Dir(srcDirBazelFileRelativePath)
+		if android.ShouldKeepExistingBuildFileForDir(srcDirBazelFileDir) {
 			// Don't ignore this existing build file
-			return nil
+			continue
 		}
-		f_bp2build := shared.JoinPath(topDir, generatedRoot, f)
-		if _, err := os.Stat(f_bp2build); err == nil {
+		correspondingBp2BuildFile := shared.JoinPath(topDir, generatedRoot, srcDirBazelFileRelativePath)
+		if _, err := os.Stat(correspondingBp2BuildFile); err == nil {
 			// If bp2build generated an alternate BUILD file, don't exclude this workspace path
 			// BUILD file clash resolution happens later in the symlink forest creation
-			return nil
+			continue
+		}
+		fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", srcDirBazelFileRelativePath)
+		paths = append(paths, srcDirBazelFileRelativePath)
 	}
-		fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", f)
-		paths = append(paths, f)
-		return nil
-	})
 
-	return paths, err
+	return paths
 }
 
 // Returns temporary symlink forest excludes necessary for bazel build //external/... (and bazel build //frameworks/...) to work
@@ -431,6 +417,22 @@ func getTemporaryExcludes() []string {
 	return excludes
 }
 
+// Read the bazel.list file that the Soong Finder already dumped earlier (hopefully)
+// It contains the locations of BUILD files, BUILD.bazel files, etc. in the source dir
+func getExistingBazelRelatedFiles(topDir string) ([]string, error) {
+	bazelFinderFile := filepath.Join(filepath.Dir(bootstrap.CmdlineArgs.ModuleListFile), "bazel.list")
+	if !filepath.IsAbs(bazelFinderFile) {
+		// Assume this was a relative path under topDir
+		bazelFinderFile = filepath.Join(topDir, bazelFinderFile)
+	}
+	data, err := ioutil.ReadFile(bazelFinderFile)
+	if err != nil {
+		return nil, err
+	}
+	files := strings.Split(strings.TrimSpace(string(data)), "\n")
+	return files, nil
+}
+
 // Run Soong in the bp2build mode. This creates a standalone context that registers
 // an alternate pipeline of mutators and singletons specifically for generating
 // Bazel BUILD files instead of Ninja files.
@@ -489,14 +491,13 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 		excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir)
 	}
 
-	// FIXME: Don't hardcode this here
-	topLevelOutDir := "out"
-
-	pathsToIgnoredBuildFiles, err := getPathsToIgnoredBuildFiles(topDir, topLevelOutDir, generatedRoot)
+	existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error walking SrcDir: '%s': %s\n", configuration.SrcDir(), err)
+		fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err)
 		os.Exit(1)
 	}
+
+	pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(topDir, generatedRoot, existingBazelRelatedFiles)
 	excludes = append(excludes, pathsToIgnoredBuildFiles...)
 
 	excludes = append(excludes, getTemporaryExcludes()...)