From e03bb6107275ac646dcd978cfeccc3de1924a066 Mon Sep 17 00:00:00 2001 From: Rupert Shuttleworth Date: Thu, 20 May 2021 19:34:50 -0400 Subject: [PATCH] Use Finder for getPathsToIgnoredBuildFiles() to try and speed up builds with slow hard drives. NOTE: This relies on https://chromium-review.googlesource.com/c/angle/angle/+/2906252 being merged into AOSP first Test: bazel build //bionic/... //external/... //frameworks/... //system/... Change-Id: Id552941274ffbc9955dfcd8f545a58f1e3368845 --- cmd/soong_build/main.go | 81 +++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 7abb67f29..0336fb668 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -17,7 +17,6 @@ package main import ( "flag" "fmt" - "io/fs" "io/ioutil" "os" "path/filepath" @@ -364,54 +363,41 @@ func touch(path string) { // - won't be overwritten by corresponding bp2build generated files // // And return their paths so they can be left out of the Bazel workspace dir (i.e. ignored) -func getPathsToIgnoredBuildFiles(topDir string, outDir string, generatedRoot string) ([]string, error) { +func getPathsToIgnoredBuildFiles(topDir string, generatedRoot string, srcDirBazelFiles []string) []string { paths := make([]string, 0) - err := filepath.WalkDir(topDir, func(fFullPath string, fDirEntry fs.DirEntry, err error) error { + for _, srcDirBazelFileRelativePath := range srcDirBazelFiles { + srcDirBazelFileFullPath := shared.JoinPath(topDir, srcDirBazelFileRelativePath) + fileInfo, err := os.Stat(srcDirBazelFileFullPath) if err != nil { - // Warn about error, but continue trying to walk the directory tree - fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fFullPath, err) - return nil + // Warn about error, but continue trying to check files + fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", srcDirBazelFileFullPath, err) + continue } - if fDirEntry.IsDir() { + if fileInfo.IsDir() { // Don't ignore entire directories - return nil + continue } - if !(fDirEntry.Name() == "BUILD" || fDirEntry.Name() == "BUILD.bazel") { + if !(fileInfo.Name() == "BUILD" || fileInfo.Name() == "BUILD.bazel") { // Don't ignore this file - it is not a build file - return nil + continue } - f := strings.TrimPrefix(fFullPath, topDir+"/") - if strings.HasPrefix(f, ".repo/") { - // Don't check for files to ignore in the .repo dir (recursively) - return fs.SkipDir - } - if strings.HasPrefix(f, outDir+"/") { - // Don't check for files to ignore in the out dir (recursively) - return fs.SkipDir - } - if strings.HasPrefix(f, generatedRoot) { - // Don't check for files to ignore in the bp2build dir (recursively) - // NOTE: This is usually under outDir - return fs.SkipDir - } - fDir := filepath.Dir(f) - if android.ShouldKeepExistingBuildFileForDir(fDir) { + srcDirBazelFileDir := filepath.Dir(srcDirBazelFileRelativePath) + if android.ShouldKeepExistingBuildFileForDir(srcDirBazelFileDir) { // Don't ignore this existing build file - return nil + continue } - f_bp2build := shared.JoinPath(topDir, generatedRoot, f) - if _, err := os.Stat(f_bp2build); err == nil { + correspondingBp2BuildFile := shared.JoinPath(topDir, generatedRoot, srcDirBazelFileRelativePath) + if _, err := os.Stat(correspondingBp2BuildFile); err == nil { // If bp2build generated an alternate BUILD file, don't exclude this workspace path // BUILD file clash resolution happens later in the symlink forest creation - return nil + continue } - fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", f) - paths = append(paths, f) - return nil - }) + fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", srcDirBazelFileRelativePath) + paths = append(paths, srcDirBazelFileRelativePath) + } - return paths, err + return paths } // Returns temporary symlink forest excludes necessary for bazel build //external/... (and bazel build //frameworks/...) to work @@ -431,6 +417,22 @@ func getTemporaryExcludes() []string { return excludes } +// Read the bazel.list file that the Soong Finder already dumped earlier (hopefully) +// It contains the locations of BUILD files, BUILD.bazel files, etc. in the source dir +func getExistingBazelRelatedFiles(topDir string) ([]string, error) { + bazelFinderFile := filepath.Join(filepath.Dir(bootstrap.CmdlineArgs.ModuleListFile), "bazel.list") + if !filepath.IsAbs(bazelFinderFile) { + // Assume this was a relative path under topDir + bazelFinderFile = filepath.Join(topDir, bazelFinderFile) + } + data, err := ioutil.ReadFile(bazelFinderFile) + if err != nil { + return nil, err + } + files := strings.Split(strings.TrimSpace(string(data)), "\n") + return files, nil +} + // Run Soong in the bp2build mode. This creates a standalone context that registers // an alternate pipeline of mutators and singletons specifically for generating // Bazel BUILD files instead of Ninja files. @@ -489,14 +491,13 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir) } - // FIXME: Don't hardcode this here - topLevelOutDir := "out" - - pathsToIgnoredBuildFiles, err := getPathsToIgnoredBuildFiles(topDir, topLevelOutDir, generatedRoot) + existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir) if err != nil { - fmt.Fprintf(os.Stderr, "Error walking SrcDir: '%s': %s\n", configuration.SrcDir(), err) + fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err) os.Exit(1) } + + pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(topDir, generatedRoot, existingBazelRelatedFiles) excludes = append(excludes, pathsToIgnoredBuildFiles...) excludes = append(excludes, getTemporaryExcludes()...)