From c541cd27fa8e8260699fa47ebd46508ba5b64789 Mon Sep 17 00:00:00 2001 From: "Lukacs T. Berki" Date: Wed, 26 Oct 2022 07:26:50 +0000 Subject: [PATCH] Create Bazel symlink forest in a separate process. This helps with incrementality a lot: the symlink forest must depend on almost every directory in the source tree so that if a new file is added or removed from *anywhere*, it is regenerated. Previously, we couldn't do this without invoking bp2build, which is quite wasteful because bp2build takes way more time than the symlink forest creation, even though we do the latter in a very suboptimal way at the moment. This means that if a source file is added or removed (which does not affect globs), we don't pay the cost of bp2build anymore. Also refactored symlink_forest.go on the side. Too much state was being passed around in arguments. This change reimplements aosp/2263423 ; the semantics of not touching an output file is the exact same as order-only inputs and the latter is a bit fewer lines of code. Test: Presubmits. Change-Id: I565c580df8a01bacf175d56747c3f50743d4a4d4 --- android/config.go | 3 ++ android/neverallow.go | 1 + bp2build/symlink_forest.go | 108 +++++++++++++++++++++---------------- cmd/soong_build/main.go | 103 ++++++++++++++++++++--------------- tests/bootstrap_test.sh | 40 +++++++++++++- tests/mixed_mode_test.sh | 47 ++++++++++++++++ ui/build/config.go | 6 ++- ui/build/soong.go | 43 ++++++++++----- 8 files changed, 245 insertions(+), 106 deletions(-) diff --git a/android/config.go b/android/config.go index 1ed405b2b..df2c7673a 100644 --- a/android/config.go +++ b/android/config.go @@ -75,6 +75,9 @@ const ( // Don't use bazel at all during module analysis. AnalysisNoBazel SoongBuildMode = iota + // Symlink fores mode: merge two directory trees into a symlink forest + SymlinkForest + // Bp2build mode: Generate BUILD files from blueprint files and exit. Bp2build diff --git a/android/neverallow.go b/android/neverallow.go index cf149b29d..d28843995 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -69,6 +69,7 @@ func AddNeverAllowRules(rules ...Rule) { func createBp2BuildRule() Rule { return NeverAllow(). With("bazel_module.bp2build_available", "true"). + NotIn("soong_tests"). // only used in tests Because("setting bp2build_available in Android.bp is not " + "supported for custom conversion, use allowlists.go instead.") } diff --git a/bp2build/symlink_forest.go b/bp2build/symlink_forest.go index 092b24039..e2b99c438 100644 --- a/bp2build/symlink_forest.go +++ b/bp2build/symlink_forest.go @@ -21,7 +21,6 @@ import ( "path/filepath" "regexp" - "android/soong/android" "android/soong/shared" ) @@ -31,14 +30,22 @@ import ( // or a directory. If excluded is true, then that file/directory should be // excluded from symlinking. Otherwise, the node is not excluded, but one of its // descendants is (otherwise the node in question would not exist) -type node struct { + +type instructionsNode struct { name string excluded bool // If false, this is just an intermediate node - children map[string]*node + children map[string]*instructionsNode +} + +type symlinkForestContext struct { + verbose bool + topdir string // $TOPDIR + deps []string // Files/directories read while constructing the forest + okay bool // Whether the forest was successfully constructed } // Ensures that the node for the given path exists in the tree and returns it. -func ensureNodeExists(root *node, path string) *node { +func ensureNodeExists(root *instructionsNode, path string) *instructionsNode { if path == "" { return root } @@ -56,15 +63,14 @@ func ensureNodeExists(root *node, path string) *node { if child, ok := dn.children[base]; ok { return child } else { - dn.children[base] = &node{base, false, make(map[string]*node)} + dn.children[base] = &instructionsNode{base, false, make(map[string]*instructionsNode)} return dn.children[base] } } -// Turns a list of paths to be excluded into a tree made of "node" objects where -// the specified paths are marked as excluded. -func treeFromExcludePathList(paths []string) *node { - result := &node{"", false, make(map[string]*node)} +// Turns a list of paths to be excluded into a tree +func instructionsFromExcludePathList(paths []string) *instructionsNode { + result := &instructionsNode{"", false, make(map[string]*instructionsNode)} for _, p := range paths { ensureNodeExists(result, p).excluded = true @@ -179,17 +185,21 @@ func isDir(path string, fi os.FileInfo) bool { // Recursively plants a symlink forest at forestDir. The symlink tree will // contain every file in buildFilesDir and srcDir excluding the files in -// exclude. Collects every directory encountered during the traversal of srcDir -// into acc. -func plantSymlinkForestRecursive(cfg android.Config, topdir string, forestDir string, buildFilesDir string, srcDir string, exclude *node, acc *[]string, okay *bool) { - if exclude != nil && exclude.excluded { +// instructions. Collects every directory encountered during the traversal of +// srcDir . +func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *instructionsNode, forestDir string, buildFilesDir string, srcDir string) { + if instructions != nil && instructions.excluded { // This directory is not needed, bail out return } - *acc = append(*acc, srcDir) - srcDirMap := readdirToMap(shared.JoinPath(topdir, srcDir)) - buildFilesMap := readdirToMap(shared.JoinPath(topdir, buildFilesDir)) + // We don't add buildFilesDir here because the bp2build files marker files is + // already a dependency which covers it. If we ever wanted to turn this into + // a generic symlink forest creation tool, we'd need to add it, too. + context.deps = append(context.deps, srcDir) + + srcDirMap := readdirToMap(shared.JoinPath(context.topdir, srcDir)) + buildFilesMap := readdirToMap(shared.JoinPath(context.topdir, buildFilesDir)) renamingBuildFile := false if _, ok := srcDirMap["BUILD"]; ok { @@ -211,7 +221,7 @@ func plantSymlinkForestRecursive(cfg android.Config, topdir string, forestDir st allEntries[n] = struct{}{} } - err := os.MkdirAll(shared.JoinPath(topdir, forestDir), 0777) + err := os.MkdirAll(shared.JoinPath(context.topdir, forestDir), 0777) if err != nil { fmt.Fprintf(os.Stderr, "Cannot mkdir '%s': %s\n", forestDir, err) os.Exit(1) @@ -230,69 +240,69 @@ func plantSymlinkForestRecursive(cfg android.Config, topdir string, forestDir st } buildFilesChild := shared.JoinPath(buildFilesDir, f) - // Descend in the exclusion tree, if there are any excludes left - var excludeChild *node = nil - if exclude != nil { + // Descend in the instruction tree if it exists + var instructionsChild *instructionsNode = nil + if instructions != nil { if f == "BUILD.bazel" && renamingBuildFile { - excludeChild = exclude.children["BUILD"] + instructionsChild = instructions.children["BUILD"] } else { - excludeChild = exclude.children[f] + instructionsChild = instructions.children[f] } } srcChildEntry, sExists := srcDirMap[f] buildFilesChildEntry, bExists := buildFilesMap[f] - if excludeChild != nil && excludeChild.excluded { + if instructionsChild != nil && instructionsChild.excluded { if bExists { - symlinkIntoForest(topdir, forestChild, buildFilesChild) + symlinkIntoForest(context.topdir, forestChild, buildFilesChild) } continue } - sDir := sExists && isDir(shared.JoinPath(topdir, srcChild), srcChildEntry) - bDir := bExists && isDir(shared.JoinPath(topdir, buildFilesChild), buildFilesChildEntry) + sDir := sExists && isDir(shared.JoinPath(context.topdir, srcChild), srcChildEntry) + bDir := bExists && isDir(shared.JoinPath(context.topdir, buildFilesChild), buildFilesChildEntry) if !sExists { - if bDir && excludeChild != nil { + if bDir && instructionsChild != nil { // Not in the source tree, but we have to exclude something from under // this subtree, so descend - plantSymlinkForestRecursive(cfg, topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) + plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild) } else { // Not in the source tree, symlink BUILD file - symlinkIntoForest(topdir, forestChild, buildFilesChild) + symlinkIntoForest(context.topdir, forestChild, buildFilesChild) } } else if !bExists { - if sDir && excludeChild != nil { + if sDir && instructionsChild != nil { // Not in the build file tree, but we have to exclude something from // under this subtree, so descend - plantSymlinkForestRecursive(cfg, topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) + plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild) } else { // Not in the build file tree, symlink source tree, carry on - symlinkIntoForest(topdir, forestChild, srcChild) + symlinkIntoForest(context.topdir, forestChild, srcChild) } } else if sDir && bDir { // Both are directories. Descend. - plantSymlinkForestRecursive(cfg, topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) + plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild) } else if !sDir && !bDir { // Neither is a directory. Merge them. - srcBuildFile := shared.JoinPath(topdir, srcChild) - generatedBuildFile := shared.JoinPath(topdir, buildFilesChild) + srcBuildFile := shared.JoinPath(context.topdir, srcChild) + generatedBuildFile := shared.JoinPath(context.topdir, buildFilesChild) // The Android.bp file that codegen used to produce `buildFilesChild` is // already a dependency, we can ignore `buildFilesChild`. - *acc = append(*acc, srcChild) - err = mergeBuildFiles(shared.JoinPath(topdir, forestChild), srcBuildFile, generatedBuildFile, cfg.IsEnvTrue("BP2BUILD_VERBOSE")) + context.deps = append(context.deps, srcChild) + err = mergeBuildFiles(shared.JoinPath(context.topdir, forestChild), srcBuildFile, generatedBuildFile, context.verbose) if err != nil { fmt.Fprintf(os.Stderr, "Error merging %s and %s: %s", srcBuildFile, generatedBuildFile, err) - *okay = false + context.okay = false } } else { // Both exist and one is a file. This is an error. fmt.Fprintf(os.Stderr, "Conflict in workspace symlink tree creation: both '%s' and '%s' exist and exactly one is a directory\n", srcChild, buildFilesChild) - *okay = false + context.okay = false } } } @@ -301,14 +311,20 @@ func plantSymlinkForestRecursive(cfg android.Config, topdir string, forestDir st // "srcDir" while excluding paths listed in "exclude". Returns the set of paths // under srcDir on which readdir() had to be called to produce the symlink // forest. -func PlantSymlinkForest(cfg android.Config, topdir string, forest string, buildFiles string, srcDir string, exclude []string) []string { - deps := make([]string, 0) +func PlantSymlinkForest(verbose bool, topdir string, forest string, buildFiles string, exclude []string) []string { + context := &symlinkForestContext{ + verbose: verbose, + topdir: topdir, + deps: make([]string, 0), + okay: true, + } + os.RemoveAll(shared.JoinPath(topdir, forest)) - excludeTree := treeFromExcludePathList(exclude) - okay := true - plantSymlinkForestRecursive(cfg, topdir, forest, buildFiles, srcDir, excludeTree, &deps, &okay) - if !okay { + + instructions := instructionsFromExcludePathList(exclude) + plantSymlinkForestRecursive(context, instructions, forest, buildFiles, ".") + if !context.okay { os.Exit(1) } - return deps + return context.deps } diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 87710c0f2..1f3507db6 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -56,6 +56,7 @@ var ( bazelQueryViewDir string bazelApiBp2buildDir string bp2buildMarker string + symlinkForestMarker string cmdlineArgs bootstrap.Args ) @@ -86,6 +87,7 @@ func init() { flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top") flag.StringVar(&bazelApiBp2buildDir, "bazel_api_bp2build_dir", "", "path to the bazel api_bp2build directory relative to --top") flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit") + flag.StringVar(&symlinkForestMarker, "symlink_forest_marker", "", "If set, create the bp2build symlink forest, touch the specified marker file, then exit") flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output") flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file") flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules") @@ -130,7 +132,9 @@ func newContext(configuration android.Config) *android.Context { func newConfig(availableEnv map[string]string) android.Config { var buildMode android.SoongBuildMode - if bp2buildMarker != "" { + if symlinkForestMarker != "" { + buildMode = android.SymlinkForest + } else if bp2buildMarker != "" { buildMode = android.Bp2build } else if bazelQueryViewDir != "" { buildMode = android.GenerateQueryView @@ -254,11 +258,10 @@ func runApiBp2build(configuration android.Config, extraNinjaDeps []string) strin // Create the symlink forest symlinkDeps := bp2build.PlantSymlinkForest( - configuration, + configuration.IsEnvTrue("BP2BUILD_VERBOSE"), topDir, workspace, bazelApiBp2buildDir, - ".", excludes) ninjaDeps = append(ninjaDeps, symlinkDeps...) @@ -345,7 +348,10 @@ func writeDepFile(outputFile string, eventHandler metrics.EventHandler, ninjaDep // or the actual Soong build for the build.ninja file. Returns the top level // output file of the specific activity. func doChosenActivity(ctx *android.Context, configuration android.Config, extraNinjaDeps []string) string { - if configuration.BuildMode == android.Bp2build { + if configuration.BuildMode == android.SymlinkForest { + runSymlinkForestCreation(configuration, extraNinjaDeps) + return symlinkForestMarker + } else if configuration.BuildMode == android.Bp2build { // Run the alternate pipeline of bp2build mutators and singleton to convert // Blueprint to BUILD files before everything else. runBp2Build(configuration, extraNinjaDeps) @@ -519,12 +525,6 @@ func touch(path string) { } } -func touchIfDoesNotExist(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - touch(path) - } -} - // Find BUILD files in the srcDir which are not in the allowlist // (android.Bp2BuildConversionAllowlist#ShouldKeepExistingBuildFileForDir) // and return their paths so they can be left out of the Bazel workspace dir (i.e. ignored) @@ -605,6 +605,54 @@ func bazelArtifacts() []string { } } +// This could in theory easily be separated into a binary that generically +// merges two directories into a symlink tree. The main obstacle is that this +// function currently depends on both Bazel-specific knowledge (the existence +// of bazel-* symlinks) and configuration (the set of BUILD.bazel files that +// should and should not be kept) +// +// Ideally, bp2build would write a file that contains instructions to the +// symlink tree creation binary. Then the latter would not need to depend on +// the very heavy-weight machinery of soong_build . +func runSymlinkForestCreation(configuration android.Config, extraNinjaDeps []string) { + eventHandler := metrics.EventHandler{} + + var ninjaDeps []string + ninjaDeps = append(ninjaDeps, extraNinjaDeps...) + + generatedRoot := shared.JoinPath(configuration.SoongOutDir(), "bp2build") + workspaceRoot := shared.JoinPath(configuration.SoongOutDir(), "workspace") + + excludes := bazelArtifacts() + + if outDir[0] != '/' { + excludes = append(excludes, outDir) + } + + existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err) + os.Exit(1) + } + + pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(configuration.Bp2buildPackageConfig, topDir, existingBazelRelatedFiles, configuration.IsEnvTrue("BP2BUILD_VERBOSE")) + excludes = append(excludes, pathsToIgnoredBuildFiles...) + excludes = append(excludes, getTemporaryExcludes()...) + + // PlantSymlinkForest() returns all the directories that were readdir()'ed. + // Such a directory SHOULD be added to `ninjaDeps` so that a child directory + // or file created/deleted under it would trigger an update of the symlink + // forest. + eventHandler.Do("symlink_forest", func() { + symlinkForestDeps := bp2build.PlantSymlinkForest( + configuration.IsEnvTrue("BP2BUILD_VERBOSE"), topDir, workspaceRoot, generatedRoot, excludes) + ninjaDeps = append(ninjaDeps, symlinkForestDeps...) + }) + + writeDepFile(symlinkForestMarker, eventHandler, ninjaDeps) + touch(shared.JoinPath(topDir, symlinkForestMarker)) +} + // Run Soong in the bp2build mode. This creates a standalone context that registers // an alternate pipeline of mutators and singletons specifically for generating // Bazel BUILD files instead of Ninja files. @@ -646,43 +694,10 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { codegenMetrics = bp2build.Codegen(codegenContext) }) - generatedRoot := shared.JoinPath(configuration.SoongOutDir(), "bp2build") - workspaceRoot := shared.JoinPath(configuration.SoongOutDir(), "workspace") - - excludes := bazelArtifacts() - - if outDir[0] != '/' { - excludes = append(excludes, outDir) - } - - existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir) - if err != nil { - fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err) - os.Exit(1) - } - - pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(configuration.Bp2buildPackageConfig, topDir, existingBazelRelatedFiles, configuration.IsEnvTrue("BP2BUILD_VERBOSE")) - excludes = append(excludes, pathsToIgnoredBuildFiles...) - - excludes = append(excludes, getTemporaryExcludes()...) - - // PlantSymlinkForest() returns all the directories that were readdir()'ed. - // Such a directory SHOULD be added to `ninjaDeps` so that a child directory - // or file created/deleted under it would trigger an update of the symlink - // forest. - eventHandler.Do("symlink_forest", func() { - symlinkForestDeps := bp2build.PlantSymlinkForest( - configuration, topDir, workspaceRoot, generatedRoot, ".", excludes) - ninjaDeps = append(ninjaDeps, symlinkForestDeps...) - }) - ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) writeDepFile(bp2buildMarker, eventHandler, ninjaDeps) - - // Create an empty bp2build marker file, if it does not already exist. - // Note the relevant rule has `restat = true` - touchIfDoesNotExist(shared.JoinPath(topDir, bp2buildMarker)) + touch(shared.JoinPath(topDir, bp2buildMarker)) }) // Only report metrics when in bp2build mode. The metrics aren't relevant diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index e92a561f1..2331eb15b 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -551,8 +551,45 @@ function test_bp2build_generates_marker_file { run_soong bp2build + if [[ ! -f "./out/soong/bp2build_files_marker" ]]; then + fail "bp2build marker file was not generated" + fi + if [[ ! -f "./out/soong/bp2build_workspace_marker" ]]; then - fail "Marker file was not generated" + fail "symlink forest marker file was not generated" + fi +} + +function test_bp2build_add_irrelevant_file { + setup + + mkdir -p a/b + touch a/b/c.txt + cat > a/b/Android.bp <<'EOF' +filegroup { + name: "c", + srcs: ["c.txt"], + bazel_module: { bp2build_available: true }, +} +EOF + + run_soong bp2build + if [[ ! -e out/soong/bp2build/a/b/BUILD.bazel ]]; then + fail "BUILD file in symlink forest was not created"; + fi + + local mtime1=$(stat -c "%y" out/soong/bp2build/a/b/BUILD.bazel) + + touch a/irrelevant.txt + run_soong bp2build + local mtime2=$(stat -c "%y" out/soong/bp2build/a/b/BUILD.bazel) + + if [[ "$mtime1" != "$mtime2" ]]; then + fail "BUILD.bazel file was regenerated" + fi + + if [[ ! -e "out/soong/workspace/a/irrelevant.txt" ]]; then + fail "New file was not symlinked into symlink forest" fi } @@ -849,6 +886,7 @@ test_bp2build_generates_marker_file test_bp2build_null_build test_bp2build_back_and_forth_null_build test_bp2build_add_android_bp +test_bp2build_add_irrelevant_file test_bp2build_add_to_glob test_bp2build_bazel_workspace_structure test_bp2build_bazel_workspace_add_file diff --git a/tests/mixed_mode_test.sh b/tests/mixed_mode_test.sh index f6fffad4b..076ec4b54 100755 --- a/tests/mixed_mode_test.sh +++ b/tests/mixed_mode_test.sh @@ -19,4 +19,51 @@ function test_bazel_smoke { run_bazel info --config=bp2build } +function test_add_irrelevant_file { + setup + create_mock_bazel + + mkdir -p soong_tests/a/b + touch soong_tests/a/b/c.txt + cat > soong_tests/a/b/Android.bp <<'EOF' +filegroup { + name: "c", + srcs: ["c.txt"], + bazel_module: { bp2build_available: true }, +} +EOF + + run_soong --bazel-mode nothing + + if [[ ! -e out/soong/bp2build/soong_tests/a/b/BUILD.bazel ]]; then + fail "BUILD.bazel not created" + fi + + if [[ ! -e out/soong/build.ninja ]]; then + fail "build.ninja not created" + fi + + local mtime_build1=$(stat -c "%y" out/soong/bp2build/soong_tests/a/b/BUILD.bazel) + local mtime_ninja1=$(stat -c "%y" out/soong/build.ninja) + + touch soong_tests/a/irrelevant.txt + + run_soong --bazel-mode nothing + local mtime_build2=$(stat -c "%y" out/soong/bp2build/soong_tests/a/b/BUILD.bazel) + local mtime_ninja2=$(stat -c "%y" out/soong/build.ninja) + + if [[ "$mtime_build1" != "$mtime_build2" ]]; then + fail "BUILD.bazel was generated" + fi + + if [[ "$mtime_ninja1" != "$mtime_ninja2" ]]; then + fail "build.ninja was regenerated" + fi + + if [[ ! -e out/soong/workspace/soong_tests/a/irrelevant.txt ]]; then + fail "new file was not symlinked" + fi +} + +test_add_irrelevant_file test_bazel_smoke diff --git a/ui/build/config.go b/ui/build/config.go index cde8d5dd1..c1c83ff2a 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -910,7 +910,11 @@ func (c *configImpl) UsedEnvFile(tag string) string { return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag) } -func (c *configImpl) Bp2BuildMarkerFile() string { +func (c *configImpl) Bp2BuildFilesMarkerFile() string { + return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker") +} + +func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string { return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker") } diff --git a/ui/build/soong.go b/ui/build/soong.go index 88e55920a..ebf7166f8 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -41,12 +41,13 @@ const ( availableEnvFile = "soong.environment.available" usedEnvFile = "soong.environment.used" - soongBuildTag = "build" - bp2buildTag = "bp2build" - jsonModuleGraphTag = "modulegraph" - queryviewTag = "queryview" - apiBp2buildTag = "api_bp2build" - soongDocsTag = "soong_docs" + soongBuildTag = "build" + bp2buildFilesTag = "bp2build_files" + bp2buildWorkspaceTag = "bp2build_workspace" + jsonModuleGraphTag = "modulegraph" + queryviewTag = "queryview" + apiBp2buildTag = "api_bp2build" + soongDocsTag = "soong_docs" // bootstrapEpoch is used to determine if an incremental build is incompatible with the current // version of bootstrap and needs cleaning before continuing the build. Increment this for @@ -235,7 +236,7 @@ func bootstrapEpochCleanup(ctx Context, config Config) { func bootstrapGlobFileList(config Config) []string { return []string{ config.NamedGlobFile(soongBuildTag), - config.NamedGlobFile(bp2buildTag), + config.NamedGlobFile(bp2buildFilesTag), config.NamedGlobFile(jsonModuleGraphTag), config.NamedGlobFile(queryviewTag), config.NamedGlobFile(apiBp2buildTag), @@ -276,20 +277,33 @@ func bootstrapBlueprint(ctx Context, config Config) { // Mixed builds call Bazel from soong_build and they therefore need the // Bazel workspace to be available. Make that so by adding a dependency on // the bp2build marker file to the action that invokes soong_build . - mainSoongBuildInvocation.Inputs = append(mainSoongBuildInvocation.Inputs, - config.Bp2BuildMarkerFile()) + mainSoongBuildInvocation.OrderOnlyInputs = append(mainSoongBuildInvocation.OrderOnlyInputs, + config.Bp2BuildWorkspaceMarkerFile()) } bp2buildInvocation := primaryBuilderInvocation( config, - bp2buildTag, - config.Bp2BuildMarkerFile(), + bp2buildFilesTag, + config.Bp2BuildFilesMarkerFile(), []string{ - "--bp2build_marker", config.Bp2BuildMarkerFile(), + "--bp2build_marker", config.Bp2BuildFilesMarkerFile(), }, fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()), ) + bp2buildWorkspaceInvocation := primaryBuilderInvocation( + config, + bp2buildWorkspaceTag, + config.Bp2BuildWorkspaceMarkerFile(), + []string{ + "--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(), + }, + fmt.Sprintf("Creating Bazel symlink forest"), + ) + + bp2buildWorkspaceInvocation.Inputs = append(bp2buildWorkspaceInvocation.Inputs, + config.Bp2BuildFilesMarkerFile()) + jsonModuleGraphInvocation := primaryBuilderInvocation( config, jsonModuleGraphTag, @@ -361,6 +375,7 @@ func bootstrapBlueprint(ctx Context, config Config) { primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{ mainSoongBuildInvocation, bp2buildInvocation, + bp2buildWorkspaceInvocation, jsonModuleGraphInvocation, queryviewInvocation, apiBp2buildInvocation, @@ -426,7 +441,7 @@ func runSoong(ctx Context, config Config) { checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongBuildTag)) if config.BazelBuildEnabled() || config.Bp2Build() { - checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(bp2buildTag)) + checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag)) } if config.JsonModuleGraph() { @@ -497,7 +512,7 @@ func runSoong(ctx Context, config Config) { } if config.Bp2Build() { - targets = append(targets, config.Bp2BuildMarkerFile()) + targets = append(targets, config.Bp2BuildWorkspaceMarkerFile()) } if config.Queryview() {