// Copyright 2017 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package build import ( "encoding/json" "errors" "fmt" "io/ioutil" "math/rand" "os" "os/exec" "os/user" "path/filepath" "runtime" "strconv" "strings" "syscall" "time" "android/soong/shared" "android/soong/ui/metrics" "google.golang.org/protobuf/proto" smpb "android/soong/ui/metrics/metrics_proto" ) const ( envConfigDir = "vendor/google/tools/soong_config" jsonSuffix = "json" ) var ( rbeRandPrefix int googleProdCredsExistCache bool ) func init() { rand.Seed(time.Now().UnixNano()) rbeRandPrefix = rand.Intn(1000) } type Config struct{ *configImpl } type configImpl struct { // Some targets that are implemented in soong_build // (bp2build, json-module-graph) are not here and have their own bits below. arguments []string goma bool environ *Environment distDir string buildDateTime string logsPrefix string // From the arguments parallel int keepGoing int verbose bool checkbuild bool dist bool jsonModuleGraph bool queryview bool reportMkMetrics bool // Collect and report mk2bp migration progress metrics. soongDocs bool skipConfig bool skipKati bool skipKatiNinja bool skipSoong bool skipNinja bool skipSoongTests bool searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces skipMetricsUpload bool buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time buildFromSourceStub bool ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built // From the product config katiArgs []string ninjaArgs []string katiSuffix string targetDevice string targetDeviceDir string sandboxConfig *SandboxConfig // Autodetected totalRAM uint64 brokenDupRules bool brokenUsesNetwork bool brokenNinjaEnvVars []string pathReplaced bool // Set by multiproduct_kati emptyNinjaFile bool metricsUploader string includeTags []string sourceRootDirs []string // Data source to write ninja weight list ninjaWeightListSource NinjaWeightListSource // This file is a detailed dump of all soong-defined modules for debugging purposes. // There's quite a bit of overlap with module-info.json and soong module graph. We // could consider merging them. moduleDebugFile string } type NinjaWeightListSource uint const ( // ninja doesn't use weight list. NOT_USED NinjaWeightListSource = iota // ninja uses weight list based on previous builds by ninja log NINJA_LOG // ninja thinks every task has the same weight. EVENLY_DISTRIBUTED // ninja uses an external custom weight list EXTERNAL_FILE // ninja uses a prioritized module list from Soong HINT_FROM_SOONG // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead. // We can assume it is an incremental build if ninja log exists. DEFAULT ) const srcDirFileCheck = "build/soong/root.bp" var buildFiles = []string{"Android.mk", "Android.bp"} type BuildAction uint const ( // Builds all of the modules and their dependencies of a specified directory, relative to the root // directory of the source tree. BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota // Builds all of the modules and their dependencies of a list of specified directories. All specified // directories are relative to the root directory of the source tree. BUILD_MODULES_IN_DIRECTORIES // Build a list of specified modules. If none was specified, simply build the whole source tree. BUILD_MODULES ) // checkTopDir validates that the current directory is at the root directory of the source tree. func checkTopDir(ctx Context) { if _, err := os.Stat(srcDirFileCheck); err != nil { if os.IsNotExist(err) { ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) } ctx.Fatalln("Error verifying tree state:", err) } } func loadEnvConfig(ctx Context, config *configImpl, bc string) error { if bc == "" { return nil } configDirs := []string{ config.OutDir(), os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"), envConfigDir, } for _, dir := range configDirs { cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix)) envVarsJSON, err := ioutil.ReadFile(cfgFile) if err != nil { continue } ctx.Verbosef("Loading config file %v\n", cfgFile) var envVars map[string]map[string]string if err := json.Unmarshal(envVarsJSON, &envVars); err != nil { fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error()) continue } for k, v := range envVars["env"] { if os.Getenv(k) != "" { continue } config.environ.Set(k, v) } ctx.Verbosef("Finished loading config file %v\n", cfgFile) break } return nil } func NewConfig(ctx Context, args ...string) Config { ret := &configImpl{ environ: OsEnvironment(), sandboxConfig: &SandboxConfig{}, ninjaWeightListSource: DEFAULT, } // Skip soong tests by default on Linux if runtime.GOOS == "linux" { ret.skipSoongTests = true } // Default matching ninja ret.parallel = runtime.NumCPU() + 2 ret.keepGoing = 1 ret.totalRAM = detectTotalRAM(ctx) ret.parseArgs(ctx, args) if ret.ninjaWeightListSource == HINT_FROM_SOONG { ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always") } else if ret.ninjaWeightListSource == DEFAULT { defaultNinjaWeightListSource := NINJA_LOG if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) { ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead") defaultNinjaWeightListSource = HINT_FROM_SOONG } else { ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG") } ret.ninjaWeightListSource = defaultNinjaWeightListSource // soong_build generates ninja hint depending on ninja log existence. // Set it "depend" to avoid soong re-run due to env variable change. ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend") } // Make sure OUT_DIR is set appropriately if outDir, ok := ret.environ.Get("OUT_DIR"); ok { ret.environ.Set("OUT_DIR", filepath.Clean(outDir)) } else { outDir := "out" if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok { if wd, err := os.Getwd(); err != nil { ctx.Fatalln("Failed to get working directory:", err) } else { outDir = filepath.Join(baseDir, filepath.Base(wd)) } } ret.environ.Set("OUT_DIR", outDir) } // loadEnvConfig needs to know what the OUT_DIR is, so it should // be called after we determine the appropriate out directory. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") if bc != "" { if err := loadEnvConfig(ctx, ret, bc); err != nil { ctx.Fatalln("Failed to parse env config files: %v", err) } if !ret.canSupportRBE() { // Explicitly set USE_RBE env variable to false when we cannot run // an RBE build to avoid ninja local execution pool issues. ret.environ.Set("USE_RBE", "false") } } if distDir, ok := ret.environ.Get("DIST_DIR"); ok { ret.distDir = filepath.Clean(distDir) } else { ret.distDir = filepath.Join(ret.OutDir(), "dist") } if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok { ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false") } if os.Getenv("GENERATE_SOONG_DEBUG") == "true" { ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json")) } ret.environ.Unset( // We're already using it "USE_SOONG_UI", // We should never use GOROOT/GOPATH from the shell environment "GOROOT", "GOPATH", // These should only come from Soong, not the environment. "CLANG", "CLANG_CXX", "CCC_CC", "CCC_CXX", // Used by the goma compiler wrapper, but should only be set by // gomacc "GOMACC_PATH", // We handle this above "OUT_DIR_COMMON_BASE", // This is handled above too, and set for individual commands later "DIST_DIR", // Variables that have caused problems in the past "BASH_ENV", "CDPATH", "DISPLAY", "GREP_OPTIONS", "JAVAC", "NDK_ROOT", "POSIXLY_CORRECT", // Drop make flags "MAKEFLAGS", "MAKELEVEL", "MFLAGS", // Set in envsetup.sh, reset in makefiles "ANDROID_JAVA_TOOLCHAIN", // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional "ANDROID_BUILD_TOP", "ANDROID_HOST_OUT", "ANDROID_PRODUCT_OUT", "ANDROID_HOST_OUT_TESTCASES", "ANDROID_TARGET_OUT_TESTCASES", "ANDROID_TOOLCHAIN", "ANDROID_TOOLCHAIN_2ND_ARCH", "ANDROID_DEV_SCRIPTS", "ANDROID_EMULATOR_PREBUILTS", "ANDROID_PRE_BUILD_PATHS", // We read it here already, don't let others share in the fun "GENERATE_SOONG_DEBUG", ) if ret.UseGoma() || ret.ForceUseGoma() { ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.") ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.") } // Tell python not to spam the source tree with .pyc files. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") tmpDir := absPath(ctx, ret.TempDir()) ret.environ.Set("TMPDIR", tmpDir) // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(), "llvm-binutils-stable/llvm-symbolizer") ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath)) // Precondition: the current directory is the top of the source tree checkTopDir(ctx) srcDir := absPath(ctx, ".") if strings.ContainsRune(srcDir, ' ') { ctx.Println("You are building in a directory whose absolute path contains a space character:") ctx.Println() ctx.Printf("%q\n", srcDir) ctx.Println() ctx.Fatalln("Directory names containing spaces are not supported") } ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ) if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') { ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") ctx.Println() ctx.Printf("%q\n", outDir) ctx.Println() ctx.Fatalln("Directory names containing spaces are not supported") } if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') { ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") ctx.Println() ctx.Printf("%q\n", distDir) ctx.Println() ctx.Fatalln("Directory names containing spaces are not supported") } // Configure Java-related variables, including adding it to $PATH java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag()) java21Home := filepath.Join("prebuilts/jdk/jdk21", ret.HostPrebuiltTag()) javaHome := func() string { if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok { return override } if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" { ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") } if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" { ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") } if toolchain21, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN"); ok && toolchain21 != "true" { ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") } return java21Home }() absJavaHome := absPath(ctx, javaHome) ret.configureLocale(ctx) newPath := []string{filepath.Join(absJavaHome, "bin")} if path, ok := ret.environ.Get("PATH"); ok && path != "" { newPath = append(newPath, path) } ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME") ret.environ.Set("JAVA_HOME", absJavaHome) ret.environ.Set("ANDROID_JAVA_HOME", javaHome) ret.environ.Set("ANDROID_JAVA8_HOME", java8Home) ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator))) // b/286885495, https://bugzilla.redhat.com/show_bug.cgi?id=2227130: some versions of Fedora include patches // to unzip to enable zipbomb detection that incorrectly handle zip64 and data descriptors and fail on large // zip files produced by soong_zip. Disable zipbomb detection. ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE") outDir := ret.OutDir() buildDateTimeFile := filepath.Join(outDir, "build_date.txt") if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" { ret.buildDateTime = buildDateTime } else { ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10) } ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) if _, ok := ret.environ.Get("BUILD_USERNAME"); !ok { username := "unknown" if u, err := user.Current(); err == nil { username = u.Username } else { ctx.Println("Failed to get current user:", err) } ret.environ.Set("BUILD_USERNAME", username) } if ret.UseRBE() { for k, v := range getRBEVars(ctx, Config{ret}) { ret.environ.Set(k, v) } } c := Config{ret} storeConfigMetrics(ctx, c) return c } // NewBuildActionConfig returns a build configuration based on the build action. The arguments are // processed based on the build action and extracts any arguments that belongs to the build action. func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config { return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) } // Prepare for getting make variables. For them to be accurate, we need to have // obtained PRODUCT_RELEASE_CONFIG_MAPS. // // Returns: // // Whether config should be called again. // // TODO: when converting product config to a declarative language, make sure // that PRODUCT_RELEASE_CONFIG_MAPS is properly handled as a separate step in // that process. func SetProductReleaseConfigMaps(ctx Context, config Config) bool { ctx.BeginTrace(metrics.RunKati, "SetProductReleaseConfigMaps") defer ctx.EndTrace() if config.SkipConfig() { // This duplicates the logic from Build to skip product config // if the user has explicitly said to. return false } releaseConfigVars := []string{ "PRODUCT_RELEASE_CONFIG_MAPS", } origValue, _ := config.environ.Get("PRODUCT_RELEASE_CONFIG_MAPS") // Get the PRODUCT_RELEASE_CONFIG_MAPS for this product, to avoid polluting the environment // when we run product config to get the rest of the make vars. releaseMapVars, err := dumpMakeVars(ctx, config, nil, releaseConfigVars, false, "") if err != nil { ctx.Fatalln("Error getting PRODUCT_RELEASE_CONFIG_MAPS:", err) } productReleaseConfigMaps := releaseMapVars["PRODUCT_RELEASE_CONFIG_MAPS"] os.Setenv("PRODUCT_RELEASE_CONFIG_MAPS", productReleaseConfigMaps) return origValue != productReleaseConfigMaps } // storeConfigMetrics selects a set of configuration information and store in // the metrics system for further analysis. func storeConfigMetrics(ctx Context, config Config) { if ctx.Metrics == nil { return } ctx.Metrics.BuildConfig(buildConfig(config)) s := &smpb.SystemResourceInfo{ TotalPhysicalMemory: proto.Uint64(config.TotalRAM()), AvailableCpus: proto.Int32(int32(runtime.NumCPU())), } ctx.Metrics.SystemResourceInfo(s) } func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource { switch s { case NINJA_LOG: return smpb.BuildConfig_NINJA_LOG.Enum() case EVENLY_DISTRIBUTED: return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum() case EXTERNAL_FILE: return smpb.BuildConfig_EXTERNAL_FILE.Enum() case HINT_FROM_SOONG: return smpb.BuildConfig_HINT_FROM_SOONG.Enum() default: return smpb.BuildConfig_NOT_USED.Enum() } } func buildConfig(config Config) *smpb.BuildConfig { c := &smpb.BuildConfig{ ForceUseGoma: proto.Bool(config.ForceUseGoma()), UseGoma: proto.Bool(config.UseGoma()), UseRbe: proto.Bool(config.UseRBE()), NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()), } c.Targets = append(c.Targets, config.arguments...) return c } // getConfigArgs processes the command arguments based on the build action and creates a set of new // arguments to be accepted by Config. func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { // The next block of code verifies that the current directory is the root directory of the source // tree. It then finds the relative path of dir based on the root directory of the source tree // and verify that dir is inside of the source tree. checkTopDir(ctx) topDir, err := os.Getwd() if err != nil { ctx.Fatalf("Error retrieving top directory: %v", err) } dir, err = filepath.EvalSymlinks(dir) if err != nil { ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err) } dir, err = filepath.Abs(dir) if err != nil { ctx.Fatalf("Unable to find absolute path %s: %v", dir, err) } relDir, err := filepath.Rel(topDir, dir) if err != nil { ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err) } // If there are ".." in the path, it's not in the source tree. if strings.Contains(relDir, "..") { ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir) } configArgs := args[:] // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules. targetNamePrefix := "MODULES-IN-" if inList("GET-INSTALL-PATH", configArgs) { targetNamePrefix = "GET-INSTALL-PATH-IN-" configArgs = removeFromList("GET-INSTALL-PATH", configArgs) } var targets []string switch action { case BUILD_MODULES: // No additional processing is required when building a list of specific modules or all modules. case BUILD_MODULES_IN_A_DIRECTORY: // If dir is the root source tree, all the modules are built of the source tree are built so // no need to find the build file. if topDir == dir { break } buildFile := findBuildFile(ctx, relDir) if buildFile == "" { ctx.Fatalf("Build file not found for %s directory", relDir) } targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} case BUILD_MODULES_IN_DIRECTORIES: newConfigArgs, dirs := splitArgs(configArgs) configArgs = newConfigArgs targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix) } // Tidy only override all other specified targets. tidyOnly := os.Getenv("WITH_TIDY_ONLY") if tidyOnly == "true" || tidyOnly == "1" { configArgs = append(configArgs, "tidy_only") } else { configArgs = append(configArgs, targets...) } return configArgs } // convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name. func convertToTarget(dir string, targetNamePrefix string) string { return targetNamePrefix + strings.ReplaceAll(dir, "/", "-") } // hasBuildFile returns true if dir contains an Android build file. func hasBuildFile(ctx Context, dir string) bool { for _, buildFile := range buildFiles { _, err := os.Stat(filepath.Join(dir, buildFile)) if err == nil { return true } if !os.IsNotExist(err) { ctx.Fatalf("Error retrieving the build file stats: %v", err) } } return false } // findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file // in the current and any sub directory of dir. If a build file is not found, traverse the path // up by one directory and repeat again until either a build file is found or reached to the root // source tree. The returned filename of build file is "Android.mk". If one was not found, a blank // string is returned. func findBuildFile(ctx Context, dir string) string { // If the string is empty or ".", assume it is top directory of the source tree. if dir == "" || dir == "." { return "" } found := false for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) { err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if found { return filepath.SkipDir } if info.IsDir() { return nil } for _, buildFile := range buildFiles { if info.Name() == buildFile { found = true return filepath.SkipDir } } return nil }) if err != nil { ctx.Fatalf("Error finding Android build file: %v", err) } if found { return filepath.Join(buildDir, "Android.mk") } } return "" } // splitArgs iterates over the arguments list and splits into two lists: arguments and directories. func splitArgs(args []string) (newArgs []string, dirs []string) { specialArgs := map[string]bool{ "showcommands": true, "snod": true, "dist": true, "checkbuild": true, } newArgs = []string{} dirs = []string{} for _, arg := range args { // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory. if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 { newArgs = append(newArgs, arg) continue } if _, ok := specialArgs[arg]; ok { newArgs = append(newArgs, arg) continue } dirs = append(dirs, arg) } return newArgs, dirs } // getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a // directory from the dirs list does not exist, a fatal error is raised. relDir is related to the // source root tree where the build action command was invoked. Each directory is validated if the // build file can be found and follows the format "dir1:target1,target2,...". Target is optional. func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) { for _, dir := range dirs { // The directory may have specified specific modules to build. ":" is the separator to separate // the directory and the list of modules. s := strings.Split(dir, ":") l := len(s) if l > 2 { // more than one ":" was specified. ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir) } dir = filepath.Join(relDir, s[0]) if _, err := os.Stat(dir); err != nil { ctx.Fatalf("couldn't find directory %s", dir) } // Verify that if there are any targets specified after ":". Each target is separated by ",". var newTargets []string if l == 2 && s[1] != "" { newTargets = strings.Split(s[1], ",") if inList("", newTargets) { ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir) } } // If there are specified targets to build in dir, an android build file must exist for the one // shot build. For the non-targets case, find the appropriate build file and build all the // modules in dir (or the closest one in the dir path). if len(newTargets) > 0 { if !hasBuildFile(ctx, dir) { ctx.Fatalf("Couldn't locate a build file from %s directory", dir) } } else { buildFile := findBuildFile(ctx, dir) if buildFile == "" { ctx.Fatalf("Build file not found for %s directory", dir) } newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} } targets = append(targets, newTargets...) } return targets } func (c *configImpl) parseArgs(ctx Context, args []string) { for i := 0; i < len(args); i++ { arg := strings.TrimSpace(args[i]) if arg == "showcommands" { c.verbose = true } else if arg == "--empty-ninja-file" { c.emptyNinjaFile = true } else if arg == "--skip-ninja" { c.skipNinja = true } else if arg == "--skip-make" { // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati, // but it does run a Kati ninja file if the .kati_enabled marker file was created // by a previous build. c.skipConfig = true c.skipKati = true } else if arg == "--soong-only" { c.skipKati = true c.skipKatiNinja = true } else if arg == "--config-only" { c.skipKati = true c.skipKatiNinja = true c.skipSoong = true } else if arg == "--skip-config" { c.skipConfig = true } else if arg == "--skip-soong-tests" { c.skipSoongTests = true } else if arg == "--no-skip-soong-tests" { c.skipSoongTests = false } else if arg == "--skip-metrics-upload" { c.skipMetricsUpload = true } else if arg == "--mk-metrics" { c.reportMkMetrics = true } else if arg == "--search-api-dir" { c.searchApiDir = true } else if strings.HasPrefix(arg, "--ninja_weight_source=") { source := strings.TrimPrefix(arg, "--ninja_weight_source=") if source == "ninja_log" { c.ninjaWeightListSource = NINJA_LOG } else if source == "evenly_distributed" { c.ninjaWeightListSource = EVENLY_DISTRIBUTED } else if source == "not_used" { c.ninjaWeightListSource = NOT_USED } else if source == "soong" { c.ninjaWeightListSource = HINT_FROM_SOONG } else if strings.HasPrefix(source, "file,") { c.ninjaWeightListSource = EXTERNAL_FILE filePath := strings.TrimPrefix(source, "file,") err := validateNinjaWeightList(filePath) if err != nil { ctx.Fatalf("Malformed weight list from %s: %s", filePath, err) } _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list")) if err != nil { ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err) } } else { ctx.Fatalf("unknown option for ninja_weight_source: %s", source) } } else if arg == "--build-from-source-stub" { c.buildFromSourceStub = true } else if strings.HasPrefix(arg, "--build-command=") { buildCmd := strings.TrimPrefix(arg, "--build-command=") // remove quotations buildCmd = strings.TrimPrefix(buildCmd, "\"") buildCmd = strings.TrimSuffix(buildCmd, "\"") ctx.Metrics.SetBuildCommand([]string{buildCmd}) } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") { buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=") val, err := strconv.ParseInt(buildTimeStr, 10, 64) if err == nil { c.buildStartedTime = val } else { ctx.Fatalf("Error parsing build-time-started-unix-millis", err) } } else if arg == "--ensure-allowlist-integrity" { c.ensureAllowlistIntegrity = true } else if len(arg) > 0 && arg[0] == '-' { parseArgNum := func(def int) int { if len(arg) > 2 { p, err := strconv.ParseUint(arg[2:], 10, 31) if err != nil { ctx.Fatalf("Failed to parse %q: %v", arg, err) } return int(p) } else if i+1 < len(args) { p, err := strconv.ParseUint(args[i+1], 10, 31) if err == nil { i++ return int(p) } } return def } if len(arg) > 1 && arg[1] == 'j' { c.parallel = parseArgNum(c.parallel) } else if len(arg) > 1 && arg[1] == 'k' { c.keepGoing = parseArgNum(0) } else { ctx.Fatalln("Unknown option:", arg) } } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 { if k == "OUT_DIR" { ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.") } c.environ.Set(k, v) } else if arg == "dist" { c.dist = true } else if arg == "json-module-graph" { c.jsonModuleGraph = true } else if arg == "queryview" { c.queryview = true } else if arg == "soong_docs" { c.soongDocs = true } else { if arg == "checkbuild" { c.checkbuild = true } c.arguments = append(c.arguments, arg) } } } func validateNinjaWeightList(weightListFilePath string) (err error) { data, err := os.ReadFile(weightListFilePath) if err != nil { return } lines := strings.Split(strings.TrimSpace(string(data)), "\n") for _, line := range lines { fields := strings.Split(line, ",") if len(fields) != 2 { return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line) } _, err = strconv.Atoi(fields[1]) if err != nil { return } } return } func (c *configImpl) configureLocale(ctx Context) { cmd := Command(ctx, Config{c}, "locale", "locale", "-a") output, err := cmd.Output() var locales []string if err == nil { locales = strings.Split(string(output), "\n") } else { // If we're unable to list the locales, let's assume en_US.UTF-8 locales = []string{"en_US.UTF-8"} ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales) } // gettext uses LANGUAGE, which is passed directly through // For LANG and LC_*, only preserve the evaluated version of // LC_MESSAGES userLang := "" if lc_all, ok := c.environ.Get("LC_ALL"); ok { userLang = lc_all } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok { userLang = lc_messages } else if lang, ok := c.environ.Get("LANG"); ok { userLang = lang } c.environ.UnsetWithPrefix("LC_") if userLang != "" { c.environ.Set("LC_MESSAGES", userLang) } // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed // for others) if inList("C.UTF-8", locales) { c.environ.Set("LANG", "C.UTF-8") } else if inList("C.utf8", locales) { // These normalize to the same thing c.environ.Set("LANG", "C.UTF-8") } else if inList("en_US.UTF-8", locales) { c.environ.Set("LANG", "en_US.UTF-8") } else if inList("en_US.utf8", locales) { // These normalize to the same thing c.environ.Set("LANG", "en_US.UTF-8") } else { ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8") } } func (c *configImpl) Environment() *Environment { return c.environ } func (c *configImpl) Arguments() []string { return c.arguments } func (c *configImpl) SoongBuildInvocationNeeded() bool { if len(c.Arguments()) > 0 { // Explicit targets requested that are not special targets like b2pbuild // or the JSON module graph return true } if !c.JsonModuleGraph() && !c.Queryview() && !c.SoongDocs() { // Command line was empty, the default Ninja target is built return true } if c.Dist() { return true } // build.ninja doesn't need to be generated return false } func (c *configImpl) OutDir() string { if outDir, ok := c.environ.Get("OUT_DIR"); ok { return outDir } return "out" } func (c *configImpl) DistDir() string { return c.distDir } func (c *configImpl) RealDistDir() string { return c.distDir } func (c *configImpl) NinjaArgs() []string { if c.skipKati { return c.arguments } return c.ninjaArgs } func (c *configImpl) SoongOutDir() string { return filepath.Join(c.OutDir(), "soong") } func (c *configImpl) ApiSurfacesOutDir() string { return filepath.Join(c.OutDir(), "api_surfaces") } func (c *configImpl) PrebuiltOS() string { switch runtime.GOOS { case "linux": return "linux-x86" case "darwin": return "darwin-x86" default: panic("Unknown GOOS") } } func (c *configImpl) HostToolDir() string { if c.SkipKatiNinja() { return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") } else { return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin") } } func (c *configImpl) NamedGlobFile(name string) string { return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja") } func (c *configImpl) UsedEnvFile(tag string) string { if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+"."+tag) } return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag) } func (c *configImpl) SoongDocsHtml() string { return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html") } func (c *configImpl) QueryviewMarkerFile() string { return shared.JoinPath(c.SoongOutDir(), "queryview.marker") } func (c *configImpl) ModuleGraphFile() string { return shared.JoinPath(c.SoongOutDir(), "module-graph.json") } func (c *configImpl) ModuleActionsFile() string { return shared.JoinPath(c.SoongOutDir(), "module-actions.json") } func (c *configImpl) TempDir() string { return shared.TempDirForOutDir(c.SoongOutDir()) } func (c *configImpl) FileListDir() string { return filepath.Join(c.OutDir(), ".module_paths") } func (c *configImpl) KatiSuffix() string { if c.katiSuffix != "" { return c.katiSuffix } panic("SetKatiSuffix has not been called") } // Checkbuild returns true if "checkbuild" was one of the build goals, which means that the // user is interested in additional checks at the expense of build time. func (c *configImpl) Checkbuild() bool { return c.checkbuild } func (c *configImpl) Dist() bool { return c.dist } func (c *configImpl) JsonModuleGraph() bool { return c.jsonModuleGraph } func (c *configImpl) Queryview() bool { return c.queryview } func (c *configImpl) SoongDocs() bool { return c.soongDocs } func (c *configImpl) IsVerbose() bool { return c.verbose } func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource { return c.ninjaWeightListSource } func (c *configImpl) SkipKati() bool { return c.skipKati } func (c *configImpl) SkipKatiNinja() bool { return c.skipKatiNinja } func (c *configImpl) SkipSoong() bool { return c.skipSoong } func (c *configImpl) SkipNinja() bool { return c.skipNinja } func (c *configImpl) SetSkipNinja(v bool) { c.skipNinja = v } func (c *configImpl) SkipConfig() bool { return c.skipConfig } func (c *configImpl) BuildFromTextStub() bool { return !c.buildFromSourceStub } func (c *configImpl) TargetProduct() string { if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { return v } panic("TARGET_PRODUCT is not defined") } func (c *configImpl) TargetProductOrErr() (string, error) { if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { return v, nil } return "", fmt.Errorf("TARGET_PRODUCT is not defined") } func (c *configImpl) TargetDevice() string { return c.targetDevice } func (c *configImpl) SetTargetDevice(device string) { c.targetDevice = device } func (c *configImpl) TargetBuildVariant() string { if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok { return v } panic("TARGET_BUILD_VARIANT is not defined") } func (c *configImpl) KatiArgs() []string { return c.katiArgs } func (c *configImpl) Parallel() int { return c.parallel } func (c *configImpl) GetSourceRootDirs() []string { return c.sourceRootDirs } func (c *configImpl) SetSourceRootDirs(i []string) { c.sourceRootDirs = i } func (c *configImpl) GetLogsPrefix() string { return c.logsPrefix } func (c *configImpl) SetLogsPrefix(prefix string) { c.logsPrefix = prefix } func (c *configImpl) HighmemParallel() int { if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok { return i } const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024 parallel := c.Parallel() if c.UseRemoteBuild() { // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention. // Return 1/16th of the size of the local pool, rounding up. return (parallel + 15) / 16 } else if c.totalRAM == 0 { // Couldn't detect the total RAM, don't restrict highmem processes. return parallel } else if c.totalRAM <= 16*1024*1024*1024 { // Less than 16GB of ram, restrict to 1 highmem processes return 1 } else if c.totalRAM <= 32*1024*1024*1024 { // Less than 32GB of ram, restrict to 2 highmem processes return 2 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel { // If less than 8GB total RAM per process, reduce the number of highmem processes return p } // No restriction on highmem processes return parallel } func (c *configImpl) TotalRAM() uint64 { return c.totalRAM } // ForceUseGoma determines whether we should override Goma deprecation // and use Goma for the current build or not. func (c *configImpl) ForceUseGoma() bool { if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok { v = strings.TrimSpace(v) if v != "" && v != "false" { return true } } return false } func (c *configImpl) UseGoma() bool { if v, ok := c.environ.Get("USE_GOMA"); ok { v = strings.TrimSpace(v) if v != "" && v != "false" { return true } } return false } func (c *configImpl) StartGoma() bool { if !c.UseGoma() { return false } if v, ok := c.environ.Get("NOSTART_GOMA"); ok { v = strings.TrimSpace(v) if v != "" && v != "false" { return false } } return true } func (c *configImpl) canSupportRBE() bool { // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since // its unlikely that we will be able to obtain necessary creds without stubby. authType, _ := c.rbeAuth() if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") { return false } return true } func (c *configImpl) UseRBE() bool { // These alternate modes of running Soong do not use RBE / reclient. if c.Queryview() || c.JsonModuleGraph() { return false } if !c.canSupportRBE() { return false } if v, ok := c.Environment().Get("USE_RBE"); ok { v = strings.TrimSpace(v) if v != "" && v != "false" { return true } } return false } func (c *configImpl) StartRBE() bool { if !c.UseRBE() { return false } if v, ok := c.environ.Get("NOSTART_RBE"); ok { v = strings.TrimSpace(v) if v != "" && v != "false" { return false } } return true } func (c *configImpl) rbeProxyLogsDir() string { for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { if v, ok := c.environ.Get(f); ok { return v } } return c.rbeTmpDir() } func (c *configImpl) rbeDownloadTmpDir() string { for _, f := range []string{"RBE_download_tmp_dir", "FLAG_download_tmp_dir"} { if v, ok := c.environ.Get(f); ok { return v } } return c.rbeTmpDir() } func (c *configImpl) rbeTmpDir() string { buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir()) return filepath.Join(buildTmpDir, "rbe") } func (c *configImpl) rbeCacheDir() string { for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} { if v, ok := c.environ.Get(f); ok { return v } } return shared.JoinPath(c.SoongOutDir(), "rbe") } func (c *configImpl) shouldCleanupRBELogsDir() bool { // Perform a log directory cleanup only when the log directory // is auto created by the build rather than user-specified. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { if _, ok := c.environ.Get(f); ok { return false } } return true } func (c *configImpl) rbeExecRoot() string { for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} { if v, ok := c.environ.Get(f); ok { return v } } wd, err := os.Getwd() if err != nil { return "" } return wd } func (c *configImpl) rbeDir() string { if v, ok := c.environ.Get("RBE_DIR"); ok { return v } return "prebuilts/remoteexecution-client/live/" } func (c *configImpl) rbeReproxy() string { for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} { if v, ok := c.environ.Get(f); ok { return v } } return filepath.Join(c.rbeDir(), "reproxy") } func (c *configImpl) rbeAuth() (string, string) { credFlags := []string{ "use_application_default_credentials", "use_gce_credentials", "credential_file", "use_google_prod_creds", } for _, cf := range credFlags { for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} { if v, ok := c.environ.Get(f); ok { v = strings.TrimSpace(v) if v != "" && v != "false" && v != "0" { return "RBE_" + cf, v } } } } return "RBE_use_application_default_credentials", "true" } func (c *configImpl) rbeSockAddr(dir string) (string, error) { // Absolute path socket addresses have a prefix of //. This should // be included in the length limit. maxNameLen := len(syscall.RawSockaddrUnix{}.Path) - 2 base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix) name := filepath.Join(dir, base) if len(name) < maxNameLen { return name, nil } name = filepath.Join("/tmp", base) if len(name) < maxNameLen { return name, nil } return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen) } // IsGooglerEnvironment returns true if the current build is running // on a Google developer machine and false otherwise. func (c *configImpl) IsGooglerEnvironment() bool { cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG" if v, ok := c.environ.Get(cf); ok { return v == "googler" } return false } // GoogleProdCredsExist determine whether credentials exist on the // Googler machine to use remote execution. func (c *configImpl) GoogleProdCredsExist() bool { if googleProdCredsExistCache { return googleProdCredsExistCache } if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil { return false } googleProdCredsExistCache = true return true } // UseRemoteBuild indicates whether to use a remote build acceleration system // to speed up the build. func (c *configImpl) UseRemoteBuild() bool { return c.UseGoma() || c.UseRBE() } // StubbyExists checks whether the stubby binary exists on the machine running // the build. func (c *configImpl) StubbyExists() bool { if _, err := exec.LookPath("stubby"); err != nil { return false } return true } // RemoteParallel controls how many remote jobs (i.e., commands which contain // gomacc) are run in parallel. Note the parallelism of all other jobs is // still limited by Parallel() func (c *configImpl) RemoteParallel() int { if !c.UseRemoteBuild() { return 0 } if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok { return i } return 500 } func (c *configImpl) SetKatiArgs(args []string) { c.katiArgs = args } func (c *configImpl) SetNinjaArgs(args []string) { c.ninjaArgs = args } func (c *configImpl) SetKatiSuffix(suffix string) { c.katiSuffix = suffix } func (c *configImpl) LastKatiSuffixFile() string { return filepath.Join(c.OutDir(), "last_kati_suffix") } func (c *configImpl) HasKatiSuffix() bool { return c.katiSuffix != "" } func (c *configImpl) KatiEnvFile() string { return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh") } func (c *configImpl) KatiBuildNinjaFile() string { return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja") } func (c *configImpl) KatiPackageNinjaFile() string { return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja") } func (c *configImpl) SoongVarsFile() string { targetProduct, err := c.TargetProductOrErr() if err != nil { return filepath.Join(c.SoongOutDir(), "soong.variables") } else { return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".variables") } } func (c *configImpl) SoongNinjaFile() string { targetProduct, err := c.TargetProductOrErr() if err != nil { return filepath.Join(c.SoongOutDir(), "build.ninja") } else { return filepath.Join(c.SoongOutDir(), "build."+targetProduct+".ninja") } } func (c *configImpl) CombinedNinjaFile() string { if c.katiSuffix == "" { return filepath.Join(c.OutDir(), "combined.ninja") } return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja") } func (c *configImpl) SoongAndroidMk() string { return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk") } func (c *configImpl) SoongMakeVarsMk() string { return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk") } func (c *configImpl) SoongBuildMetrics() string { return filepath.Join(c.LogsDir(), "soong_build_metrics.pb") } func (c *configImpl) ProductOut() string { return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice()) } func (c *configImpl) DevicePreviousProductConfig() string { return filepath.Join(c.ProductOut(), "previous_build_config.mk") } func (c *configImpl) KatiPackageMkDir() string { return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging") } func (c *configImpl) hostOutRoot() string { return filepath.Join(c.OutDir(), "host") } func (c *configImpl) HostOut() string { return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag()) } // This probably needs to be multi-valued, so not exporting it for now func (c *configImpl) hostCrossOut() string { if runtime.GOOS == "linux" { return filepath.Join(c.hostOutRoot(), "windows-x86") } else { return "" } } func (c *configImpl) HostPrebuiltTag() string { if runtime.GOOS == "linux" { return "linux-x86" } else if runtime.GOOS == "darwin" { return "darwin-x86" } else { panic("Unsupported OS") } } func (c *configImpl) PrebuiltBuildTool(name string) string { if v, ok := c.environ.Get("SANITIZE_HOST"); ok { if sanitize := strings.Fields(v); inList("address", sanitize) { asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name) if _, err := os.Stat(asan); err == nil { return asan } } } return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name) } func (c *configImpl) SetBuildBrokenDupRules(val bool) { c.brokenDupRules = val } func (c *configImpl) BuildBrokenDupRules() bool { return c.brokenDupRules } func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) { c.brokenUsesNetwork = val } func (c *configImpl) BuildBrokenUsesNetwork() bool { return c.brokenUsesNetwork } func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) { c.brokenNinjaEnvVars = val } func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string { return c.brokenNinjaEnvVars } func (c *configImpl) SetTargetDeviceDir(dir string) { c.targetDeviceDir = dir } func (c *configImpl) TargetDeviceDir() string { return c.targetDeviceDir } func (c *configImpl) BuildDateTime() string { return c.buildDateTime } func (c *configImpl) MetricsUploaderApp() string { return c.metricsUploader } // LogsDir returns the absolute path to the logs directory where build log and // metrics files are located. By default, the logs directory is the out // directory. If the argument dist is specified, the logs directory // is /logs. func (c *configImpl) LogsDir() string { dir := c.OutDir() if c.Dist() { // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files dir = filepath.Join(c.RealDistDir(), "logs") } absDir, err := filepath.Abs(dir) if err != nil { fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error()) os.Exit(1) } return absDir } // MkFileMetrics returns the file path for make-related metrics. func (c *configImpl) MkMetrics() string { return filepath.Join(c.LogsDir(), "mk_metrics.pb") } func (c *configImpl) SetEmptyNinjaFile(v bool) { c.emptyNinjaFile = v } func (c *configImpl) EmptyNinjaFile() bool { return c.emptyNinjaFile } func (c *configImpl) SkipMetricsUpload() bool { return c.skipMetricsUpload } func (c *configImpl) EnsureAllowlistIntegrity() bool { return c.ensureAllowlistIntegrity } // Returns a Time object if one was passed via a command-line flag. // Otherwise returns the passed default. func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time { if c.buildStartedTime == 0 { return defaultTime } return time.UnixMilli(c.buildStartedTime) } func GetMetricsUploader(topDir string, env *Environment) string { if p, ok := env.Get("METRICS_UPLOADER"); ok { metricsUploader := filepath.Join(topDir, p) if _, err := os.Stat(metricsUploader); err == nil { return metricsUploader } } return "" }