8d63d56ac6
Bug: 271526845 Test: manually invoke internal protobuf presubmit Change-Id: I79d1cc027e422fa1ca6106f5b45ef11045f6d486
1657 lines
46 KiB
Go
1657 lines
46 KiB
Go
// Copyright 2017 Google Inc. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package build
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"android/soong/shared"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
smpb "android/soong/ui/metrics/metrics_proto"
|
|
)
|
|
|
|
const (
|
|
envConfigDir = "vendor/google/tools/soong_config"
|
|
jsonSuffix = "json"
|
|
|
|
configFetcher = "vendor/google/tools/soong/expconfigfetcher"
|
|
envConfigFetchTimeout = 10 * time.Second
|
|
)
|
|
|
|
var (
|
|
rbeRandPrefix int
|
|
googleProdCredsExistCache bool
|
|
)
|
|
|
|
func init() {
|
|
rand.Seed(time.Now().UnixNano())
|
|
rbeRandPrefix = rand.Intn(1000)
|
|
}
|
|
|
|
type Config struct{ *configImpl }
|
|
|
|
type configImpl struct {
|
|
// Some targets that are implemented in soong_build
|
|
// (bp2build, json-module-graph) are not here and have their own bits below.
|
|
arguments []string
|
|
goma bool
|
|
environ *Environment
|
|
distDir string
|
|
buildDateTime string
|
|
logsPrefix string
|
|
|
|
// From the arguments
|
|
parallel int
|
|
keepGoing int
|
|
verbose bool
|
|
checkbuild bool
|
|
dist bool
|
|
jsonModuleGraph bool
|
|
apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs
|
|
bp2build bool
|
|
queryview bool
|
|
reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
|
|
soongDocs bool
|
|
multitreeBuild bool // This is a multitree build.
|
|
skipConfig bool
|
|
skipKati bool
|
|
skipKatiNinja bool
|
|
skipSoong bool
|
|
skipNinja bool
|
|
skipSoongTests bool
|
|
searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
|
|
skipMetricsUpload bool
|
|
buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
|
|
|
|
// From the product config
|
|
katiArgs []string
|
|
ninjaArgs []string
|
|
katiSuffix string
|
|
targetDevice string
|
|
targetDeviceDir string
|
|
sandboxConfig *SandboxConfig
|
|
|
|
// Autodetected
|
|
totalRAM uint64
|
|
|
|
brokenDupRules bool
|
|
brokenUsesNetwork bool
|
|
brokenNinjaEnvVars []string
|
|
|
|
pathReplaced bool
|
|
|
|
bazelProdMode bool
|
|
bazelDevMode bool
|
|
bazelStagingMode bool
|
|
|
|
// Set by multiproduct_kati
|
|
emptyNinjaFile bool
|
|
|
|
metricsUploader string
|
|
|
|
bazelForceEnabledModules string
|
|
|
|
includeTags []string
|
|
|
|
// Data source to write ninja weight list
|
|
ninjaWeightListSource NinjaWeightListSource
|
|
}
|
|
|
|
type NinjaWeightListSource uint
|
|
|
|
const (
|
|
// ninja doesn't use weight list.
|
|
NOT_USED NinjaWeightListSource = iota
|
|
// ninja uses weight list based on previous builds by ninja log
|
|
NINJA_LOG
|
|
// ninja thinks every task has the same weight.
|
|
EVENLY_DISTRIBUTED
|
|
)
|
|
const srcDirFileCheck = "build/soong/root.bp"
|
|
|
|
var buildFiles = []string{"Android.mk", "Android.bp"}
|
|
|
|
type BuildAction uint
|
|
|
|
const (
|
|
// Builds all of the modules and their dependencies of a specified directory, relative to the root
|
|
// directory of the source tree.
|
|
BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
|
|
|
|
// Builds all of the modules and their dependencies of a list of specified directories. All specified
|
|
// directories are relative to the root directory of the source tree.
|
|
BUILD_MODULES_IN_DIRECTORIES
|
|
|
|
// Build a list of specified modules. If none was specified, simply build the whole source tree.
|
|
BUILD_MODULES
|
|
)
|
|
|
|
// checkTopDir validates that the current directory is at the root directory of the source tree.
|
|
func checkTopDir(ctx Context) {
|
|
if _, err := os.Stat(srcDirFileCheck); err != nil {
|
|
if os.IsNotExist(err) {
|
|
ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
|
|
}
|
|
ctx.Fatalln("Error verifying tree state:", err)
|
|
}
|
|
}
|
|
|
|
// fetchEnvConfig optionally fetches a configuration file that can then subsequently be
|
|
// loaded into Soong environment to control certain aspects of build behavior (e.g., enabling RBE).
|
|
// If a configuration file already exists on disk, the fetch is run in the background
|
|
// so as to NOT block the rest of the build execution.
|
|
func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error {
|
|
configName := envConfigName + "." + jsonSuffix
|
|
expConfigFetcher := &smpb.ExpConfigFetcher{Filename: &configName}
|
|
defer func() {
|
|
ctx.Metrics.ExpConfigFetcher(expConfigFetcher)
|
|
}()
|
|
if !config.GoogleProdCredsExist() {
|
|
status := smpb.ExpConfigFetcher_MISSING_GCERT
|
|
expConfigFetcher.Status = &status
|
|
return nil
|
|
}
|
|
|
|
s, err := os.Stat(configFetcher)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if s.Mode()&0111 == 0 {
|
|
status := smpb.ExpConfigFetcher_ERROR
|
|
expConfigFetcher.Status = &status
|
|
return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode())
|
|
}
|
|
|
|
configExists := false
|
|
outConfigFilePath := filepath.Join(config.OutDir(), configName)
|
|
if _, err := os.Stat(outConfigFilePath); err == nil {
|
|
configExists = true
|
|
}
|
|
|
|
tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout)
|
|
fetchStart := time.Now()
|
|
cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir(),
|
|
"-output_config_name", configName)
|
|
if err := cmd.Start(); err != nil {
|
|
status := smpb.ExpConfigFetcher_ERROR
|
|
expConfigFetcher.Status = &status
|
|
return err
|
|
}
|
|
|
|
fetchCfg := func() error {
|
|
if err := cmd.Wait(); err != nil {
|
|
status := smpb.ExpConfigFetcher_ERROR
|
|
expConfigFetcher.Status = &status
|
|
return err
|
|
}
|
|
fetchEnd := time.Now()
|
|
expConfigFetcher.Micros = proto.Uint64(uint64(fetchEnd.Sub(fetchStart).Microseconds()))
|
|
expConfigFetcher.Filename = proto.String(outConfigFilePath)
|
|
|
|
if _, err := os.Stat(outConfigFilePath); err != nil {
|
|
status := smpb.ExpConfigFetcher_NO_CONFIG
|
|
expConfigFetcher.Status = &status
|
|
return err
|
|
}
|
|
status := smpb.ExpConfigFetcher_CONFIG
|
|
expConfigFetcher.Status = &status
|
|
return nil
|
|
}
|
|
|
|
// If a config file does not exist, wait for the config file to be fetched. Otherwise
|
|
// fetch the config file in the background and return immediately.
|
|
if !configExists {
|
|
defer cancel()
|
|
return fetchCfg()
|
|
}
|
|
|
|
go func() {
|
|
defer cancel()
|
|
if err := fetchCfg(); err != nil {
|
|
ctx.Verbosef("Failed to fetch config file %v: %v\n", configName, err)
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
|
|
if bc == "" {
|
|
return nil
|
|
}
|
|
|
|
configDirs := []string{
|
|
config.OutDir(),
|
|
os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
|
|
envConfigDir,
|
|
}
|
|
for _, dir := range configDirs {
|
|
cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
|
|
envVarsJSON, err := ioutil.ReadFile(cfgFile)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
ctx.Verbosef("Loading config file %v\n", cfgFile)
|
|
var envVars map[string]map[string]string
|
|
if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
|
|
fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
|
|
continue
|
|
}
|
|
for k, v := range envVars["env"] {
|
|
if os.Getenv(k) != "" {
|
|
continue
|
|
}
|
|
config.environ.Set(k, v)
|
|
}
|
|
ctx.Verbosef("Finished loading config file %v\n", cfgFile)
|
|
break
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func defaultBazelProdMode(cfg *configImpl) bool {
|
|
// Environment flag to disable Bazel for users which experience
|
|
// broken bazel-handled builds, or significant performance regressions.
|
|
if cfg.IsBazelMixedBuildForceDisabled() {
|
|
return false
|
|
}
|
|
// Darwin-host builds are currently untested with Bazel.
|
|
if runtime.GOOS == "darwin" {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func UploadOnlyConfig(ctx Context, _ ...string) Config {
|
|
ret := &configImpl{
|
|
environ: OsEnvironment(),
|
|
sandboxConfig: &SandboxConfig{},
|
|
}
|
|
srcDir := absPath(ctx, ".")
|
|
bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
|
|
if err := loadEnvConfig(ctx, ret, bc); err != nil {
|
|
ctx.Fatalln("Failed to parse env config files: %v", err)
|
|
}
|
|
ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
|
|
return Config{ret}
|
|
}
|
|
|
|
func NewConfig(ctx Context, args ...string) Config {
|
|
ret := &configImpl{
|
|
environ: OsEnvironment(),
|
|
sandboxConfig: &SandboxConfig{},
|
|
}
|
|
|
|
// Default matching ninja
|
|
ret.parallel = runtime.NumCPU() + 2
|
|
ret.keepGoing = 1
|
|
|
|
ret.totalRAM = detectTotalRAM(ctx)
|
|
ret.parseArgs(ctx, args)
|
|
// Make sure OUT_DIR is set appropriately
|
|
if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
|
|
ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
|
|
} else {
|
|
outDir := "out"
|
|
if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
|
|
if wd, err := os.Getwd(); err != nil {
|
|
ctx.Fatalln("Failed to get working directory:", err)
|
|
} else {
|
|
outDir = filepath.Join(baseDir, filepath.Base(wd))
|
|
}
|
|
}
|
|
ret.environ.Set("OUT_DIR", outDir)
|
|
}
|
|
|
|
// loadEnvConfig needs to know what the OUT_DIR is, so it should
|
|
// be called after we determine the appropriate out directory.
|
|
bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
|
|
|
|
if bc != "" {
|
|
if err := fetchEnvConfig(ctx, ret, bc); err != nil {
|
|
ctx.Verbosef("Failed to fetch config file: %v\n", err)
|
|
}
|
|
if err := loadEnvConfig(ctx, ret, bc); err != nil {
|
|
ctx.Fatalln("Failed to parse env config files: %v", err)
|
|
}
|
|
}
|
|
|
|
if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
|
|
ret.distDir = filepath.Clean(distDir)
|
|
} else {
|
|
ret.distDir = filepath.Join(ret.OutDir(), "dist")
|
|
}
|
|
|
|
if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
|
|
ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
|
|
}
|
|
|
|
ret.environ.Unset(
|
|
// We're already using it
|
|
"USE_SOONG_UI",
|
|
|
|
// We should never use GOROOT/GOPATH from the shell environment
|
|
"GOROOT",
|
|
"GOPATH",
|
|
|
|
// These should only come from Soong, not the environment.
|
|
"CLANG",
|
|
"CLANG_CXX",
|
|
"CCC_CC",
|
|
"CCC_CXX",
|
|
|
|
// Used by the goma compiler wrapper, but should only be set by
|
|
// gomacc
|
|
"GOMACC_PATH",
|
|
|
|
// We handle this above
|
|
"OUT_DIR_COMMON_BASE",
|
|
|
|
// This is handled above too, and set for individual commands later
|
|
"DIST_DIR",
|
|
|
|
// Variables that have caused problems in the past
|
|
"BASH_ENV",
|
|
"CDPATH",
|
|
"DISPLAY",
|
|
"GREP_OPTIONS",
|
|
"JAVAC",
|
|
"NDK_ROOT",
|
|
"POSIXLY_CORRECT",
|
|
|
|
// Drop make flags
|
|
"MAKEFLAGS",
|
|
"MAKELEVEL",
|
|
"MFLAGS",
|
|
|
|
// Set in envsetup.sh, reset in makefiles
|
|
"ANDROID_JAVA_TOOLCHAIN",
|
|
|
|
// Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
|
|
"ANDROID_BUILD_TOP",
|
|
"ANDROID_HOST_OUT",
|
|
"ANDROID_PRODUCT_OUT",
|
|
"ANDROID_HOST_OUT_TESTCASES",
|
|
"ANDROID_TARGET_OUT_TESTCASES",
|
|
"ANDROID_TOOLCHAIN",
|
|
"ANDROID_TOOLCHAIN_2ND_ARCH",
|
|
"ANDROID_DEV_SCRIPTS",
|
|
"ANDROID_EMULATOR_PREBUILTS",
|
|
"ANDROID_PRE_BUILD_PATHS",
|
|
)
|
|
|
|
if ret.UseGoma() || ret.ForceUseGoma() {
|
|
ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
|
|
ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
|
|
}
|
|
|
|
// Tell python not to spam the source tree with .pyc files.
|
|
ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
|
|
|
|
tmpDir := absPath(ctx, ret.TempDir())
|
|
ret.environ.Set("TMPDIR", tmpDir)
|
|
|
|
// Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
|
|
symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
|
|
"llvm-binutils-stable/llvm-symbolizer")
|
|
ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
|
|
|
|
// Precondition: the current directory is the top of the source tree
|
|
checkTopDir(ctx)
|
|
|
|
srcDir := absPath(ctx, ".")
|
|
if strings.ContainsRune(srcDir, ' ') {
|
|
ctx.Println("You are building in a directory whose absolute path contains a space character:")
|
|
ctx.Println()
|
|
ctx.Printf("%q\n", srcDir)
|
|
ctx.Println()
|
|
ctx.Fatalln("Directory names containing spaces are not supported")
|
|
}
|
|
|
|
ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
|
|
|
|
if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
|
|
ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
|
|
ctx.Println()
|
|
ctx.Printf("%q\n", outDir)
|
|
ctx.Println()
|
|
ctx.Fatalln("Directory names containing spaces are not supported")
|
|
}
|
|
|
|
if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
|
|
ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
|
|
ctx.Println()
|
|
ctx.Printf("%q\n", distDir)
|
|
ctx.Println()
|
|
ctx.Fatalln("Directory names containing spaces are not supported")
|
|
}
|
|
|
|
// Configure Java-related variables, including adding it to $PATH
|
|
java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
|
|
java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
|
|
java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag())
|
|
java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag())
|
|
javaHome := func() string {
|
|
if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
|
|
return override
|
|
}
|
|
if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
|
|
ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
|
|
}
|
|
if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
|
|
ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.")
|
|
}
|
|
return java17Home
|
|
}()
|
|
absJavaHome := absPath(ctx, javaHome)
|
|
|
|
ret.configureLocale(ctx)
|
|
|
|
newPath := []string{filepath.Join(absJavaHome, "bin")}
|
|
if path, ok := ret.environ.Get("PATH"); ok && path != "" {
|
|
newPath = append(newPath, path)
|
|
}
|
|
|
|
ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
|
|
ret.environ.Set("JAVA_HOME", absJavaHome)
|
|
ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
|
|
ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
|
|
ret.environ.Set("ANDROID_JAVA9_HOME", java9Home)
|
|
ret.environ.Set("ANDROID_JAVA11_HOME", java11Home)
|
|
ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
|
|
|
|
if ret.MultitreeBuild() {
|
|
ret.environ.Set("MULTITREE_BUILD", "true")
|
|
}
|
|
|
|
outDir := ret.OutDir()
|
|
buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
|
|
if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
|
|
ret.buildDateTime = buildDateTime
|
|
} else {
|
|
ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
|
|
}
|
|
|
|
ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
|
|
|
|
if ret.UseRBE() {
|
|
for k, v := range getRBEVars(ctx, Config{ret}) {
|
|
ret.environ.Set(k, v)
|
|
}
|
|
}
|
|
|
|
bpd := ret.BazelMetricsDir()
|
|
if err := os.RemoveAll(bpd); err != nil {
|
|
ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
|
|
}
|
|
|
|
c := Config{ret}
|
|
storeConfigMetrics(ctx, c)
|
|
return c
|
|
}
|
|
|
|
// NewBuildActionConfig returns a build configuration based on the build action. The arguments are
|
|
// processed based on the build action and extracts any arguments that belongs to the build action.
|
|
func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
|
|
return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
|
|
}
|
|
|
|
// storeConfigMetrics selects a set of configuration information and store in
|
|
// the metrics system for further analysis.
|
|
func storeConfigMetrics(ctx Context, config Config) {
|
|
if ctx.Metrics == nil {
|
|
return
|
|
}
|
|
|
|
ctx.Metrics.BuildConfig(buildConfig(config))
|
|
|
|
s := &smpb.SystemResourceInfo{
|
|
TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
|
|
AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
|
|
}
|
|
ctx.Metrics.SystemResourceInfo(s)
|
|
}
|
|
|
|
func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
|
|
switch s {
|
|
case NINJA_LOG:
|
|
return smpb.BuildConfig_NINJA_LOG.Enum()
|
|
case EVENLY_DISTRIBUTED:
|
|
return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
|
|
default:
|
|
return smpb.BuildConfig_NOT_USED.Enum()
|
|
}
|
|
}
|
|
|
|
func buildConfig(config Config) *smpb.BuildConfig {
|
|
c := &smpb.BuildConfig{
|
|
ForceUseGoma: proto.Bool(config.ForceUseGoma()),
|
|
UseGoma: proto.Bool(config.UseGoma()),
|
|
UseRbe: proto.Bool(config.UseRBE()),
|
|
BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
|
|
ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
|
|
NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
|
|
}
|
|
c.Targets = append(c.Targets, config.arguments...)
|
|
|
|
return c
|
|
}
|
|
|
|
// getConfigArgs processes the command arguments based on the build action and creates a set of new
|
|
// arguments to be accepted by Config.
|
|
func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
|
|
// The next block of code verifies that the current directory is the root directory of the source
|
|
// tree. It then finds the relative path of dir based on the root directory of the source tree
|
|
// and verify that dir is inside of the source tree.
|
|
checkTopDir(ctx)
|
|
topDir, err := os.Getwd()
|
|
if err != nil {
|
|
ctx.Fatalf("Error retrieving top directory: %v", err)
|
|
}
|
|
dir, err = filepath.EvalSymlinks(dir)
|
|
if err != nil {
|
|
ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
|
|
}
|
|
dir, err = filepath.Abs(dir)
|
|
if err != nil {
|
|
ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
|
|
}
|
|
relDir, err := filepath.Rel(topDir, dir)
|
|
if err != nil {
|
|
ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
|
|
}
|
|
// If there are ".." in the path, it's not in the source tree.
|
|
if strings.Contains(relDir, "..") {
|
|
ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
|
|
}
|
|
|
|
configArgs := args[:]
|
|
|
|
// If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
|
|
// GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
|
|
targetNamePrefix := "MODULES-IN-"
|
|
if inList("GET-INSTALL-PATH", configArgs) {
|
|
targetNamePrefix = "GET-INSTALL-PATH-IN-"
|
|
configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
|
|
}
|
|
|
|
var targets []string
|
|
|
|
switch action {
|
|
case BUILD_MODULES:
|
|
// No additional processing is required when building a list of specific modules or all modules.
|
|
case BUILD_MODULES_IN_A_DIRECTORY:
|
|
// If dir is the root source tree, all the modules are built of the source tree are built so
|
|
// no need to find the build file.
|
|
if topDir == dir {
|
|
break
|
|
}
|
|
|
|
buildFile := findBuildFile(ctx, relDir)
|
|
if buildFile == "" {
|
|
ctx.Fatalf("Build file not found for %s directory", relDir)
|
|
}
|
|
targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
|
|
case BUILD_MODULES_IN_DIRECTORIES:
|
|
newConfigArgs, dirs := splitArgs(configArgs)
|
|
configArgs = newConfigArgs
|
|
targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
|
|
}
|
|
|
|
// Tidy only override all other specified targets.
|
|
tidyOnly := os.Getenv("WITH_TIDY_ONLY")
|
|
if tidyOnly == "true" || tidyOnly == "1" {
|
|
configArgs = append(configArgs, "tidy_only")
|
|
} else {
|
|
configArgs = append(configArgs, targets...)
|
|
}
|
|
|
|
return configArgs
|
|
}
|
|
|
|
// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
|
|
func convertToTarget(dir string, targetNamePrefix string) string {
|
|
return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
|
|
}
|
|
|
|
// hasBuildFile returns true if dir contains an Android build file.
|
|
func hasBuildFile(ctx Context, dir string) bool {
|
|
for _, buildFile := range buildFiles {
|
|
_, err := os.Stat(filepath.Join(dir, buildFile))
|
|
if err == nil {
|
|
return true
|
|
}
|
|
if !os.IsNotExist(err) {
|
|
ctx.Fatalf("Error retrieving the build file stats: %v", err)
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
|
|
// in the current and any sub directory of dir. If a build file is not found, traverse the path
|
|
// up by one directory and repeat again until either a build file is found or reached to the root
|
|
// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
|
|
// string is returned.
|
|
func findBuildFile(ctx Context, dir string) string {
|
|
// If the string is empty or ".", assume it is top directory of the source tree.
|
|
if dir == "" || dir == "." {
|
|
return ""
|
|
}
|
|
|
|
found := false
|
|
for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
|
|
err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found {
|
|
return filepath.SkipDir
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
for _, buildFile := range buildFiles {
|
|
if info.Name() == buildFile {
|
|
found = true
|
|
return filepath.SkipDir
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
ctx.Fatalf("Error finding Android build file: %v", err)
|
|
}
|
|
|
|
if found {
|
|
return filepath.Join(buildDir, "Android.mk")
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
|
|
func splitArgs(args []string) (newArgs []string, dirs []string) {
|
|
specialArgs := map[string]bool{
|
|
"showcommands": true,
|
|
"snod": true,
|
|
"dist": true,
|
|
"checkbuild": true,
|
|
}
|
|
|
|
newArgs = []string{}
|
|
dirs = []string{}
|
|
|
|
for _, arg := range args {
|
|
// It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
|
|
if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
|
|
newArgs = append(newArgs, arg)
|
|
continue
|
|
}
|
|
|
|
if _, ok := specialArgs[arg]; ok {
|
|
newArgs = append(newArgs, arg)
|
|
continue
|
|
}
|
|
|
|
dirs = append(dirs, arg)
|
|
}
|
|
|
|
return newArgs, dirs
|
|
}
|
|
|
|
// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
|
|
// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
|
|
// source root tree where the build action command was invoked. Each directory is validated if the
|
|
// build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
|
|
func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
|
|
for _, dir := range dirs {
|
|
// The directory may have specified specific modules to build. ":" is the separator to separate
|
|
// the directory and the list of modules.
|
|
s := strings.Split(dir, ":")
|
|
l := len(s)
|
|
if l > 2 { // more than one ":" was specified.
|
|
ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
|
|
}
|
|
|
|
dir = filepath.Join(relDir, s[0])
|
|
if _, err := os.Stat(dir); err != nil {
|
|
ctx.Fatalf("couldn't find directory %s", dir)
|
|
}
|
|
|
|
// Verify that if there are any targets specified after ":". Each target is separated by ",".
|
|
var newTargets []string
|
|
if l == 2 && s[1] != "" {
|
|
newTargets = strings.Split(s[1], ",")
|
|
if inList("", newTargets) {
|
|
ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
|
|
}
|
|
}
|
|
|
|
// If there are specified targets to build in dir, an android build file must exist for the one
|
|
// shot build. For the non-targets case, find the appropriate build file and build all the
|
|
// modules in dir (or the closest one in the dir path).
|
|
if len(newTargets) > 0 {
|
|
if !hasBuildFile(ctx, dir) {
|
|
ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
|
|
}
|
|
} else {
|
|
buildFile := findBuildFile(ctx, dir)
|
|
if buildFile == "" {
|
|
ctx.Fatalf("Build file not found for %s directory", dir)
|
|
}
|
|
newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
|
|
}
|
|
|
|
targets = append(targets, newTargets...)
|
|
}
|
|
|
|
return targets
|
|
}
|
|
|
|
func (c *configImpl) parseArgs(ctx Context, args []string) {
|
|
for i := 0; i < len(args); i++ {
|
|
arg := strings.TrimSpace(args[i])
|
|
if arg == "showcommands" {
|
|
c.verbose = true
|
|
} else if arg == "--empty-ninja-file" {
|
|
c.emptyNinjaFile = true
|
|
} else if arg == "--skip-ninja" {
|
|
c.skipNinja = true
|
|
} else if arg == "--skip-make" {
|
|
// TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
|
|
// but it does run a Kati ninja file if the .kati_enabled marker file was created
|
|
// by a previous build.
|
|
c.skipConfig = true
|
|
c.skipKati = true
|
|
} else if arg == "--skip-kati" {
|
|
// TODO: remove --skip-kati once module builds have been migrated to --song-only
|
|
c.skipKati = true
|
|
} else if arg == "--soong-only" {
|
|
c.skipKati = true
|
|
c.skipKatiNinja = true
|
|
} else if arg == "--config-only" {
|
|
c.skipKati = true
|
|
c.skipKatiNinja = true
|
|
c.skipSoong = true
|
|
} else if arg == "--skip-config" {
|
|
c.skipConfig = true
|
|
} else if arg == "--skip-soong-tests" {
|
|
c.skipSoongTests = true
|
|
} else if arg == "--skip-metrics-upload" {
|
|
c.skipMetricsUpload = true
|
|
} else if arg == "--mk-metrics" {
|
|
c.reportMkMetrics = true
|
|
} else if arg == "--multitree-build" {
|
|
c.multitreeBuild = true
|
|
} else if arg == "--bazel-mode" {
|
|
c.bazelProdMode = true
|
|
} else if arg == "--bazel-mode-dev" {
|
|
c.bazelDevMode = true
|
|
} else if arg == "--bazel-mode-staging" {
|
|
c.bazelStagingMode = true
|
|
} else if arg == "--search-api-dir" {
|
|
c.searchApiDir = true
|
|
} else if strings.HasPrefix(arg, "--ninja_weight_source=") {
|
|
source := strings.TrimPrefix(arg, "--ninja_weight_source=")
|
|
if source == "ninja_log" {
|
|
c.ninjaWeightListSource = NINJA_LOG
|
|
} else if source == "evenly_distributed" {
|
|
c.ninjaWeightListSource = EVENLY_DISTRIBUTED
|
|
} else if source == "not_used" {
|
|
c.ninjaWeightListSource = NOT_USED
|
|
} else {
|
|
ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
|
|
}
|
|
} else if strings.HasPrefix(arg, "--build-command=") {
|
|
buildCmd := strings.TrimPrefix(arg, "--build-command=")
|
|
// remove quotations
|
|
buildCmd = strings.TrimPrefix(buildCmd, "\"")
|
|
buildCmd = strings.TrimSuffix(buildCmd, "\"")
|
|
ctx.Metrics.SetBuildCommand([]string{buildCmd})
|
|
} else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
|
|
c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
|
|
} else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
|
|
buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
|
|
val, err := strconv.ParseInt(buildTimeStr, 10, 64)
|
|
if err == nil {
|
|
c.buildStartedTime = val
|
|
} else {
|
|
ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
|
|
}
|
|
} else if len(arg) > 0 && arg[0] == '-' {
|
|
parseArgNum := func(def int) int {
|
|
if len(arg) > 2 {
|
|
p, err := strconv.ParseUint(arg[2:], 10, 31)
|
|
if err != nil {
|
|
ctx.Fatalf("Failed to parse %q: %v", arg, err)
|
|
}
|
|
return int(p)
|
|
} else if i+1 < len(args) {
|
|
p, err := strconv.ParseUint(args[i+1], 10, 31)
|
|
if err == nil {
|
|
i++
|
|
return int(p)
|
|
}
|
|
}
|
|
return def
|
|
}
|
|
|
|
if len(arg) > 1 && arg[1] == 'j' {
|
|
c.parallel = parseArgNum(c.parallel)
|
|
} else if len(arg) > 1 && arg[1] == 'k' {
|
|
c.keepGoing = parseArgNum(0)
|
|
} else {
|
|
ctx.Fatalln("Unknown option:", arg)
|
|
}
|
|
} else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
|
|
if k == "OUT_DIR" {
|
|
ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
|
|
}
|
|
c.environ.Set(k, v)
|
|
} else if arg == "dist" {
|
|
c.dist = true
|
|
} else if arg == "json-module-graph" {
|
|
c.jsonModuleGraph = true
|
|
} else if arg == "bp2build" {
|
|
c.bp2build = true
|
|
} else if arg == "api_bp2build" {
|
|
c.apiBp2build = true
|
|
} else if arg == "queryview" {
|
|
c.queryview = true
|
|
} else if arg == "soong_docs" {
|
|
c.soongDocs = true
|
|
} else {
|
|
if arg == "checkbuild" {
|
|
c.checkbuild = true
|
|
}
|
|
c.arguments = append(c.arguments, arg)
|
|
}
|
|
}
|
|
if (!c.bazelProdMode) && (!c.bazelDevMode) && (!c.bazelStagingMode) {
|
|
c.bazelProdMode = defaultBazelProdMode(c)
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) configureLocale(ctx Context) {
|
|
cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
|
|
output, err := cmd.Output()
|
|
|
|
var locales []string
|
|
if err == nil {
|
|
locales = strings.Split(string(output), "\n")
|
|
} else {
|
|
// If we're unable to list the locales, let's assume en_US.UTF-8
|
|
locales = []string{"en_US.UTF-8"}
|
|
ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
|
|
}
|
|
|
|
// gettext uses LANGUAGE, which is passed directly through
|
|
|
|
// For LANG and LC_*, only preserve the evaluated version of
|
|
// LC_MESSAGES
|
|
userLang := ""
|
|
if lc_all, ok := c.environ.Get("LC_ALL"); ok {
|
|
userLang = lc_all
|
|
} else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
|
|
userLang = lc_messages
|
|
} else if lang, ok := c.environ.Get("LANG"); ok {
|
|
userLang = lang
|
|
}
|
|
|
|
c.environ.UnsetWithPrefix("LC_")
|
|
|
|
if userLang != "" {
|
|
c.environ.Set("LC_MESSAGES", userLang)
|
|
}
|
|
|
|
// The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
|
|
// for others)
|
|
if inList("C.UTF-8", locales) {
|
|
c.environ.Set("LANG", "C.UTF-8")
|
|
} else if inList("C.utf8", locales) {
|
|
// These normalize to the same thing
|
|
c.environ.Set("LANG", "C.UTF-8")
|
|
} else if inList("en_US.UTF-8", locales) {
|
|
c.environ.Set("LANG", "en_US.UTF-8")
|
|
} else if inList("en_US.utf8", locales) {
|
|
// These normalize to the same thing
|
|
c.environ.Set("LANG", "en_US.UTF-8")
|
|
} else {
|
|
ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) Environment() *Environment {
|
|
return c.environ
|
|
}
|
|
|
|
func (c *configImpl) Arguments() []string {
|
|
return c.arguments
|
|
}
|
|
|
|
func (c *configImpl) SoongBuildInvocationNeeded() bool {
|
|
if len(c.Arguments()) > 0 {
|
|
// Explicit targets requested that are not special targets like b2pbuild
|
|
// or the JSON module graph
|
|
return true
|
|
}
|
|
|
|
if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() && !c.ApiBp2build() {
|
|
// Command line was empty, the default Ninja target is built
|
|
return true
|
|
}
|
|
|
|
// bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
|
|
if c.Dist() && !c.Bp2Build() {
|
|
return true
|
|
}
|
|
|
|
// build.ninja doesn't need to be generated
|
|
return false
|
|
}
|
|
|
|
func (c *configImpl) OutDir() string {
|
|
if outDir, ok := c.environ.Get("OUT_DIR"); ok {
|
|
return outDir
|
|
}
|
|
return "out"
|
|
}
|
|
|
|
func (c *configImpl) DistDir() string {
|
|
return c.distDir
|
|
}
|
|
|
|
func (c *configImpl) RealDistDir() string {
|
|
return c.distDir
|
|
}
|
|
|
|
func (c *configImpl) NinjaArgs() []string {
|
|
if c.skipKati {
|
|
return c.arguments
|
|
}
|
|
return c.ninjaArgs
|
|
}
|
|
|
|
func (c *configImpl) BazelOutDir() string {
|
|
return filepath.Join(c.OutDir(), "bazel")
|
|
}
|
|
|
|
func (c *configImpl) bazelOutputBase() string {
|
|
return filepath.Join(c.BazelOutDir(), "output")
|
|
}
|
|
|
|
func (c *configImpl) SoongOutDir() string {
|
|
return filepath.Join(c.OutDir(), "soong")
|
|
}
|
|
|
|
func (c *configImpl) ApiSurfacesOutDir() string {
|
|
return filepath.Join(c.OutDir(), "api_surfaces")
|
|
}
|
|
|
|
func (c *configImpl) PrebuiltOS() string {
|
|
switch runtime.GOOS {
|
|
case "linux":
|
|
return "linux-x86"
|
|
case "darwin":
|
|
return "darwin-x86"
|
|
default:
|
|
panic("Unknown GOOS")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) HostToolDir() string {
|
|
if c.SkipKatiNinja() {
|
|
return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
|
|
} else {
|
|
return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) NamedGlobFile(name string) string {
|
|
return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
|
|
}
|
|
|
|
func (c *configImpl) UsedEnvFile(tag string) string {
|
|
return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
|
|
}
|
|
|
|
func (c *configImpl) Bp2BuildFilesMarkerFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
|
|
}
|
|
|
|
func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
|
|
}
|
|
|
|
func (c *configImpl) SoongDocsHtml() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
|
|
}
|
|
|
|
func (c *configImpl) QueryviewMarkerFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
|
|
}
|
|
|
|
func (c *configImpl) ApiBp2buildMarkerFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "api_bp2build.marker")
|
|
}
|
|
|
|
func (c *configImpl) ModuleGraphFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
|
|
}
|
|
|
|
func (c *configImpl) ModuleActionsFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
|
|
}
|
|
|
|
func (c *configImpl) TempDir() string {
|
|
return shared.TempDirForOutDir(c.SoongOutDir())
|
|
}
|
|
|
|
func (c *configImpl) FileListDir() string {
|
|
return filepath.Join(c.OutDir(), ".module_paths")
|
|
}
|
|
|
|
func (c *configImpl) KatiSuffix() string {
|
|
if c.katiSuffix != "" {
|
|
return c.katiSuffix
|
|
}
|
|
panic("SetKatiSuffix has not been called")
|
|
}
|
|
|
|
// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
|
|
// user is interested in additional checks at the expense of build time.
|
|
func (c *configImpl) Checkbuild() bool {
|
|
return c.checkbuild
|
|
}
|
|
|
|
func (c *configImpl) Dist() bool {
|
|
return c.dist
|
|
}
|
|
|
|
func (c *configImpl) JsonModuleGraph() bool {
|
|
return c.jsonModuleGraph
|
|
}
|
|
|
|
func (c *configImpl) Bp2Build() bool {
|
|
return c.bp2build
|
|
}
|
|
|
|
func (c *configImpl) ApiBp2build() bool {
|
|
return c.apiBp2build
|
|
}
|
|
|
|
func (c *configImpl) Queryview() bool {
|
|
return c.queryview
|
|
}
|
|
|
|
func (c *configImpl) SoongDocs() bool {
|
|
return c.soongDocs
|
|
}
|
|
|
|
func (c *configImpl) IsVerbose() bool {
|
|
return c.verbose
|
|
}
|
|
|
|
func (c *configImpl) MultitreeBuild() bool {
|
|
return c.multitreeBuild
|
|
}
|
|
|
|
func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
|
|
return c.ninjaWeightListSource
|
|
}
|
|
|
|
func (c *configImpl) SkipKati() bool {
|
|
return c.skipKati
|
|
}
|
|
|
|
func (c *configImpl) SkipKatiNinja() bool {
|
|
return c.skipKatiNinja
|
|
}
|
|
|
|
func (c *configImpl) SkipSoong() bool {
|
|
return c.skipSoong
|
|
}
|
|
|
|
func (c *configImpl) SkipNinja() bool {
|
|
return c.skipNinja
|
|
}
|
|
|
|
func (c *configImpl) SetSkipNinja(v bool) {
|
|
c.skipNinja = v
|
|
}
|
|
|
|
func (c *configImpl) SkipConfig() bool {
|
|
return c.skipConfig
|
|
}
|
|
|
|
func (c *configImpl) TargetProduct() string {
|
|
if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
|
|
return v
|
|
}
|
|
panic("TARGET_PRODUCT is not defined")
|
|
}
|
|
|
|
func (c *configImpl) TargetDevice() string {
|
|
return c.targetDevice
|
|
}
|
|
|
|
func (c *configImpl) SetTargetDevice(device string) {
|
|
c.targetDevice = device
|
|
}
|
|
|
|
func (c *configImpl) TargetBuildVariant() string {
|
|
if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
|
|
return v
|
|
}
|
|
panic("TARGET_BUILD_VARIANT is not defined")
|
|
}
|
|
|
|
func (c *configImpl) KatiArgs() []string {
|
|
return c.katiArgs
|
|
}
|
|
|
|
func (c *configImpl) Parallel() int {
|
|
return c.parallel
|
|
}
|
|
|
|
func (c *configImpl) GetIncludeTags() []string {
|
|
return c.includeTags
|
|
}
|
|
|
|
func (c *configImpl) SetIncludeTags(i []string) {
|
|
c.includeTags = i
|
|
}
|
|
|
|
func (c *configImpl) GetLogsPrefix() string {
|
|
return c.logsPrefix
|
|
}
|
|
|
|
func (c *configImpl) SetLogsPrefix(prefix string) {
|
|
c.logsPrefix = prefix
|
|
}
|
|
|
|
func (c *configImpl) HighmemParallel() int {
|
|
if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
|
|
return i
|
|
}
|
|
|
|
const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
|
|
parallel := c.Parallel()
|
|
if c.UseRemoteBuild() {
|
|
// Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
|
|
// is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
|
|
// to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
|
|
// Return 1/16th of the size of the local pool, rounding up.
|
|
return (parallel + 15) / 16
|
|
} else if c.totalRAM == 0 {
|
|
// Couldn't detect the total RAM, don't restrict highmem processes.
|
|
return parallel
|
|
} else if c.totalRAM <= 16*1024*1024*1024 {
|
|
// Less than 16GB of ram, restrict to 1 highmem processes
|
|
return 1
|
|
} else if c.totalRAM <= 32*1024*1024*1024 {
|
|
// Less than 32GB of ram, restrict to 2 highmem processes
|
|
return 2
|
|
} else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
|
|
// If less than 8GB total RAM per process, reduce the number of highmem processes
|
|
return p
|
|
}
|
|
// No restriction on highmem processes
|
|
return parallel
|
|
}
|
|
|
|
func (c *configImpl) TotalRAM() uint64 {
|
|
return c.totalRAM
|
|
}
|
|
|
|
// ForceUseGoma determines whether we should override Goma deprecation
|
|
// and use Goma for the current build or not.
|
|
func (c *configImpl) ForceUseGoma() bool {
|
|
if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (c *configImpl) UseGoma() bool {
|
|
if v, ok := c.environ.Get("USE_GOMA"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (c *configImpl) StartGoma() bool {
|
|
if !c.UseGoma() {
|
|
return false
|
|
}
|
|
|
|
if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c *configImpl) UseRBE() bool {
|
|
if v, ok := c.Environment().Get("USE_RBE"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (c *configImpl) BazelBuildEnabled() bool {
|
|
return c.bazelProdMode || c.bazelDevMode || c.bazelStagingMode
|
|
}
|
|
|
|
func (c *configImpl) StartRBE() bool {
|
|
if !c.UseRBE() {
|
|
return false
|
|
}
|
|
|
|
if v, ok := c.environ.Get("NOSTART_RBE"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c *configImpl) rbeProxyLogsDir() string {
|
|
for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
return v
|
|
}
|
|
}
|
|
buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
|
|
return filepath.Join(buildTmpDir, "rbe")
|
|
}
|
|
|
|
func (c *configImpl) shouldCleanupRBELogsDir() bool {
|
|
// Perform a log directory cleanup only when the log directory
|
|
// is auto created by the build rather than user-specified.
|
|
for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
|
|
if _, ok := c.environ.Get(f); ok {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c *configImpl) rbeExecRoot() string {
|
|
for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
return v
|
|
}
|
|
}
|
|
wd, err := os.Getwd()
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return wd
|
|
}
|
|
|
|
func (c *configImpl) rbeDir() string {
|
|
if v, ok := c.environ.Get("RBE_DIR"); ok {
|
|
return v
|
|
}
|
|
return "prebuilts/remoteexecution-client/live/"
|
|
}
|
|
|
|
func (c *configImpl) rbeReproxy() string {
|
|
for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
return v
|
|
}
|
|
}
|
|
return filepath.Join(c.rbeDir(), "reproxy")
|
|
}
|
|
|
|
func (c *configImpl) rbeAuth() (string, string) {
|
|
credFlags := []string{
|
|
"use_application_default_credentials",
|
|
"use_gce_credentials",
|
|
"credential_file",
|
|
"use_google_prod_creds",
|
|
}
|
|
for _, cf := range credFlags {
|
|
for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" && v != "0" {
|
|
return "RBE_" + cf, v
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return "RBE_use_application_default_credentials", "true"
|
|
}
|
|
|
|
func (c *configImpl) rbeSockAddr(dir string) (string, error) {
|
|
maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
|
|
base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
|
|
|
|
name := filepath.Join(dir, base)
|
|
if len(name) < maxNameLen {
|
|
return name, nil
|
|
}
|
|
|
|
name = filepath.Join("/tmp", base)
|
|
if len(name) < maxNameLen {
|
|
return name, nil
|
|
}
|
|
|
|
return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
|
|
}
|
|
|
|
// IsGooglerEnvironment returns true if the current build is running
|
|
// on a Google developer machine and false otherwise.
|
|
func (c *configImpl) IsGooglerEnvironment() bool {
|
|
cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
|
|
if v, ok := c.environ.Get(cf); ok {
|
|
return v == "googler"
|
|
}
|
|
return false
|
|
}
|
|
|
|
// GoogleProdCredsExist determine whether credentials exist on the
|
|
// Googler machine to use remote execution.
|
|
func (c *configImpl) GoogleProdCredsExist() bool {
|
|
if googleProdCredsExistCache {
|
|
return googleProdCredsExistCache
|
|
}
|
|
if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil {
|
|
return false
|
|
}
|
|
googleProdCredsExistCache = true
|
|
return true
|
|
}
|
|
|
|
// UseRemoteBuild indicates whether to use a remote build acceleration system
|
|
// to speed up the build.
|
|
func (c *configImpl) UseRemoteBuild() bool {
|
|
return c.UseGoma() || c.UseRBE()
|
|
}
|
|
|
|
// StubbyExists checks whether the stubby binary exists on the machine running
|
|
// the build.
|
|
func (c *configImpl) StubbyExists() bool {
|
|
if _, err := exec.LookPath("stubby"); err != nil {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// RemoteParallel controls how many remote jobs (i.e., commands which contain
|
|
// gomacc) are run in parallel. Note the parallelism of all other jobs is
|
|
// still limited by Parallel()
|
|
func (c *configImpl) RemoteParallel() int {
|
|
if !c.UseRemoteBuild() {
|
|
return 0
|
|
}
|
|
if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
|
|
return i
|
|
}
|
|
return 500
|
|
}
|
|
|
|
func (c *configImpl) SetKatiArgs(args []string) {
|
|
c.katiArgs = args
|
|
}
|
|
|
|
func (c *configImpl) SetNinjaArgs(args []string) {
|
|
c.ninjaArgs = args
|
|
}
|
|
|
|
func (c *configImpl) SetKatiSuffix(suffix string) {
|
|
c.katiSuffix = suffix
|
|
}
|
|
|
|
func (c *configImpl) LastKatiSuffixFile() string {
|
|
return filepath.Join(c.OutDir(), "last_kati_suffix")
|
|
}
|
|
|
|
func (c *configImpl) HasKatiSuffix() bool {
|
|
return c.katiSuffix != ""
|
|
}
|
|
|
|
func (c *configImpl) KatiEnvFile() string {
|
|
return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
|
|
}
|
|
|
|
func (c *configImpl) KatiBuildNinjaFile() string {
|
|
return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
|
|
}
|
|
|
|
func (c *configImpl) KatiPackageNinjaFile() string {
|
|
return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
|
|
}
|
|
|
|
func (c *configImpl) SoongVarsFile() string {
|
|
return filepath.Join(c.SoongOutDir(), "soong.variables")
|
|
}
|
|
|
|
func (c *configImpl) SoongNinjaFile() string {
|
|
return filepath.Join(c.SoongOutDir(), "build.ninja")
|
|
}
|
|
|
|
func (c *configImpl) CombinedNinjaFile() string {
|
|
if c.katiSuffix == "" {
|
|
return filepath.Join(c.OutDir(), "combined.ninja")
|
|
}
|
|
return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
|
|
}
|
|
|
|
func (c *configImpl) SoongAndroidMk() string {
|
|
return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
|
|
}
|
|
|
|
func (c *configImpl) SoongMakeVarsMk() string {
|
|
return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
|
|
}
|
|
|
|
func (c *configImpl) ProductOut() string {
|
|
return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
|
|
}
|
|
|
|
func (c *configImpl) DevicePreviousProductConfig() string {
|
|
return filepath.Join(c.ProductOut(), "previous_build_config.mk")
|
|
}
|
|
|
|
func (c *configImpl) KatiPackageMkDir() string {
|
|
return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
|
|
}
|
|
|
|
func (c *configImpl) hostOutRoot() string {
|
|
return filepath.Join(c.OutDir(), "host")
|
|
}
|
|
|
|
func (c *configImpl) HostOut() string {
|
|
return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
|
|
}
|
|
|
|
// This probably needs to be multi-valued, so not exporting it for now
|
|
func (c *configImpl) hostCrossOut() string {
|
|
if runtime.GOOS == "linux" {
|
|
return filepath.Join(c.hostOutRoot(), "windows-x86")
|
|
} else {
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) HostPrebuiltTag() string {
|
|
if runtime.GOOS == "linux" {
|
|
return "linux-x86"
|
|
} else if runtime.GOOS == "darwin" {
|
|
return "darwin-x86"
|
|
} else {
|
|
panic("Unsupported OS")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) PrebuiltBuildTool(name string) string {
|
|
if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
|
|
if sanitize := strings.Fields(v); inList("address", sanitize) {
|
|
asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
|
|
if _, err := os.Stat(asan); err == nil {
|
|
return asan
|
|
}
|
|
}
|
|
}
|
|
return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
|
|
}
|
|
|
|
func (c *configImpl) SetBuildBrokenDupRules(val bool) {
|
|
c.brokenDupRules = val
|
|
}
|
|
|
|
func (c *configImpl) BuildBrokenDupRules() bool {
|
|
return c.brokenDupRules
|
|
}
|
|
|
|
func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
|
|
c.brokenUsesNetwork = val
|
|
}
|
|
|
|
func (c *configImpl) BuildBrokenUsesNetwork() bool {
|
|
return c.brokenUsesNetwork
|
|
}
|
|
|
|
func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
|
|
c.brokenNinjaEnvVars = val
|
|
}
|
|
|
|
func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
|
|
return c.brokenNinjaEnvVars
|
|
}
|
|
|
|
func (c *configImpl) SetTargetDeviceDir(dir string) {
|
|
c.targetDeviceDir = dir
|
|
}
|
|
|
|
func (c *configImpl) TargetDeviceDir() string {
|
|
return c.targetDeviceDir
|
|
}
|
|
|
|
func (c *configImpl) BuildDateTime() string {
|
|
return c.buildDateTime
|
|
}
|
|
|
|
func (c *configImpl) MetricsUploaderApp() string {
|
|
return c.metricsUploader
|
|
}
|
|
|
|
// LogsDir returns the absolute path to the logs directory where build log and
|
|
// metrics files are located. By default, the logs directory is the out
|
|
// directory. If the argument dist is specified, the logs directory
|
|
// is <dist_dir>/logs.
|
|
func (c *configImpl) LogsDir() string {
|
|
dir := c.OutDir()
|
|
if c.Dist() {
|
|
// Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
|
|
dir = filepath.Join(c.RealDistDir(), "logs")
|
|
}
|
|
absDir, err := filepath.Abs(dir)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
|
|
os.Exit(1)
|
|
}
|
|
return absDir
|
|
}
|
|
|
|
// BazelMetricsDir returns the <logs dir>/bazel_metrics directory
|
|
// where the bazel profiles are located.
|
|
func (c *configImpl) BazelMetricsDir() string {
|
|
return filepath.Join(c.LogsDir(), "bazel_metrics")
|
|
}
|
|
|
|
// MkFileMetrics returns the file path for make-related metrics.
|
|
func (c *configImpl) MkMetrics() string {
|
|
return filepath.Join(c.LogsDir(), "mk_metrics.pb")
|
|
}
|
|
|
|
func (c *configImpl) SetEmptyNinjaFile(v bool) {
|
|
c.emptyNinjaFile = v
|
|
}
|
|
|
|
func (c *configImpl) EmptyNinjaFile() bool {
|
|
return c.emptyNinjaFile
|
|
}
|
|
|
|
func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
|
|
return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
|
|
}
|
|
|
|
func (c *configImpl) IsPersistentBazelEnabled() bool {
|
|
return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
|
|
}
|
|
|
|
func (c *configImpl) BazelModulesForceEnabledByFlag() string {
|
|
return c.bazelForceEnabledModules
|
|
}
|
|
|
|
func (c *configImpl) SkipMetricsUpload() bool {
|
|
return c.skipMetricsUpload
|
|
}
|
|
|
|
// Returns a Time object if one was passed via a command-line flag.
|
|
// Otherwise returns the passed default.
|
|
func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
|
|
if c.buildStartedTime == 0 {
|
|
return defaultTime
|
|
}
|
|
return time.UnixMilli(c.buildStartedTime)
|
|
}
|
|
|
|
func GetMetricsUploader(topDir string, env *Environment) string {
|
|
if p, ok := env.Get("METRICS_UPLOADER"); ok {
|
|
metricsUploader := filepath.Join(topDir, p)
|
|
if _, err := os.Stat(metricsUploader); err == nil {
|
|
return metricsUploader
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|