Keep all Soong command line arguments in a single structure

Define a structure android.CmdArgs and keep all soong_build command line
arguments used to build a configuration in it. Some of them related to BazelMode
were previously kept in Blueprint's bootstrap.Args where they do not belong.
This simplifies the code and lays the gound work for future simplifying changes.

Test: treehugger
Change-Id: I995871f4e6b0e95cc9df65696ad2020df221d8c8
This commit is contained in:
Sasha Smundak 2022-12-12 21:23:34 -08:00
parent 841fc797c1
commit af5ca926be
2 changed files with 98 additions and 110 deletions

View file

@ -21,7 +21,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
@ -70,6 +69,26 @@ type Config struct {
type SoongBuildMode int
type CmdArgs struct {
bootstrap.Args
RunGoTests bool
OutDir string
SoongOutDir string
SymlinkForestMarker string
Bp2buildMarker string
BazelQueryViewDir string
BazelApiBp2buildDir string
ModuleGraphFile string
ModuleActionsFile string
DocFile string
BazelMode bool
BazelModeDev bool
BazelModeStaging bool
BazelForceEnabledModules string
}
// Build modes that soong_build can run as.
const (
// Don't use bazel at all during module analysis.
@ -307,7 +326,7 @@ func saveToConfigFile(config *productVariables, filename string) error {
return fmt.Errorf("cannot marshal config data: %s", err.Error())
}
f, err := ioutil.TempFile(filepath.Dir(filename), "config")
f, err := os.CreateTemp(filepath.Dir(filename), "config")
if err != nil {
return fmt.Errorf("cannot create empty config file %s: %s", filename, err.Error())
}
@ -404,20 +423,19 @@ func NullConfig(outDir, soongOutDir string) Config {
// NewConfig creates a new Config object. The srcDir argument specifies the path
// to the root source directory. It also loads the config file, if found.
func NewConfig(moduleListFile string, buildMode SoongBuildMode, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string,
bazelForceEnabledModules []string) (Config, error) {
func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) {
// Make a config with default options.
config := &config{
ProductVariablesFileName: filepath.Join(soongOutDir, productVariablesFileName),
ProductVariablesFileName: filepath.Join(cmdArgs.SoongOutDir, productVariablesFileName),
env: availableEnv,
outDir: outDir,
soongOutDir: soongOutDir,
runGoTests: runGoTests,
outDir: cmdArgs.OutDir,
soongOutDir: cmdArgs.SoongOutDir,
runGoTests: cmdArgs.RunGoTests,
multilibConflicts: make(map[ArchType]bool),
moduleListFile: moduleListFile,
moduleListFile: cmdArgs.ModuleListFile,
fs: pathtools.NewOsFs(absSrcDir),
mixedBuildDisabledModules: make(map[string]struct{}),
mixedBuildEnabledModules: make(map[string]struct{}),
@ -430,7 +448,7 @@ func NewConfig(moduleListFile string, buildMode SoongBuildMode, runGoTests bool,
// Soundness check of the build and source directories. This won't catch strange
// configurations with symlinks, but at least checks the obvious case.
absBuildDir, err := filepath.Abs(soongOutDir)
absBuildDir, err := filepath.Abs(cmdArgs.SoongOutDir)
if err != nil {
return Config{}, err
}
@ -450,7 +468,7 @@ func NewConfig(moduleListFile string, buildMode SoongBuildMode, runGoTests bool,
return Config{}, err
}
KatiEnabledMarkerFile := filepath.Join(soongOutDir, ".soong.kati_enabled")
KatiEnabledMarkerFile := filepath.Join(cmdArgs.SoongOutDir, ".soong.kati_enabled")
if _, err := os.Stat(absolutePath(KatiEnabledMarkerFile)); err == nil {
config.katiEnabled = true
}
@ -503,11 +521,32 @@ func NewConfig(moduleListFile string, buildMode SoongBuildMode, runGoTests bool,
config.AndroidFirstDeviceTarget = FirstTarget(config.Targets[Android], "lib64", "lib32")[0]
}
config.BuildMode = buildMode
if cmdArgs.SymlinkForestMarker != "" {
config.BuildMode = SymlinkForest
} else if cmdArgs.Bp2buildMarker != "" {
config.BuildMode = Bp2build
} else if cmdArgs.BazelQueryViewDir != "" {
config.BuildMode = GenerateQueryView
} else if cmdArgs.BazelApiBp2buildDir != "" {
config.BuildMode = ApiBp2build
} else if cmdArgs.ModuleGraphFile != "" {
config.BuildMode = GenerateModuleGraph
} else if cmdArgs.DocFile != "" {
config.BuildMode = GenerateDocFile
} else if cmdArgs.BazelModeDev {
config.BuildMode = BazelDevMode
} else if cmdArgs.BazelMode {
config.BuildMode = BazelProdMode
} else if cmdArgs.BazelModeStaging {
config.BuildMode = BazelStagingMode
} else {
config.BuildMode = AnalysisNoBazel
}
config.BazelContext, err = NewBazelContext(config)
config.Bp2buildPackageConfig = GetBp2BuildAllowList()
for _, module := range bazelForceEnabledModules {
for _, module := range strings.Split(cmdArgs.BazelForceEnabledModules, ",") {
config.bazelForceEnabledModules[module] = struct{}{}
}
@ -1150,7 +1189,7 @@ func (c *config) DexpreoptGlobalConfig(ctx PathContext) ([]byte, error) {
return nil, nil
}
ctx.AddNinjaFileDeps(path.String())
return ioutil.ReadFile(absolutePath(path.String()))
return os.ReadFile(absolutePath(path.String()))
}
func (c *deviceConfig) WithDexpreopt() bool {
@ -1169,7 +1208,7 @@ func (c *config) HasMultilibConflict(arch ArchType) bool {
return c.multilibConflicts[arch]
}
func (c *config) PrebuiltHiddenApiDir(ctx PathContext) string {
func (c *config) PrebuiltHiddenApiDir(_ PathContext) string {
return String(c.productVariables.PrebuiltHiddenApiDir)
}

View file

@ -18,7 +18,6 @@ import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
@ -38,38 +37,26 @@ import (
var (
topDir string
outDir string
soongOutDir string
availableEnvFile string
usedEnvFile string
runGoTests bool
globFile string
globListDir string
delveListen string
delvePath string
moduleGraphFile string
moduleActionsFile string
docFile string
bazelQueryViewDir string
bazelApiBp2buildDir string
bp2buildMarker string
symlinkForestMarker string
cmdlineArgs bootstrap.Args
cmdlineArgs android.CmdArgs
)
func init() {
// Flags that make sense in every mode
flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
flag.StringVar(&soongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
flag.StringVar(&cmdlineArgs.SoongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
flag.StringVar(&outDir, "out", "", "the ninja builddir directory")
flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
// Debug flags
@ -81,13 +68,13 @@ func init() {
flag.BoolVar(&cmdlineArgs.NoGC, "nogc", false, "turn off GC for debugging")
// Flags representing various modes soong_build can run in
flag.StringVar(&moduleGraphFile, "module_graph_file", "", "JSON module graph file to output")
flag.StringVar(&moduleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules")
flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
flag.StringVar(&bazelApiBp2buildDir, "bazel_api_bp2build_dir", "", "path to the bazel api_bp2build directory relative to --top")
flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
flag.StringVar(&symlinkForestMarker, "symlink_forest_marker", "", "If set, create the bp2build symlink forest, touch the specified marker file, then exit")
flag.StringVar(&cmdlineArgs.ModuleGraphFile, "module_graph_file", "", "JSON module graph file to output")
flag.StringVar(&cmdlineArgs.ModuleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules")
flag.StringVar(&cmdlineArgs.DocFile, "soong_docs", "", "build documentation file to output")
flag.StringVar(&cmdlineArgs.BazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
flag.StringVar(&cmdlineArgs.BazelApiBp2buildDir, "bazel_api_bp2build_dir", "", "path to the bazel api_bp2build directory relative to --top")
flag.StringVar(&cmdlineArgs.Bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
flag.StringVar(&cmdlineArgs.SymlinkForestMarker, "symlink_forest_marker", "", "If set, create the bp2build symlink forest, touch the specified marker file, then exit")
flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output")
flag.StringVar(&cmdlineArgs.BazelForceEnabledModules, "bazel-force-enabled-modules", "", "additional modules to build with Bazel. Comma-delimited")
flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file")
@ -95,9 +82,9 @@ func init() {
flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
flag.BoolVar(&cmdlineArgs.BazelModeDev, "bazel-mode-dev", false, "use bazel for analysis of a large number of modules (less stable)")
// Flags that probably shouldn't be flags of soong_build but we haven't found
// Flags that probably shouldn't be flags of soong_build, but we haven't found
// the time to remove them yet
flag.BoolVar(&runGoTests, "t", false, "build and run go tests during bootstrap")
flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
// Disable deterministic randomization in the protobuf package, so incremental
// builds with unrelated Soong changes don't trigger large rebuilds (since we
@ -118,43 +105,6 @@ func newContext(configuration android.Config) *android.Context {
return ctx
}
func newConfig(availableEnv map[string]string) android.Config {
var buildMode android.SoongBuildMode
var bazelForceEnabledModules []string
if len(cmdlineArgs.BazelForceEnabledModules) > 0 {
bazelForceEnabledModules = strings.Split(cmdlineArgs.BazelForceEnabledModules, ",")
}
if symlinkForestMarker != "" {
buildMode = android.SymlinkForest
} else if bp2buildMarker != "" {
buildMode = android.Bp2build
} else if bazelQueryViewDir != "" {
buildMode = android.GenerateQueryView
} else if bazelApiBp2buildDir != "" {
buildMode = android.ApiBp2build
} else if moduleGraphFile != "" {
buildMode = android.GenerateModuleGraph
} else if docFile != "" {
buildMode = android.GenerateDocFile
} else if cmdlineArgs.BazelModeDev {
buildMode = android.BazelDevMode
} else if cmdlineArgs.BazelMode {
buildMode = android.BazelProdMode
} else if cmdlineArgs.BazelModeStaging {
buildMode = android.BazelStagingMode
} else {
buildMode = android.AnalysisNoBazel
}
configuration, err := android.NewConfig(cmdlineArgs.ModuleListFile, buildMode, runGoTests, outDir, soongOutDir, availableEnv, bazelForceEnabledModules)
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
}
return configuration
}
// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a
// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata
// for modules that should be handled by Bazel.
@ -166,7 +116,7 @@ func runMixedModeBuild(configuration android.Config, ctx *android.Context, extra
return configuration.BazelContext.InvokeBazel(configuration, ctx)
}
ctx.SetBeforePrepareBuildActionsHook(bazelHook)
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, ctx.Context, configuration)
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, bootstrap.DoEverything, ctx.Context, configuration)
ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
bazelPaths, err := readBazelPaths(configuration)
@ -215,7 +165,7 @@ func runApiBp2build(configuration android.Config, ctx *android.Context, extraNin
}
// Run the loading and analysis phase
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs,
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args,
bootstrap.StopBeforePrepareBuildActions,
ctx.Context,
configuration)
@ -227,7 +177,7 @@ func runApiBp2build(configuration android.Config, ctx *android.Context, extraNin
// Run codegen to generate BUILD files
codegenContext := bp2build.NewCodegenContext(configuration, ctx, bp2build.ApiBp2build)
absoluteApiBp2buildDir := shared.JoinPath(topDir, bazelApiBp2buildDir)
absoluteApiBp2buildDir := shared.JoinPath(topDir, cmdlineArgs.BazelApiBp2buildDir)
if err := createBazelWorkspace(codegenContext, absoluteApiBp2buildDir); err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
@ -254,7 +204,7 @@ func runApiBp2build(configuration android.Config, ctx *android.Context, extraNin
// Android.bp files for api surfaces are mounted to out/, but out/ should not be a
// dep for api_bp2build.
// Otherwise api_bp2build will be run every single time
// Otherwise, api_bp2build will be run every single time
excludes = append(excludes, configuration.OutDir())
// Create the symlink forest
@ -262,7 +212,7 @@ func runApiBp2build(configuration android.Config, ctx *android.Context, extraNin
configuration.IsEnvTrue("BP2BUILD_VERBOSE"),
topDir,
workspace,
bazelApiBp2buildDir,
cmdlineArgs.BazelApiBp2buildDir,
excludes)
ninjaDeps = append(ninjaDeps, symlinkDeps...)
@ -308,9 +258,9 @@ func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandl
}
}
func writeJsonModuleGraphAndActions(ctx *android.Context, graphPath string, actionsPath string) {
graphFile, graphErr := os.Create(shared.JoinPath(topDir, graphPath))
actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, actionsPath))
func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) {
graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile))
actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleActionsFile))
if graphErr != nil || actionsErr != nil {
fmt.Fprintf(os.Stderr, "Graph err: %s, actions err: %s", graphErr, actionsErr)
os.Exit(1)
@ -390,7 +340,7 @@ func runSoongOnlyBuild(configuration android.Config, ctx *android.Context, extra
stopBefore = bootstrap.DoEverything
}
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, stopBefore, ctx.Context, configuration)
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, configuration)
ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration)
@ -398,24 +348,24 @@ func runSoongOnlyBuild(configuration android.Config, ctx *android.Context, extra
// Convert the Soong module graph into Bazel BUILD files.
if configuration.BuildMode == android.GenerateQueryView {
queryviewMarkerFile := bazelQueryViewDir + ".marker"
runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx)
queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"
runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, configuration, ctx)
writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)
return queryviewMarkerFile
} else if configuration.BuildMode == android.GenerateModuleGraph {
writeJsonModuleGraphAndActions(ctx, moduleGraphFile, moduleActionsFile)
writeDepFile(moduleGraphFile, ctx.EventHandler, ninjaDeps)
return moduleGraphFile
writeJsonModuleGraphAndActions(ctx, cmdlineArgs)
writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)
return cmdlineArgs.ModuleGraphFile
} else if configuration.BuildMode == android.GenerateDocFile {
// TODO: we could make writeDocs() return the list of documentation files
// written and add them to the .d file. Then soong_docs would be re-run
// whenever one is deleted.
if err := writeDocs(ctx, shared.JoinPath(topDir, docFile)); err != nil {
if err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile)); err != nil {
fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err)
os.Exit(1)
}
writeDepFile(docFile, ctx.EventHandler, ninjaDeps)
return docFile
writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)
return cmdlineArgs.DocFile
} else {
// The actual output (build.ninja) was written in the RunBlueprint() call
// above
@ -459,17 +409,16 @@ func main() {
android.InitSandbox(topDir)
availableEnv := parseAvailableEnv()
configuration := newConfig(availableEnv)
extraNinjaDeps := []string{
configuration.ProductVariablesFileName,
usedEnvFile,
configuration, err := android.NewConfig(cmdlineArgs, availableEnv)
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
}
if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
configuration.SetAllowMissingDependencies()
}
extraNinjaDeps := []string{configuration.ProductVariablesFileName, usedEnvFile}
if shared.IsDebugging() {
// Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
// enabled even if it completed successfully.
@ -612,7 +561,7 @@ func getExistingBazelRelatedFiles(topDir string) ([]string, error) {
// Assume this was a relative path under topDir
bazelFinderFile = filepath.Join(topDir, bazelFinderFile)
}
data, err := ioutil.ReadFile(bazelFinderFile)
data, err := os.ReadFile(bazelFinderFile)
if err != nil {
return nil, err
}
@ -649,8 +598,8 @@ func runSymlinkForestCreation(configuration android.Config, ctx *android.Context
excludes := bazelArtifacts()
if outDir[0] != '/' {
excludes = append(excludes, outDir)
if cmdlineArgs.OutDir[0] != '/' {
excludes = append(excludes, cmdlineArgs.OutDir)
}
existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir)
@ -672,8 +621,8 @@ func runSymlinkForestCreation(configuration android.Config, ctx *android.Context
ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
})
writeDepFile(symlinkForestMarker, ctx.EventHandler, ninjaDeps)
touch(shared.JoinPath(topDir, symlinkForestMarker))
writeDepFile(cmdlineArgs.SymlinkForestMarker, ctx.EventHandler, ninjaDeps)
touch(shared.JoinPath(topDir, cmdlineArgs.SymlinkForestMarker))
})
codegenMetrics := bp2build.ReadCodegenMetrics(metricsDir)
if codegenMetrics == nil {
@ -685,7 +634,7 @@ func runSymlinkForestCreation(configuration android.Config, ctx *android.Context
}
writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
return symlinkForestMarker
return cmdlineArgs.SymlinkForestMarker
}
// Run Soong in the bp2build mode. This creates a standalone context that registers
@ -710,7 +659,7 @@ func runBp2Build(configuration android.Config, ctx *android.Context, extraNinjaD
// from the regular Modules.
ctx.EventHandler.Do("bootstrap", func() {
blueprintArgs := cmdlineArgs
bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.StopBeforePrepareBuildActions, ctx.Context, configuration)
bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs.Args, bootstrap.StopBeforePrepareBuildActions, ctx.Context, configuration)
ninjaDeps = append(ninjaDeps, bootstrapDeps...)
})
@ -726,8 +675,8 @@ func runBp2Build(configuration android.Config, ctx *android.Context, extraNinjaD
ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
writeDepFile(bp2buildMarker, ctx.EventHandler, ninjaDeps)
touch(shared.JoinPath(topDir, bp2buildMarker))
writeDepFile(cmdlineArgs.Bp2buildMarker, ctx.EventHandler, ninjaDeps)
touch(shared.JoinPath(topDir, cmdlineArgs.Bp2buildMarker))
})
// Only report metrics when in bp2build mode. The metrics aren't relevant
@ -737,7 +686,7 @@ func runBp2Build(configuration android.Config, ctx *android.Context, extraNinjaD
codegenMetrics.Print()
}
writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
return bp2buildMarker
return cmdlineArgs.Bp2buildMarker
}
// Write Bp2Build metrics into $LOG_DIR