tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Refactor mixed build allowlist handling"

Browse source
This commit is contained in:
Christopher Parsons 2022-08-23 17:21:16 +00:00 committed by Gerrit Code Review
commit 5a2a4814af
12 changed files with 144 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
android/bazel.go
View file

@ -231,16 +231,18 @@ type bp2BuildConversionAllowlist struct {
// when they have the same type as one listed.
moduleTypeAlwaysConvert map[string]bool
// Per-module denylist to always opt modules out of both bp2build and mixed builds.
// Per-module denylist to always opt modules out of bp2build conversion.
moduleDoNotConvert map[string]bool
// Per-module denylist of cc_library modules to only generate the static
// variant if their shared variant isn't ready or buildable by Bazel.
ccLibraryStaticOnly map[string]bool
}
// Per-module denylist to opt modules out of mixed builds. Such modules will
// still be generated via bp2build.
mixedBuildsDisabled map[string]bool
// GenerateCcLibraryStaticOnly returns whether a cc_library module should only
// generate a static version of itself based on the current global configuration.
func (a bp2BuildConversionAllowlist) GenerateCcLibraryStaticOnly(moduleName string) bool {
return a.ccLibraryStaticOnly[moduleName]
}
// NewBp2BuildAllowlist creates a new, empty bp2BuildConversionAllowlist
@ -253,7 +255,6 @@ func NewBp2BuildAllowlist() bp2BuildConversionAllowlist {
map[string]bool{},
map[string]bool{},
map[string]bool{},
map[string]bool{},
}
}
@ -329,43 +330,24 @@ func (a bp2BuildConversionAllowlist) SetCcLibraryStaticOnlyList(ccLibraryStaticO
return a
}
// SetMixedBuildsDisabledList copies the entries from mixedBuildsDisabled into the allowlist
func (a bp2BuildConversionAllowlist) SetMixedBuildsDisabledList(mixedBuildsDisabled []string) bp2BuildConversionAllowlist {
if a.mixedBuildsDisabled == nil {
a.mixedBuildsDisabled = map[string]bool{}
}
for _, m := range mixedBuildsDisabled {
a.mixedBuildsDisabled[m] = true
}
return a
}
var bp2BuildAllowListKey = NewOnceKey("Bp2BuildAllowlist")
var bp2buildAllowlist OncePer
func getBp2BuildAllowList() bp2BuildConversionAllowlist {
func GetBp2BuildAllowList() bp2BuildConversionAllowlist {
return bp2buildAllowlist.Once(bp2BuildAllowListKey, func() interface{} {
return NewBp2BuildAllowlist().SetDefaultConfig(allowlists.Bp2buildDefaultConfig).
SetKeepExistingBuildFile(allowlists.Bp2buildKeepExistingBuildFile).
SetModuleAlwaysConvertList(allowlists.Bp2buildModuleAlwaysConvertList).
SetModuleTypeAlwaysConvertList(allowlists.Bp2buildModuleTypeAlwaysConvertList).
SetModuleDoNotConvertList(allowlists.Bp2buildModuleDoNotConvertList).
SetCcLibraryStaticOnlyList(allowlists.Bp2buildCcLibraryStaticOnlyList).
SetMixedBuildsDisabledList(allowlists.MixedBuildsDisabledList)
SetCcLibraryStaticOnlyList(allowlists.Bp2buildCcLibraryStaticOnlyList)
}).(bp2BuildConversionAllowlist)
}
// GenerateCcLibraryStaticOnly returns whether a cc_library module should only
// generate a static version of itself based on the current global configuration.
func GenerateCcLibraryStaticOnly(moduleName string) bool {
return getBp2BuildAllowList().ccLibraryStaticOnly[moduleName]
}
// ShouldKeepExistingBuildFileForDir returns whether an existing BUILD file should be
// added to the build symlink forest based on the current global configuration.
func ShouldKeepExistingBuildFileForDir(dir string) bool {
return shouldKeepExistingBuildFileForDir(getBp2BuildAllowList(), dir)
return shouldKeepExistingBuildFileForDir(GetBp2BuildAllowList(), dir)
}
func shouldKeepExistingBuildFileForDir(allowlist bp2BuildConversionAllowlist, dir string) bool {
@ -405,20 +387,10 @@ func mixedBuildPossible(ctx BaseModuleContext) bool {
if !ctx.Module().Enabled() {
return false
}
if !ctx.Config().BazelContext.BazelEnabled() {
return false
}
if !convertedToBazel(ctx, ctx.Module()) {
return false
}
if GenerateCcLibraryStaticOnly(ctx.Module().Name()) {
// Don't use partially-converted cc_library targets in mixed builds,
// since mixed builds would generally rely on both static and shared
// variants of a cc_library.
return false
}
return !getBp2BuildAllowList().mixedBuildsDisabled[ctx.Module().Name()]
return ctx.Config().BazelContext.BazelAllowlisted(ctx.Module().Name())
}
// ConvertedToBazel returns whether this module has been converted (with bp2build or manually) to Bazel.

61
android/bazel_handler.go
View file

@ -27,6 +27,7 @@ import (
"strings"
"sync"
"android/soong/android/allowlists"
"android/soong/bazel/cquery"
"android/soong/shared"
@ -142,8 +143,11 @@ type BazelContext interface {
// queued in the BazelContext.
InvokeBazel(config Config) error
// Returns true if bazel is enabled for the given configuration.
BazelEnabled() bool
// Returns true if Bazel handling is enabled for the module with the given name.
// Note that this only implies "bazel mixed build" allowlisting. The caller
// should independently verify the module is eligible for Bazel handling
// (for example, that it is MixedBuildBuildable).
BazelAllowlisted(moduleName string) bool
// Returns the bazel output base (the root directory for all bazel intermediate outputs).
OutputBase() string
@ -183,6 +187,17 @@ type bazelContext struct {
// Depsets which should be used for Bazel's build statements.
depsets []bazel.AqueryDepset
// Per-module allowlist/denylist functionality to control whether analysis of
// modules are handled by Bazel. For modules which do not have a Bazel definition
// (or do not sufficiently support bazel handling via MixedBuildBuildable),
// this allowlist will have no effect, even if the module is explicitly allowlisted here.
// Per-module denylist to opt modules out of bazel handling.
bazelDisabledModules map[string]bool
// Per-module allowlist to opt modules in to bazel handling.
bazelEnabledModules map[string]bool
// If true, modules are bazel-enabled by default, unless present in bazelDisabledModules.
modulesDefaultToBazel bool
}
var _ BazelContext = &bazelContext{}
@ -229,7 +244,7 @@ func (m MockBazelContext) InvokeBazel(_ Config) error {
panic("unimplemented")
}
func (m MockBazelContext) BazelEnabled() bool {
func (m MockBazelContext) BazelAllowlisted(moduleName string) bool {
return true
}
@ -315,7 +330,7 @@ func (m noopBazelContext) OutputBase() string {
return ""
}
func (n noopBazelContext) BazelEnabled() bool {
func (n noopBazelContext) BazelAllowlisted(moduleName string) bool {
return false
}
@ -328,9 +343,7 @@ func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset {
}
func NewBazelContext(c *config) (BazelContext, error) {
// TODO(cparsons): Assess USE_BAZEL=1 instead once "mixed Soong/Bazel builds"
// are production ready.
if !c.IsEnvTrue("USE_BAZEL_ANALYSIS") {
if !c.IsMixedBuildsEnabled() {
return noopBazelContext{}, nil
}
@ -338,10 +351,26 @@ func NewBazelContext(c *config) (BazelContext, error) {
if err != nil {
return nil, err
}
// TODO(cparsons): Use a different allowlist depending on prod vs. dev
// bazel mode.
disabledModules := map[string]bool{}
// Don't use partially-converted cc_library targets in mixed builds,
// since mixed builds would generally rely on both static and shared
// variants of a cc_library.
for staticOnlyModule, _ := range GetBp2BuildAllowList().ccLibraryStaticOnly {
disabledModules[staticOnlyModule] = true
}
for _, disabledDevModule := range allowlists.MixedBuildsDisabledList {
disabledModules[disabledDevModule] = true
}
return &bazelContext{
bazelRunner: &builtinBazelRunner{},
paths: p,
requests: make(map[cqueryKey]bool),
bazelRunner: &builtinBazelRunner{},
paths: p,
requests: make(map[cqueryKey]bool),
modulesDefaultToBazel: true,
bazelDisabledModules: disabledModules,
}, nil
}
@ -386,8 +415,14 @@ func (p *bazelPaths) BazelMetricsDir() string {
return p.metricsDir
}
func (context *bazelContext) BazelEnabled() bool {
return true
func (context *bazelContext) BazelAllowlisted(moduleName string) bool {
if context.bazelDisabledModules[moduleName] {
return false
}
if context.bazelEnabledModules[moduleName] {
return true
}
return context.modulesDefaultToBazel
}
func pwdPrefix() string {
@ -851,7 +886,7 @@ type bazelSingleton struct{}
func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
// bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled.
if !ctx.Config().BazelContext.BazelEnabled() {
if !ctx.Config().IsMixedBuildsEnabled() {
return
}

7
android/bazel_test.go
View file

@ -389,7 +389,7 @@ func TestBp2BuildAllowlist(t *testing.T) {
}
func TestBp2buildAllowList(t *testing.T) {
allowlist := getBp2BuildAllowList()
allowlist := GetBp2BuildAllowList()
for k, v := range allowlists.Bp2buildDefaultConfig {
if allowlist.defaultConfig[k] != v {
t.Errorf("bp2build default config of %s: expected: %v, got: %v", k, v, allowlist.defaultConfig[k])
@ -415,9 +415,4 @@ func TestBp2buildAllowList(t *testing.T) {
t.Errorf("bp2build cc library static only of %s: expected: true, got: %v", k, allowlist.ccLibraryStaticOnly[k])
}
}
for _, k := range allowlists.MixedBuildsDisabledList {
if !allowlist.mixedBuildsDisabled[k] {
t.Errorf("bp2build mix build disabled of %s: expected: true, got: %v", k, allowlist.mixedBuildsDisabled[k])
}
}
}

59
android/config.go
View file

@ -68,6 +68,38 @@ type Config struct {
*config
}
type SoongBuildMode int
// Build modes that soong_build can run as.
const (
// Don't use bazel at all during module analysis.
AnalysisNoBazel SoongBuildMode = iota
// Bp2build mode: Generate BUILD files from blueprint files and exit.
Bp2build
// Generate BUILD files which faithfully represent the dependency graph of
// blueprint modules. Individual BUILD targets will not, however, faitfhully
// express build semantics.
GenerateQueryView
// Create a JSON representation of the module graph and exit.
GenerateModuleGraph
// Generate a documentation file for module type definitions and exit.
GenerateDocFile
// Use bazel during analysis of many allowlisted build modules. The allowlist
// is considered a "developer mode" allowlist, as some modules may be
// allowlisted on an experimental basis.
BazelDevMode
// Use bazel during analysis of build modules from an allowlist carefully
// curated by the build team to be proven stable.
// TODO(cparsons): Implement this mode.
BazelProdMode
)
// SoongOutDir returns the build output directory for the configuration.
func (c Config) SoongOutDir() string {
return c.soongOutDir
@ -157,7 +189,7 @@ type config struct {
fs pathtools.FileSystem
mockBpList string
runningAsBp2Build bool
BuildMode SoongBuildMode
bp2buildPackageConfig bp2BuildConversionAllowlist
Bp2buildSoongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions
@ -171,6 +203,12 @@ type config struct {
OncePer
// These fields are only used for metrics collection. A module should be added
// to these maps only if its implementation supports Bazel handling in mixed
// builds. A module being in the "enabled" list indicates that there is a
// variant of that module for which bazel-handling actually took place.
// A module being in the "disabled" list indicates that there is a variant of
// that module for which bazel-handling was denied.
mixedBuildsLock sync.Mutex
mixedBuildEnabledModules map[string]struct{}
mixedBuildDisabledModules map[string]struct{}
@ -346,7 +384,7 @@ func NullConfig(outDir, soongOutDir string) Config {
// NewConfig creates a new Config object. The srcDir argument specifies the path
// to the root source directory. It also loads the config file, if found.
func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string) (Config, error) {
func NewConfig(moduleListFile string, buildMode SoongBuildMode, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string) (Config, error) {
// Make a config with default options.
config := &config{
ProductVariablesFileName: filepath.Join(soongOutDir, productVariablesFileName),
@ -443,8 +481,17 @@ func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir strin
config.AndroidFirstDeviceTarget = FirstTarget(config.Targets[Android], "lib64", "lib32")[0]
}
// Checking USE_BAZEL_ANALYSIS must be done here instead of in the caller, so
// that we can invoke IsEnvTrue (which also registers the env var as a
// dependency of the build).
// TODO(cparsons): Remove this hack once USE_BAZEL_ANALYSIS is removed.
if buildMode == AnalysisNoBazel && config.IsEnvTrue("USE_BAZEL_ANALYSIS") {
buildMode = BazelDevMode
}
config.BuildMode = buildMode
config.BazelContext, err = NewBazelContext(config)
config.bp2buildPackageConfig = getBp2BuildAllowList()
config.bp2buildPackageConfig = GetBp2BuildAllowList()
return Config{config}, err
}
@ -479,6 +526,12 @@ func (c *config) mockFileSystem(bp string, fs map[string][]byte) {
c.mockBpList = blueprint.MockModuleListFile
}
// Returns true if "Bazel builds" is enabled. In this mode, part of build
// analysis is handled by Bazel.
func (c *config) IsMixedBuildsEnabled() bool {
return c.BuildMode == BazelProdMode || c.BuildMode == BazelDevMode
}
func (c *config) SetAllowMissingDependencies() {
c.productVariables.Allow_missing_dependencies = proptools.BoolPtr(true)
}

2
android/defaults.go
View file

@ -448,7 +448,7 @@ func (defaultable *DefaultableModuleBase) applyDefaults(ctx TopDownMutatorContex
}
for _, defaults := range defaultsList {
if ctx.Config().runningAsBp2Build {
if ctx.Config().BuildMode == Bp2build {
applyNamespacedVariableDefaults(defaults, ctx)
}

3
android/module.go
View file

@ -2367,9 +2367,6 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
}
func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) {
if !ctx.Config().BazelContext.BazelEnabled() {
return nil, false
}
if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok {
if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) {
return mixedBuildMod, true

4
android/register.go
View file

@ -164,10 +164,6 @@ func NewContext(config Config) *Context {
return ctx
}
func (ctx *Context) SetRunningAsBp2build() {
ctx.config.runningAsBp2Build = true
}
// RegisterForBazelConversion registers an alternate shadow pipeline of
// singletons, module types and mutators to register for converting Blueprint
// files to semantically equivalent BUILD files.

4
android/soong_config_modules.go
View file

@ -382,7 +382,7 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s
defer r.Close()
mtDef, errs := soongconfig.Parse(r, from)
if ctx.Config().runningAsBp2Build {
if ctx.Config().BuildMode == Bp2build {
ctx.Config().Bp2buildSoongConfigDefinitions.AddVars(*mtDef)
}
@ -398,7 +398,7 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s
for name, moduleType := range mtDef.ModuleTypes {
factory := globalModuleTypes[moduleType.BaseModuleType]
if factory != nil {
factories[name] = configModuleFactory(factory, moduleType, ctx.Config().runningAsBp2Build)
factories[name] = configModuleFactory(factory, moduleType, ctx.Config().BuildMode == Bp2build)
} else {
reportErrors(ctx, from,
fmt.Errorf("missing global module type factory for %q", moduleType.BaseModuleType))

2
android/testing.go
View file

@ -457,7 +457,7 @@ func (ctx *TestContext) Register() {
// RegisterForBazelConversion prepares a test context for bp2build conversion.
func (ctx *TestContext) RegisterForBazelConversion() {
ctx.SetRunningAsBp2build()
ctx.config.BuildMode = Bp2build
RegisterMutatorsForBazelConversion(ctx.Context, ctx.bp2buildPreArch)
}

2
cc/bp2build.go
View file

@ -983,7 +983,7 @@ func bp2BuildParseExportedIncludes(ctx android.BazelConversionPathContext, modul
func bazelLabelForStaticModule(ctx android.BazelConversionPathContext, m blueprint.Module) string {
label := android.BazelModuleLabel(ctx, m)
if ccModule, ok := m.(*Module); ok && ccModule.typ() == fullLibrary && !android.GenerateCcLibraryStaticOnly(m.Name()) {
if ccModule, ok := m.(*Module); ok && ccModule.typ() == fullLibrary && !android.GetBp2BuildAllowList().GenerateCcLibraryStaticOnly(m.Name()) {
label += "_bp2build_cc_library_static"
}
return label

2
cc/library.go
View file

@ -282,7 +282,7 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) {
// For some cc_library modules, their static variants are ready to be
// converted, but not their shared variants. For these modules, delegate to
// the cc_library_static bp2build converter temporarily instead.
if android.GenerateCcLibraryStaticOnly(ctx.Module().Name()) {
if android.GetBp2BuildAllowList().GenerateCcLibraryStaticOnly(ctx.Module().Name()) {
sharedOrStaticLibraryBp2Build(ctx, m, true)
return
}

49
cmd/soong_build/main.go
View file

@ -121,7 +121,21 @@ func newContext(configuration android.Config) *android.Context {
}
func newConfig(availableEnv map[string]string) android.Config {
configuration, err := android.NewConfig(cmdlineArgs.ModuleListFile, runGoTests, outDir, soongOutDir, availableEnv)
var buildMode android.SoongBuildMode
if bp2buildMarker != "" {
buildMode = android.Bp2build
} else if bazelQueryViewDir != "" {
buildMode = android.GenerateQueryView
} else if moduleGraphFile != "" {
buildMode = android.GenerateModuleGraph
} else if docFile != "" {
buildMode = android.GenerateDocFile
} else {
buildMode = android.AnalysisNoBazel
}
configuration, err := android.NewConfig(cmdlineArgs.ModuleListFile, buildMode, runGoTests, outDir, soongOutDir, availableEnv)
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
@ -221,52 +235,40 @@ func writeDepFile(outputFile string, eventHandler metrics.EventHandler, ninjaDep
// or the actual Soong build for the build.ninja file. Returns the top level
// output file of the specific activity.
func doChosenActivity(ctx *android.Context, configuration android.Config, extraNinjaDeps []string, logDir string) string {
mixedModeBuild := configuration.BazelContext.BazelEnabled()
generateBazelWorkspace := bp2buildMarker != ""
generateQueryView := bazelQueryViewDir != ""
generateModuleGraphFile := moduleGraphFile != ""
generateDocFile := docFile != ""
if generateBazelWorkspace {
if configuration.BuildMode == android.Bp2build {
// Run the alternate pipeline of bp2build mutators and singleton to convert
// Blueprint to BUILD files before everything else.
runBp2Build(configuration, extraNinjaDeps)
return bp2buildMarker
}
blueprintArgs := cmdlineArgs
if mixedModeBuild {
} else if configuration.IsMixedBuildsEnabled() {
runMixedModeBuild(configuration, ctx, extraNinjaDeps)
} else {
var stopBefore bootstrap.StopBefore
if generateModuleGraphFile {
if configuration.BuildMode == android.GenerateModuleGraph {
stopBefore = bootstrap.StopBeforeWriteNinja
} else if generateQueryView {
stopBefore = bootstrap.StopBeforePrepareBuildActions
} else if generateDocFile {
} else if configuration.BuildMode == android.GenerateQueryView || configuration.BuildMode == android.GenerateDocFile {
stopBefore = bootstrap.StopBeforePrepareBuildActions
} else {
stopBefore = bootstrap.DoEverything
}
ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, stopBefore, ctx.Context, configuration)
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, stopBefore, ctx.Context, configuration)
ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration)
ninjaDeps = append(ninjaDeps, globListFiles...)
// Convert the Soong module graph into Bazel BUILD files.
if generateQueryView {
if configuration.BuildMode == android.GenerateQueryView {
queryviewMarkerFile := bazelQueryViewDir + ".marker"
runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx)
writeDepFile(queryviewMarkerFile, *ctx.EventHandler, ninjaDeps)
return queryviewMarkerFile
} else if generateModuleGraphFile {
} else if configuration.BuildMode == android.GenerateModuleGraph {
writeJsonModuleGraphAndActions(ctx, moduleGraphFile, moduleActionsFile)
writeDepFile(moduleGraphFile, *ctx.EventHandler, ninjaDeps)
return moduleGraphFile
} else if generateDocFile {
} else if configuration.BuildMode == android.GenerateDocFile {
// TODO: we could make writeDocs() return the list of documentation files
// written and add them to the .d file. Then soong_docs would be re-run
// whenever one is deleted.
@ -491,11 +493,6 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
// conversion for Bazel conversion.
bp2buildCtx := android.NewContext(configuration)
// Soong internals like LoadHooks behave differently when running as
// bp2build. This is the bit to differentiate between Soong-as-Soong and
// Soong-as-bp2build.
bp2buildCtx.SetRunningAsBp2build()
// Propagate "allow misssing dependencies" bit. This is normally set in
// newContext(), but we create bp2buildCtx without calling that method.
bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())