Merge "Bp2build support for multiple product configs"

This commit is contained in:
Cole Faust 2023-01-07 03:09:08 +00:00 committed by Gerrit Code Review
commit 501a7976be
13 changed files with 222 additions and 45 deletions

View file

@ -205,6 +205,9 @@ type bazelContext struct {
bazelEnabledModules map[string]bool
// If true, modules are bazel-enabled by default, unless present in bazelDisabledModules.
modulesDefaultToBazel bool
targetProduct string
targetBuildVariant string
}
var _ BazelContext = &bazelContext{}
@ -460,6 +463,18 @@ func NewBazelContext(c *config) (BazelContext, error) {
if len(missing) > 0 {
return nil, fmt.Errorf("missing required env vars to use bazel: %s", missing)
}
targetBuildVariant := "user"
if c.Eng() {
targetBuildVariant = "eng"
} else if c.Debuggable() {
targetBuildVariant = "userdebug"
}
targetProduct := "unknown"
if c.HasDeviceProduct() {
targetProduct = c.DeviceProduct()
}
return &bazelContext{
bazelRunner: &builtinBazelRunner{},
paths: &paths,
@ -467,6 +482,8 @@ func NewBazelContext(c *config) (BazelContext, error) {
modulesDefaultToBazel: c.BuildMode == BazelDevMode,
bazelEnabledModules: enabledModules,
bazelDisabledModules: disabledModules,
targetProduct: targetProduct,
targetBuildVariant: targetBuildVariant,
}, nil
}
@ -563,9 +580,9 @@ func (r *builtinBazelRunner) createBazelCommand(paths *bazelPaths, runName bazel
// The actual platform values here may be overridden by configuration
// transitions from the buildroot.
fmt.Sprintf("--extra_toolchains=%s", "//prebuilts/clang/host/linux-x86:all"),
// This should be parameterized on the host OS, but let's restrict to linux
// to keep things simple for now.
fmt.Sprintf("--host_platform=%s", "//build/bazel/platforms:linux_x86_64"),
// We don't need to set --host_platforms because it's set in bazelrc files
// that the bazel shell script wrapper passes
// Explicitly disable downloading rules (such as canonical C++ and Java rules) from the network.
"--experimental_repository_disable_download",
@ -610,8 +627,12 @@ func (context *bazelContext) mainBzlFileContents() []byte {
#####################################################
def _config_node_transition_impl(settings, attr):
if attr.os == "android" and attr.arch == "target":
target = "{PRODUCT}-{VARIANT}"
else:
target = "{PRODUCT}-{VARIANT}_%s_%s" % (attr.os, attr.arch)
return {
"//command_line_option:platforms": "@//build/bazel/platforms:%s_%s" % (attr.os, attr.arch),
"//command_line_option:platforms": "@soong_injection//product_config_platforms/products/{PRODUCT}-{VARIANT}:%s" % target,
}
_config_node_transition = transition(
@ -658,7 +679,12 @@ phony_root = rule(
attrs = {"deps" : attr.label_list()},
)
`
return []byte(contents)
productReplacer := strings.NewReplacer(
"{PRODUCT}", context.targetProduct,
"{VARIANT}", context.targetBuildVariant)
return []byte(productReplacer.Replace(contents))
}
func (context *bazelContext) mainBuildFileContents() []byte {
@ -780,26 +806,24 @@ def json_encode(input):
t = type(p)
if t == "string" or t == "int":
return repr(p)
fail("unsupported value '%%s' of type '%%s'" %% (p, type(p)))
fail("unsupported value '%s' of type '%s'" % (p, type(p)))
def encode_list(list):
return "[%%s]" %% ", ".join([encode_primitive(item) for item in list])
return "[%s]" % ", ".join([encode_primitive(item) for item in list])
def encode_list_or_primitive(v):
return encode_list(v) if type(v) == "list" else encode_primitive(v)
if type(input) == "dict":
# TODO(juu): the result is read line by line so can't use '\n' yet
kv_pairs = [("%%s: %%s" %% (encode_primitive(k), encode_list_or_primitive(v))) for (k, v) in input.items()]
return "{ %%s }" %% ", ".join(kv_pairs)
kv_pairs = [("%s: %s" % (encode_primitive(k), encode_list_or_primitive(v))) for (k, v) in input.items()]
return "{ %s }" % ", ".join(kv_pairs)
else:
return encode_list_or_primitive(input)
# Label Map Section
%s
{LABEL_REGISTRATION_MAP_SECTION}
# Function Def Section
%s
{FUNCTION_DEF_SECTION}
def get_arch(target):
# TODO(b/199363072): filegroups and file targets aren't associated with any
@ -811,22 +835,26 @@ def get_arch(target):
# File targets do not have buildoptions. File targets aren't associated with
# any specific platform architecture in mixed builds, so use the host.
return "x86_64|linux"
platforms = build_options(target)["//command_line_option:platforms"]
platforms = buildoptions["//command_line_option:platforms"]
if len(platforms) != 1:
# An individual configured target should have only one platform architecture.
# Note that it's fine for there to be multiple architectures for the same label,
# but each is its own configured target.
fail("expected exactly 1 platform for " + str(target.label) + " but got " + str(platforms))
platform_name = build_options(target)["//command_line_option:platforms"][0].name
platform_name = platforms[0].name
if platform_name == "host":
return "HOST"
if not platform_name.startswith("{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}"):
fail("expected platform name of the form '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_android_<arch>' or '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
platform_name = platform_name.removeprefix("{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}").removeprefix("_")
if not platform_name:
return "target|android"
elif platform_name.startswith("android_"):
return platform_name[len("android_"):] + "|" + platform_name[:len("android_")-1]
return platform_name.removeprefix("android_") + "|android"
elif platform_name.startswith("linux_"):
return platform_name[len("linux_"):] + "|" + platform_name[:len("linux_")-1]
return platform_name.removeprefix("linux_") + "|linux"
else:
fail("expected platform name of the form 'android_<arch>' or 'linux_<arch>', but was " + str(platforms))
return "UNKNOWN"
fail("expected platform name of the form '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_android_<arch>' or '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
def format(target):
id_string = str(target.label) + "|" + get_arch(target)
@ -835,15 +863,20 @@ def format(target):
if id_string.startswith("//"):
id_string = "@" + id_string
# Main switch section
%s
{MAIN_SWITCH_SECTION}
# This target was not requested via cquery, and thus must be a dependency
# of a requested target.
return id_string + ">>NONE"
`
replacer := strings.NewReplacer(
"{TARGET_PRODUCT}", context.targetProduct,
"{TARGET_BUILD_VARIANT}", context.targetBuildVariant,
"{LABEL_REGISTRATION_MAP_SECTION}", labelRegistrationMapSection,
"{FUNCTION_DEF_SECTION}", functionDefSection,
"{MAIN_SWITCH_SECTION}", mainSwitchSection)
return []byte(fmt.Sprintf(formatString, labelRegistrationMapSection, functionDefSection,
mainSwitchSection))
return []byte(replacer.Replace(formatString))
}
// Returns a path containing build-related metadata required for interfacing

View file

@ -769,6 +769,13 @@ func (c *config) DeviceProduct() string {
return *c.productVariables.DeviceProduct
}
// HasDeviceProduct returns if the build has a product. A build will not
// necessarily have a product when --skip-config is passed to soong, like it is
// in prebuilts/build-tools/build-prebuilts.sh
func (c *config) HasDeviceProduct() bool {
return c.productVariables.DeviceProduct != nil
}
func (c *config) DeviceResourceOverlays() []string {
return c.productVariables.DeviceResourceOverlays
}

View file

@ -8,6 +8,7 @@ bootstrap_go_package {
srcs: [
"androidbp_to_build_templates.go",
"bp2build.go",
"bp2build_product_config.go",
"build_conversion.go",
"bzl_conversion.go",
"configurability.go",

View file

@ -45,7 +45,14 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics {
bp2buildFiles := CreateBazelFiles(ctx.Config(), nil, res.buildFileToTargets, ctx.mode)
writeFiles(ctx, bp2buildDir, bp2buildFiles)
productConfigFiles, err := CreateProductConfigFiles(ctx)
if err != nil {
fmt.Printf("ERROR: %s", err.Error())
os.Exit(1)
}
soongInjectionDir := android.PathForOutput(ctx, bazel.SoongInjectionDirName)
writeFiles(ctx, soongInjectionDir, productConfigFiles)
writeFiles(ctx, soongInjectionDir, CreateSoongInjectionFiles(ctx.Config(), res.metrics))
return &res.metrics

View file

@ -0,0 +1,124 @@
package bp2build
import (
"fmt"
"os"
"path/filepath"
"strings"
)
func CreateProductConfigFiles(
ctx *CodegenContext) ([]BazelFile, error) {
cfg := &ctx.config
targetProduct := "unknown"
if cfg.HasDeviceProduct() {
targetProduct = cfg.DeviceProduct()
}
targetBuildVariant := "user"
if cfg.Eng() {
targetBuildVariant = "eng"
} else if cfg.Debuggable() {
targetBuildVariant = "userdebug"
}
productVariablesFileName := cfg.ProductVariablesFileName
if !strings.HasPrefix(productVariablesFileName, "/") {
productVariablesFileName = filepath.Join(ctx.topDir, productVariablesFileName)
}
bytes, err := os.ReadFile(productVariablesFileName)
if err != nil {
return nil, err
}
// TODO(b/249685973): the name is product_config_platforms because product_config
// was already used for other files. Deduplicate them.
currentProductFolder := fmt.Sprintf("product_config_platforms/products/%s-%s", targetProduct, targetBuildVariant)
productReplacer := strings.NewReplacer(
"{PRODUCT}", targetProduct,
"{VARIANT}", targetBuildVariant,
"{PRODUCT_FOLDER}", currentProductFolder)
result := []BazelFile{
newFile(
currentProductFolder,
"soong.variables.bzl",
`variables = json.decode("""`+strings.ReplaceAll(string(bytes), "\\", "\\\\")+`""")`),
newFile(
currentProductFolder,
"BUILD",
productReplacer.Replace(`
package(default_visibility=[
"@soong_injection//product_config_platforms:__subpackages__",
"@//build/bazel/product_config:__subpackages__",
])
load(":soong.variables.bzl", _soong_variables = "variables")
load("@//build/bazel/product_config:utils.bzl", "android_product")
android_product(
name = "{PRODUCT}-{VARIANT}",
soong_variables = _soong_variables,
)
`)),
newFile(
"product_config_platforms",
"BUILD.bazel",
productReplacer.Replace(`
package(default_visibility = [
"@//build/bazel/product_config:__subpackages__",
"@soong_injection//product_config_platforms:__subpackages__",
])
# TODO(b/249685973): Remove this. It was only added for a platform_mappings file,
# which can possibly be replaced with autogenerating the platform_mappings file,
# or removing that file entirely.
alias(
name = "current_android_platform",
# TODO: When we start generating the platforms for more than just the
# currently lunched, product, turn this into a select with an arm for each product.
actual = "@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}",
)
alias(
name = "product_vars",
actual = select({
# TODO: When we start generating the platforms for more than just the
# currently lunched, product, this select should have an arm for each product.
"@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_constraint_value": "@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_product_vars",
}),
)
`)),
newFile(
"product_config_platforms",
"common.bazelrc",
productReplacer.Replace(`
build --platforms @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_linux_x86_64
build:android --platforms=@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}
build:linux_x86_64 --platforms=@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_linux_x86_64
build:linux_bionic_x86_64 --platforms=@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_linux_bionic_x86_64
`)),
newFile(
"product_config_platforms",
"linux.bazelrc",
productReplacer.Replace(`
build --host_platform @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_linux_x86_64
`)),
newFile(
"product_config_platforms",
"darwin.bazelrc",
productReplacer.Replace(`
build --host_platform @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_darwin_x86_64
`)),
newFile(
"product_config_platforms",
"platform_mappings",
productReplacer.Replace(`
flags:
--cpu=k8
@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}
`)),
}
return result, nil
}

View file

@ -140,6 +140,7 @@ type CodegenContext struct {
mode CodegenMode
additionalDeps []string
unconvertedDepMode unconvertedDepsMode
topDir string
}
func (ctx *CodegenContext) Mode() CodegenMode {
@ -208,7 +209,7 @@ func (ctx *CodegenContext) Context() *android.Context { return ctx.context }
// NewCodegenContext creates a wrapper context that conforms to PathContext for
// writing BUILD files in the output directory.
func NewCodegenContext(config android.Config, context *android.Context, mode CodegenMode) *CodegenContext {
func NewCodegenContext(config android.Config, context *android.Context, mode CodegenMode, topDir string) *CodegenContext {
var unconvertedDeps unconvertedDepsMode
if config.IsEnvTrue("BP2BUILD_ERROR_UNCONVERTED") {
unconvertedDeps = errorModulesUnconvertedDeps
@ -218,6 +219,7 @@ func NewCodegenContext(config android.Config, context *android.Context, mode Cod
config: config,
mode: mode,
unconvertedDepMode: unconvertedDeps,
topDir: topDir,
}
}

View file

@ -209,7 +209,7 @@ func TestGenerateSoongModuleTargets(t *testing.T) {
_, errs = ctx.PrepareBuildActions(config)
android.FailIfErrored(t, errs)
codegenCtx := NewCodegenContext(config, ctx.Context, QueryView)
codegenCtx := NewCodegenContext(config, ctx.Context, QueryView, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, dir)
android.FailIfErrored(t, err)
if actualCount, expectedCount := len(bazelTargets), 1; actualCount != expectedCount {
@ -530,7 +530,7 @@ custom {
return
}
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, dir)
android.FailIfErrored(t, err)
@ -903,7 +903,7 @@ load("//build/bazel/rules:rules.bzl", "my_library")`,
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, dir)
android.FailIfErrored(t, err)
if actualCount := len(bazelTargets); actualCount != testCase.expectedBazelTargetCount {
@ -1156,7 +1156,7 @@ func TestAllowlistingBp2buildTargetsExplicitly(t *testing.T) {
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, dir)
android.FailIfErrored(t, err)
if actualCount := len(bazelTargets); actualCount != testCase.expectedCount {
@ -1263,7 +1263,7 @@ filegroup { name: "opt-out-h", bazel_module: { bp2build_available: false } }
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
// For each directory, test that the expected number of generated targets is correct.
for dir, expectedCount := range testCase.expectedCount {
@ -1398,7 +1398,7 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) {
if testCase.Dir != "" {
checkDir = testCase.Dir
}
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, checkDir)
android.FailIfErrored(t, err)
bazelTargets.sort()

View file

@ -106,7 +106,7 @@ func setup(builddir string, tcSize float64) testConfig {
ctx := android.NewTestContext(config)
registerCustomModuleForBp2buildConversion(ctx)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
return testConfig{
config,
ctx,

View file

@ -200,7 +200,7 @@ func (b *bazelTestRunner) PostParseProcessor(result android.CustomTestResult) {
return
}
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
res, errs := GenerateBazelTargets(codegenCtx, false)
if bazelResult.CollateErrs(errs) {
return

View file

@ -134,7 +134,7 @@ func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string {
func runQueryView(queryviewDir, queryviewMarker string, ctx *android.Context) {
ctx.EventHandler.Begin("queryview")
defer ctx.EventHandler.End("queryview")
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.QueryView)
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.QueryView, topDir)
err := createBazelWorkspace(codegenContext, shared.JoinPath(topDir, queryviewDir))
maybeQuit(err, "")
touch(shared.JoinPath(topDir, queryviewMarker))
@ -169,7 +169,7 @@ func runApiBp2build(ctx *android.Context, extraNinjaDeps []string) string {
ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
// Run codegen to generate BUILD files
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.ApiBp2build)
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.ApiBp2build, topDir)
absoluteApiBp2buildDir := shared.JoinPath(topDir, cmdlineArgs.BazelApiBp2buildDir)
err := createBazelWorkspace(codegenContext, absoluteApiBp2buildDir)
maybeQuit(err, "")
@ -586,7 +586,7 @@ func runBp2Build(ctx *android.Context, extraNinjaDeps []string, metricsDir strin
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion codegenMetrics to the user.
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build)
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build, topDir)
ctx.EventHandler.Do("codegen", func() {
codegenMetrics = bp2build.Codegen(codegenContext)
})

View file

@ -67,23 +67,26 @@ call_bazel build --config=bp2build --config=ci --config=android \
//packages/modules/adb/apex:com.android.adbd \
//system/timezone/apex:com.android.tzdata \
//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex
BAZEL_ADBD="$(realpath $(call_bazel cquery --config=bp2build --config=android --config=ci --output=files //packages/modules/adb/apex:com.android.adbd))"
BAZEL_TZDATA="$(realpath $(call_bazel cquery --config=bp2build --config=android --config=ci --output=files //system/timezone/apex:com.android.tzdata))"
BAZEL_MINIMAL="$(realpath $(call_bazel cquery --config=bp2build --config=android --config=ci --output=files //build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex))"
# # Build debugfs separately, as it's not a dep of apexer, but needs to be an explicit arg.
call_bazel build --config=bp2build --config=linux_x86_64 //external/e2fsprogs/debugfs //system/apex/tools:deapexer
DEBUGFS_PATH="$BAZEL_OUT/linux_x86_64-opt/bin/external/e2fsprogs/debugfs/debugfs"
DEAPEXER="$BAZEL_OUT/linux_x86_64-opt/bin/system/apex/tools/deapexer --debugfs_path=$DEBUGFS_PATH"
DEBUGFS_PATH="$(realpath $(call_bazel cquery --config=bp2build --config=linux_x86_64 --config=ci --output=files //external/e2fsprogs/debugfs))"
DEAPEXER="$(realpath $(call_bazel cquery --config=bp2build --config=linux_x86_64 --config=ci --output=files //system/apex/tools:deapexer))"
DEAPEXER="$DEAPEXER --debugfs_path=$DEBUGFS_PATH"
#######
# Tests
#######
function compare_deapexer_list() {
local APEX_DIR=$1; shift
local BAZEL_APEX=$1; shift
local APEX=$1; shift
# Compare the outputs of `deapexer list`, which lists the contents of the apex filesystem image.
local SOONG_APEX="$SOONG_OUTPUT_DIR/$APEX"
local BAZEL_APEX="$BAZEL_OUT/android_target-opt/bin/$APEX_DIR/$APEX"
local SOONG_LIST="$OUTPUT_DIR/soong.list"
local BAZEL_LIST="$OUTPUT_DIR/bazel.list"
@ -108,6 +111,6 @@ function compare_deapexer_list() {
fi
}
compare_deapexer_list packages/modules/adb/apex com.android.adbd.apex
compare_deapexer_list system/timezone/apex com.android.tzdata.apex
compare_deapexer_list build/bazel/examples/apex/minimal build.bazel.examples.apex.minimal.apex
compare_deapexer_list "${BAZEL_ADBD}" com.android.adbd.apex
compare_deapexer_list "${BAZEL_TZDATA}" com.android.tzdata.apex
compare_deapexer_list "${BAZEL_MINIMAL}" build.bazel.examples.apex.minimal.apex

View file

@ -140,9 +140,9 @@ EOF
# NOTE: We don't actually use the extra BUILD file for anything here
run_bazel build --config=android --config=bp2build --config=ci //foo/...
local the_answer_file="bazel-out/android_target-opt/bin/foo/convertible_soong_module/the_answer.txt"
local the_answer_file="$(find -L bazel-out -name the_answer.txt)"
if [[ ! -f "${the_answer_file}" ]]; then
fail "Expected '${the_answer_file}' to be generated, but was missing"
fail "Expected the_answer.txt to be generated, but was missing"
fi
if ! grep 42 "${the_answer_file}"; then
fail "Expected to find 42 in '${the_answer_file}'"

View file

@ -107,7 +107,7 @@ function setup {
info "Running test case \e[96;1m${FUNCNAME[1]}\e[0m"
cd "$MOCK_TOP"
tar xzf "$WARMED_UP_MOCK_TOP"
tar xzf "$WARMED_UP_MOCK_TOP" --warning=no-timestamp
}
# shellcheck disable=SC2120