Bp2build support for multiple product configs

Create a
build/bazel/product_config/generated/products/<product_name>/BUILD
file that contains the platform definitions needed for
a particular product. Currently we just create it for the
current lunch target, but the idea is that eventually when
all product config is in starlark, all the products will
have their platform definitions in the tree at once.

Bug: 249685973
Test: Presubmits
Change-Id: I08c82ff28dcf62f09d3b1d2e3186a6b961e12f6e
This commit is contained in:
Cole Faust 2022-11-08 18:14:01 -08:00
parent 059285911d
commit b85d1a15cc
13 changed files with 222 additions and 45 deletions

View file

@ -205,6 +205,9 @@ type bazelContext struct {
bazelEnabledModules map[string]bool
// If true, modules are bazel-enabled by default, unless present in bazelDisabledModules.
modulesDefaultToBazel bool
targetProduct string
targetBuildVariant string
}
var _ BazelContext = &bazelContext{}
@ -460,6 +463,18 @@ func NewBazelContext(c *config) (BazelContext, error) {
if len(missing) > 0 {
return nil, fmt.Errorf("missing required env vars to use bazel: %s", missing)
}
targetBuildVariant := "user"
if c.Eng() {
targetBuildVariant = "eng"
} else if c.Debuggable() {
targetBuildVariant = "userdebug"
}
targetProduct := "unknown"
if c.HasDeviceProduct() {
targetProduct = c.DeviceProduct()
}
return &bazelContext{
bazelRunner: &builtinBazelRunner{},
paths: &paths,
@ -467,6 +482,8 @@ func NewBazelContext(c *config) (BazelContext, error) {
modulesDefaultToBazel: c.BuildMode == BazelDevMode,
bazelEnabledModules: enabledModules,
bazelDisabledModules: disabledModules,
targetProduct: targetProduct,
targetBuildVariant: targetBuildVariant,
}, nil
}
@ -563,9 +580,9 @@ func (r *builtinBazelRunner) createBazelCommand(paths *bazelPaths, runName bazel
// The actual platform values here may be overridden by configuration
// transitions from the buildroot.
fmt.Sprintf("--extra_toolchains=%s", "//prebuilts/clang/host/linux-x86:all"),
// This should be parameterized on the host OS, but let's restrict to linux
// to keep things simple for now.
fmt.Sprintf("--host_platform=%s", "//build/bazel/platforms:linux_x86_64"),
// We don't need to set --host_platforms because it's set in bazelrc files
// that the bazel shell script wrapper passes
// Explicitly disable downloading rules (such as canonical C++ and Java rules) from the network.
"--experimental_repository_disable_download",
@ -610,8 +627,12 @@ func (context *bazelContext) mainBzlFileContents() []byte {
#####################################################
def _config_node_transition_impl(settings, attr):
if attr.os == "android" and attr.arch == "target":
target = "{PRODUCT}-{VARIANT}"
else:
target = "{PRODUCT}-{VARIANT}_%s_%s" % (attr.os, attr.arch)
return {
"//command_line_option:platforms": "@//build/bazel/platforms:%s_%s" % (attr.os, attr.arch),
"//command_line_option:platforms": "@soong_injection//product_config_platforms/products/{PRODUCT}-{VARIANT}:%s" % target,
}
_config_node_transition = transition(
@ -658,7 +679,12 @@ phony_root = rule(
attrs = {"deps" : attr.label_list()},
)
`
return []byte(contents)
productReplacer := strings.NewReplacer(
"{PRODUCT}", context.targetProduct,
"{VARIANT}", context.targetBuildVariant)
return []byte(productReplacer.Replace(contents))
}
func (context *bazelContext) mainBuildFileContents() []byte {
@ -780,26 +806,24 @@ def json_encode(input):
t = type(p)
if t == "string" or t == "int":
return repr(p)
fail("unsupported value '%%s' of type '%%s'" %% (p, type(p)))
fail("unsupported value '%s' of type '%s'" % (p, type(p)))
def encode_list(list):
return "[%%s]" %% ", ".join([encode_primitive(item) for item in list])
return "[%s]" % ", ".join([encode_primitive(item) for item in list])
def encode_list_or_primitive(v):
return encode_list(v) if type(v) == "list" else encode_primitive(v)
if type(input) == "dict":
# TODO(juu): the result is read line by line so can't use '\n' yet
kv_pairs = [("%%s: %%s" %% (encode_primitive(k), encode_list_or_primitive(v))) for (k, v) in input.items()]
return "{ %%s }" %% ", ".join(kv_pairs)
kv_pairs = [("%s: %s" % (encode_primitive(k), encode_list_or_primitive(v))) for (k, v) in input.items()]
return "{ %s }" % ", ".join(kv_pairs)
else:
return encode_list_or_primitive(input)
# Label Map Section
%s
{LABEL_REGISTRATION_MAP_SECTION}
# Function Def Section
%s
{FUNCTION_DEF_SECTION}
def get_arch(target):
# TODO(b/199363072): filegroups and file targets aren't associated with any
@ -811,22 +835,26 @@ def get_arch(target):
# File targets do not have buildoptions. File targets aren't associated with
# any specific platform architecture in mixed builds, so use the host.
return "x86_64|linux"
platforms = build_options(target)["//command_line_option:platforms"]
platforms = buildoptions["//command_line_option:platforms"]
if len(platforms) != 1:
# An individual configured target should have only one platform architecture.
# Note that it's fine for there to be multiple architectures for the same label,
# but each is its own configured target.
fail("expected exactly 1 platform for " + str(target.label) + " but got " + str(platforms))
platform_name = build_options(target)["//command_line_option:platforms"][0].name
platform_name = platforms[0].name
if platform_name == "host":
return "HOST"
if not platform_name.startswith("{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}"):
fail("expected platform name of the form '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_android_<arch>' or '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
platform_name = platform_name.removeprefix("{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}").removeprefix("_")
if not platform_name:
return "target|android"
elif platform_name.startswith("android_"):
return platform_name[len("android_"):] + "|" + platform_name[:len("android_")-1]
return platform_name.removeprefix("android_") + "|android"
elif platform_name.startswith("linux_"):
return platform_name[len("linux_"):] + "|" + platform_name[:len("linux_")-1]
return platform_name.removeprefix("linux_") + "|linux"
else:
fail("expected platform name of the form 'android_<arch>' or 'linux_<arch>', but was " + str(platforms))
return "UNKNOWN"
fail("expected platform name of the form '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_android_<arch>' or '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
def format(target):
id_string = str(target.label) + "|" + get_arch(target)
@ -835,15 +863,20 @@ def format(target):
if id_string.startswith("//"):
id_string = "@" + id_string
# Main switch section
%s
{MAIN_SWITCH_SECTION}
# This target was not requested via cquery, and thus must be a dependency
# of a requested target.
return id_string + ">>NONE"
`
replacer := strings.NewReplacer(
"{TARGET_PRODUCT}", context.targetProduct,
"{TARGET_BUILD_VARIANT}", context.targetBuildVariant,
"{LABEL_REGISTRATION_MAP_SECTION}", labelRegistrationMapSection,
"{FUNCTION_DEF_SECTION}", functionDefSection,
"{MAIN_SWITCH_SECTION}", mainSwitchSection)
return []byte(fmt.Sprintf(formatString, labelRegistrationMapSection, functionDefSection,
mainSwitchSection))
return []byte(replacer.Replace(formatString))
}
// Returns a path containing build-related metadata required for interfacing

View file

@ -769,6 +769,13 @@ func (c *config) DeviceProduct() string {
return *c.productVariables.DeviceProduct
}
// HasDeviceProduct returns if the build has a product. A build will not
// necessarily have a product when --skip-config is passed to soong, like it is
// in prebuilts/build-tools/build-prebuilts.sh
func (c *config) HasDeviceProduct() bool {
return c.productVariables.DeviceProduct != nil
}
func (c *config) DeviceResourceOverlays() []string {
return c.productVariables.DeviceResourceOverlays
}

View file

@ -8,6 +8,7 @@ bootstrap_go_package {
srcs: [
"androidbp_to_build_templates.go",
"bp2build.go",
"bp2build_product_config.go",
"build_conversion.go",
"bzl_conversion.go",
"configurability.go",

View file

@ -45,7 +45,14 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics {
bp2buildFiles := CreateBazelFiles(ctx.Config(), nil, res.buildFileToTargets, ctx.mode)
writeFiles(ctx, bp2buildDir, bp2buildFiles)
productConfigFiles, err := CreateProductConfigFiles(ctx)
if err != nil {
fmt.Printf("ERROR: %s", err.Error())
os.Exit(1)
}
soongInjectionDir := android.PathForOutput(ctx, bazel.SoongInjectionDirName)
writeFiles(ctx, soongInjectionDir, productConfigFiles)
writeFiles(ctx, soongInjectionDir, CreateSoongInjectionFiles(ctx.Config(), res.metrics))
return &res.metrics

View file

@ -0,0 +1,124 @@
package bp2build
import (
"fmt"
"os"
"path/filepath"
"strings"
)
func CreateProductConfigFiles(
ctx *CodegenContext) ([]BazelFile, error) {
cfg := &ctx.config
targetProduct := "unknown"
if cfg.HasDeviceProduct() {
targetProduct = cfg.DeviceProduct()
}
targetBuildVariant := "user"
if cfg.Eng() {
targetBuildVariant = "eng"
} else if cfg.Debuggable() {
targetBuildVariant = "userdebug"
}
productVariablesFileName := cfg.ProductVariablesFileName
if !strings.HasPrefix(productVariablesFileName, "/") {
productVariablesFileName = filepath.Join(ctx.topDir, productVariablesFileName)
}
bytes, err := os.ReadFile(productVariablesFileName)
if err != nil {
return nil, err
}
// TODO(b/249685973): the name is product_config_platforms because product_config
// was already used for other files. Deduplicate them.
currentProductFolder := fmt.Sprintf("product_config_platforms/products/%s-%s", targetProduct, targetBuildVariant)
productReplacer := strings.NewReplacer(
"{PRODUCT}", targetProduct,
"{VARIANT}", targetBuildVariant,
"{PRODUCT_FOLDER}", currentProductFolder)
result := []BazelFile{
newFile(
currentProductFolder,
"soong.variables.bzl",
`variables = json.decode("""`+strings.ReplaceAll(string(bytes), "\\", "\\\\")+`""")`),
newFile(
currentProductFolder,
"BUILD",
productReplacer.Replace(`
package(default_visibility=[
"@soong_injection//product_config_platforms:__subpackages__",
"@//build/bazel/product_config:__subpackages__",
])
load(":soong.variables.bzl", _soong_variables = "variables")
load("@//build/bazel/product_config:utils.bzl", "android_product")
android_product(
name = "{PRODUCT}-{VARIANT}",
soong_variables = _soong_variables,
)
`)),
newFile(
"product_config_platforms",
"BUILD.bazel",
productReplacer.Replace(`
package(default_visibility = [
"@//build/bazel/product_config:__subpackages__",
"@soong_injection//product_config_platforms:__subpackages__",
])
# TODO(b/249685973): Remove this. It was only added for a platform_mappings file,
# which can possibly be replaced with autogenerating the platform_mappings file,
# or removing that file entirely.
alias(
name = "current_android_platform",
# TODO: When we start generating the platforms for more than just the
# currently lunched, product, turn this into a select with an arm for each product.
actual = "@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}",
)
alias(
name = "product_vars",
actual = select({
# TODO: When we start generating the platforms for more than just the
# currently lunched, product, this select should have an arm for each product.
"@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_constraint_value": "@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_product_vars",
}),
)
`)),
newFile(
"product_config_platforms",
"common.bazelrc",
productReplacer.Replace(`
build --platforms @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_linux_x86_64
build:android --platforms=@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}
build:linux_x86_64 --platforms=@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_linux_x86_64
build:linux_bionic_x86_64 --platforms=@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_linux_bionic_x86_64
`)),
newFile(
"product_config_platforms",
"linux.bazelrc",
productReplacer.Replace(`
build --host_platform @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_linux_x86_64
`)),
newFile(
"product_config_platforms",
"darwin.bazelrc",
productReplacer.Replace(`
build --host_platform @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_darwin_x86_64
`)),
newFile(
"product_config_platforms",
"platform_mappings",
productReplacer.Replace(`
flags:
--cpu=k8
@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}
`)),
}
return result, nil
}

View file

@ -140,6 +140,7 @@ type CodegenContext struct {
mode CodegenMode
additionalDeps []string
unconvertedDepMode unconvertedDepsMode
topDir string
}
func (ctx *CodegenContext) Mode() CodegenMode {
@ -208,7 +209,7 @@ func (ctx *CodegenContext) Context() *android.Context { return ctx.context }
// NewCodegenContext creates a wrapper context that conforms to PathContext for
// writing BUILD files in the output directory.
func NewCodegenContext(config android.Config, context *android.Context, mode CodegenMode) *CodegenContext {
func NewCodegenContext(config android.Config, context *android.Context, mode CodegenMode, topDir string) *CodegenContext {
var unconvertedDeps unconvertedDepsMode
if config.IsEnvTrue("BP2BUILD_ERROR_UNCONVERTED") {
unconvertedDeps = errorModulesUnconvertedDeps
@ -218,6 +219,7 @@ func NewCodegenContext(config android.Config, context *android.Context, mode Cod
config: config,
mode: mode,
unconvertedDepMode: unconvertedDeps,
topDir: topDir,
}
}

View file

@ -209,7 +209,7 @@ func TestGenerateSoongModuleTargets(t *testing.T) {
_, errs = ctx.PrepareBuildActions(config)
android.FailIfErrored(t, errs)
codegenCtx := NewCodegenContext(config, ctx.Context, QueryView)
codegenCtx := NewCodegenContext(config, ctx.Context, QueryView, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, dir)
android.FailIfErrored(t, err)
if actualCount, expectedCount := len(bazelTargets), 1; actualCount != expectedCount {
@ -530,7 +530,7 @@ custom {
return
}
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, dir)
android.FailIfErrored(t, err)
@ -903,7 +903,7 @@ load("//build/bazel/rules:rules.bzl", "my_library")`,
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, dir)
android.FailIfErrored(t, err)
if actualCount := len(bazelTargets); actualCount != testCase.expectedBazelTargetCount {
@ -1156,7 +1156,7 @@ func TestAllowlistingBp2buildTargetsExplicitly(t *testing.T) {
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, dir)
android.FailIfErrored(t, err)
if actualCount := len(bazelTargets); actualCount != testCase.expectedCount {
@ -1263,7 +1263,7 @@ filegroup { name: "opt-out-h", bazel_module: { bp2build_available: false } }
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
// For each directory, test that the expected number of generated targets is correct.
for dir, expectedCount := range testCase.expectedCount {
@ -1398,7 +1398,7 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) {
if testCase.Dir != "" {
checkDir = testCase.Dir
}
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
bazelTargets, err := generateBazelTargetsForDir(codegenCtx, checkDir)
android.FailIfErrored(t, err)
bazelTargets.sort()

View file

@ -106,7 +106,7 @@ func setup(builddir string, tcSize float64) testConfig {
ctx := android.NewTestContext(config)
registerCustomModuleForBp2buildConversion(ctx)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
return testConfig{
config,
ctx,

View file

@ -200,7 +200,7 @@ func (b *bazelTestRunner) PostParseProcessor(result android.CustomTestResult) {
return
}
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
res, errs := GenerateBazelTargets(codegenCtx, false)
if bazelResult.CollateErrs(errs) {
return

View file

@ -134,7 +134,7 @@ func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string {
func runQueryView(queryviewDir, queryviewMarker string, ctx *android.Context) {
ctx.EventHandler.Begin("queryview")
defer ctx.EventHandler.End("queryview")
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.QueryView)
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.QueryView, topDir)
err := createBazelWorkspace(codegenContext, shared.JoinPath(topDir, queryviewDir))
maybeQuit(err, "")
touch(shared.JoinPath(topDir, queryviewMarker))
@ -169,7 +169,7 @@ func runApiBp2build(ctx *android.Context, extraNinjaDeps []string) string {
ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
// Run codegen to generate BUILD files
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.ApiBp2build)
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.ApiBp2build, topDir)
absoluteApiBp2buildDir := shared.JoinPath(topDir, cmdlineArgs.BazelApiBp2buildDir)
err := createBazelWorkspace(codegenContext, absoluteApiBp2buildDir)
maybeQuit(err, "")
@ -586,7 +586,7 @@ func runBp2Build(ctx *android.Context, extraNinjaDeps []string, metricsDir strin
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion codegenMetrics to the user.
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build)
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build, topDir)
ctx.EventHandler.Do("codegen", func() {
codegenMetrics = bp2build.Codegen(codegenContext)
})

View file

@ -67,23 +67,26 @@ call_bazel build --config=bp2build --config=ci --config=android \
//packages/modules/adb/apex:com.android.adbd \
//system/timezone/apex:com.android.tzdata \
//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex
BAZEL_ADBD="$(realpath $(call_bazel cquery --config=bp2build --config=android --config=ci --output=files //packages/modules/adb/apex:com.android.adbd))"
BAZEL_TZDATA="$(realpath $(call_bazel cquery --config=bp2build --config=android --config=ci --output=files //system/timezone/apex:com.android.tzdata))"
BAZEL_MINIMAL="$(realpath $(call_bazel cquery --config=bp2build --config=android --config=ci --output=files //build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex))"
# # Build debugfs separately, as it's not a dep of apexer, but needs to be an explicit arg.
call_bazel build --config=bp2build --config=linux_x86_64 //external/e2fsprogs/debugfs //system/apex/tools:deapexer
DEBUGFS_PATH="$BAZEL_OUT/linux_x86_64-opt/bin/external/e2fsprogs/debugfs/debugfs"
DEAPEXER="$BAZEL_OUT/linux_x86_64-opt/bin/system/apex/tools/deapexer --debugfs_path=$DEBUGFS_PATH"
DEBUGFS_PATH="$(realpath $(call_bazel cquery --config=bp2build --config=linux_x86_64 --config=ci --output=files //external/e2fsprogs/debugfs))"
DEAPEXER="$(realpath $(call_bazel cquery --config=bp2build --config=linux_x86_64 --config=ci --output=files //system/apex/tools:deapexer))"
DEAPEXER="$DEAPEXER --debugfs_path=$DEBUGFS_PATH"
#######
# Tests
#######
function compare_deapexer_list() {
local APEX_DIR=$1; shift
local BAZEL_APEX=$1; shift
local APEX=$1; shift
# Compare the outputs of `deapexer list`, which lists the contents of the apex filesystem image.
local SOONG_APEX="$SOONG_OUTPUT_DIR/$APEX"
local BAZEL_APEX="$BAZEL_OUT/android_target-opt/bin/$APEX_DIR/$APEX"
local SOONG_LIST="$OUTPUT_DIR/soong.list"
local BAZEL_LIST="$OUTPUT_DIR/bazel.list"
@ -108,6 +111,6 @@ function compare_deapexer_list() {
fi
}
compare_deapexer_list packages/modules/adb/apex com.android.adbd.apex
compare_deapexer_list system/timezone/apex com.android.tzdata.apex
compare_deapexer_list build/bazel/examples/apex/minimal build.bazel.examples.apex.minimal.apex
compare_deapexer_list "${BAZEL_ADBD}" com.android.adbd.apex
compare_deapexer_list "${BAZEL_TZDATA}" com.android.tzdata.apex
compare_deapexer_list "${BAZEL_MINIMAL}" build.bazel.examples.apex.minimal.apex

View file

@ -140,9 +140,9 @@ EOF
# NOTE: We don't actually use the extra BUILD file for anything here
run_bazel build --config=android --config=bp2build --config=ci //foo/...
local the_answer_file="bazel-out/android_target-opt/bin/foo/convertible_soong_module/the_answer.txt"
local the_answer_file="$(find -L bazel-out -name the_answer.txt)"
if [[ ! -f "${the_answer_file}" ]]; then
fail "Expected '${the_answer_file}' to be generated, but was missing"
fail "Expected the_answer.txt to be generated, but was missing"
fi
if ! grep 42 "${the_answer_file}"; then
fail "Expected to find 42 in '${the_answer_file}'"

View file

@ -107,7 +107,7 @@ function setup {
info "Running test case \e[96;1m${FUNCNAME[1]}\e[0m"
cd "$MOCK_TOP"
tar xzf "$WARMED_UP_MOCK_TOP"
tar xzf "$WARMED_UP_MOCK_TOP" --warning=no-timestamp
}
# shellcheck disable=SC2120