From 6054cdf3b13c7cfe23a23e409bb3672fb5873da5 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Tue, 12 Sep 2023 10:07:07 -0700 Subject: [PATCH] Generate android_certificate_directory Previously, partners were required to add an android_certificate_directory filegroup in their certificate directories, and allowlist that BUILD file. Now, we generate the filegroup automatically. We're using a different name, generated_android_certificate_directory, to avoid conflicts with already-checked-in filegroups. Bug: 285777389 Test: b test //build/bazel/rules/apex/... Change-Id: Ib1bde487acd79d58368faf0aad02ded0bcdaceb4 --- bp2build/bp2build.go | 36 ++++----- bp2build/bp2build_product_config.go | 117 +++++++++++++++++----------- bp2build/conversion.go | 17 +++- bp2build/conversion_test.go | 49 ++++++++++-- 4 files changed, 141 insertions(+), 78 deletions(-) diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go index 5f7b382f0..6c9d90330 100644 --- a/bp2build/bp2build.go +++ b/bp2build/bp2build.go @@ -82,15 +82,25 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics { os.Exit(1) } var bp2buildFiles []BazelFile + productConfig, err := createProductConfigFiles(ctx, res.metrics) ctx.Context().EventHandler.Do("CreateBazelFile", func() { - bp2buildFiles = CreateBazelFiles(nil, res.buildFileToTargets, ctx.mode) + allTargets := make(map[string]BazelTargets) + for k, v := range res.buildFileToTargets { + allTargets[k] = append(allTargets[k], v...) + } + for k, v := range productConfig.bp2buildTargets { + allTargets[k] = append(allTargets[k], v...) + } + bp2buildFiles = CreateBazelFiles(nil, allTargets, ctx.mode) }) - injectionFiles, additionalBp2buildFiles, err := CreateSoongInjectionDirFiles(ctx, res.metrics) + bp2buildFiles = append(bp2buildFiles, productConfig.bp2buildFiles...) + injectionFiles, err := createSoongInjectionDirFiles(ctx, res.metrics) if err != nil { fmt.Printf("%s\n", err.Error()) os.Exit(1) } - bp2buildFiles = append(bp2buildFiles, additionalBp2buildFiles...) + injectionFiles = append(injectionFiles, productConfig.injectionFiles...) + writeFiles(ctx, bp2buildDir, bp2buildFiles) // Delete files under the bp2build root which weren't just written. An // alternative would have been to delete the whole directory and write these @@ -109,26 +119,6 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics { return &res.metrics } -// Wrapper function that will be responsible for all files in soong_injection directory -// This includes -// 1. config value(s) that are hardcoded in Soong -// 2. product_config variables -func CreateSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) ([]BazelFile, []BazelFile, error) { - var ret []BazelFile - - productConfigInjectionFiles, productConfigBp2BuildDirFiles, err := CreateProductConfigFiles(ctx, metrics) - if err != nil { - return nil, nil, err - } - ret = append(ret, productConfigInjectionFiles...) - injectionFiles, err := soongInjectionFiles(ctx.Config(), metrics) - if err != nil { - return nil, nil, err - } - ret = append(injectionFiles, ret...) - return ret, productConfigBp2BuildDirFiles, nil -} - // Get the output directory and create it if it doesn't exist. func getOrCreateOutputDir(outputDir android.OutputPath, ctx android.PathContext, dir string) android.OutputPath { dirPath := outputDir.Join(ctx, dir) diff --git a/bp2build/bp2build_product_config.go b/bp2build/bp2build_product_config.go index 20355d736..50b83582a 100644 --- a/bp2build/bp2build_product_config.go +++ b/bp2build/bp2build_product_config.go @@ -12,14 +12,19 @@ import ( "android/soong/android/soongconfig" "android/soong/starlark_import" - "github.com/google/blueprint" "github.com/google/blueprint/proptools" "go.starlark.net/starlark" ) -func CreateProductConfigFiles( +type createProductConfigFilesResult struct { + injectionFiles []BazelFile + bp2buildFiles []BazelFile + bp2buildTargets map[string]BazelTargets +} + +func createProductConfigFiles( ctx *CodegenContext, - metrics CodegenMetrics) ([]BazelFile, []BazelFile, error) { + metrics CodegenMetrics) (createProductConfigFilesResult, error) { cfg := &ctx.config targetProduct := "unknown" if cfg.HasDeviceProduct() { @@ -32,29 +37,22 @@ func CreateProductConfigFiles( targetBuildVariant = "userdebug" } + var res createProductConfigFilesResult + productVariablesFileName := cfg.ProductVariablesFileName if !strings.HasPrefix(productVariablesFileName, "/") { productVariablesFileName = filepath.Join(ctx.topDir, productVariablesFileName) } productVariablesBytes, err := os.ReadFile(productVariablesFileName) if err != nil { - return nil, nil, err + return res, err } productVariables := android.ProductVariables{} err = json.Unmarshal(productVariablesBytes, &productVariables) if err != nil { - return nil, nil, err + return res, err } - // Visit all modules to determine the list of ndk libraries - // This list will be used to add additional flags for cc stub generation - ndkLibsStringFormatted := []string{} - ctx.Context().VisitAllModules(func(m blueprint.Module) { - if ctx.Context().ModuleType(m) == "ndk_library" { - ndkLibsStringFormatted = append(ndkLibsStringFormatted, fmt.Sprintf(`"%s"`, m.Name())) // name will be `"libc.ndk"` - } - }) - // TODO(b/249685973): the name is product_config_platforms because product_config // was already used for other files. Deduplicate them. currentProductFolder := fmt.Sprintf("product_config_platforms/products/%s-%s", targetProduct, targetBuildVariant) @@ -64,25 +62,36 @@ func CreateProductConfigFiles( "{VARIANT}", targetBuildVariant, "{PRODUCT_FOLDER}", currentProductFolder) - platformMappingContent, err := platformMappingContent( - productReplacer.Replace("@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}"), - &productVariables, - ctx.Config().Bp2buildSoongConfigDefinitions, - metrics.convertedModulePathMap) - if err != nil { - return nil, nil, err - } - productsForTestingMap, err := starlark_import.GetStarlarkValue[map[string]map[string]starlark.Value]("products_for_testing") if err != nil { - return nil, nil, err + return res, err } productsForTesting := android.SortedKeys(productsForTestingMap) for i := range productsForTesting { productsForTesting[i] = fmt.Sprintf(" \"@//build/bazel/tests/products:%s\",", productsForTesting[i]) } - injectionDirFiles := []BazelFile{ + productLabelsToVariables := make(map[string]*android.ProductVariables) + productLabelsToVariables[productReplacer.Replace("@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}")] = &productVariables + for product, productVariablesStarlark := range productsForTestingMap { + productVariables, err := starlarkMapToProductVariables(productVariablesStarlark) + if err != nil { + return res, err + } + productLabelsToVariables["@//build/bazel/tests/products:"+product] = &productVariables + } + + res.bp2buildTargets = createTargets(productLabelsToVariables) + + platformMappingContent, err := platformMappingContent( + productLabelsToVariables, + ctx.Config().Bp2buildSoongConfigDefinitions, + metrics.convertedModulePathMap) + if err != nil { + return res, err + } + + res.injectionFiles = []BazelFile{ newFile( currentProductFolder, "soong.variables.bzl", @@ -164,30 +173,21 @@ build --host_platform @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_lin productReplacer.Replace(` build --host_platform @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_darwin_x86_64 `)), - newFile( - "cc_toolchain", - "ndk_libs.bzl", - fmt.Sprintf("ndk_libs = [%v]", strings.Join(ndkLibsStringFormatted, ", ")), - ), } - bp2buildDirFiles := []BazelFile{ + res.bp2buildFiles = []BazelFile{ newFile( "", "platform_mappings", platformMappingContent), } - return injectionDirFiles, bp2buildDirFiles, nil + + return res, nil } func platformMappingContent( - mainProductLabel string, - mainProductVariables *android.ProductVariables, + productLabelToVariables map[string]*android.ProductVariables, soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions, convertedModulePathMap map[string]string) (string, error) { - productsForTesting, err := starlark_import.GetStarlarkValue[map[string]map[string]starlark.Value]("products_for_testing") - if err != nil { - return "", err - } var result strings.Builder mergedConvertedModulePathMap := make(map[string]string) @@ -203,13 +203,8 @@ func platformMappingContent( } result.WriteString("platforms:\n") - platformMappingSingleProduct(mainProductLabel, mainProductVariables, soongConfigDefinitions, mergedConvertedModulePathMap, &result) - for product, productVariablesStarlark := range productsForTesting { - productVariables, err := starlarkMapToProductVariables(productVariablesStarlark) - if err != nil { - return "", err - } - platformMappingSingleProduct("@//build/bazel/tests/products:"+product, &productVariables, soongConfigDefinitions, mergedConvertedModulePathMap, &result) + for productLabel, productVariables := range productLabelToVariables { + platformMappingSingleProduct(productLabel, productVariables, soongConfigDefinitions, mergedConvertedModulePathMap, &result) } return result.String(), nil } @@ -248,7 +243,7 @@ func platformMappingSingleProduct( defaultAppCertificateFilegroup := "//build/bazel/utils:empty_filegroup" if proptools.String(productVariables.DefaultAppCertificate) != "" { - defaultAppCertificateFilegroup = "@//" + filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) + ":android_certificate_directory" + defaultAppCertificateFilegroup = "@//" + filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) + ":generated_android_certificate_directory" } for _, suffix := range bazelPlatformSuffixes { @@ -419,3 +414,33 @@ func starlarkMapToProductVariables(in map[string]starlark.Value) (android.Produc return result, nil } + +func createTargets(productLabelsToVariables map[string]*android.ProductVariables) map[string]BazelTargets { + res := make(map[string]BazelTargets) + var allDefaultAppCertificateDirs []string + for _, productVariables := range productLabelsToVariables { + if proptools.String(productVariables.DefaultAppCertificate) != "" { + d := filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) + if !android.InList(d, allDefaultAppCertificateDirs) { + allDefaultAppCertificateDirs = append(allDefaultAppCertificateDirs, d) + } + } + } + for _, dir := range allDefaultAppCertificateDirs { + content := fmt.Sprintf(ruleTargetTemplate, "filegroup", "generated_android_certificate_directory", propsToAttributes(map[string]string{ + "srcs": `glob([ + "*.pk8", + "*.pem", + "*.avbpubkey", + ])`, + "visibility": `["//visibility:public"]`, + })) + res[dir] = append(res[dir], BazelTarget{ + name: "generated_android_certificate_directory", + packageName: dir, + content: content, + ruleClass: "filegroup", + }) + } + return res +} diff --git a/bp2build/conversion.go b/bp2build/conversion.go index da4b5cf1f..c69723587 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -15,6 +15,7 @@ import ( rust_config "android/soong/rust/config" "android/soong/starlark_fmt" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -24,16 +25,28 @@ type BazelFile struct { Contents string } -// PRIVATE: Use CreateSoongInjectionDirFiles instead -func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) ([]BazelFile, error) { +// createSoongInjectionDirFiles returns most of the files to write to the soong_injection directory. +// Some other files also come from CreateProductConfigFiles +func createSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) ([]BazelFile, error) { + cfg := ctx.Config() var files []BazelFile files = append(files, newFile("android", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package. files = append(files, newFile("android", "constants.bzl", android.BazelCcToolchainVars(cfg))) + // Visit all modules to determine the list of ndk libraries + // This list will be used to add additional flags for cc stub generation + ndkLibsStringFormatted := []string{} + ctx.Context().VisitAllModules(func(m blueprint.Module) { + if ctx.Context().ModuleType(m) == "ndk_library" { + ndkLibsStringFormatted = append(ndkLibsStringFormatted, fmt.Sprintf(`"%s"`, m.Name())) // name will be `"libc.ndk"` + } + }) + files = append(files, newFile("cc_toolchain", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package. files = append(files, newFile("cc_toolchain", "config_constants.bzl", cc_config.BazelCcToolchainVars(cfg))) files = append(files, newFile("cc_toolchain", "sanitizer_constants.bzl", cc.BazelCcSanitizerToolchainVars(cfg))) + files = append(files, newFile("cc_toolchain", "ndk_libs.bzl", fmt.Sprintf("ndk_libs = [%v]", strings.Join(ndkLibsStringFormatted, ", ")))) files = append(files, newFile("java_toolchain", GeneratedBuildFileName, "")) // Creates a //java_toolchain package. files = append(files, newFile("java_toolchain", "constants.bzl", java_config.BazelJavaToolchainVars(cfg))) diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index 89dd38ef0..6b100772c 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -83,7 +83,8 @@ func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) { func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { testConfig := android.TestConfig("", make(map[string]string), "", make(map[string][]byte)) - files, err := soongInjectionFiles(testConfig, CreateCodegenMetrics()) + codegenCtx := NewCodegenContext(testConfig, android.NewTestContext(testConfig).Context, Bp2Build, "") + files, err := createSoongInjectionDirFiles(codegenCtx, CreateCodegenMetrics()) if err != nil { t.Error(err) } @@ -104,6 +105,10 @@ func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { dir: "cc_toolchain", basename: "config_constants.bzl", }, + { + dir: "cc_toolchain", + basename: "ndk_libs.bzl", + }, { dir: "cc_toolchain", basename: "sanitizer_constants.bzl", @@ -182,15 +187,45 @@ func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { }, } - if len(files) != len(expectedFilePaths) { - t.Errorf("Expected %d file, got %d", len(expectedFilePaths), len(files)) + less := func(a bazelFilepath, b bazelFilepath) bool { + return a.dir+"/"+a.basename < b.dir+"/"+b.basename } - for i := range files { - actualFile, expectedFile := files[i], expectedFilePaths[i] + fileToFilepath := func(a BazelFile) bazelFilepath { + return bazelFilepath{basename: a.Basename, dir: a.Dir} + } - if actualFile.Dir != expectedFile.dir || actualFile.Basename != expectedFile.basename { - t.Errorf("Did not find expected file %s/%s", actualFile.Dir, actualFile.Basename) + sort.Slice(expectedFilePaths, func(i, j int) bool { + return less(expectedFilePaths[i], expectedFilePaths[j]) + }) + sort.Slice(files, func(i, j int) bool { + return less(fileToFilepath(files[i]), fileToFilepath(files[j])) + }) + + i := 0 + j := 0 + for i < len(expectedFilePaths) && j < len(files) { + expectedFile, actualFile := expectedFilePaths[i], files[j] + + if actualFile.Dir == expectedFile.dir && actualFile.Basename == expectedFile.basename { + i++ + j++ + } else if less(expectedFile, fileToFilepath(actualFile)) { + t.Errorf("Did not find expected file %s/%s", expectedFile.dir, expectedFile.basename) + i++ + } else { + t.Errorf("Found unexpected file %s/%s", actualFile.Dir, actualFile.Basename) + j++ } } + for i < len(expectedFilePaths) { + expectedFile := expectedFilePaths[i] + t.Errorf("Did not find expected file %s/%s", expectedFile.dir, expectedFile.basename) + i++ + } + for j < len(files) { + actualFile := files[j] + t.Errorf("Found unexpected file %s/%s", actualFile.Dir, actualFile.Basename) + j++ + } }