From 5ba7e479d19205f8930375020cb8a81876482eea Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Wed, 15 Jul 2020 10:06:41 +0000 Subject: [PATCH] Create a AOSP Bazel overlay workspace with Soong The Bazel overlay is a directory at out/soong/bazel_overlay that replicates the layout of the AOSP Soong module tree, but as a Bazel workspace. Each Soong module variant is represented as a BUILD target created with the `soong_module` rule. To create this overlay, run `m bazel_overlay`. A `soong_module` target can depend on other `soong_module` targets. These dependencies replicate each module's `directDeps` in the Blueprint graph, just before `PrepareBuildActions`. This enables users to use bazel query as a way to introspect the Soong module graph. For example, - Direct reverse dependencies of //bionic/libc:generated_android_ids in //bionic/libc/...: $ bazel query 'rdeps(//bionic/libc/..., //bionic/libc:generated_android_ids, 1)' //bionic/libc:libc_bionic_ndk--android_recovery_arm_armv7-a-neon_static //bionic/libc:libc_bionic_ndk--android_ramdisk_arm_armv7-a-neon_static //bionic/libc:libc_bionic_ndk--android_arm_armv7-a-neon_static_com.android.runtime //bionic/libc:libc_bionic_ndk--android_arm_armv7-a-neon_static //bionic/libc:generated_android_ids - Why does com.android.runtime depend on lzma? $ bazel query 'somepath(//bionic/apex:com.android.runtime--android_common_com.android.runtime_image, //external/lzma/...)' //bionic/apex:com.android.runtime--android_common_com.android.runtime_image //bionic/libc/malloc_debug:libc_malloc_debug--android_arm_armv7-a-neon_shared_com.android.runtime //system/core/libunwindstack:libunwindstack--android_arm_armv7-a-neon_shared_com.android.runtime //external/lzma/C:liblzma--android_arm_armv7-a-neon_shared_com.android.runtime - What does the dep graph of //bionic/libc:crtbegin_so look like? $ bazel query 'deps(//bionic/libc:crtbegin_so--android_arm_armv7-a-neon)' --output=graph > graph.in && dot -Tpng < graph.in > graph.png https://photos.app.goo.gl/DfsdoFRNsRjGwTmy8 Test: croot && m bazel_overlay && cd out/soong/bazel_overlay && bazel query //... && bazel query 'rdeps(//bionic/libc/..., //bionic/libc:generated_android_ids, 1)' Signed-off-by: Jingwen Chen Change-Id: I3bf40309bfb2d963bb8a688706385a57ee304c37# --- android/Android.bp | 1 + android/bazel_overlay.go | 76 ++++++++++++++ cmd/soong_build/Android.bp | 1 + cmd/soong_build/bazel_overlay.go | 173 +++++++++++++++++++++++++++++++ cmd/soong_build/main.go | 21 +++- 5 files changed, 269 insertions(+), 3 deletions(-) create mode 100644 android/bazel_overlay.go create mode 100644 cmd/soong_build/bazel_overlay.go diff --git a/android/Android.bp b/android/Android.bp index 50faa4436..ff14a70e1 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "apex.go", "api_levels.go", "arch.go", + "bazel_overlay.go", "config.go", "csuite_config.go", "defaults.go", diff --git a/android/bazel_overlay.go b/android/bazel_overlay.go new file mode 100644 index 000000000..a0342826d --- /dev/null +++ b/android/bazel_overlay.go @@ -0,0 +1,76 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "os" + "strings" + + "github.com/google/blueprint" +) + +// The Bazel Overlay singleton is responsible for generating the Ninja actions +// for calling the soong_build primary builder in the main build.ninja file. +func init() { + RegisterSingletonType("bazel_overlay", BazelOverlaySingleton) +} + +func BazelOverlaySingleton() Singleton { + return &bazelOverlaySingleton{} +} + +type bazelOverlaySingleton struct{} + +func (c *bazelOverlaySingleton) GenerateBuildActions(ctx SingletonContext) { + // Create a build and rule statement, using the Bazel overlay's WORKSPACE + // file as the output file marker. + var deps Paths + moduleListFilePath := pathForBuildToolDep(ctx, ctx.Config().moduleListFile) + deps = append(deps, moduleListFilePath) + deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().ProductVariablesFileName)) + + bazelOverlayDirectory := PathForOutput(ctx, "bazel_overlay") + bazelOverlayWorkspaceFile := bazelOverlayDirectory.Join(ctx, "WORKSPACE") + primaryBuilder := primaryBuilderPath(ctx) + bazelOverlay := ctx.Rule(pctx, "bazelOverlay", + blueprint.RuleParams{ + Command: fmt.Sprintf( + "rm -rf ${outDir}/* && %s --bazel_overlay_dir ${outDir} %s && echo WORKSPACE: `cat %s` > ${outDir}/.overlay-depfile.d", + primaryBuilder.String(), + strings.Join(os.Args[1:], " "), + moduleListFilePath.String(), // Use the contents of Android.bp.list as the depfile. + ), + CommandDeps: []string{primaryBuilder.String()}, + Description: fmt.Sprintf( + "Creating the Bazel overlay workspace with %s at $outDir", + primaryBuilder.Base()), + Deps: blueprint.DepsGCC, + Depfile: "${outDir}/.overlay-depfile.d", + }, + "outDir") + + ctx.Build(pctx, BuildParams{ + Rule: bazelOverlay, + Output: bazelOverlayWorkspaceFile, + Inputs: deps, + Args: map[string]string{ + "outDir": bazelOverlayDirectory.String(), + }, + }) + + // Add a phony target for building the bazel overlay + ctx.Phony("bazel_overlay", bazelOverlayWorkspaceFile) +} diff --git a/cmd/soong_build/Android.bp b/cmd/soong_build/Android.bp index b559bac86..7b8352b6a 100644 --- a/cmd/soong_build/Android.bp +++ b/cmd/soong_build/Android.bp @@ -26,6 +26,7 @@ bootstrap_go_binary { srcs: [ "main.go", "writedocs.go", + "bazel_overlay.go", ], primaryBuilder: true, } diff --git a/cmd/soong_build/bazel_overlay.go b/cmd/soong_build/bazel_overlay.go new file mode 100644 index 000000000..e37c163d4 --- /dev/null +++ b/cmd/soong_build/bazel_overlay.go @@ -0,0 +1,173 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "android/soong/android" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/google/blueprint" +) + +const ( + soongModuleLoad = `package(default_visibility = ["//visibility:public"]) +load("//:soong_module.bzl", "soong_module") +` + + // A BUILD file target snippet representing a Soong module + soongModuleTarget = `soong_module( + name = "%s", + module_name = "%s", + module_type = "%s", + module_variant = "%s", + deps = [ + %s + ], +) +` + + // The soong_module rule implementation in a .bzl file + soongModuleBzl = ` +SoongModuleInfo = provider( + fields = { + "name": "Name of module", + "type": "Type of module", + "variant": "Variant of module", + }, +) + +def _soong_module_impl(ctx): + return [ + SoongModuleInfo( + name = ctx.attr.module_name, + type = ctx.attr.module_type, + variant = ctx.attr.module_variant, + ), + ] + +soong_module = rule( + implementation = _soong_module_impl, + attrs = { + "module_name": attr.string(mandatory = True), + "module_type": attr.string(mandatory = True), + "module_variant": attr.string(), + "deps": attr.label_list(providers = [SoongModuleInfo]), + }, +) +` +) + +func targetNameWithVariant(c *blueprint.Context, logicModule blueprint.Module) string { + name := "" + if c.ModuleSubDir(logicModule) != "" { + name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule) + } else { + name = c.ModuleName(logicModule) + } + + return strings.Replace(name, "//", "", 1) +} + +func qualifiedTargetLabel(c *blueprint.Context, logicModule blueprint.Module) string { + return "//" + + packagePath(c, logicModule) + + ":" + + targetNameWithVariant(c, logicModule) +} + +func packagePath(c *blueprint.Context, logicModule blueprint.Module) string { + return filepath.Dir(c.BlueprintFile(logicModule)) +} + +func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error { + blueprintCtx := ctx.Context + blueprintCtx.VisitAllModules(func(module blueprint.Module) { + buildFile, err := buildFileForModule(blueprintCtx, module) + if err != nil { + panic(err) + } + + // TODO(b/163018919): DirectDeps can have duplicate (module, variant) + // items, if the modules are added using different DependencyTag. Figure + // out the implications of that. + depLabels := map[string]bool{} + blueprintCtx.VisitDirectDeps(module, func(depModule blueprint.Module) { + depLabels[qualifiedTargetLabel(blueprintCtx, depModule)] = true + }) + + var depLabelList string + for depLabel, _ := range depLabels { + depLabelList += "\"" + depLabel + "\",\n " + } + buildFile.Write([]byte( + fmt.Sprintf( + soongModuleTarget, + targetNameWithVariant(blueprintCtx, module), + blueprintCtx.ModuleName(module), + blueprintCtx.ModuleType(module), + // misleading name, this actually returns the variant. + blueprintCtx.ModuleSubDir(module), + depLabelList))) + buildFile.Close() + }) + + if err := writeReadOnlyFile(bazelOverlayDir, "WORKSPACE", ""); err != nil { + return err + } + + if err := writeReadOnlyFile(bazelOverlayDir, "BUILD", ""); err != nil { + return err + } + + return writeReadOnlyFile(bazelOverlayDir, "soong_module.bzl", soongModuleBzl) +} + +func buildFileForModule(ctx *blueprint.Context, module blueprint.Module) (*os.File, error) { + // Create nested directories for the BUILD file + dirPath := filepath.Join(bazelOverlayDir, packagePath(ctx, module)) + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + os.MkdirAll(dirPath, os.ModePerm) + } + // Open the file for appending, and create it if it doesn't exist + f, err := os.OpenFile( + filepath.Join(dirPath, "BUILD.bazel"), + os.O_APPEND|os.O_CREATE|os.O_WRONLY, + 0644) + if err != nil { + return nil, err + } + + // If the file is empty, add the load statement for the `soong_module` rule + fi, err := f.Stat() + if err != nil { + return nil, err + } + if fi.Size() == 0 { + f.Write([]byte(soongModuleLoad + "\n")) + } + + return f, nil +} + +// The overlay directory should be read-only, sufficient for bazel query. +func writeReadOnlyFile(dir string, baseName string, content string) error { + workspaceFile := filepath.Join(bazelOverlayDir, baseName) + // 0444 is read-only + return ioutil.WriteFile(workspaceFile, []byte(content), 0444) +} diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 532d9e486..01a39a216 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -26,11 +26,13 @@ import ( ) var ( - docFile string + docFile string + bazelOverlayDir string ) func init() { flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output") + flag.StringVar(&bazelOverlayDir, "bazel_overlay_dir", "", "path to the bazel overlay directory") } func newNameResolver(config android.Config) *android.NameResolver { @@ -65,7 +67,7 @@ func main() { os.Exit(1) } - if docFile != "" { + if !shouldPrepareBuildActions() { configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) } @@ -85,6 +87,13 @@ func main() { bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...) + if bazelOverlayDir != "" { + if err := createBazelOverlay(ctx, bazelOverlayDir); err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } + } + if docFile != "" { if err := writeDocs(ctx, docFile); err != nil { fmt.Fprintf(os.Stderr, "%s", err) @@ -94,7 +103,7 @@ func main() { // TODO(ccross): make this a command line argument. Requires plumbing through blueprint // to affect the command line of the primary builder. - if docFile == "" { + if shouldPrepareBuildActions() { metricsFile := filepath.Join(bootstrap.BuildDir, "soong_build_metrics.pb") err = android.WriteMetrics(configuration, metricsFile) if err != nil { @@ -103,3 +112,9 @@ func main() { } } } + +func shouldPrepareBuildActions() bool { + // If we're writing soong_docs or bazel_overlay, don't write build.ninja or + // collect metrics. + return docFile == "" && bazelOverlayDir == "" +}