tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

bp2build: add a simple UI to report migration progress.

Browse source 
Sample output:

[bp2build] cc_library_headers: 5 targets
[bp2build] cc_object: 5 targets
[bp2build] filegroup: 4 targets
[bp2build] genrule: 4 targets
[bp2build] sh_binary: 1 targets
[bp2build] Generated 19 total BUILD targets from 39270 Android.bp modules.

This CL adds an additional CodegenMetrics return value to
GenerateBazelTargets calls, which are called from bp2build, queryview,
and their tests. For this UI, we only want to use it for bp2build, and
not queryview or tests, since it's not useful for the former, and can
pollute the CLI for the latter.

Test: build/bazel/scripts/milestone-2/demo.sh
Change-Id: Ic84307a1ed1a25e360c9b23459e5449d932bc2e7
This commit is contained in:
Jingwen Chen 2021-02-19 00:48:40 -05:00
parent f4dd965933
commit 164e0867fc
10 changed files with 107 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
bp2build/Android.bp
View file

@ -11,6 +11,7 @@ bootstrap_go_package {
"build_conversion.go",
"bzl_conversion.go",
"conversion.go",
"metrics.go",
],
deps: [
"soong-android",

6
bp2build/bp2build.go
View file

@ -22,13 +22,13 @@ import (
// The Bazel bp2build code generator is responsible for writing .bzl files that are equivalent to
// Android.bp files that are capable of being built with Bazel.
func Codegen(ctx CodegenContext) {
func Codegen(ctx CodegenContext) CodegenMetrics {
outputDir := android.PathForOutput(ctx, "bp2build")
android.RemoveAllOutputDir(outputDir)
ruleShims := CreateRuleShims(android.ModuleTypeFactories())
buildToTargets := GenerateBazelTargets(ctx.Context(), ctx.mode)
buildToTargets, metrics := GenerateBazelTargets(ctx)
filesToWrite := CreateBazelFiles(ruleShims, buildToTargets, ctx.mode)
for _, f := range filesToWrite {
@ -36,6 +36,8 @@ func Codegen(ctx CodegenContext) {
fmt.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err)
}
}
return metrics
}
func writeFile(outputDir android.OutputPath, ctx android.PathContext, f BazelFile) error {

43
bp2build/build_conversion.go
View file

@ -105,6 +105,10 @@ type CodegenContext struct {
mode CodegenMode
}
func (c *CodegenContext) Mode() CodegenMode {
return c.mode
}
// CodegenMode is an enum to differentiate code-generation modes.
type CodegenMode int
@ -160,33 +164,54 @@ func propsToAttributes(props map[string]string) string {
return attributes
}
func GenerateBazelTargets(ctx bpToBuildContext, codegenMode CodegenMode) map[string]BazelTargets {
func GenerateBazelTargets(ctx CodegenContext) (map[string]BazelTargets, CodegenMetrics) {
buildFileToTargets := make(map[string]BazelTargets)
ctx.VisitAllModules(func(m blueprint.Module) {
dir := ctx.ModuleDir(m)
// Simple metrics tracking for bp2build
totalModuleCount := 0
ruleClassCount := make(map[string]int)
bpCtx := ctx.Context()
bpCtx.VisitAllModules(func(m blueprint.Module) {
dir := bpCtx.ModuleDir(m)
var t BazelTarget
switch codegenMode {
switch ctx.Mode() {
case Bp2Build:
if _, ok := m.(android.BazelTargetModule); !ok {
// Only include regular Soong modules (non-BazelTargetModules) into the total count.
totalModuleCount += 1
return
}
t = generateBazelTarget(ctx, m)
t = generateBazelTarget(bpCtx, m)
ruleClassCount[t.ruleClass] += 1
case QueryView:
// Blocklist certain module types from being generated.
if canonicalizeModuleType(ctx.ModuleType(m)) == "package" {
if canonicalizeModuleType(bpCtx.ModuleType(m)) == "package" {
// package module name contain slashes, and thus cannot
// be mapped cleanly to a bazel label.
return
}
t = generateSoongModuleTarget(ctx, m)
t = generateSoongModuleTarget(bpCtx, m)
default:
panic(fmt.Errorf("Unknown code-generation mode: %s", codegenMode))
panic(fmt.Errorf("Unknown code-generation mode: %s", ctx.Mode()))
}
buildFileToTargets[dir] = append(buildFileToTargets[dir], t)
})
return buildFileToTargets
metrics := CodegenMetrics{
TotalModuleCount: totalModuleCount,
RuleClassCount: ruleClassCount,
}
return buildFileToTargets, metrics
}
// Helper method for tests to easily access the targets in a dir.
func GenerateBazelTargetsForDir(codegenCtx CodegenContext, dir string) BazelTargets {
bazelTargetsMap, _ := GenerateBazelTargets(codegenCtx)
return bazelTargetsMap[dir]
}
// Helper method to trim quotes around strings.

22
bp2build/build_conversion_test.go
View file

@ -194,6 +194,7 @@ func TestGenerateSoongModuleTargets(t *testing.T) {
for _, testCase := range testCases {
config := android.TestConfig(buildDir, nil, testCase.bp, nil)
ctx := android.NewTestContext(config)
ctx.RegisterModuleType("custom", customModuleFactory)
ctx.Register()
@ -202,7 +203,8 @@ func TestGenerateSoongModuleTargets(t *testing.T) {
_, errs = ctx.PrepareBuildActions(config)
android.FailIfErrored(t, errs)
bazelTargets := GenerateBazelTargets(ctx.Context.Context, QueryView)[dir]
codegenCtx := NewCodegenContext(config, *ctx.Context, QueryView)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, dir)
if actualCount, expectedCount := len(bazelTargets), 1; actualCount != expectedCount {
t.Fatalf("Expected %d bazel target, got %d", expectedCount, actualCount)
}
@ -245,6 +247,7 @@ func TestGenerateBazelTargetModules(t *testing.T) {
for _, testCase := range testCases {
config := android.TestConfig(buildDir, nil, testCase.bp, nil)
ctx := android.NewTestContext(config)
ctx.RegisterModuleType("custom", customModuleFactory)
ctx.RegisterBp2BuildMutator("custom", customBp2BuildMutator)
ctx.RegisterForBazelConversion()
@ -258,7 +261,9 @@ func TestGenerateBazelTargetModules(t *testing.T) {
continue
}
bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[dir]
codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, dir)
if actualCount, expectedCount := len(bazelTargets), 1; actualCount != expectedCount {
t.Errorf("Expected %d bazel target, got %d", expectedCount, actualCount)
} else {
@ -415,7 +420,8 @@ load("//build/bazel/rules:rules.bzl", "my_library")`,
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[dir]
codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, dir)
if actualCount := len(bazelTargets); actualCount != testCase.expectedBazelTargetCount {
t.Fatalf("Expected %d bazel target, got %d", testCase.expectedBazelTargetCount, actualCount)
}
@ -904,7 +910,9 @@ genrule {
if testCase.dir != "" {
checkDir = testCase.dir
}
bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[checkDir]
codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, checkDir)
if actualCount, expectedCount := len(bazelTargets), len(testCase.expectedBazelTargets); actualCount != expectedCount {
t.Errorf("%s: Expected %d bazel target, got %d", testCase.description, expectedCount, actualCount)
} else {
@ -1118,7 +1126,8 @@ genrule {
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[dir]
codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, dir)
if actualCount := len(bazelTargets); actualCount != 1 {
t.Fatalf("%s: Expected 1 bazel target, got %d", testCase.description, actualCount)
}
@ -1205,7 +1214,8 @@ func TestAllowlistingBp2buildTargets(t *testing.T) {
_, errs = ctx.ResolveDependencies(config)
android.FailIfErrored(t, errs)
bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[dir]
codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, dir)
if actualCount := len(bazelTargets); actualCount != testCase.expectedCount {
t.Fatalf("%s: Expected %d bazel target, got %d", testCase.description, testCase.expectedCount, actualCount)
}

3
bp2build/cc_library_headers_conversion_test.go
View file

@ -202,7 +202,8 @@ cc_library_headers {
if testCase.dir != "" {
checkDir = testCase.dir
}
bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[checkDir]
codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, checkDir)
if actualCount, expectedCount := len(bazelTargets), len(testCase.expectedBazelTargets); actualCount != expectedCount {
t.Errorf("%s: Expected %d bazel target, got %d", testCase.description, expectedCount, actualCount)
} else {

3
bp2build/cc_object_conversion_test.go
View file

@ -166,7 +166,8 @@ cc_defaults {
continue
}
bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[dir]
codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, dir)
if actualCount, expectedCount := len(bazelTargets), len(testCase.expectedBazelTargets); actualCount != expectedCount {
fmt.Println(bazelTargets)
t.Errorf("%s: Expected %d bazel target, got %d", testCase.description, expectedCount, actualCount)

30
bp2build/metrics.go Normal file
View file

@ -0,0 +1,30 @@
package bp2build
import (
"android/soong/android"
"fmt"
)
// Simple metrics struct to collect information about a Blueprint to BUILD
// conversion process.
type CodegenMetrics struct {
// Total number of Soong/Blueprint modules
TotalModuleCount int
// Counts of generated Bazel targets per Bazel rule class
RuleClassCount map[string]int
}
// Print the codegen metrics to stdout.
func (metrics CodegenMetrics) Print() {
generatedTargetCount := 0
for _, ruleClass := range android.SortedStringKeys(metrics.RuleClassCount) {
count := metrics.RuleClassCount[ruleClass]
fmt.Printf("[bp2build] %s: %d targets\n", ruleClass, count)
generatedTargetCount += count
}
fmt.Printf(
"[bp2build] Generated %d total BUILD targets from %d Android.bp modules.\n",
generatedTargetCount,
metrics.TotalModuleCount)
}

3
bp2build/sh_conversion_test.go
View file

@ -115,7 +115,8 @@ func TestShBinaryBp2Build(t *testing.T) {
if testCase.dir != "" {
checkDir = testCase.dir
}
bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[checkDir]
codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build)
bazelTargets := GenerateBazelTargetsForDir(codegenCtx, checkDir)
if actualCount, expectedCount := len(bazelTargets), len(testCase.expectedBazelTargets); actualCount != expectedCount {
t.Errorf("%s: Expected %d bazel target, got %d", testCase.description, expectedCount, actualCount)
} else {

14
cmd/soong_build/main.go
View file

@ -134,7 +134,9 @@ func main() {
// Convert the Soong module graph into Bazel BUILD files.
if bazelQueryViewDir != "" {
if err := createBazelQueryView(ctx, bazelQueryViewDir); err != nil {
// Run the code-generation phase to convert BazelTargetModules to BUILD files.
codegenContext := bp2build.NewCodegenContext(configuration, *ctx, bp2build.QueryView)
if err := createBazelQueryView(codegenContext, bazelQueryViewDir); err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
}
@ -188,9 +190,15 @@ func runBp2Build(srcDir string, configuration android.Config) {
// from the regular Modules.
bootstrap.Main(bp2buildCtx.Context, configuration, extraNinjaDeps...)
// Run the code-generation phase to convert BazelTargetModules to BUILD files.
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion metrics to the user.
codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build)
bp2build.Codegen(codegenContext)
metrics := bp2build.Codegen(codegenContext)
// Only report metrics when in bp2build mode. The metrics aren't relevant
// for queryview, since that's a total repo-wide conversion and there's a
// 1:1 mapping for each module.
metrics.Print()
// Workarounds to support running bp2build in a clean AOSP checkout with no
// prior builds, and exiting early as soon as the BUILD files get generated,

7
cmd/soong_build/queryview.go
View file

@ -22,9 +22,12 @@ import (
"path/filepath"
)
func createBazelQueryView(ctx *android.Context, bazelQueryViewDir string) error {
func createBazelQueryView(ctx bp2build.CodegenContext, bazelQueryViewDir string) error {
ruleShims := bp2build.CreateRuleShims(android.ModuleTypeFactories())
buildToTargets := bp2build.GenerateBazelTargets(*ctx, bp2build.QueryView)
// Ignore metrics reporting for queryview, since queryview is already a full-repo
// conversion and can use data from bazel query directly.
buildToTargets, _ := bp2build.GenerateBazelTargets(ctx)
filesToWrite := bp2build.CreateBazelFiles(ruleShims, buildToTargets, bp2build.QueryView)
for _, f := range filesToWrite {