tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Initial implementation of the bazel sandwich" into main

Browse source
This commit is contained in:
Treehugger Robot 2023-07-31 23:00:55 +00:00 committed by Gerrit Code Review
commit 61a27f80cd
8 changed files with 323 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
android/allowlists/allowlists.go
View file

@ -1643,4 +1643,14 @@ var (
"art_": DEFAULT_PRIORITIZED_WEIGHT,
"ndk_library": DEFAULT_PRIORITIZED_WEIGHT,
}
BazelSandwichTargets = []struct {
Label string
Host bool
}{
{
Label: "//build/bazel/examples/partitions:system_image",
Host: false,
},
}
)

89
android/bazel_handler.go
View file

@ -186,6 +186,8 @@ type BazelContext interface {
// Returns the depsets defined in Bazel's aquery response.
AqueryDepsets() []bazel.AqueryDepset
QueueBazelSandwichCqueryRequests(config Config) error
}
type bazelRunner interface {
@ -264,6 +266,10 @@ func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequ
m.BazelRequests[key] = true
}
func (m MockBazelContext) QueueBazelSandwichCqueryRequests(config Config) error {
panic("unimplemented")
}
func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) {
result, ok := m.LabelToOutputFiles[label]
if !ok {
@ -424,6 +430,10 @@ func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configK
panic("unimplemented")
}
func (n noopBazelContext) QueueBazelSandwichCqueryRequests(config Config) error {
panic("unimplemented")
}
func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) {
panic("unimplemented")
}
@ -1042,6 +1052,45 @@ var (
allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd}
)
func GetBazelSandwichCqueryRequests(config Config) ([]cqueryKey, error) {
result := make([]cqueryKey, 0, len(allowlists.BazelSandwichTargets))
// Note that bazel "targets" are different from soong "targets", the bazel targets are
// synonymous with soong modules, and soong targets are a configuration a module is built in.
for _, target := range allowlists.BazelSandwichTargets {
var soongTarget Target
if target.Host {
soongTarget = config.BuildOSTarget
} else {
soongTarget = config.AndroidCommonTarget
}
result = append(result, cqueryKey{
label: target.Label,
requestType: cquery.GetOutputFiles,
configKey: configKey{
arch: soongTarget.Arch.String(),
osType: soongTarget.Os,
},
})
}
return result, nil
}
// QueueBazelSandwichCqueryRequests queues cquery requests for all the bazel labels in
// bazel_sandwich_targets. These will later be given phony targets so that they can be built on the
// command line.
func (context *mixedBuildBazelContext) QueueBazelSandwichCqueryRequests(config Config) error {
requests, err := GetBazelSandwichCqueryRequests(config)
if err != nil {
return err
}
for _, request := range requests {
context.QueueBazelRequest(request.label, request.requestType, request.configKey)
}
return nil
}
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext.
func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error {
@ -1255,6 +1304,11 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__")
bazelOutDir := path.Join(executionRoot, "bazel-out")
rel, err := filepath.Rel(ctx.Config().OutDir(), executionRoot)
if err != nil {
ctx.Errorf("%s", err.Error())
}
dotdotsToOutRoot := strings.Repeat("../", strings.Count(rel, "/")+1)
for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
// nil build statements are a valid case where we do not create an action because it is
// unnecessary or handled by other processing
@ -1286,7 +1340,8 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
})
}
}
createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset)
createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset, dotdotsToOutRoot)
desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
rule.Build(fmt.Sprintf("bazel %d", index), desc)
continue
@ -1331,6 +1386,24 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
panic(fmt.Sprintf("unhandled build statement: %v", buildStatement))
}
}
// Create phony targets for all the bazel sandwich output files
requests, err := GetBazelSandwichCqueryRequests(ctx.Config())
if err != nil {
ctx.Errorf(err.Error())
}
for _, request := range requests {
files, err := ctx.Config().BazelContext.GetOutputFiles(request.label, request.configKey)
if err != nil {
ctx.Errorf(err.Error())
}
filesAsPaths := make([]Path, 0, len(files))
for _, file := range files {
filesAsPaths = append(filesAsPaths, PathForBazelOut(ctx, file))
}
ctx.Phony("bazel_sandwich", filesAsPaths...)
}
ctx.Phony("checkbuild", PathForPhony(ctx, "bazel_sandwich"))
}
// Returns a out dir path for a sandboxed mixed build action
@ -1344,7 +1417,7 @@ func intermediatePathForSboxMixedBuildAction(ctx PathContext, statement *bazel.B
}
// Register bazel-owned build statements (obtained from the aquery invocation).
func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset) {
func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset, dotdotsToOutRoot string) {
// executionRoot is the action cwd.
if buildStatement.ShouldRunInSbox {
// mkdir -p ensures that the directory exists when run via sbox
@ -1367,14 +1440,17 @@ func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement
cmd.Flag(pair.Key + "=" + pair.Value)
}
command := buildStatement.Command
command = strings.ReplaceAll(command, "{DOTDOTS_TO_OUTPUT_ROOT}", dotdotsToOutRoot)
// The actual Bazel action.
if len(buildStatement.Command) > 16*1024 {
if len(command) > 16*1024 {
commandFile := PathForBazelOut(ctx, buildStatement.OutputPaths[0]+".sh")
WriteFileRule(ctx, commandFile, buildStatement.Command)
WriteFileRule(ctx, commandFile, command)
cmd.Text("bash").Text(buildStatement.OutputPaths[0] + ".sh").Implicit(commandFile)
} else {
cmd.Text(buildStatement.Command)
cmd.Text(command)
}
for _, outputPath := range buildStatement.OutputPaths {
@ -1403,6 +1479,9 @@ func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement
cmd.Implicit(PathForPhony(ctx, otherDepsetName))
}
}
for _, implicitPath := range buildStatement.ImplicitDeps {
cmd.Implicit(PathForArbitraryOutput(ctx, implicitPath))
}
if depfile := buildStatement.Depfile; depfile != nil {
// The paths in depfile are relative to `executionRoot`.

4
android/bazel_handler_test.go
View file

@ -181,7 +181,7 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) {
cmd := RuleBuilderCommand{}
ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{})
createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "")
if actual, expected := cmd.buf.String(), testCase.command; expected != actual {
t.Errorf("expected: [%s], actual: [%s]", expected, actual)
}
@ -224,7 +224,7 @@ func TestMixedBuildSandboxedAction(t *testing.T) {
cmd := RuleBuilderCommand{}
ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{})
createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "")
// Assert that the output is generated in an intermediate directory
// fe05bcdcdc4928012781a5f1a2a77cbb5398e106 is the sha1 checksum of "one"
if actual, expected := cmd.outputs[0].String(), "out/soong/mixed_build_sbox_intermediates/fe05bcdcdc4928012781a5f1a2a77cbb5398e106/test/exec_root/one"; expected != actual {

20
android/paths.go
View file

@ -1029,16 +1029,16 @@ func (p basePath) withRel(rel string) basePath {
return p
}
func (p basePath) RelativeToTop() Path {
ensureTestOnly()
return p
}
// SourcePath is a Path representing a file path rooted from SrcDir
type SourcePath struct {
basePath
}
func (p SourcePath) RelativeToTop() Path {
ensureTestOnly()
return p
}
var _ Path = SourcePath{}
func (p SourcePath) withRel(rel string) SourcePath {
@ -1126,6 +1126,16 @@ func PathForSource(ctx PathContext, pathComponents ...string) SourcePath {
return path
}
// PathForArbitraryOutput creates a path for the given components. Unlike PathForOutput,
// the path is relative to the root of the output folder, not the out/soong folder.
func PathForArbitraryOutput(ctx PathContext, pathComponents ...string) Path {
p, err := validatePath(pathComponents...)
if err != nil {
reportPathError(ctx, err)
}
return basePath{path: filepath.Join(ctx.Config().OutDir(), p)}
}
// MaybeExistentPathForSource joins the provided path components and validates that the result
// neither escapes the source dir nor is in the out dir.
// It does not validate whether the path exists.

76
bazel/aquery.go
View file

@ -17,15 +17,15 @@ package bazel
import (
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"path/filepath"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
"reflect"
"sort"
"strings"
"sync"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
"github.com/google/blueprint/metrics"
"github.com/google/blueprint/proptools"
"google.golang.org/protobuf/proto"
@ -119,6 +119,10 @@ type BuildStatement struct {
// If ShouldRunInSbox is true, Soong will use sbox to created an isolated environment
// and run the mixed build action there
ShouldRunInSbox bool
// A list of files to add as implicit deps to the outputs of this BuildStatement.
// Unlike most properties in BuildStatement, these paths must be relative to the root of
// the whole out/ folder, instead of relative to ctx.Config().BazelContext.OutputBase()
ImplicitDeps []string
}
// A helper type for aquery processing which facilitates retrieval of path IDs from their
@ -581,6 +585,72 @@ func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry *ana
}, nil
}
type bazelSandwichJson struct {
Target string `json:"target"`
DependOnTarget *bool `json:"depend_on_target,omitempty"`
ImplicitDeps []string `json:"implicit_deps"`
}
func (a *aqueryArtifactHandler) unresolvedSymlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
outputPaths, depfile, err := a.getOutputPaths(actionEntry)
if err != nil {
return nil, err
}
if len(actionEntry.InputDepSetIds) != 0 || len(outputPaths) != 1 {
return nil, fmt.Errorf("expected 0 inputs and 1 output to symlink action, got: input %q, output %q", actionEntry.InputDepSetIds, outputPaths)
}
target := actionEntry.UnresolvedSymlinkTarget
if target == "" {
return nil, fmt.Errorf("expected an unresolved_symlink_target, but didn't get one")
}
if filepath.Clean(target) != target {
return nil, fmt.Errorf("expected %q, got %q", filepath.Clean(target), target)
}
if strings.HasPrefix(target, "/") {
return nil, fmt.Errorf("no absolute symlinks allowed: %s", target)
}
out := outputPaths[0]
outDir := filepath.Dir(out)
var implicitDeps []string
if strings.HasPrefix(target, "bazel_sandwich:") {
j := bazelSandwichJson{}
err := json.Unmarshal([]byte(target[len("bazel_sandwich:"):]), &j)
if err != nil {
return nil, err
}
if proptools.BoolDefault(j.DependOnTarget, true) {
implicitDeps = append(implicitDeps, j.Target)
}
implicitDeps = append(implicitDeps, j.ImplicitDeps...)
dotDotsToReachCwd := ""
if outDir != "." {
dotDotsToReachCwd = strings.Repeat("../", strings.Count(outDir, "/")+1)
}
target = proptools.ShellEscapeIncludingSpaces(j.Target)
target = "{DOTDOTS_TO_OUTPUT_ROOT}" + dotDotsToReachCwd + target
} else {
target = proptools.ShellEscapeIncludingSpaces(target)
}
outDir = proptools.ShellEscapeIncludingSpaces(outDir)
out = proptools.ShellEscapeIncludingSpaces(out)
// Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, target)
symlinkPaths := outputPaths[:]
buildStatement := &BuildStatement{
Command: command,
Depfile: depfile,
OutputPaths: outputPaths,
Env: actionEntry.EnvironmentVariables,
Mnemonic: actionEntry.Mnemonic,
SymlinkPaths: symlinkPaths,
ImplicitDeps: implicitDeps,
}
return buildStatement, nil
}
func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
outputPaths, depfile, err := a.getOutputPaths(actionEntry)
if err != nil {
@ -690,6 +760,8 @@ func (a *aqueryArtifactHandler) actionToBuildStatement(actionEntry *analysis_v2_
return a.fileWriteActionBuildStatement(actionEntry)
case "SymlinkTree":
return a.symlinkTreeActionBuildStatement(actionEntry)
case "UnresolvedSymlink":
return a.unresolvedSymlinkActionBuildStatement(actionEntry)
}
if len(actionEntry.Arguments) < 1 {

133
bazel/aquery_test.go
View file

@ -357,9 +357,11 @@ func TestDepfiles(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
if expected := 1; len(actual) != expected {
t.Fatalf("Expected %d build statements, got %d", expected, len(actual))
return
}
bs := actual[0]
@ -544,6 +546,7 @@ func TestSymlinkTree(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{
&BuildStatement{
@ -756,9 +759,11 @@ func TestMiddlemenAction(t *testing.T) {
actualBuildStatements, actualDepsets, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
if expected := 2; len(actualBuildStatements) != expected {
t.Fatalf("Expected %d build statements, got %d %#v", expected, len(actualBuildStatements), actualBuildStatements)
return
}
expectedDepsetFiles := [][]string{
@ -859,6 +864,7 @@ func TestSimpleSymlink(t *testing.T) {
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
expectedBuildStatements := []*BuildStatement{
@ -907,6 +913,7 @@ func TestSymlinkQuotesPaths(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
expectedBuildStatements := []*BuildStatement{
@ -1017,6 +1024,7 @@ func TestTemplateExpandActionSubstitutions(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
expectedBuildStatements := []*BuildStatement{
@ -1088,6 +1096,7 @@ func TestFileWrite(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{
&BuildStatement{
@ -1126,6 +1135,7 @@ func TestSourceSymlinkManifest(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{
&BuildStatement{
@ -1136,6 +1146,126 @@ func TestSourceSymlinkManifest(t *testing.T) {
}, actual)
}
func TestUnresolvedSymlink(t *testing.T) {
const inputString = `
{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 }
],
"actions": [{
"target_id": 1,
"action_key": "x",
"mnemonic": "UnresolvedSymlink",
"configuration_id": 1,
"output_ids": [1],
"primary_output_id": 1,
"execution_platform": "//build/bazel/platforms:linux_x86_64",
"unresolved_symlink_target": "symlink/target"
}],
"path_fragments": [
{ "id": 1, "label": "path/to/symlink" }
]
}
`
data, err := JsonToActionGraphContainer(inputString)
if err != nil {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{{
Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf symlink/target path/to/symlink",
OutputPaths: []string{"path/to/symlink"},
Mnemonic: "UnresolvedSymlink",
SymlinkPaths: []string{"path/to/symlink"},
}}, actual)
}
func TestUnresolvedSymlinkBazelSandwich(t *testing.T) {
const inputString = `
{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 }
],
"actions": [{
"target_id": 1,
"action_key": "x",
"mnemonic": "UnresolvedSymlink",
"configuration_id": 1,
"output_ids": [1],
"primary_output_id": 1,
"execution_platform": "//build/bazel/platforms:linux_x86_64",
"unresolved_symlink_target": "bazel_sandwich:{\"target\":\"target/product/emulator_x86_64/system\"}"
}],
"path_fragments": [
{ "id": 1, "label": "path/to/symlink" }
]
}
`
data, err := JsonToActionGraphContainer(inputString)
if err != nil {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{{
Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink",
OutputPaths: []string{"path/to/symlink"},
Mnemonic: "UnresolvedSymlink",
SymlinkPaths: []string{"path/to/symlink"},
ImplicitDeps: []string{"target/product/emulator_x86_64/system"},
}}, actual)
}
func TestUnresolvedSymlinkBazelSandwichWithAlternativeDeps(t *testing.T) {
const inputString = `
{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 }
],
"actions": [{
"target_id": 1,
"action_key": "x",
"mnemonic": "UnresolvedSymlink",
"configuration_id": 1,
"output_ids": [1],
"primary_output_id": 1,
"execution_platform": "//build/bazel/platforms:linux_x86_64",
"unresolved_symlink_target": "bazel_sandwich:{\"depend_on_target\":false,\"implicit_deps\":[\"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp\"],\"target\":\"target/product/emulator_x86_64/system\"}"
}],
"path_fragments": [
{ "id": 1, "label": "path/to/symlink" }
]
}
`
data, err := JsonToActionGraphContainer(inputString)
if err != nil {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{{
Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink",
OutputPaths: []string{"path/to/symlink"},
Mnemonic: "UnresolvedSymlink",
SymlinkPaths: []string{"path/to/symlink"},
// Note that the target of the symlink, target/product/emulator_x86_64/system, is not listed here
ImplicitDeps: []string{"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp"},
}}, actual)
}
func assertError(t *testing.T, err error, expected string) {
t.Helper()
if err == nil {
@ -1201,6 +1331,9 @@ func buildStatementEquals(first *BuildStatement, second *BuildStatement) string
if !reflect.DeepEqual(sortedStrings(first.SymlinkPaths), sortedStrings(second.SymlinkPaths)) {
return "SymlinkPaths"
}
if !reflect.DeepEqual(sortedStrings(first.ImplicitDeps), sortedStrings(second.ImplicitDeps)) {
return "ImplicitDeps"
}
if first.Depfile != second.Depfile {
return "Depfile"
}

1
bp2build/bp2build_product_config.go
View file

@ -97,6 +97,7 @@ load("@//build/bazel/product_config:android_product.bzl", "android_product")
android_product(
name = "mixed_builds_product-{VARIANT}",
soong_variables = _soong_variables,
extra_constraints = ["@//build/bazel/platforms:mixed_builds"],
)
`)),
newFile(

4
cmd/soong_build/main.go
View file

@ -121,6 +121,10 @@ func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string {
defer ctx.EventHandler.End("mixed_build")
bazelHook := func() error {
err := ctx.Config().BazelContext.QueueBazelSandwichCqueryRequests(ctx.Config())
if err != nil {
return err
}
return ctx.Config().BazelContext.InvokeBazel(ctx.Config(), ctx)
}
ctx.SetBeforePrepareBuildActionsHook(bazelHook)