tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge changes from topic "build_go_source_mixed_builds" into main

Browse source 
* changes:
  Delete aliases to prebuilts
  Add functionality to sandbox mixed build actions
This commit is contained in:
Spandan Das 2023-07-19 17:21:25 +00:00 committed by Gerrit Code Review
commit 958ca02582
7 changed files with 137 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
android/bazel_handler.go
View file

@ -16,6 +16,8 @@ package android
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"fmt"
"os"
"path"
@ -1222,7 +1224,11 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
ctx.AddNinjaFileDeps(file)
}
depsetHashToDepset := map[string]bazel.AqueryDepset{}
for _, depset := range ctx.Config().BazelContext.AqueryDepsets() {
depsetHashToDepset[depset.ContentHash] = depset
var outputs []Path
var orderOnlies []Path
for _, depsetDepHash := range depset.TransitiveDepSetHashes {
@ -1257,7 +1263,30 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
}
if len(buildStatement.Command) > 0 {
rule := NewRuleBuilder(pctx, ctx)
createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx)
intermediateDir, intermediateDirHash := intermediatePathForSboxMixedBuildAction(ctx, buildStatement)
if buildStatement.ShouldRunInSbox {
// Create a rule to build the output inside a sandbox
// This will create two changes of working directory
// 1. From ANDROID_BUILD_TOP to sbox top
// 2. From sbox top to a a synthetic mixed build execution root relative to it
// Finally, the outputs will be copied to intermediateDir
rule.Sbox(intermediateDir,
PathForOutput(ctx, "mixed_build_sbox_intermediates", intermediateDirHash+".textproto")).
SandboxInputs().
// Since we will cd to mixed build execution root, set sbox's out subdir to empty
// Without this, we will try to copy from $SBOX_SANDBOX_DIR/out/out/bazel/output/execroot/__main__/...
SetSboxOutDirDirAsEmpty()
// Create another set of rules to copy files from the intermediate dir to mixed build execution root
for _, outputPath := range buildStatement.OutputPaths {
ctx.Build(pctx, BuildParams{
Rule: CpIfChanged,
Input: intermediateDir.Join(ctx, executionRoot, outputPath),
Output: PathForBazelOut(ctx, outputPath),
})
}
}
createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset)
desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
rule.Build(fmt.Sprintf("bazel %d", index), desc)
continue
@ -1304,10 +1333,25 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
}
}
// Returns a out dir path for a sandboxed mixed build action
func intermediatePathForSboxMixedBuildAction(ctx PathContext, statement *bazel.BuildStatement) (OutputPath, string) {
// An artifact can be generated by a single buildstatement.
// Use the hash of the first artifact to create a unique path
uniqueDir := sha1.New()
uniqueDir.Write([]byte(statement.OutputPaths[0]))
uniqueDirHashString := hex.EncodeToString(uniqueDir.Sum(nil))
return PathForOutput(ctx, "mixed_build_sbox_intermediates", uniqueDirHashString), uniqueDirHashString
}
// Register bazel-owned build statements (obtained from the aquery invocation).
func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext) {
func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset) {
// executionRoot is the action cwd.
cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot))
if buildStatement.ShouldRunInSbox {
// mkdir -p ensures that the directory exists when run via sbox
cmd.Text(fmt.Sprintf("mkdir -p '%s' && cd '%s' &&", executionRoot, executionRoot))
} else {
cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot))
}
// Remove old outputs, as some actions might not rerun if the outputs are detected.
if len(buildStatement.OutputPaths) > 0 {
@ -1334,7 +1378,16 @@ func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement
}
for _, outputPath := range buildStatement.OutputPaths {
cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
if buildStatement.ShouldRunInSbox {
// The full path has three components that get joined together
// 1. intermediate output dir that `sbox` will place the artifacts at
// 2. mixed build execution root
// 3. artifact path returned by aquery
intermediateDir, _ := intermediatePathForSboxMixedBuildAction(ctx, buildStatement)
cmd.ImplicitOutput(intermediateDir.Join(ctx, executionRoot, outputPath))
} else {
cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
}
}
for _, inputPath := range buildStatement.OrderOnlyInputs {
cmd.OrderOnly(PathForBazelOut(ctx, inputPath))
@ -1343,8 +1396,15 @@ func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement
cmd.Implicit(PathForBazelOut(ctx, inputPath))
}
for _, inputDepsetHash := range buildStatement.InputDepsetHashes {
otherDepsetName := bazelDepsetName(inputDepsetHash)
cmd.Implicit(PathForPhony(ctx, otherDepsetName))
if buildStatement.ShouldRunInSbox {
// Bazel depsets are phony targets that are used to group files.
// We need to copy the grouped files into the sandbox
ds, _ := depsetHashToDepset[inputDepsetHash]
cmd.Implicits(PathsForBazelOut(ctx, ds.DirectArtifacts))
} else {
otherDepsetName := bazelDepsetName(inputDepsetHash)
cmd.Implicit(PathForPhony(ctx, otherDepsetName))
}
}
if depfile := buildStatement.Depfile; depfile != nil {

51
android/bazel_handler_test.go
View file

@ -181,13 +181,62 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) {
cmd := RuleBuilderCommand{}
ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx)
createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{})
if actual, expected := cmd.buf.String(), testCase.command; expected != actual {
t.Errorf("expected: [%s], actual: [%s]", expected, actual)
}
}
}
func TestMixedBuildSandboxedAction(t *testing.T) {
input := `{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 },
{ "id": 2, "path_fragment_id": 2 }],
"actions": [{
"target_Id": 1,
"action_Key": "x",
"mnemonic": "x",
"arguments": ["touch", "foo"],
"input_dep_set_ids": [1],
"output_Ids": [1],
"primary_output_id": 1
}],
"dep_set_of_files": [
{ "id": 1, "direct_artifact_ids": [1, 2] }],
"path_fragments": [
{ "id": 1, "label": "one" },
{ "id": 2, "label": "two" }]
}`
data, err := JsonToActionGraphContainer(input)
if err != nil {
t.Error(err)
}
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{aqueryCmd: string(data)})
err = bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("TestMixedBuildSandboxedAction did not expect error invoking Bazel, but got %s", err)
}
statement := bazelContext.BuildStatementsToRegister()[0]
statement.ShouldRunInSbox = true
cmd := RuleBuilderCommand{}
ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{})
// Assert that the output is generated in an intermediate directory
// fe05bcdcdc4928012781a5f1a2a77cbb5398e106 is the sha1 checksum of "one"
if actual, expected := cmd.outputs[0].String(), "out/soong/mixed_build_sbox_intermediates/fe05bcdcdc4928012781a5f1a2a77cbb5398e106/test/exec_root/one"; expected != actual {
t.Errorf("expected: [%s], actual: [%s]", expected, actual)
}
// Assert the actual command remains unchanged inside the sandbox
if actual, expected := cmd.buf.String(), "mkdir -p 'test/exec_root' && cd 'test/exec_root' && rm -rf 'one' && touch foo"; expected != actual {
t.Errorf("expected: [%s], actual: [%s]", expected, actual)
}
}
func TestCoverageFlagsAfterInvokeBazel(t *testing.T) {
testConfig.productVariables.ClangCoverage = boolPtr(true)

12
android/rule_builder.go
View file

@ -53,6 +53,7 @@ type RuleBuilder struct {
remoteable RemoteRuleSupports
rbeParams *remoteexec.REParams
outDir WritablePath
sboxOutSubDir string
sboxTools bool
sboxInputs bool
sboxManifestPath WritablePath
@ -65,9 +66,18 @@ func NewRuleBuilder(pctx PackageContext, ctx BuilderContext) *RuleBuilder {
pctx: pctx,
ctx: ctx,
temporariesSet: make(map[WritablePath]bool),
sboxOutSubDir: sboxOutSubDir,
}
}
// SetSboxOutDirDirAsEmpty sets the out subdirectory to an empty string
// This is useful for sandboxing actions that change the execution root to a path in out/ (e.g mixed builds)
// For such actions, SetSboxOutDirDirAsEmpty ensures that the path does not become $SBOX_SANDBOX_DIR/out/out/bazel/output/execroot/__main__/...
func (rb *RuleBuilder) SetSboxOutDirDirAsEmpty() *RuleBuilder {
rb.sboxOutSubDir = ""
return rb
}
// RuleBuilderInstall is a tuple of install from and to locations.
type RuleBuilderInstall struct {
From Path
@ -585,7 +595,7 @@ func (r *RuleBuilder) Build(name string, desc string) {
for _, output := range outputs {
rel := Rel(r.ctx, r.outDir.String(), output.String())
command.CopyAfter = append(command.CopyAfter, &sbox_proto.Copy{
From: proto.String(filepath.Join(sboxOutSubDir, rel)),
From: proto.String(filepath.Join(r.sboxOutSubDir, rel)),
To: proto.String(output.String()),
})
}

9
bazel/aquery.go
View file

@ -117,6 +117,9 @@ type BuildStatement struct {
InputPaths []string
OrderOnlyInputs []string
FileContents string
// If ShouldRunInSbox is true, Soong will use sbox to created an isolated environment
// and run the mixed build action there
ShouldRunInSbox bool
}
// A helper type for aquery processing which facilitates retrieval of path IDs from their
@ -517,6 +520,12 @@ func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry *analysis
Env: actionEntry.EnvironmentVariables,
Mnemonic: actionEntry.Mnemonic,
}
if buildStatement.Mnemonic == "GoToolchainBinaryBuild" {
// Unlike b's execution root, mixed build execution root contains a symlink to prebuilts/go
// This causes issues for `GOCACHE=$(mktemp -d) go build ...`
// To prevent this, sandbox this action in mixed builds as well
buildStatement.ShouldRunInSbox = true
}
return buildStatement, nil
}

13
bp2build/build_conversion.go
View file

@ -471,17 +471,6 @@ func generateBazelTargetsGoBinary(ctx *android.Context, g *bootstrap.GoBinary, g
return []BazelTarget{binTarget}, nil
}
var (
// TODO - b/284483729: Remove this denyilst
// Temporary denylist of go binaries that are currently used in mixed builds
// This denylist allows us to rollout bp2build converters for go targets without affecting mixed builds
goBinaryDenylist = []string{
"soong_zip",
"zip2zip",
"bazel_notice_gen",
}
)
func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (conversionResults, []error) {
buildFileToTargets := make(map[string]BazelTargets)
@ -574,7 +563,7 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
targets, targetErrs = generateBazelTargetsGoPackage(bpCtx, glib, nameToGoLibMap)
errs = append(errs, targetErrs...)
metrics.IncrementRuleClassCount("go_library")
} else if gbin, ok := m.(*bootstrap.GoBinary); ok && !android.InList(m.Name(), goBinaryDenylist) {
} else if gbin, ok := m.(*bootstrap.GoBinary); ok {
targets, targetErrs = generateBazelTargetsGoBinary(bpCtx, gbin, nameToGoLibMap)
errs = append(errs, targetErrs...)
metrics.IncrementRuleClassCount("go_binary")

18
cmd/zip2zip/BUILD.bazel
View file

@ -1,18 +0,0 @@
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
alias(
name = "zip2zip",
actual = "//prebuilts/build-tools:linux-x86/bin/zip2zip",
)

20
zip/cmd/BUILD.bazel
View file

@ -1,20 +0,0 @@
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO(b/194644518): Switch to the source version when Bazel can build go
# binaries.
alias(
name = "soong_zip",
actual = "//prebuilts/build-tools:linux-x86/bin/soong_zip",
)