tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Collect more metrics for aquery handling" am: 80936bd15f

Browse source 
Original change: https://android-review.googlesource.com/c/platform/build/soong/+/2431534

Change-Id: I583e1913f14cb9ad9ad5ce6b7180d55d581ee7e7
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Treehugger Robot 2023-02-14 19:51:29 +00:00 committed by Automerger Merge Worker
commit 43002dd9e9
4 changed files with 145 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
android/bazel_handler.go
View file

@ -32,6 +32,7 @@ import (
"android/soong/starlark_fmt"
"github.com/google/blueprint"
"github.com/google/blueprint/metrics"
"android/soong/bazel"
)
@ -135,6 +136,10 @@ func (c cqueryKey) String() string {
return fmt.Sprintf("cquery(%s,%s,%s)", c.label, c.requestType.Name(), c.configKey)
}
type invokeBazelContext interface {
GetEventHandler() *metrics.EventHandler
}
// BazelContext is a context object useful for interacting with Bazel during
// the course of a build. Use of Bazel to evaluate part of the build graph
// is referred to as a "mixed build". (Some modules are managed by Soong,
@ -171,7 +176,7 @@ type BazelContext interface {
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext. The ctx argument is optional and is only
// used for performance data collection
InvokeBazel(config Config, ctx *Context) error
InvokeBazel(config Config, ctx invokeBazelContext) error
// Returns true if Bazel handling is enabled for the module with the given name.
// Note that this only implies "bazel mixed build" allowlisting. The caller
@ -191,7 +196,7 @@ type BazelContext interface {
type bazelRunner interface {
createBazelCommand(config Config, paths *bazelPaths, runName bazel.RunName, command bazelCommand, extraFlags ...string) *exec.Cmd
issueBazelCommand(bazelCmd *exec.Cmd) (output string, errorMessage string, error error)
issueBazelCommand(bazelCmd *exec.Cmd, eventHandler *metrics.EventHandler) (output string, errorMessage string, error error)
}
type bazelPaths struct {
@ -299,7 +304,7 @@ func (m MockBazelContext) GetCcUnstrippedInfo(label string, _ configKey) (cquery
return result, nil
}
func (m MockBazelContext) InvokeBazel(_ Config, _ *Context) error {
func (m MockBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
panic("unimplemented")
}
@ -417,7 +422,7 @@ func (n noopBazelContext) GetCcUnstrippedInfo(_ string, _ configKey) (cquery.CcU
panic("implement me")
}
func (n noopBazelContext) InvokeBazel(_ Config, _ *Context) error {
func (n noopBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
panic("unimplemented")
}
@ -595,7 +600,7 @@ func (r *mockBazelRunner) createBazelCommand(_ Config, _ *bazelPaths, _ bazel.Ru
return cmd
}
func (r *mockBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd) (string, string, error) {
func (r *mockBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, _ *metrics.EventHandler) (string, string, error) {
if command, ok := r.tokens[bazelCmd]; ok {
return r.bazelCommandResults[command], "", nil
}
@ -608,7 +613,9 @@ type builtinBazelRunner struct{}
// Returns (stdout, stderr, error). The first and second return values are strings
// containing the stdout and stderr of the run command, and an error is returned if
// the invocation returned an error code.
func (r *builtinBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd) (string, string, error) {
func (r *builtinBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, eventHandler *metrics.EventHandler) (string, string, error) {
eventHandler.Begin("bazel command")
defer eventHandler.End("bazel command")
stderr := &bytes.Buffer{}
bazelCmd.Stderr = stderr
if output, err := bazelCmd.Output(); err != nil {
@ -985,11 +992,10 @@ var (
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext.
func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx *Context) error {
if ctx != nil {
ctx.EventHandler.Begin("bazel")
defer ctx.EventHandler.End("bazel")
}
func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error {
eventHandler := ctx.GetEventHandler()
eventHandler.Begin("bazel")
defer eventHandler.End("bazel")
if metricsDir := context.paths.BazelMetricsDir(); metricsDir != "" {
if err := os.MkdirAll(metricsDir, 0777); err != nil {
@ -1012,11 +1018,10 @@ func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx *Context)
return nil
}
func (context *mixedBuildBazelContext) runCquery(config Config, ctx *Context) error {
if ctx != nil {
ctx.EventHandler.Begin("cquery")
defer ctx.EventHandler.End("cquery")
}
func (context *mixedBuildBazelContext) runCquery(config Config, ctx invokeBazelContext) error {
eventHandler := ctx.GetEventHandler()
eventHandler.Begin("cquery")
defer eventHandler.End("cquery")
soongInjectionPath := absolutePath(context.paths.injectedFilesDir())
mixedBuildsPath := filepath.Join(soongInjectionPath, "mixed_builds")
if _, err := os.Stat(mixedBuildsPath); os.IsNotExist(err) {
@ -1041,7 +1046,7 @@ func (context *mixedBuildBazelContext) runCquery(config Config, ctx *Context) er
cqueryCommandWithFlag := context.createBazelCommand(config, context.paths, bazel.CqueryBuildRootRunName, cqueryCmd,
"--output=starlark", "--starlark:file="+absolutePath(cqueryFileRelpath))
cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCommandWithFlag)
cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCommandWithFlag, eventHandler)
if cqueryErr != nil {
return cqueryErr
}
@ -1075,11 +1080,10 @@ func writeFileBytesIfChanged(path string, contents []byte, perm os.FileMode) err
return nil
}
func (context *mixedBuildBazelContext) runAquery(config Config, ctx *Context) error {
if ctx != nil {
ctx.EventHandler.Begin("aquery")
defer ctx.EventHandler.End("aquery")
}
func (context *mixedBuildBazelContext) runAquery(config Config, ctx invokeBazelContext) error {
eventHandler := ctx.GetEventHandler()
eventHandler.Begin("aquery")
defer eventHandler.End("aquery")
// Issue an aquery command to retrieve action information about the bazel build tree.
//
// Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's
@ -1105,23 +1109,22 @@ func (context *mixedBuildBazelContext) runAquery(config Config, ctx *Context) er
}
}
aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(config, context.paths, bazel.AqueryBuildRootRunName, aqueryCmd,
extraFlags...))
extraFlags...), eventHandler)
if err != nil {
return err
}
context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput))
context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput), eventHandler)
return err
}
func (context *mixedBuildBazelContext) generateBazelSymlinks(config Config, ctx *Context) error {
if ctx != nil {
ctx.EventHandler.Begin("symlinks")
defer ctx.EventHandler.End("symlinks")
}
func (context *mixedBuildBazelContext) generateBazelSymlinks(config Config, ctx invokeBazelContext) error {
eventHandler := ctx.GetEventHandler()
eventHandler.Begin("symlinks")
defer eventHandler.End("symlinks")
// Issue a build command of the phony root to generate symlink forests for dependencies of the
// Bazel build. This is necessary because aquery invocations do not generate this symlink forest,
// but some of symlinks may be required to resolve source dependencies of the build.
_, _, err := context.issueBazelCommand(context.createBazelCommand(config, context.paths, bazel.BazelBuildPhonyRootRunName, buildCmd))
_, _, err := context.issueBazelCommand(context.createBazelCommand(config, context.paths, bazel.BazelBuildPhonyRootRunName, buildCmd), eventHandler)
return err
}

15
android/bazel_handler_test.go
View file

@ -11,11 +11,18 @@ import (
"android/soong/bazel/cquery"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
"github.com/google/blueprint/metrics"
"google.golang.org/protobuf/proto"
)
var testConfig = TestConfig("out", nil, "", nil)
type testInvokeBazelContext struct{}
func (t *testInvokeBazelContext) GetEventHandler() *metrics.EventHandler {
return &metrics.EventHandler{}
}
func TestRequestResultsAfterInvokeBazel(t *testing.T) {
label := "@//foo:bar"
cfg := configKey{"arm64_armv8-a", Android}
@ -23,7 +30,7 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) {
bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot, 2)"}: `@//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`,
})
bazelContext.QueueBazelRequest(label, cquery.GetOutputFiles, cfg)
err := bazelContext.InvokeBazel(testConfig, nil)
err := bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}
@ -37,7 +44,7 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) {
func TestInvokeBazelWritesBazelFiles(t *testing.T) {
bazelContext, baseDir := testBazelContext(t, map[bazelCommand]string{})
err := bazelContext.InvokeBazel(testConfig, nil)
err := bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}
@ -118,7 +125,7 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) {
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{
bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: string(data)})
err = bazelContext.InvokeBazel(testConfig, nil)
err = bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("testCase #%d: did not expect error invoking Bazel, but got %s", i+1, err)
}
@ -197,7 +204,7 @@ func TestBazelRequestsSorted(t *testing.T) {
func verifyExtraFlags(t *testing.T, config Config, expected string) string {
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{})
err := bazelContext.InvokeBazel(config, nil)
err := bazelContext.InvokeBazel(config, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}

28
bazel/aquery.go
View file

@ -23,9 +23,11 @@ import (
"sort"
"strings"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
"github.com/google/blueprint/metrics"
"github.com/google/blueprint/proptools"
"google.golang.org/protobuf/proto"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
)
type artifactId int
@ -313,7 +315,7 @@ func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) (
// action graph, as described by the given action graph json proto.
// BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets
// are one-to-one with Bazel's depSetOfFiles objects.
func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDepset, error) {
func AqueryBuildStatements(aqueryJsonProto []byte, eventHandler *metrics.EventHandler) ([]BuildStatement, []AqueryDepset, error) {
aqueryProto := &analysis_v2_proto.ActionGraphContainer{}
err := proto.Unmarshal(aqueryJsonProto, aqueryProto)
if err != nil {
@ -387,12 +389,21 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDe
ParentId: pathFragmentId(protoPathFragments.ParentId)})
}
aqueryHandler, err := newAqueryHandler(aqueryResult)
var aqueryHandler *aqueryArtifactHandler
{
eventHandler.Begin("init_handler")
defer eventHandler.End("init_handler")
aqueryHandler, err = newAqueryHandler(aqueryResult)
if err != nil {
return nil, nil, err
}
}
var buildStatements []BuildStatement
{
eventHandler.Begin("build_statements")
defer eventHandler.End("build_statements")
for _, actionEntry := range aqueryResult.Actions {
if shouldSkipAction(actionEntry) {
continue
@ -408,7 +419,7 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDe
} else if actionEntry.isSymlinkTreeAction() {
buildStatement, err = aqueryHandler.symlinkTreeActionBuildStatement(actionEntry)
} else if len(actionEntry.Arguments) < 1 {
return nil, nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic)
err = fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic)
} else {
buildStatement, err = aqueryHandler.normalActionBuildStatement(actionEntry)
}
@ -418,9 +429,13 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDe
}
buildStatements = append(buildStatements, buildStatement)
}
}
depsetsByHash := map[string]AqueryDepset{}
var depsets []AqueryDepset
{
eventHandler.Begin("depsets")
defer eventHandler.End("depsets")
for _, aqueryDepset := range aqueryHandler.depsetIdToAqueryDepset {
if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey {
// Two depsets collide on hash. Ensure that their contents are identical.
@ -432,7 +447,9 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDe
depsets = append(depsets, aqueryDepset)
}
}
}
eventHandler.Do("build_statement_sort", func() {
// Build Statements and depsets must be sorted by their content hash to
// preserve determinism between builds (this will result in consistent ninja file
// output). Note they are not sorted by their original IDs nor their Bazel ordering,
@ -453,9 +470,12 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDe
// There may be multiple outputs, but the output ordering is deterministic.
return outputs_i[0] < outputs_j[0]
})
})
eventHandler.Do("depset_sort", func() {
sort.Slice(depsets, func(i, j int) bool {
return depsets[i].ContentHash < depsets[j].ContentHash
})
})
return buildStatements, depsets, nil
}

46
bazel/aquery_test.go
View file

@ -21,8 +21,10 @@ import (
"sort"
"testing"
"google.golang.org/protobuf/proto"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
"github.com/google/blueprint/metrics"
"google.golang.org/protobuf/proto"
)
func TestAqueryMultiArchGenrule(t *testing.T) {
@ -136,7 +138,7 @@ func TestAqueryMultiArchGenrule(t *testing.T) {
t.Error(err)
return
}
actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data)
actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
var expectedBuildStatements []BuildStatement
for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} {
expectedBuildStatements = append(expectedBuildStatements,
@ -195,7 +197,7 @@ func TestInvalidOutputId(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, "undefined outputId 3")
}
@ -226,7 +228,7 @@ func TestInvalidInputDepsetIdFromAction(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, "undefined (not even empty) input depsetId 2")
}
@ -257,7 +259,7 @@ func TestInvalidInputDepsetIdFromDepset(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, "undefined input depsetId 42 (referenced by depsetId 1)")
}
@ -288,7 +290,7 @@ func TestInvalidInputArtifactId(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, "undefined input artifactId 3")
}
@ -319,7 +321,7 @@ func TestInvalidPathFragmentId(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, "undefined path fragment id 3")
}
@ -352,7 +354,7 @@ func TestDepfiles(t *testing.T) {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data)
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
}
@ -402,7 +404,7 @@ func TestMultipleDepfiles(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, `found multiple potential depfiles "two.d", "other.d"`)
}
@ -483,7 +485,7 @@ func TestTransitiveInputDepsets(t *testing.T) {
t.Error(err)
return
}
actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data)
actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
expectedBuildStatements := []BuildStatement{
{
@ -538,7 +540,7 @@ func TestSymlinkTree(t *testing.T) {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data)
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
}
@ -594,7 +596,7 @@ func TestBazelOutRemovalFromInputDepsets(t *testing.T) {
t.Error(err)
return
}
actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data)
actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
if len(actualDepsets) != 1 {
t.Errorf("expected 1 depset but found %#v", actualDepsets)
return
@ -681,7 +683,7 @@ func TestBazelOutRemovalFromTransitiveInputDepsets(t *testing.T) {
t.Error(err)
return
}
actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data)
actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
if len(actualDepsets) != 0 {
t.Errorf("expected 0 depsets but found %#v", actualDepsets)
return
@ -748,7 +750,7 @@ func TestMiddlemenAction(t *testing.T) {
t.Error(err)
return
}
actualBuildStatements, actualDepsets, err := AqueryBuildStatements(data)
actualBuildStatements, actualDepsets, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
}
@ -845,7 +847,7 @@ func TestSimpleSymlink(t *testing.T) {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data)
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
@ -894,7 +896,7 @@ func TestSymlinkQuotesPaths(t *testing.T) {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data)
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
}
@ -940,7 +942,7 @@ func TestSymlinkMultipleInputs(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file" "other_file"], output ["symlink"]`)
}
@ -971,7 +973,7 @@ func TestSymlinkMultipleOutputs(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, "undefined outputId 2")
}
@ -1004,7 +1006,7 @@ func TestTemplateExpandActionSubstitutions(t *testing.T) {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data)
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
}
@ -1046,7 +1048,7 @@ func TestTemplateExpandActionNoOutput(t *testing.T) {
t.Error(err)
return
}
_, _, err = AqueryBuildStatements(data)
_, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
assertError(t, err, `Expect 1 output to template expand action, got: output []`)
}
@ -1074,7 +1076,7 @@ func TestFileWrite(t *testing.T) {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data)
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
}
@ -1111,7 +1113,7 @@ func TestSourceSymlinkManifest(t *testing.T) {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data)
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
}