tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Batch cquery requests for mixed builds

Browse source 
This adds an extra step to mixed builds: creating a master BUILD/bzl
file pair to facilitate running a single cquery command to analyze
all soong->bazel edges in a single request.

Test: Mixed build tested with aosp/1441774, verified ninja outputs
depend on `bazel-out/` intermediates
Test: Manually verified contents of master BUILD and bzl files

Change-Id: I04803bcc91ac4182578f505b3f42893061ddd167
This commit is contained in:
Chris Parsons 2020-10-23 16:48:08 -04:00 committed by Christopher Parsons
parent a03f8fe6da
commit b0f8ac4ff0
1 changed files with 114 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
android/bazel_handler.go
View file

@ -29,10 +29,16 @@ import (
"github.com/google/blueprint/bootstrap"
)
type CqueryRequestType int
const (
getAllFiles CqueryRequestType = iota
)
// Map key to describe bazel cquery requests.
type cqueryKey struct {
label string
starlarkExpr string
label string
requestType CqueryRequestType
}
type BazelContext interface {
@ -61,6 +67,7 @@ type bazelContext struct {
bazelPath string
outputBase string
workspaceDir string
buildDir string
requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel
requestMutex sync.Mutex // requests can be written in parallel
@ -96,8 +103,7 @@ func (m MockBazelContext) BazelEnabled() bool {
var _ BazelContext = MockBazelContext{}
func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
starlarkExpr := "', '.join([f.path for f in target.files.to_list()])"
result, ok := bazelCtx.cquery(label, starlarkExpr)
result, ok := bazelCtx.cquery(label, getAllFiles)
if ok {
bazelOutput := strings.TrimSpace(result)
return strings.Split(bazelOutput, ", "), true
@ -125,7 +131,7 @@ func NewBazelContext(c *config) (BazelContext, error) {
return noopBazelContext{}, nil
}
bazelCtx := bazelContext{requests: make(map[cqueryKey]bool)}
bazelCtx := bazelContext{buildDir: c.buildDir, requests: make(map[cqueryKey]bool)}
missingEnvVars := []string{}
if len(c.Getenv("BAZEL_HOME")) > 1 {
bazelCtx.homeDir = c.Getenv("BAZEL_HOME")
@ -163,8 +169,8 @@ func (context *bazelContext) BazelEnabled() bool {
// If the given request was already made (and the results are available), then
// returns (result, true). If the request is queued but no results are available,
// then returns ("", false).
func (context *bazelContext) cquery(label string, starlarkExpr string) (string, bool) {
key := cqueryKey{label, starlarkExpr}
func (context *bazelContext) cquery(label string, requestType CqueryRequestType) (string, bool) {
key := cqueryKey{label, requestType}
if result, ok := context.results[key]; ok {
return result, true
} else {
@ -186,7 +192,8 @@ func pwdPrefix() string {
func (context *bazelContext) issueBazelCommand(command string, labels []string,
extraFlags ...string) (string, error) {
cmdFlags := []string{"--output_base=" + context.outputBase, command}
cmdFlags := []string{"--bazelrc=build/bazel/common.bazelrc",
"--output_base=" + context.outputBase, command}
cmdFlags = append(cmdFlags, labels...)
cmdFlags = append(cmdFlags, extraFlags...)
@ -204,27 +211,113 @@ func (context *bazelContext) issueBazelCommand(command string, labels []string,
}
}
func (context *bazelContext) mainBzlFileContents() []byte {
contents := `
# This file is generated by soong_build. Do not edit.
def _mixed_build_root_impl(ctx):
return [DefaultInfo(files = depset(ctx.files.deps))]
mixed_build_root = rule(
implementation = _mixed_build_root_impl,
attrs = {"deps" : attr.label_list()},
)
`
return []byte(contents)
}
func (context *bazelContext) mainBuildFileContents() []byte {
formatString := `
# This file is generated by soong_build. Do not edit.
load(":main.bzl", "mixed_build_root")
mixed_build_root(name = "buildroot",
deps = [%s],
)
`
var buildRootDeps []string = nil
for val, _ := range context.requests {
buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", val.label))
}
buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
return []byte(fmt.Sprintf(formatString, buildRootDepsString))
}
func (context *bazelContext) cqueryStarlarkFileContents() []byte {
formatString := `
# This file is generated by soong_build. Do not edit.
getAllFilesLabels = {
%s
}
def format(target):
if str(target.label) in getAllFilesLabels:
return str(target.label) + ">>" + ', '.join([f.path for f in target.files.to_list()])
else:
# This target was not requested via cquery, and thus must be a dependency
# of a requested target.
return ""
`
var buildRootDeps []string = nil
// TODO(cparsons): Sort by request type instead of assuming all requests
// are of GetAllFiles type.
for val, _ := range context.requests {
buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", val.label))
}
buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
return []byte(fmt.Sprintf(formatString, buildRootDepsString))
}
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext.
func (context *bazelContext) InvokeBazel() error {
context.results = make(map[cqueryKey]string)
var labels []string
var cqueryOutput string
var err error
err = ioutil.WriteFile(
absolutePath(filepath.Join(context.buildDir, "main.bzl")),
context.mainBzlFileContents(), 0666)
if err != nil {
return err
}
err = ioutil.WriteFile(
absolutePath(filepath.Join(context.buildDir, "BUILD.bazel")),
context.mainBuildFileContents(), 0666)
if err != nil {
return err
}
cquery_file_relpath := filepath.Join(context.buildDir, "buildroot.cquery")
err = ioutil.WriteFile(
absolutePath(cquery_file_relpath),
context.cqueryStarlarkFileContents(), 0666)
if err != nil {
return err
}
buildroot_label := fmt.Sprintf("//%s:buildroot", context.buildDir)
cqueryOutput, err = context.issueBazelCommand("cquery",
[]string{fmt.Sprintf("deps(%s)", buildroot_label)},
"--output=starlark",
"--starlark:file="+cquery_file_relpath)
if err != nil {
return err
}
cqueryResults := map[string]string{}
for _, outputLine := range strings.Split(cqueryOutput, "\n") {
if strings.Contains(outputLine, ">>") {
splitLine := strings.SplitN(outputLine, ">>", 2)
cqueryResults[splitLine[0]] = splitLine[1]
}
}
for val, _ := range context.requests {
labels = append(labels, val.label)
// TODO(cparsons): Combine requests into a batch cquery request.
// TODO(cparsons): Use --query_file to avoid command line limits.
cqueryOutput, err = context.issueBazelCommand("cquery", []string{val.label},
"--output=starlark",
"--starlark:expr="+val.starlarkExpr)
if err != nil {
return err
if cqueryResult, ok := cqueryResults[val.label]; ok {
context.results[val] = string(cqueryResult)
} else {
context.results[val] = string(cqueryOutput)
return fmt.Errorf("missing result for bazel target %s", val.label)
}
}
@ -233,7 +326,7 @@ func (context *bazelContext) InvokeBazel() error {
// bazel actions should either be added to the Ninja file and executed later,
// or bazel should handle execution.
// TODO(cparsons): Use --target_pattern_file to avoid command line limits.
_, err = context.issueBazelCommand("build", labels)
_, err = context.issueBazelCommand("build", []string{buildroot_label})
if err != nil {
return err