tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Allow remote execution of link actions."

Browse source
This commit is contained in:
Ramy Medhat 2020-04-17 13:59:26 +00:00 committed by Gerrit Code Review
commit 383683559c
6 changed files with 305 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
Android.bp
View file

@ -132,6 +132,7 @@ bootstrap_go_package {
pkgPath: "android/soong/cc/config",
deps: [
"soong-android",
"soong-remoteexec",
],
srcs: [
"cc/config/clang.go",
@ -547,6 +548,22 @@ bootstrap_go_package {
pluginFor: ["soong_build"],
}
bootstrap_go_package {
name: "soong-remoteexec",
pkgPath: "android/soong/remoteexec",
deps: [
"blueprint",
"soong-android",
],
srcs: [
"remoteexec/remoteexec.go",
],
testSrcs: [
"remoteexec/remoteexec_test.go",
],
pluginFor: ["soong_build"],
}
//
// Defaults to enable various configurations of host bionic
//

47
cc/builder.go
View file

@ -29,6 +29,7 @@ import (
"android/soong/android"
"android/soong/cc/config"
"android/soong/remoteexec"
)
const (
@ -62,7 +63,7 @@ var (
},
"ccCmd", "cFlags")
ld = pctx.AndroidStaticRule("ld",
ld, ldRE = remoteexec.StaticRules(pctx, "ld",
blueprint.RuleParams{
Command: "$ldCmd ${crtBegin} @${out}.rsp " +
"${libFlags} ${crtEnd} -o ${out} ${ldFlags} ${extraLibFlags}",
@ -72,16 +73,28 @@ var (
// clang -Wl,--out-implib doesn't update its output file if it hasn't changed.
Restat: true,
},
"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags", "extraLibFlags")
&remoteexec.REParams{Labels: map[string]string{"type": "link", "tool": "clang"},
ExecStrategy: "${config.RECXXLinksExecStrategy}",
Inputs: []string{"${out}.rsp"},
RSPFile: "${out}.rsp",
OutputFiles: []string{"${out}"},
ToolchainInputs: []string{"$ldCmd"},
Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"},
}, []string{"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags", "extraLibFlags"}, nil)
partialLd = pctx.AndroidStaticRule("partialLd",
partialLd, partialLdRE = remoteexec.StaticRules(pctx, "partialLd",
blueprint.RuleParams{
// Without -no-pie, clang 7.0 adds -pie to link Android files,
// but -r and -pie cannot be used together.
Command: "$ldCmd -fuse-ld=lld -nostdlib -no-pie -Wl,-r ${in} -o ${out} ${ldFlags}",
CommandDeps: []string{"$ldCmd"},
},
"ldCmd", "ldFlags")
}, &remoteexec.REParams{
Labels: map[string]string{"type": "link", "tool": "clang"},
ExecStrategy: "${config.RECXXLinksExecStrategy}", Inputs: []string{"$inCommaList"},
OutputFiles: []string{"${out}"},
ToolchainInputs: []string{"$ldCmd"},
Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"},
}, []string{"ldCmd", "ldFlags"}, []string{"inCommaList"})
ar = pctx.AndroidStaticRule("ar",
blueprint.RuleParams{
@ -262,6 +275,7 @@ func init() {
}
pctx.HostBinToolVariable("SoongZipCmd", "soong_zip")
pctx.Import("android/soong/remoteexec")
}
type builderFlags struct {
@ -657,8 +671,13 @@ func TransformObjToDynamicBinary(ctx android.ModuleContext,
deps = append(deps, crtBegin.Path(), crtEnd.Path())
}
rule := ld
if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") {
rule = ldRE
}
ctx.Build(pctx, android.BuildParams{
Rule: ld,
Rule: rule,
Description: "link " + outputFile.Base(),
Output: outputFile,
ImplicitOutputs: implicitOutputs,
@ -798,16 +817,22 @@ func TransformObjsToObj(ctx android.ModuleContext, objFiles android.Paths,
ldCmd := "${config.ClangBin}/clang++"
rule := partialLd
args := map[string]string{
"ldCmd": ldCmd,
"ldFlags": flags.globalLdFlags + " " + flags.localLdFlags,
}
if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") {
rule = partialLdRE
args["inCommaList"] = strings.Join(objFiles.Strings(), ",")
}
ctx.Build(pctx, android.BuildParams{
Rule: partialLd,
Rule: rule,
Description: "link " + outputFile.Base(),
Output: outputFile,
Inputs: objFiles,
Implicits: deps,
Args: map[string]string{
"ldCmd": ldCmd,
"ldFlags": flags.globalLdFlags + " " + flags.localLdFlags,
},
Args: args,
})
}

13
cc/config/global.go
View file

@ -18,6 +18,7 @@ import (
"strings"
"android/soong/android"
"android/soong/remoteexec"
)
var (
@ -255,6 +256,9 @@ func init() {
}
return ""
})
pctx.VariableFunc("RECXXLinksPool", envOverrideFunc("RBE_CXX_LINKS_POOL", remoteexec.DefaultPool))
pctx.VariableFunc("RECXXLinksExecStrategy", envOverrideFunc("RBE_CXX_LINKS_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
}
var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS)
@ -268,3 +272,12 @@ func bionicHeaders(kernelArch string) string {
"-isystem bionic/libc/kernel/android/uapi",
}, " ")
}
func envOverrideFunc(envVar, defaultVal string) func(ctx android.PackageVarContext) string {
return func(ctx android.PackageVarContext) string {
if override := ctx.Config().Getenv(envVar); override != "" {
return override
}
return defaultVal
}
}

155
remoteexec/remoteexec.go Normal file
View file

@ -0,0 +1,155 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remoteexec
import (
"sort"
"strings"
"android/soong/android"
"github.com/google/blueprint"
)
const (
// ContainerImageKey is the key identifying the container image in the platform spec.
ContainerImageKey = "container-image"
// PoolKey is the key identifying the pool to use for remote execution.
PoolKey = "Pool"
// DefaultImage is the default container image used for Android remote execution. The
// image was built with the Dockerfile at
// https://android.googlesource.com/platform/prebuilts/remoteexecution-client/+/refs/heads/master/docker/Dockerfile
DefaultImage = "docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:582efb38f0c229ea39952fff9e132ccbe183e14869b39888010dacf56b360d62"
// DefaultWrapperPath is the default path to the remote execution wrapper.
DefaultWrapperPath = "prebuilts/remoteexecution-client/live/rewrapper"
// DefaultPool is the name of the pool to use for remote execution when none is specified.
DefaultPool = "default"
// LocalExecStrategy is the exec strategy to indicate that the action should be run locally.
LocalExecStrategy = "local"
// RemoteExecStrategy is the exec strategy to indicate that the action should be run
// remotely.
RemoteExecStrategy = "remote"
// RemoteLocalFallbackExecStrategy is the exec strategy to indicate that the action should
// be run remotely and fallback to local execution if remote fails.
RemoteLocalFallbackExecStrategy = "remote_local_fallback"
)
var (
defaultLabels = map[string]string{"type": "tool"}
defaultExecStrategy = LocalExecStrategy
pctx = android.NewPackageContext("android/soong/remoteexec")
)
// REParams holds information pertinent to the remote execution of a rule.
type REParams struct {
// Platform is the key value pair used for remotely executing the action.
Platform map[string]string
// Labels is a map of labels that identify the rule.
Labels map[string]string
// ExecStrategy is the remote execution strategy: remote, local, or remote_local_fallback.
ExecStrategy string
// Inputs is a list of input paths or ninja variables.
Inputs []string
// RSPFile is the name of the ninja variable used by the rule as a placeholder for an rsp
// input.
RSPFile string
// OutputFiles is a list of output file paths or ninja variables as placeholders for rule
// outputs.
OutputFiles []string
// ToolchainInputs is a list of paths or ninja variables pointing to the location of
// toolchain binaries used by the rule.
ToolchainInputs []string
}
func init() {
pctx.VariableFunc("Wrapper", func(ctx android.PackageVarContext) string {
if override := ctx.Config().Getenv("RBE_WRAPPER"); override != "" {
return override
}
return DefaultWrapperPath
})
}
// Generate the remote execution wrapper template to be added as a prefix to the rule's command.
func (r *REParams) Template() string {
template := "${remoteexec.Wrapper}"
var kvs []string
labels := r.Labels
if len(labels) == 0 {
labels = defaultLabels
}
for k, v := range labels {
kvs = append(kvs, k+"="+v)
}
sort.Strings(kvs)
template += " --labels=" + strings.Join(kvs, ",")
var platform []string
for k, v := range r.Platform {
if v == "" {
continue
}
platform = append(platform, k+"="+v)
}
if _, ok := r.Platform[ContainerImageKey]; !ok {
platform = append(platform, ContainerImageKey+"="+DefaultImage)
}
if platform != nil {
sort.Strings(platform)
template += " --platform=\"" + strings.Join(platform, ",") + "\""
}
strategy := r.ExecStrategy
if strategy == "" {
strategy = defaultExecStrategy
}
template += " --exec_strategy=" + strategy
if len(r.Inputs) > 0 {
template += " --inputs=" + strings.Join(r.Inputs, ",")
}
if r.RSPFile != "" {
template += " --input_list_paths=" + r.RSPFile
}
if len(r.OutputFiles) > 0 {
template += " --output_files=" + strings.Join(r.OutputFiles, ",")
}
if len(r.ToolchainInputs) > 0 {
template += " --toolchain_inputs=" + strings.Join(r.ToolchainInputs, ",")
}
return template + " -- "
}
// StaticRules returns a pair of rules based on the given RuleParams, where the first rule is a
// locally executable rule and the second rule is a remotely executable rule.
func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams *REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) {
ruleParamsRE := ruleParams
ruleParamsRE.Command = reParams.Template() + ruleParamsRE.Command
return ctx.AndroidStaticRule(name, ruleParams, commonArgs...),
ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...)
}

83
remoteexec/remoteexec_test.go Normal file
View file

@ -0,0 +1,83 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remoteexec
import (
"fmt"
"testing"
)
func TestTemplate(t *testing.T) {
tests := []struct {
name string
params *REParams
want string
}{
{
name: "basic",
params: &REParams{
Labels: map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"},
Inputs: []string{"$in"},
OutputFiles: []string{"$out"},
Platform: map[string]string{
ContainerImageKey: DefaultImage,
PoolKey: "default",
},
},
want: fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=local --inputs=$in --output_files=$out -- ", DefaultImage),
},
{
name: "all params",
params: &REParams{
Labels: map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"},
Inputs: []string{"$in"},
OutputFiles: []string{"$out"},
ExecStrategy: "remote",
RSPFile: "$out.rsp",
ToolchainInputs: []string{"clang++"},
Platform: map[string]string{
ContainerImageKey: DefaultImage,
PoolKey: "default",
},
},
want: fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=remote --inputs=$in --input_list_paths=$out.rsp --output_files=$out --toolchain_inputs=clang++ -- ", DefaultImage),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := test.params.Template(); got != test.want {
t.Errorf("Template() returned\n%s\nwant\n%s", got, test.want)
}
})
}
}
func TestTemplateDeterminism(t *testing.T) {
r := &REParams{
Labels: map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"},
Inputs: []string{"$in"},
OutputFiles: []string{"$out"},
Platform: map[string]string{
ContainerImageKey: DefaultImage,
PoolKey: "default",
},
}
want := fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=local --inputs=$in --output_files=$out -- ", DefaultImage)
for i := 0; i < 1000; i++ {
if got := r.Template(); got != want {
t.Fatalf("Template() returned\n%s\nwant\n%s", got, want)
}
}
}

1
ui/build/dumpvars.go
View file

@ -190,6 +190,7 @@ func runMakeProductConfig(ctx Context, config Config) {
// compiler wrappers set up by make
"CC_WRAPPER",
"CXX_WRAPPER",
"RBE_WRAPPER",
"JAVAC_WRAPPER",
"R8_WRAPPER",
"D8_WRAPPER",