Allow remote execution of link actions.

This CL adds a remoteexec package that allows adding a configurable RBE
prefix to the template.

Change-Id: I6b4ea773f6bae5706415a1830f822db7decca92f
Test: built aosp crosshatch userdebug with and without RBE_CXX_LINKS.
Merged-In: Ica920c3d7f79f2996210b9cbd448126451c1707c
Bug: b/154820732

Android.bp

@@ -130,6 +130,7 @@ bootstrap_go_package {
     pkgPath: "android/soong/cc/config",
     deps: [
         "soong-android",
+        "soong-remoteexec",
     ],
     srcs: [
         "cc/config/clang.go",
@@ -546,6 +547,22 @@ bootstrap_go_package {
     pluginFor: ["soong_build"],
 }
 
+bootstrap_go_package {
+    name: "soong-remoteexec",
+    pkgPath: "android/soong/remoteexec",
+    deps: [
+        "blueprint",
+        "soong-android",
+    ],
+    srcs: [
+        "remoteexec/remoteexec.go",
+    ],
+    testSrcs: [
+        "remoteexec/remoteexec_test.go",
+    ],
+    pluginFor: ["soong_build"],
+}
+
 //
 // Defaults to enable various configurations of host bionic
 //

cc/builder.go

@@ -29,6 +29,7 @@ import (
 
 	"android/soong/android"
 	"android/soong/cc/config"
+	"android/soong/remoteexec"
 )
 
 const (
@@ -62,7 +63,7 @@ var (
 		},
 		"ccCmd", "cFlags")
 
-	ld = pctx.AndroidStaticRule("ld",
+	ld, ldRE = remoteexec.StaticRules(pctx, "ld",
 		blueprint.RuleParams{
 			Command: "$ldCmd ${crtBegin} @${out}.rsp " +
 				"${libFlags} ${crtEnd} -o ${out} ${ldFlags} ${extraLibFlags}",
@@ -72,16 +73,28 @@ var (
 			// clang -Wl,--out-implib doesn't update its output file if it hasn't changed.
 			Restat: true,
 		},
-		"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags", "extraLibFlags")
+		&remoteexec.REParams{Labels: map[string]string{"type": "link", "tool": "clang"},
+			ExecStrategy:    "${config.RECXXLinksExecStrategy}",
+			Inputs:          []string{"${out}.rsp"},
+			RSPFile:         "${out}.rsp",
+			OutputFiles:     []string{"${out}"},
+			ToolchainInputs: []string{"$ldCmd"},
+			Platform:        map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"},
+		}, []string{"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags", "extraLibFlags"}, nil)
 
-	partialLd = pctx.AndroidStaticRule("partialLd",
+	partialLd, partialLdRE = remoteexec.StaticRules(pctx, "partialLd",
 		blueprint.RuleParams{
 			// Without -no-pie, clang 7.0 adds -pie to link Android files,
 			// but -r and -pie cannot be used together.
 			Command:     "$ldCmd -fuse-ld=lld -nostdlib -no-pie -Wl,-r ${in} -o ${out} ${ldFlags}",
 			CommandDeps: []string{"$ldCmd"},
-		},
-		"ldCmd", "ldFlags")
+		}, &remoteexec.REParams{
+			Labels:       map[string]string{"type": "link", "tool": "clang"},
+			ExecStrategy: "${config.RECXXLinksExecStrategy}", Inputs: []string{"$inCommaList"},
+			OutputFiles:     []string{"${out}"},
+			ToolchainInputs: []string{"$ldCmd"},
+			Platform:        map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"},
+		}, []string{"ldCmd", "ldFlags"}, []string{"inCommaList"})
 
 	ar = pctx.AndroidStaticRule("ar",
 		blueprint.RuleParams{
@@ -262,6 +275,7 @@ func init() {
 	}
 
 	pctx.HostBinToolVariable("SoongZipCmd", "soong_zip")
+	pctx.Import("android/soong/remoteexec")
 }
 
 type builderFlags struct {
@@ -657,8 +671,13 @@ func TransformObjToDynamicBinary(ctx android.ModuleContext,
 		deps = append(deps, crtBegin.Path(), crtEnd.Path())
 	}
 
+	rule := ld
+	if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") {
+		rule = ldRE
+	}
+
 	ctx.Build(pctx, android.BuildParams{
-		Rule:            ld,
+		Rule:            rule,
 		Description:     "link " + outputFile.Base(),
 		Output:          outputFile,
 		ImplicitOutputs: implicitOutputs,
@@ -798,16 +817,22 @@ func TransformObjsToObj(ctx android.ModuleContext, objFiles android.Paths,
 
 	ldCmd := "${config.ClangBin}/clang++"
 
+	rule := partialLd
+	args := map[string]string{
+		"ldCmd":   ldCmd,
+		"ldFlags": flags.globalLdFlags + " " + flags.localLdFlags,
+	}
+	if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") {
+		rule = partialLdRE
+		args["inCommaList"] = strings.Join(objFiles.Strings(), ",")
+	}
 	ctx.Build(pctx, android.BuildParams{
-		Rule:        partialLd,
+		Rule:        rule,
 		Description: "link " + outputFile.Base(),
 		Output:      outputFile,
 		Inputs:      objFiles,
 		Implicits:   deps,
-		Args: map[string]string{
-			"ldCmd":   ldCmd,
-			"ldFlags": flags.globalLdFlags + " " + flags.localLdFlags,
-		},
+		Args:        args,
 	})
 }
 

cc/config/global.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 
 	"android/soong/android"
+	"android/soong/remoteexec"
 )
 
 var (
@@ -255,6 +256,9 @@ func init() {
 		}
 		return ""
 	})
+
+	pctx.VariableFunc("RECXXLinksPool", envOverrideFunc("RBE_CXX_LINKS_POOL", remoteexec.DefaultPool))
+	pctx.VariableFunc("RECXXLinksExecStrategy", envOverrideFunc("RBE_CXX_LINKS_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
 }
 
 var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS)
@@ -268,3 +272,12 @@ func bionicHeaders(kernelArch string) string {
 		"-isystem bionic/libc/kernel/android/uapi",
 	}, " ")
 }
+
+func envOverrideFunc(envVar, defaultVal string) func(ctx android.PackageVarContext) string {
+	return func(ctx android.PackageVarContext) string {
+		if override := ctx.Config().Getenv(envVar); override != "" {
+			return override
+		}
+		return defaultVal
+	}
+}

remoteexec/remoteexec.go

@@ -0,0 +1,155 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remoteexec
+
+import (
+	"sort"
+	"strings"
+
+	"android/soong/android"
+
+	"github.com/google/blueprint"
+)
+
+const (
+	// ContainerImageKey is the key identifying the container image in the platform spec.
+	ContainerImageKey = "container-image"
+
+	// PoolKey is the key identifying the pool to use for remote execution.
+	PoolKey = "Pool"
+
+	// DefaultImage is the default container image used for Android remote execution. The
+	// image was built with the Dockerfile at
+	// https://android.googlesource.com/platform/prebuilts/remoteexecution-client/+/refs/heads/master/docker/Dockerfile
+	DefaultImage = "docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:582efb38f0c229ea39952fff9e132ccbe183e14869b39888010dacf56b360d62"
+
+	// DefaultWrapperPath is the default path to the remote execution wrapper.
+	DefaultWrapperPath = "prebuilts/remoteexecution-client/live/rewrapper"
+
+	// DefaultPool is the name of the pool to use for remote execution when none is specified.
+	DefaultPool = "default"
+
+	// LocalExecStrategy is the exec strategy to indicate that the action should be run locally.
+	LocalExecStrategy = "local"
+
+	// RemoteExecStrategy is the exec strategy to indicate that the action should be run
+	// remotely.
+	RemoteExecStrategy = "remote"
+
+	// RemoteLocalFallbackExecStrategy is the exec strategy to indicate that the action should
+	// be run remotely and fallback to local execution if remote fails.
+	RemoteLocalFallbackExecStrategy = "remote_local_fallback"
+)
+
+var (
+	defaultLabels       = map[string]string{"type": "tool"}
+	defaultExecStrategy = LocalExecStrategy
+	pctx                = android.NewPackageContext("android/soong/remoteexec")
+)
+
+// REParams holds information pertinent to the remote execution of a rule.
+type REParams struct {
+	// Platform is the key value pair used for remotely executing the action.
+	Platform map[string]string
+	// Labels is a map of labels that identify the rule.
+	Labels map[string]string
+	// ExecStrategy is the remote execution strategy: remote, local, or remote_local_fallback.
+	ExecStrategy string
+	// Inputs is a list of input paths or ninja variables.
+	Inputs []string
+	// RSPFile is the name of the ninja variable used by the rule as a placeholder for an rsp
+	// input.
+	RSPFile string
+	// OutputFiles is a list of output file paths or ninja variables as placeholders for rule
+	// outputs.
+	OutputFiles []string
+	// ToolchainInputs is a list of paths or ninja variables pointing to the location of
+	// toolchain binaries used by the rule.
+	ToolchainInputs []string
+}
+
+func init() {
+	pctx.VariableFunc("Wrapper", func(ctx android.PackageVarContext) string {
+		if override := ctx.Config().Getenv("RBE_WRAPPER"); override != "" {
+			return override
+		}
+		return DefaultWrapperPath
+	})
+}
+
+// Generate the remote execution wrapper template to be added as a prefix to the rule's command.
+func (r *REParams) Template() string {
+	template := "${remoteexec.Wrapper}"
+
+	var kvs []string
+	labels := r.Labels
+	if len(labels) == 0 {
+		labels = defaultLabels
+	}
+	for k, v := range labels {
+		kvs = append(kvs, k+"="+v)
+	}
+	sort.Strings(kvs)
+	template += " --labels=" + strings.Join(kvs, ",")
+
+	var platform []string
+	for k, v := range r.Platform {
+		if v == "" {
+			continue
+		}
+		platform = append(platform, k+"="+v)
+	}
+	if _, ok := r.Platform[ContainerImageKey]; !ok {
+		platform = append(platform, ContainerImageKey+"="+DefaultImage)
+	}
+	if platform != nil {
+		sort.Strings(platform)
+		template += " --platform=\"" + strings.Join(platform, ",") + "\""
+	}
+
+	strategy := r.ExecStrategy
+	if strategy == "" {
+		strategy = defaultExecStrategy
+	}
+	template += " --exec_strategy=" + strategy
+
+	if len(r.Inputs) > 0 {
+		template += " --inputs=" + strings.Join(r.Inputs, ",")
+	}
+
+	if r.RSPFile != "" {
+		template += " --input_list_paths=" + r.RSPFile
+	}
+
+	if len(r.OutputFiles) > 0 {
+		template += " --output_files=" + strings.Join(r.OutputFiles, ",")
+	}
+
+	if len(r.ToolchainInputs) > 0 {
+		template += " --toolchain_inputs=" + strings.Join(r.ToolchainInputs, ",")
+	}
+
+	return template + " -- "
+}
+
+// StaticRules returns a pair of rules based on the given RuleParams, where the first rule is a
+// locally executable rule and the second rule is a remotely executable rule.
+func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams *REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) {
+	ruleParamsRE := ruleParams
+	ruleParamsRE.Command = reParams.Template() + ruleParamsRE.Command
+
+	return ctx.AndroidStaticRule(name, ruleParams, commonArgs...),
+		ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...)
+}

remoteexec/remoteexec_test.go

@@ -0,0 +1,83 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remoteexec
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestTemplate(t *testing.T) {
+	tests := []struct {
+		name   string
+		params *REParams
+		want   string
+	}{
+		{
+			name: "basic",
+			params: &REParams{
+				Labels:      map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"},
+				Inputs:      []string{"$in"},
+				OutputFiles: []string{"$out"},
+				Platform: map[string]string{
+					ContainerImageKey: DefaultImage,
+					PoolKey:           "default",
+				},
+			},
+			want: fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=local --inputs=$in --output_files=$out -- ", DefaultImage),
+		},
+		{
+			name: "all params",
+			params: &REParams{
+				Labels:          map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"},
+				Inputs:          []string{"$in"},
+				OutputFiles:     []string{"$out"},
+				ExecStrategy:    "remote",
+				RSPFile:         "$out.rsp",
+				ToolchainInputs: []string{"clang++"},
+				Platform: map[string]string{
+					ContainerImageKey: DefaultImage,
+					PoolKey:           "default",
+				},
+			},
+			want: fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=remote --inputs=$in --input_list_paths=$out.rsp --output_files=$out --toolchain_inputs=clang++ -- ", DefaultImage),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := test.params.Template(); got != test.want {
+				t.Errorf("Template() returned\n%s\nwant\n%s", got, test.want)
+			}
+		})
+	}
+}
+
+func TestTemplateDeterminism(t *testing.T) {
+	r := &REParams{
+		Labels:      map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"},
+		Inputs:      []string{"$in"},
+		OutputFiles: []string{"$out"},
+		Platform: map[string]string{
+			ContainerImageKey: DefaultImage,
+			PoolKey:           "default",
+		},
+	}
+	want := fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=local --inputs=$in --output_files=$out -- ", DefaultImage)
+	for i := 0; i < 1000; i++ {
+		if got := r.Template(); got != want {
+			t.Fatalf("Template() returned\n%s\nwant\n%s", got, want)
+		}
+	}
+}

ui/build/dumpvars.go

@@ -190,6 +190,7 @@ func runMakeProductConfig(ctx Context, config Config) {
 		// compiler wrappers set up by make
 		"CC_WRAPPER",
 		"CXX_WRAPPER",
+		"RBE_WRAPPER",
 		"JAVAC_WRAPPER",
 		"R8_WRAPPER",
 		"D8_WRAPPER",