From 366afc5011ca300d6bd14246c29e08b77387a12d Mon Sep 17 00:00:00 2001 From: Kousik Kumar Date: Wed, 20 May 2020 11:27:16 -0700 Subject: [PATCH] Add support for remote-execution / caching of jar/zip actions Test: Ran a build with `RBE_JAR="true" RBE_JAR_EXEC_STRATEGY="remote" RBE_ZIP="true" RBE_ZIP_EXEC_STRATEGY="remote" ... use_rbe m` and that succeeded. Bug: b/156765207 Change-Id: Ia17b532fbb282be414adf879914870082dd0841b --- java/app_builder.go | 14 ++++++++--- java/builder.go | 51 +++++++++++++++++++++++++++---------- java/config/config.go | 2 ++ java/dex.go | 54 ++++++++++++++++++++++++++-------------- java/java.go | 14 ++++++++--- remoteexec/remoteexec.go | 16 ++++++++++++ 6 files changed, 112 insertions(+), 39 deletions(-) diff --git a/java/app_builder.go b/java/app_builder.go index c3918d8e8..014bd54f1 100644 --- a/java/app_builder.go +++ b/java/app_builder.go @@ -232,14 +232,20 @@ func TransformJniLibsToJar(ctx android.ModuleContext, outputFile android.Writabl "-f", j.path.String()) } + rule := zip + args := map[string]string{ + "jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "), + } + if ctx.Config().IsEnvTrue("RBE_ZIP") { + rule = zipRE + args["implicits"] = strings.Join(deps.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: zip, + Rule: rule, Description: "zip jni libs", Output: outputFile, Implicits: deps, - Args: map[string]string{ - "jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "), - }, + Args: args, }) } diff --git a/java/builder.go b/java/builder.go index b6cb2aeeb..a27e5c390 100644 --- a/java/builder.go +++ b/java/builder.go @@ -40,17 +40,17 @@ var ( // (if the rule produces .class files) or a .srcjar file (if the rule produces .java files). // .srcjar files are unzipped into a temporary directory when compiled with javac. // TODO(b/143658984): goma can't handle the --system argument to javac. - javac, javacRE = remoteexec.StaticRules(pctx, "javac", + javac, javacRE = remoteexec.MultiCommandStaticRules(pctx, "javac", blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + `(if [ -s $srcJarDir/list ] || [ -s $out.rsp ] ; then ` + - `${config.SoongJavacWrapper} $reTemplate${config.JavacCmd} ` + + `${config.SoongJavacWrapper} $javaTemplate${config.JavacCmd} ` + `${config.JavacHeapFlags} ${config.JavacVmFlags} ${config.CommonJdkFlags} ` + `$processorpath $processor $javacFlags $bootClasspath $classpath ` + `-source $javaVersion -target $javaVersion ` + `-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list ; fi ) && ` + - `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` + + `$zipTemplate${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` + `rm -rf "$srcJarDir"`, CommandDeps: []string{ "${config.JavacCmd}", @@ -60,10 +60,19 @@ var ( CommandOrderOnly: []string{"${config.SoongJavacWrapper}"}, Rspfile: "$out.rsp", RspfileContent: "$in", - }, &remoteexec.REParams{ - Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"}, - ExecStrategy: "${config.REJavacExecStrategy}", - Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, map[string]*remoteexec.REParams{ + "$javaTemplate": &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"}, + ExecStrategy: "${config.REJavacExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, + "$zipTemplate": &remoteexec.REParams{ + Labels: map[string]string{"type": "tool", "name": "soong_zip"}, + Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, + OutputFiles: []string{"$out"}, + ExecStrategy: "${config.REJavacExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, }, []string{"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir", "outDir", "annoDir", "javaVersion"}, nil) @@ -144,23 +153,35 @@ var ( Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, }, []string{"javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion"}, []string{"implicits"}) - jar = pctx.AndroidStaticRule("jar", + jar, jarRE = remoteexec.StaticRules(pctx, "jar", blueprint.RuleParams{ - Command: `${config.SoongZipCmd} -jar -o $out @$out.rsp`, + Command: `$reTemplate${config.SoongZipCmd} -jar -o $out @$out.rsp`, CommandDeps: []string{"${config.SoongZipCmd}"}, Rspfile: "$out.rsp", RspfileContent: "$jarArgs", }, - "jarArgs") + &remoteexec.REParams{ + ExecStrategy: "${config.REJarExecStrategy}", + Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"$out"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"jarArgs"}, nil) - zip = pctx.AndroidStaticRule("zip", + zip, zipRE = remoteexec.StaticRules(pctx, "zip", blueprint.RuleParams{ Command: `${config.SoongZipCmd} -o $out @$out.rsp`, CommandDeps: []string{"${config.SoongZipCmd}"}, Rspfile: "$out.rsp", RspfileContent: "$jarArgs", }, - "jarArgs") + &remoteexec.REParams{ + ExecStrategy: "${config.REZipExecStrategy}", + Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp", "$implicits"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"$out"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"jarArgs"}, []string{"implicits"}) combineJar = pctx.AndroidStaticRule("combineJar", blueprint.RuleParams{ @@ -457,8 +478,12 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab func TransformResourcesToJar(ctx android.ModuleContext, outputFile android.WritablePath, jarArgs []string, deps android.Paths) { + rule := jar + if ctx.Config().IsEnvTrue("RBE_JAR") { + rule = jarRE + } ctx.Build(pctx, android.BuildParams{ - Rule: jar, + Rule: rule, Description: "jar", Output: outputFile, Implicits: deps, diff --git a/java/config/config.go b/java/config/config.go index 4173659a4..edaed2a10 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -150,6 +150,8 @@ func init() { pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.VariableFunc("RESignApkExecStrategy", remoteexec.EnvOverrideFunc("RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REJarExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAR_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REZipExecStrategy", remoteexec.EnvOverrideFunc("RBE_ZIP_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar") diff --git a/java/dex.go b/java/dex.go index e2a8e7eae..9e61e95ad 100644 --- a/java/dex.go +++ b/java/dex.go @@ -24,48 +24,66 @@ import ( "android/soong/remoteexec" ) -var d8, d8RE = remoteexec.StaticRules(pctx, "d8", +var d8, d8RE = remoteexec.MultiCommandStaticRules(pctx, "d8", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + - `$reTemplate${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` + - `${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + + `$d8Template${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` + + `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + `${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`, CommandDeps: []string{ "${config.D8Cmd}", "${config.SoongZipCmd}", "${config.MergeZipsCmd}", }, - }, &remoteexec.REParams{ - Labels: map[string]string{"type": "compile", "compiler": "d8"}, - Inputs: []string{"${config.D8Jar}"}, - ExecStrategy: "${config.RED8ExecStrategy}", - ToolchainInputs: []string{"${config.JavaCmd}"}, - Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, map[string]*remoteexec.REParams{ + "$d8Template": &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "compiler": "d8"}, + Inputs: []string{"${config.D8Jar}"}, + ExecStrategy: "${config.RED8ExecStrategy}", + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, + "$zipTemplate": &remoteexec.REParams{ + Labels: map[string]string{"type": "tool", "name": "soong_zip"}, + Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, + OutputFiles: []string{"$outDir/classes.dex.jar"}, + ExecStrategy: "${config.RED8ExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, }, []string{"outDir", "d8Flags", "zipFlags"}, nil) -var r8, r8RE = remoteexec.StaticRules(pctx, "r8", +var r8, r8RE = remoteexec.MultiCommandStaticRules(pctx, "r8", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + `rm -f "$outDict" && ` + - `$reTemplate${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` + + `$r8Template${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` + `--force-proguard-compatibility ` + `--no-data-resources ` + `-printmapping $outDict ` + `$r8Flags && ` + `touch "$outDict" && ` + - `${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + + `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + `${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`, CommandDeps: []string{ "${config.R8Cmd}", "${config.SoongZipCmd}", "${config.MergeZipsCmd}", }, - }, &remoteexec.REParams{ - Labels: map[string]string{"type": "compile", "compiler": "r8"}, - Inputs: []string{"$implicits", "${config.R8Jar}"}, - ExecStrategy: "${config.RER8ExecStrategy}", - ToolchainInputs: []string{"${config.JavaCmd}"}, - Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, map[string]*remoteexec.REParams{ + "$r8Template": &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "compiler": "r8"}, + Inputs: []string{"$implicits", "${config.R8Jar}"}, + ExecStrategy: "${config.RER8ExecStrategy}", + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, + "$zipTemplate": &remoteexec.REParams{ + Labels: map[string]string{"type": "tool", "name": "soong_zip"}, + Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, + OutputFiles: []string{"$outDir/classes.dex.jar"}, + ExecStrategy: "${config.RER8ExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, }, []string{"outDir", "outDict", "r8Flags", "zipFlags"}, []string{"implicits"}) func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string { diff --git a/java/java.go b/java/java.go index 466132e1e..a54ca767e 100644 --- a/java/java.go +++ b/java/java.go @@ -1456,13 +1456,19 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { serviceFile := file.String() zipargs = append(zipargs, "-C", filepath.Dir(serviceFile), "-f", serviceFile) } + rule := zip + args := map[string]string{ + "jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "), + } + if ctx.Config().IsEnvTrue("RBE_ZIP") { + rule = zipRE + args["implicits"] = strings.Join(services.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: zip, + Rule: rule, Output: servicesJar, Implicits: services, - Args: map[string]string{ - "jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "), - }, + Args: args, }) jars = append(jars, servicesJar) } diff --git a/remoteexec/remoteexec.go b/remoteexec/remoteexec.go index 11fc2826a..d6e2c0a75 100644 --- a/remoteexec/remoteexec.go +++ b/remoteexec/remoteexec.go @@ -178,6 +178,22 @@ func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.R ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...) } +// MultiCommandStaticRules returns a pair of rules based on the given RuleParams, where the first +// rule is a locally executable rule and the second rule is a remotely executable rule. This +// function supports multiple remote execution wrappers placed in the template when commands are +// chained together with &&. commonArgs are args used for both the local and remotely executable +// rules. reArgs are args used only for remote execution. +func MultiCommandStaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams map[string]*REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) { + ruleParamsRE := ruleParams + for k, v := range reParams { + ruleParams.Command = strings.ReplaceAll(ruleParams.Command, k, "") + ruleParamsRE.Command = strings.ReplaceAll(ruleParamsRE.Command, k, v.Template()) + } + + return ctx.AndroidStaticRule(name, ruleParams, commonArgs...), + ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...) +} + // EnvOverrideFunc retrieves a variable func that evaluates to the value of the given environment // variable if set, otherwise the given default. func EnvOverrideFunc(envVar, defaultVal string) func(ctx android.PackageVarContext) string {