tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Support Javac sharding in Soong.

Browse source 
Test: m clean && m -j java and java_test.go

Change-Id: I110a0ff029448d3319aed2788d25d90a6c1a7fa0
This commit is contained in:
Nan Zhang 2017-11-02 13:28:15 -07:00
parent 9c7dcfd72c
commit 61eaedbddf
4 changed files with 143 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
README.md
View file

@ -61,6 +61,7 @@ Variables and properties are strongly typed, variables dynamically based on the
first assignment, and properties statically by the module type. The supported
types are:
* Bool (`true` or `false`)
* Integers (`int`)
* Strings (`"string"`)
* Lists of strings (`["string1", "string2"]`)
* Maps (`{key1: "value1", key2: ["value2"]}`)
@ -71,8 +72,9 @@ trailing commas after the last value.
### Operators
Strings, lists of strings, and maps can be appended using the `+` operator.
Appending a map produces the union of keys in both maps, appending the values
of any keys that are present in both maps.
Integers can be summed up using the `+` operator. Appending a map produces the
union of keys in both maps, appending the values of any keys that are present
in both maps.
### Defaults modules

35
java/builder.go
View file

@ -19,6 +19,8 @@ package java
// functions.
import (
"path/filepath"
"strconv"
"strings"
"github.com/google/blueprint"
@ -211,12 +213,16 @@ func TransformKotlinToClasses(ctx android.ModuleContext, outputFile android.Writ
})
}
func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath,
srcFiles, srcJars android.Paths,
flags javaBuilderFlags, deps android.Paths) {
func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, shardIdx int,
srcFiles, srcJars android.Paths, flags javaBuilderFlags, deps android.Paths) {
transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, deps,
"javac", "javac", javac)
// Compile java sources into .class files
desc := "javac"
if shardIdx >= 0 {
desc += strconv.Itoa(shardIdx)
}
transformJavaToClasses(ctx, outputFile, shardIdx, srcFiles, srcJars, flags, deps, "javac", desc, javac)
}
func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath,
@ -226,7 +232,7 @@ func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath,
ctx.ModuleErrorf("cannot build with Error Prone, missing external/error_prone?")
}
transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, nil,
transformJavaToClasses(ctx, outputFile, -1, srcFiles, srcJars, flags, nil,
"errorprone", "errorprone", errorprone)
}
@ -275,7 +281,7 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.
// suffix will be appended to various intermediate files and directories to avoid collisions when
// this function is called twice in the same module directory.
func transformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath,
srcFiles, srcJars android.Paths,
shardIdx int, srcFiles, srcJars android.Paths,
flags javaBuilderFlags, deps android.Paths,
intermediatesDir, desc string, rule blueprint.Rule) {
@ -298,6 +304,15 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab
deps = append(deps, flags.classpath...)
srcJarDir := "srcjars"
outDir := "classes"
annoDir := "anno"
if shardIdx >= 0 {
shardDir := "shard" + strconv.Itoa(shardIdx)
srcJarDir = filepath.Join(shardDir, srcJarDir)
outDir = filepath.Join(shardDir, outDir)
annoDir = filepath.Join(shardDir, annoDir)
}
ctx.Build(pctx, android.BuildParams{
Rule: rule,
Description: desc,
@ -309,9 +324,9 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab
"bootClasspath": bootClasspath,
"classpath": flags.classpath.FormJavaClassPath("-classpath"),
"srcJars": strings.Join(srcJars.Strings(), " "),
"srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, "srcjars").String(),
"outDir": android.PathForModuleOut(ctx, intermediatesDir, "classes").String(),
"annoDir": android.PathForModuleOut(ctx, intermediatesDir, "anno").String(),
"srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, srcJarDir).String(),
"outDir": android.PathForModuleOut(ctx, intermediatesDir, outDir).String(),
"annoDir": android.PathForModuleOut(ctx, intermediatesDir, annoDir).String(),
"javaVersion": flags.javaVersion,
},
})

52
java/java.go
View file

@ -117,6 +117,9 @@ type CompilerProperties struct {
// List of classes to pass to javac to use as annotation processors
Annotation_processor_classes []string
// The number of Java source entries each Javac instance can process
Javac_shard_size *int64
Openjdk9 struct {
// List of source files that should only be used when passing -source 1.9
Srcs []string
@ -355,6 +358,18 @@ func hasSrcExt(srcs []string, ext string) bool {
return false
}
func shardPaths(paths android.Paths, shardSize int) []android.Paths {
ret := make([]android.Paths, 0, (len(paths)+shardSize-1)/shardSize)
for len(paths) > shardSize {
ret = append(ret, paths[0:shardSize])
paths = paths[shardSize:]
}
if len(paths) > 0 {
ret = append(ret, paths)
}
return ret
}
func (j *Module) hasSrcExt(ext string) bool {
return hasSrcExt(j.properties.Srcs, ext)
}
@ -571,7 +586,17 @@ func (j *Module) compile(ctx android.ModuleContext) {
}
}
enable_sharding := false
if ctx.Device() && !ctx.AConfig().IsEnvFalse("TURBINE_ENABLED") {
if j.properties.Javac_shard_size != nil && *(j.properties.Javac_shard_size) > 0 {
enable_sharding = true
if len(j.properties.Annotation_processors) != 0 ||
len(j.properties.Annotation_processor_classes) != 0 {
ctx.PropertyErrorf("javac_shard_size",
"%q cannot be set when annotation processors are enabled.",
j.properties.Javac_shard_size)
}
}
// If sdk jar is java module, then directly return classesJar as header.jar
if j.Name() != "android_stubs_current" && j.Name() != "android_system_stubs_current" &&
j.Name() != "android_test_stubs_current" {
@ -590,18 +615,35 @@ func (j *Module) compile(ctx android.ModuleContext) {
// TODO(ccross): Once we always compile with javac9 we may be able to conditionally
// enable error-prone without affecting the output class files.
errorprone := android.PathForModuleOut(ctx, "errorprone", jarName)
RunErrorProne(ctx, errorprone, javaSrcFiles, srcJars, flags)
RunErrorProne(ctx, errorprone, uniqueSrcFiles, srcJars, flags)
extraJarDeps = append(extraJarDeps, errorprone)
}
// Compile java sources into .class files
if enable_sharding {
flags.classpath.AddPaths([]android.Path{j.headerJarFile})
shardSize := int(*(j.properties.Javac_shard_size))
var shardSrcs []android.Paths
if len(uniqueSrcFiles) > 0 {
shardSrcs = shardPaths(uniqueSrcFiles, shardSize)
for idx, shardSrc := range shardSrcs {
classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(idx))
TransformJavaToClasses(ctx, classes, idx, shardSrc, nil, flags, extraJarDeps)
jars = append(jars, classes)
}
}
if len(srcJars) > 0 {
classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(len(shardSrcs)))
TransformJavaToClasses(ctx, classes, len(shardSrcs), nil, srcJars, flags, extraJarDeps)
jars = append(jars, classes)
}
} else {
classes := android.PathForModuleOut(ctx, "javac", jarName)
TransformJavaToClasses(ctx, classes, javaSrcFiles, srcJars, flags, extraJarDeps)
TransformJavaToClasses(ctx, classes, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps)
jars = append(jars, classes)
}
if ctx.Failed() {
return
}
jars = append(jars, classes)
}
dirArgs, dirDeps := ResourceDirsToJarArgs(ctx, j.properties.Java_resource_dirs, j.properties.Exclude_java_resource_dirs)

66
java/java_test.go
View file

@ -22,6 +22,7 @@ import (
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
)
@ -669,6 +670,71 @@ func TestKotlin(t *testing.T) {
}
}
func TestTurbine(t *testing.T) {
ctx := testJava(t, `
java_library {
name: "foo",
srcs: ["a.java"],
}
java_library {
name: "bar",
srcs: ["b.java"],
static_libs: ["foo"],
}
java_library {
name: "baz",
srcs: ["c.java"],
libs: ["bar"],
sdk_version: "14",
}
`)
fooTurbine := ctx.ModuleForTests("foo", "android_common").Rule("turbine")
barTurbine := ctx.ModuleForTests("bar", "android_common").Rule("turbine")
barJavac := ctx.ModuleForTests("bar", "android_common").Rule("javac")
barTurbineCombined := ctx.ModuleForTests("bar", "android_common").Description("for turbine")
bazJavac := ctx.ModuleForTests("baz", "android_common").Rule("javac")
if len(fooTurbine.Inputs) != 1 || fooTurbine.Inputs[0].String() != "a.java" {
t.Errorf(`foo inputs %v != ["a.java"]`, fooTurbine.Inputs)
}
fooHeaderJar := filepath.Join(buildDir, ".intermediates", "foo", "android_common", "turbine-combined", "foo.jar")
if !strings.Contains(barTurbine.Args["classpath"], fooHeaderJar) {
t.Errorf("bar turbine classpath %v does not contain %q", barTurbine.Args["classpath"], fooHeaderJar)
}
if !strings.Contains(barJavac.Args["classpath"], fooHeaderJar) {
t.Errorf("bar javac classpath %v does not contain %q", barJavac.Args["classpath"], fooHeaderJar)
}
if len(barTurbineCombined.Inputs) != 2 || barTurbineCombined.Inputs[1].String() != fooHeaderJar {
t.Errorf("bar turbine combineJar inputs %v does not contain %q", barTurbineCombined.Inputs, fooHeaderJar)
}
if !strings.Contains(bazJavac.Args["classpath"], "prebuilts/sdk/14/android.jar") {
t.Errorf("baz javac classpath %v does not contain %q", bazJavac.Args["classpath"],
"prebuilts/sdk/14/android.jar")
}
}
func TestSharding(t *testing.T) {
ctx := testJava(t, `
java_library {
name: "bar",
srcs: ["a.java","b.java","c.java"],
javac_shard_size: 1
}
`)
barHeaderJar := filepath.Join(buildDir, ".intermediates", "bar", "android_common", "turbine-combined", "bar.jar")
for i := 0; i < 3; i++ {
barJavac := ctx.ModuleForTests("bar", "android_common").Description("javac" + strconv.Itoa(i))
if !strings.Contains(barJavac.Args["classpath"], barHeaderJar) {
t.Errorf("bar javac classpath %v does not contain %q", barJavac.Args["classpath"], barHeaderJar)
}
}
}
func fail(t *testing.T, errs []error) {
if len(errs) > 0 {
for _, err := range errs {