tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Generate .bzl rule definitions for every module type in Soong, and surface module properties as attributes."

Browse source
This commit is contained in:
Jingwen Chen 2020-09-14 07:27:08 +00:00 committed by Gerrit Code Review
commit c977a21419
3 changed files with 460 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

305
cmd/soong_build/bazel_overlay.go
View file

@ -24,16 +24,19 @@ import (
"strings"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap/bpdoc"
"github.com/google/blueprint/proptools"
)
const (
// The default `load` preamble for every generated BUILD file.
soongModuleLoad = `package(default_visibility = ["//visibility:public"])
load("//:soong_module.bzl", "soong_module")
`
// A BUILD file target snippet representing a Soong module
// A macro call in the BUILD file representing a Soong module, with space
// for expanding more attributes.
soongModuleTarget = `soong_module(
name = "%s",
module_name = "%s",
@ -42,24 +45,24 @@ load("//:soong_module.bzl", "soong_module")
module_deps = %s,
%s)`
// The soong_module rule implementation in a .bzl file
soongModuleBzl = `SoongModuleInfo = provider(
// A simple provider to mark and differentiate Soong module rule shims from
// regular Bazel rules. Every Soong module rule shim returns a
// SoongModuleInfo provider, and can only depend on rules returning
// SoongModuleInfo in the `module_deps` attribute.
providersBzl = `SoongModuleInfo = provider(
fields = {
"name": "Name of module",
"type": "Type of module",
"variant": "Variant of module",
},
)
`
def _merge_dicts(*dicts):
"""Adds a list of dictionaries into a single dictionary."""
// The soong_module rule implementation in a .bzl file.
soongModuleBzl = `
%s
# If keys are repeated in multiple dictionaries, the latter one "wins".
result = {}
for d in dicts:
result.update(d)
return result
load(":providers.bzl", "SoongModuleInfo")
def _generic_soong_module_impl(ctx):
return [
@ -70,37 +73,31 @@ def _generic_soong_module_impl(ctx):
),
]
_COMMON_ATTRS = {
"module_name": attr.string(mandatory = True),
"module_type": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
}
generic_soong_module = rule(
implementation = _generic_soong_module_impl,
attrs = _COMMON_ATTRS,
)
# TODO(jingwen): auto generate Soong module shims
def _soong_filegroup_impl(ctx):
return [SoongModuleInfo(),]
soong_filegroup = rule(
implementation = _soong_filegroup_impl,
# Matches https://cs.android.com/android/platform/superproject/+/master:build/soong/android/filegroup.go;l=25-40;drc=6a6478d49e78703ba22a432c41d819c8df79ef6c
attrs = _merge_dicts(_COMMON_ATTRS, {
"srcs": attr.string_list(doc = "srcs lists files that will be included in this filegroup"),
"exclude_srcs": attr.string_list(),
"path": attr.string(doc = "The base path to the files. May be used by other modules to determine which portion of the path to use. For example, when a filegroup is used as data in a cc_test rule, the base path is stripped off the path and the remaining path is used as the installation directory."),
"export_to_make_var": attr.string(doc = "Create a make variable with the specified name that contains the list of files in the filegroup, relative to the root of the source tree."),
})
attrs = {
"module_name": attr.string(mandatory = True),
"module_type": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
},
)
soong_module_rule_map = {
"filegroup": soong_filegroup,
}
%s}
_SUPPORTED_TYPES = ["bool", "int", "string"]
def _is_supported_type(value):
if type(value) in _SUPPORTED_TYPES:
return True
elif type(value) == "list":
supported = True
for v in value:
supported = supported and type(v) in _SUPPORTED_TYPES
return supported
else:
return False
# soong_module is a macro that supports arbitrary kwargs, and uses module_type to
# expand to the right underlying shim.
@ -118,12 +115,76 @@ def soong_module(name, module_type, **kwargs):
module_deps = kwargs.pop("module_deps", []),
)
else:
supported_kwargs = dict()
for key, value in kwargs.items():
if _is_supported_type(value):
supported_kwargs[key] = value
soong_module_rule(
name = name,
module_type = module_type,
**kwargs,
**supported_kwargs,
)
`
// A rule shim for representing a Soong module type and its properties.
moduleRuleShim = `
def _%[1]s_impl(ctx):
return [SoongModuleInfo()]
%[1]s = rule(
implementation = _%[1]s_impl,
attrs = %[2]s
)
`
)
var (
// An allowlist of prop types that are surfaced from module props to rule
// attributes. (nested) dictionaries are notably absent here, because while
// Soong supports multi value typed and nested dictionaries, Bazel's rule
// attr() API supports only single-level string_dicts.
allowedPropTypes = map[string]bool{
"int": true, // e.g. 42
"bool": true, // e.g. True
"string_list": true, // e.g. ["a", "b"]
"string": true, // e.g. "a"
}
// TODO(b/166563303): Specific properties of some module types aren't
// recognized by the documentation generator. As a workaround, hardcode a
// mapping of the module type to prop name to prop type here, and ultimately
// fix the documentation generator to also parse these properties correctly.
additionalPropTypes = map[string]map[string]string{
// sdk and module_exports props are created at runtime using reflection.
// bpdocs isn't wired up to read runtime generated structs.
"sdk": {
"java_header_libs": "string_list",
"java_sdk_libs": "string_list",
"java_system_modules": "string_list",
"native_header_libs": "string_list",
"native_libs": "string_list",
"native_objects": "string_list",
"native_shared_libs": "string_list",
"native_static_libs": "string_list",
},
"module_exports": {
"java_libs": "string_list",
"java_tests": "string_list",
"native_binaries": "string_list",
"native_shared_libs": "string_list",
},
}
// Certain module property names are blocklisted/ignored here, for the reasons commented.
ignoredPropNames = map[string]bool{
"name": true, // redundant, since this is explicitly generated for every target
"from": true, // reserved keyword
"in": true, // reserved keyword
"arch": true, // interface prop type is not supported yet.
"multilib": true, // interface prop type is not supported yet.
"target": true, // interface prop type is not supported yet.
"visibility": true, // Bazel has native visibility semantics. Handle later.
"features": true, // There is already a built-in attribute 'features' which cannot be overridden.
}
)
func targetNameWithVariant(c *blueprint.Context, logicModule blueprint.Module) string {
@ -206,9 +267,7 @@ func prettyPrint(propertyValue reflect.Value, indent int) (string, error) {
structProps := extractStructProperties(propertyValue, indent)
for _, k := range android.SortedStringKeys(structProps) {
ret += makeIndent(indent + 1)
ret += "\"" + k + "\": "
ret += structProps[k]
ret += ",\n"
ret += fmt.Sprintf("%q: %s,\n", k, structProps[k])
}
ret += makeIndent(indent)
ret += "}"
@ -223,6 +282,10 @@ func prettyPrint(propertyValue reflect.Value, indent int) (string, error) {
return ret, nil
}
// Converts a reflected property struct value into a map of property names and property values,
// which each property value correctly pretty-printed and indented at the right nest level,
// since property structs can be nested. In Starlark, nested structs are represented as nested
// dicts: https://docs.bazel.build/skylark/lib/dict.html
func extractStructProperties(structValue reflect.Value, indent int) map[string]string {
if structValue.Kind() != reflect.Struct {
panic(fmt.Errorf("Expected a reflect.Struct type, but got %s", structValue.Kind()))
@ -296,6 +359,102 @@ func extractModuleProperties(aModule android.Module) map[string]string {
return ret
}
// FIXME(b/168089390): In Bazel, rules ending with "_test" needs to be marked as
// testonly = True, forcing other rules that depend on _test rules to also be
// marked as testonly = True. This semantic constraint is not present in Soong.
// To work around, rename "*_test" rules to "*_test_".
func canonicalizeModuleType(moduleName string) string {
if strings.HasSuffix(moduleName, "_test") {
return moduleName + "_"
}
return moduleName
}
type RuleShim struct {
// The rule class shims contained in a bzl file. e.g. ["cc_object", "cc_library", ..]
rules []string
// The generated string content of the bzl file.
content string
}
// Create <module>.bzl containing Bazel rule shims for every module type available in Soong and
// user-specified Go plugins.
//
// This function reuses documentation generation APIs to ensure parity between modules-as-docs
// and modules-as-code, including the names and types of module properties.
func createRuleShims(packages []*bpdoc.Package) (map[string]RuleShim, error) {
var propToAttr func(prop bpdoc.Property, propName string) string
propToAttr = func(prop bpdoc.Property, propName string) string {
// dots are not allowed in Starlark attribute names. Substitute them with double underscores.
propName = strings.ReplaceAll(propName, ".", "__")
if !shouldGenerateAttribute(propName) {
return ""
}
// Canonicalize and normalize module property types to Bazel attribute types
starlarkAttrType := prop.Type
if starlarkAttrType == "list of strings" {
starlarkAttrType = "string_list"
} else if starlarkAttrType == "int64" {
starlarkAttrType = "int"
} else if starlarkAttrType == "" {
var attr string
for _, nestedProp := range prop.Properties {
nestedAttr := propToAttr(nestedProp, propName+"__"+nestedProp.Name)
if nestedAttr != "" {
// TODO(b/167662930): Fix nested props resulting in too many attributes.
// Let's still generate these, but comment them out.
attr += "# " + nestedAttr
}
}
return attr
}
if !allowedPropTypes[starlarkAttrType] {
return ""
}
return fmt.Sprintf(" %q: attr.%s(),\n", propName, starlarkAttrType)
}
ruleShims := map[string]RuleShim{}
for _, pkg := range packages {
content := "load(\":providers.bzl\", \"SoongModuleInfo\")\n"
bzlFileName := strings.ReplaceAll(pkg.Path, "android/soong/", "")
bzlFileName = strings.ReplaceAll(bzlFileName, ".", "_")
bzlFileName = strings.ReplaceAll(bzlFileName, "/", "_")
rules := []string{}
for _, moduleTypeTemplate := range moduleTypeDocsToTemplates(pkg.ModuleTypes) {
attrs := `{
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
`
for _, prop := range moduleTypeTemplate.Properties {
attrs += propToAttr(prop, prop.Name)
}
for propName, propType := range additionalPropTypes[moduleTypeTemplate.Name] {
attrs += fmt.Sprintf(" %q: attr.%s(),\n", propName, propType)
}
attrs += " },"
rule := canonicalizeModuleType(moduleTypeTemplate.Name)
content += fmt.Sprintf(moduleRuleShim, rule, attrs)
rules = append(rules, rule)
}
ruleShims[bzlFileName] = RuleShim{content: content, rules: rules}
}
return ruleShims, nil
}
func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
blueprintCtx := ctx.Context
blueprintCtx.VisitAllModules(func(module blueprint.Module) {
@ -316,21 +475,50 @@ func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
return err
}
return writeReadOnlyFile(bazelOverlayDir, "soong_module.bzl", soongModuleBzl)
if err := writeReadOnlyFile(bazelOverlayDir, "providers.bzl", providersBzl); err != nil {
return err
}
packages, err := getPackages(ctx)
if err != nil {
return err
}
ruleShims, err := createRuleShims(packages)
if err != nil {
return err
}
for bzlFileName, ruleShim := range ruleShims {
if err := writeReadOnlyFile(bazelOverlayDir, bzlFileName+".bzl", ruleShim.content); err != nil {
return err
}
}
return writeReadOnlyFile(bazelOverlayDir, "soong_module.bzl", generateSoongModuleBzl(ruleShims))
}
var ignoredProps map[string]bool = map[string]bool{
"name": true, // redundant, since this is explicitly generated for every target
"from": true, // reserved keyword
"in": true, // reserved keyword
"arch": true, // interface prop type is not supported yet.
"multilib": true, // interface prop type is not supported yet.
"target": true, // interface prop type is not supported yet.
"visibility": true, // Bazel has native visibility semantics. Handle later.
// Generate the content of soong_module.bzl with the rule shim load statements
// and mapping of module_type to rule shim map for every module type in Soong.
func generateSoongModuleBzl(bzlLoads map[string]RuleShim) string {
var loadStmts string
var moduleRuleMap string
for bzlFileName, ruleShim := range bzlLoads {
loadStmt := "load(\"//:"
loadStmt += bzlFileName
loadStmt += ".bzl\""
for _, rule := range ruleShim.rules {
loadStmt += fmt.Sprintf(", %q", rule)
moduleRuleMap += " \"" + rule + "\": " + rule + ",\n"
}
loadStmt += ")\n"
loadStmts += loadStmt
}
return fmt.Sprintf(soongModuleBzl, loadStmts, moduleRuleMap)
}
func shouldGenerateAttribute(prop string) bool {
return !ignoredProps[prop]
return !ignoredPropNames[prop]
}
// props is an unsorted map. This function ensures that
@ -367,9 +555,7 @@ func generateSoongModuleTarget(
depLabelList := "[\n"
for depLabel, _ := range depLabels {
depLabelList += " \""
depLabelList += depLabel
depLabelList += "\",\n"
depLabelList += fmt.Sprintf(" %q,\n", depLabel)
}
depLabelList += " ]"
@ -377,7 +563,7 @@ func generateSoongModuleTarget(
soongModuleTarget,
targetNameWithVariant(blueprintCtx, module),
blueprintCtx.ModuleName(module),
blueprintCtx.ModuleType(module),
canonicalizeModuleType(blueprintCtx.ModuleType(module)),
blueprintCtx.ModuleSubDir(module),
depLabelList,
attributes)
@ -410,11 +596,12 @@ func buildFileForModule(ctx *blueprint.Context, module blueprint.Module) (*os.Fi
return f, nil
}
// The overlay directory should be read-only, sufficient for bazel query.
// The overlay directory should be read-only, sufficient for bazel query. The files
// are not intended to be edited by end users.
func writeReadOnlyFile(dir string, baseName string, content string) error {
workspaceFile := filepath.Join(bazelOverlayDir, baseName)
pathToFile := filepath.Join(bazelOverlayDir, baseName)
// 0444 is read-only
return ioutil.WriteFile(workspaceFile, []byte(content), 0444)
return ioutil.WriteFile(pathToFile, []byte(content), 0444)
}
func isZero(value reflect.Value) bool {

209
cmd/soong_build/bazel_overlay_test.go
View file

@ -18,7 +18,10 @@ import (
"android/soong/android"
"io/ioutil"
"os"
"strings"
"testing"
"github.com/google/blueprint/bootstrap/bpdoc"
)
var buildDir string
@ -253,3 +256,209 @@ func TestGenerateBazelOverlayFromBlueprint(t *testing.T) {
}
}
}
func createPackageFixtures() []*bpdoc.Package {
properties := []bpdoc.Property{
bpdoc.Property{
Name: "int64_prop",
Type: "int64",
},
bpdoc.Property{
Name: "int_prop",
Type: "int",
},
bpdoc.Property{
Name: "bool_prop",
Type: "bool",
},
bpdoc.Property{
Name: "string_prop",
Type: "string",
},
bpdoc.Property{
Name: "string_list_prop",
Type: "list of strings",
},
bpdoc.Property{
Name: "nested_prop",
Type: "",
Properties: []bpdoc.Property{
bpdoc.Property{
Name: "int_prop",
Type: "int",
},
bpdoc.Property{
Name: "bool_prop",
Type: "bool",
},
bpdoc.Property{
Name: "string_prop",
Type: "string",
},
},
},
bpdoc.Property{
Name: "unknown_type",
Type: "unknown",
},
}
fooPropertyStruct := &bpdoc.PropertyStruct{
Name: "FooProperties",
Properties: properties,
}
moduleTypes := []*bpdoc.ModuleType{
&bpdoc.ModuleType{
Name: "foo_library",
PropertyStructs: []*bpdoc.PropertyStruct{
fooPropertyStruct,
},
},
&bpdoc.ModuleType{
Name: "foo_binary",
PropertyStructs: []*bpdoc.PropertyStruct{
fooPropertyStruct,
},
},
&bpdoc.ModuleType{
Name: "foo_test",
PropertyStructs: []*bpdoc.PropertyStruct{
fooPropertyStruct,
},
},
}
return [](*bpdoc.Package){
&bpdoc.Package{
Name: "foo_language",
Path: "android/soong/foo",
ModuleTypes: moduleTypes,
},
}
}
func TestGenerateModuleRuleShims(t *testing.T) {
ruleShims, err := createRuleShims(createPackageFixtures())
if err != nil {
panic(err)
}
if len(ruleShims) != 1 {
t.Errorf("Expected to generate 1 rule shim, but got %d", len(ruleShims))
}
fooRuleShim := ruleShims["foo"]
expectedRules := []string{"foo_binary", "foo_library", "foo_test_"}
if len(fooRuleShim.rules) != 3 {
t.Errorf("Expected 3 rules, but got %d", len(fooRuleShim.rules))
}
for i, rule := range fooRuleShim.rules {
if rule != expectedRules[i] {
t.Errorf("Expected rule shim to contain %s, but got %s", expectedRules[i], rule)
}
}
expectedBzl := `load(":providers.bzl", "SoongModuleInfo")
def _foo_binary_impl(ctx):
return [SoongModuleInfo()]
foo_binary = rule(
implementation = _foo_binary_impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"int64_prop": attr.int(),
"int_prop": attr.int(),
# "nested_prop__int_prop": attr.int(),
# "nested_prop__bool_prop": attr.bool(),
# "nested_prop__string_prop": attr.string(),
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
},
)
def _foo_library_impl(ctx):
return [SoongModuleInfo()]
foo_library = rule(
implementation = _foo_library_impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"int64_prop": attr.int(),
"int_prop": attr.int(),
# "nested_prop__int_prop": attr.int(),
# "nested_prop__bool_prop": attr.bool(),
# "nested_prop__string_prop": attr.string(),
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
},
)
def _foo_test__impl(ctx):
return [SoongModuleInfo()]
foo_test_ = rule(
implementation = _foo_test__impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"int64_prop": attr.int(),
"int_prop": attr.int(),
# "nested_prop__int_prop": attr.int(),
# "nested_prop__bool_prop": attr.bool(),
# "nested_prop__string_prop": attr.string(),
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
},
)
`
if fooRuleShim.content != expectedBzl {
t.Errorf(
"Expected the generated rule shim bzl to be:\n%s\nbut got:\n%s",
expectedBzl,
fooRuleShim.content)
}
}
func TestGenerateSoongModuleBzl(t *testing.T) {
ruleShims, err := createRuleShims(createPackageFixtures())
if err != nil {
panic(err)
}
actualSoongModuleBzl := generateSoongModuleBzl(ruleShims)
expectedLoad := "load(\"//:foo.bzl\", \"foo_binary\", \"foo_library\", \"foo_test_\")"
expectedRuleMap := `soong_module_rule_map = {
"foo_binary": foo_binary,
"foo_library": foo_library,
"foo_test_": foo_test_,
}`
if !strings.Contains(actualSoongModuleBzl, expectedLoad) {
t.Errorf(
"Generated soong_module.bzl:\n\n%s\n\n"+
"Could not find the load statement in the generated soong_module.bzl:\n%s",
actualSoongModuleBzl,
expectedLoad)
}
if !strings.Contains(actualSoongModuleBzl, expectedRuleMap) {
t.Errorf(
"Generated soong_module.bzl:\n\n%s\n\n"+
"Could not find the module -> rule map in the generated soong_module.bzl:\n%s",
actualSoongModuleBzl,
expectedRuleMap)
}
}

7
cmd/soong_build/writedocs.go
View file

@ -95,14 +95,17 @@ func moduleTypeDocsToTemplates(moduleTypeList []*bpdoc.ModuleType) []moduleTypeT
return result
}
func writeDocs(ctx *android.Context, filename string) error {
func getPackages(ctx *android.Context) ([]*bpdoc.Package, error) {
moduleTypeFactories := android.ModuleTypeFactories()
bpModuleTypeFactories := make(map[string]reflect.Value)
for moduleType, factory := range moduleTypeFactories {
bpModuleTypeFactories[moduleType] = reflect.ValueOf(factory)
}
return bootstrap.ModuleTypeDocs(ctx.Context, bpModuleTypeFactories)
}
packages, err := bootstrap.ModuleTypeDocs(ctx.Context, bpModuleTypeFactories)
func writeDocs(ctx *android.Context, filename string) error {
packages, err := getPackages(ctx)
if err != nil {
return err
}