From e8303bd0ae8b631ad92a9c438b8cf23410d65526 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Wed, 16 Feb 2022 09:02:48 -0500 Subject: [PATCH] Export variant/features via soong_injection Test: build/bazel/ci/bp2build.sh Change-Id: If70043dc9d020d5f4a243ba4b192b99c9c7e8563 --- android/arch.go | 49 ++++++++++++++++---- android/arch_list.go | 31 ++++--------- android/arch_test.go | 6 +-- bp2build/conversion.go | 2 + bp2build/conversion_test.go | 4 ++ cc/config/arm64_device.go | 42 +++++++++-------- cc/config/arm_device.go | 17 ++++--- cc/config/bp2build.go | 89 +++++++++++++++++++++++++++++++------ cc/config/bp2build_test.go | 47 +++++++++++++++++++- cc/config/x86_64_device.go | 10 ----- 10 files changed, 214 insertions(+), 83 deletions(-) diff --git a/android/arch.go b/android/arch.go index 0f07b26c9..8aa8d4043 100644 --- a/android/arch.go +++ b/android/arch.go @@ -22,6 +22,7 @@ import ( "strings" "android/soong/bazel" + "android/soong/starlark_fmt" "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" @@ -1794,14 +1795,9 @@ func decodeArch(os OsType, arch string, archVariant, cpuVariant *string, abi []s } } - if a.ArchVariant == "" { - // Set ArchFeatures from the default arch features. - if featureMap, ok := defaultArchFeatureMap[os]; ok { - a.ArchFeatures = featureMap[archType] - } - } else { - // Set ArchFeatures from the arch type. - if featureMap, ok := archFeatureMap[archType]; ok { + // Set ArchFeatures from the arch type. for Android OS, other os-es do not specify features + if os == Android { + if featureMap, ok := androidArchFeatureMap[archType]; ok { a.ArchFeatures = featureMap[a.ArchVariant] } } @@ -2239,3 +2235,40 @@ func mergeStructs(ctx ArchVariantContext, propertyStructs []reflect.Value, prope return value } + +func printArchTypeStarlarkDict(dict map[ArchType][]string) string { + valDict := make(map[string]string, len(dict)) + for k, v := range dict { + valDict[k.String()] = starlark_fmt.PrintStringList(v, 1) + } + return starlark_fmt.PrintDict(valDict, 0) +} + +func printArchTypeNestedStarlarkDict(dict map[ArchType]map[string][]string) string { + valDict := make(map[string]string, len(dict)) + for k, v := range dict { + valDict[k.String()] = starlark_fmt.PrintStringListDict(v, 1) + } + return starlark_fmt.PrintDict(valDict, 0) +} + +func StarlarkArchConfigurations() string { + return fmt.Sprintf(` +_arch_to_variants = %s + +_arch_to_cpu_variants = %s + +_arch_to_features = %s + +_android_arch_feature_for_arch_variant = %s + +arch_to_variants = _arch_to_variants +arch_to_cpu_variants = _arch_to_cpu_variants +arch_to_features = _arch_to_features +android_arch_feature_for_arch_variants = _android_arch_feature_for_arch_variant +`, printArchTypeStarlarkDict(archVariants), + printArchTypeStarlarkDict(cpuVariants), + printArchTypeStarlarkDict(archFeatures), + printArchTypeNestedStarlarkDict(androidArchFeatureMap), + ) +} diff --git a/android/arch_list.go b/android/arch_list.go index 79ad4afac..cbf8e7a02 100644 --- a/android/arch_list.go +++ b/android/arch_list.go @@ -14,8 +14,6 @@ package android -import "fmt" - var archVariants = map[ArchType][]string{ Arm: { "armv7-a", @@ -128,7 +126,7 @@ var archFeatures = map[ArchType][]string{ }, } -var archFeatureMap = map[ArchType]map[string][]string{ +var androidArchFeatureMap = map[ArchType]map[string][]string{ Arm: { "armv7-a-neon": { "neon", @@ -279,6 +277,13 @@ var archFeatureMap = map[ArchType]map[string][]string{ }, }, X86_64: { + "" /*default */ : { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "popcnt", + }, "amberlake": { "ssse3", "sse4", @@ -398,23 +403,3 @@ var archFeatureMap = map[ArchType]map[string][]string{ }, }, } - -var defaultArchFeatureMap = map[OsType]map[ArchType][]string{} - -// RegisterDefaultArchVariantFeatures is called by files that define Toolchains to specify the -// arch features that are available for the default arch variant. It must be called from an -// init() function. -func RegisterDefaultArchVariantFeatures(os OsType, arch ArchType, features ...string) { - checkCalledFromInit() - - for _, feature := range features { - if !InList(feature, archFeatures[arch]) { - panic(fmt.Errorf("Invalid feature %q for arch %q variant \"\"", feature, arch)) - } - } - - if defaultArchFeatureMap[os] == nil { - defaultArchFeatureMap[os] = make(map[ArchType][]string) - } - defaultArchFeatureMap[os][arch] = features -} diff --git a/android/arch_test.go b/android/arch_test.go index a82832115..7caf8378f 100644 --- a/android/arch_test.go +++ b/android/arch_test.go @@ -491,11 +491,9 @@ func TestArchProperties(t *testing.T) { arch: { arm: { a: ["arm"], - armv7_a_neon: { a: ["armv7_a_neon"] }, }, arm64: { a: ["arm64"], - armv8_a: { a: ["armv8_a"] }, }, x86: { a: ["x86"] }, x86_64: { a: ["x86_64"] }, @@ -552,12 +550,12 @@ func TestArchProperties(t *testing.T) { { module: "foo", variant: "android_arm64_armv8-a", - property: []string{"root", "linux", "bionic", "android", "android64", "arm64", "armv8_a", "lib64", "android_arm64"}, + property: []string{"root", "linux", "bionic", "android", "android64", "arm64", "lib64", "android_arm64"}, }, { module: "foo", variant: "android_arm_armv7-a-neon", - property: []string{"root", "linux", "bionic", "android", "android64", "arm", "armv7_a_neon", "lib32", "android_arm"}, + property: []string{"root", "linux", "bionic", "android", "android64", "arm", "lib32", "android_arm"}, }, }, }, diff --git a/bp2build/conversion.go b/bp2build/conversion.go index 96c12d3e1..3ab846b41 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -28,6 +28,8 @@ func CreateSoongInjectionFiles(cfg android.Config, metrics CodegenMetrics) []Baz files = append(files, newFile("product_config", "soong_config_variables.bzl", cfg.Bp2buildSoongConfigDefinitions.String())) + files = append(files, newFile("product_config", "arch_configuration.bzl", android.StarlarkArchConfigurations())) + apiLevelsContent, err := json.Marshal(android.GetApiLevelsMap(cfg)) if err != nil { panic(err) diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index 629ca9bba..c94a9236e 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -102,6 +102,10 @@ func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { dir: "product_config", basename: "soong_config_variables.bzl", }, + { + dir: "product_config", + basename: "arch_configuration.bzl", + }, { dir: "api_levels", basename: GeneratedBuildFileName, diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go index 979c82525..4d0ae1a12 100644 --- a/cc/config/arm64_device.go +++ b/cc/config/arm64_device.go @@ -104,19 +104,22 @@ func init() { exportStringListStaticVariable("Arm64Cflags", arm64Cflags) exportStringListStaticVariable("Arm64Cppflags", arm64Cppflags) - exportedStringListDictVars.Set("Arm64ArchVariantCflags", arm64ArchVariantCflags) - exportedStringListDictVars.Set("Arm64CpuVariantCflags", arm64CpuVariantCflags) + exportedVariableReferenceDictVars.Set("Arm64ArchVariantCflags", arm64ArchVariantCflagsVar) + exportedVariableReferenceDictVars.Set("Arm64CpuVariantCflags", arm64CpuVariantCflagsVar) + exportedVariableReferenceDictVars.Set("Arm64CpuVariantLdflags", arm64CpuVariantLdflags) - pctx.StaticVariable("Arm64Armv8ACflags", strings.Join(arm64ArchVariantCflags["armv8-a"], " ")) - pctx.StaticVariable("Arm64Armv8ABranchProtCflags", strings.Join(arm64ArchVariantCflags["armv8-a-branchprot"], " ")) - pctx.StaticVariable("Arm64Armv82ACflags", strings.Join(arm64ArchVariantCflags["armv8-2a"], " ")) - pctx.StaticVariable("Arm64Armv82ADotprodCflags", strings.Join(arm64ArchVariantCflags["armv8-2a-dotprod"], " ")) + exportStringListStaticVariable("Arm64Armv8ACflags", arm64ArchVariantCflags["armv8-a"]) + exportStringListStaticVariable("Arm64Armv8ABranchProtCflags", arm64ArchVariantCflags["armv8-a-branchprot"]) + exportStringListStaticVariable("Arm64Armv82ACflags", arm64ArchVariantCflags["armv8-2a"]) + exportStringListStaticVariable("Arm64Armv82ADotprodCflags", arm64ArchVariantCflags["armv8-2a-dotprod"]) - pctx.StaticVariable("Arm64CortexA53Cflags", strings.Join(arm64CpuVariantCflags["cortex-a53"], " ")) - pctx.StaticVariable("Arm64CortexA55Cflags", strings.Join(arm64CpuVariantCflags["cortex-a55"], " ")) - pctx.StaticVariable("Arm64KryoCflags", strings.Join(arm64CpuVariantCflags["kryo"], " ")) - pctx.StaticVariable("Arm64ExynosM1Cflags", strings.Join(arm64CpuVariantCflags["exynos-m1"], " ")) - pctx.StaticVariable("Arm64ExynosM2Cflags", strings.Join(arm64CpuVariantCflags["exynos-m2"], " ")) + exportStringListStaticVariable("Arm64CortexA53Cflags", arm64CpuVariantCflags["cortex-a53"]) + exportStringListStaticVariable("Arm64CortexA55Cflags", arm64CpuVariantCflags["cortex-a55"]) + exportStringListStaticVariable("Arm64KryoCflags", arm64CpuVariantCflags["kryo"]) + exportStringListStaticVariable("Arm64ExynosM1Cflags", arm64CpuVariantCflags["exynos-m1"]) + exportStringListStaticVariable("Arm64ExynosM2Cflags", arm64CpuVariantCflags["exynos-m2"]) + + exportStringListStaticVariable("Arm64FixCortexA53Ldflags", []string{"-Wl,--fix-cortex-a53-843419"}) } var ( @@ -128,7 +131,6 @@ var ( } arm64CpuVariantCflagsVar = map[string]string{ - "": "", "cortex-a53": "${config.Arm64CortexA53Cflags}", "cortex-a55": "${config.Arm64CortexA55Cflags}", "cortex-a72": "${config.Arm64CortexA53Cflags}", @@ -140,6 +142,15 @@ var ( "exynos-m1": "${config.Arm64ExynosM1Cflags}", "exynos-m2": "${config.Arm64ExynosM2Cflags}", } + + arm64CpuVariantLdflags = map[string]string{ + "cortex-a53": "${config.Arm64FixCortexA53Ldflags}", + "cortex-a72": "${config.Arm64FixCortexA53Ldflags}", + "cortex-a73": "${config.Arm64FixCortexA53Ldflags}", + "kryo": "${config.Arm64FixCortexA53Ldflags}", + "exynos-m1": "${config.Arm64FixCortexA53Ldflags}", + "exynos-m2": "${config.Arm64FixCortexA53Ldflags}", + } ) type toolchainArm64 struct { @@ -214,12 +225,7 @@ func arm64ToolchainFactory(arch android.Arch) Toolchain { toolchainCflags = append(toolchainCflags, variantOrDefault(arm64CpuVariantCflagsVar, arch.CpuVariant)) - var extraLdflags string - switch arch.CpuVariant { - case "cortex-a53", "cortex-a72", "cortex-a73", "kryo", "exynos-m1", "exynos-m2": - extraLdflags = "-Wl,--fix-cortex-a53-843419" - } - + extraLdflags := variantOrDefault(arm64CpuVariantLdflags, arch.CpuVariant) return &toolchainArm64{ ldflags: strings.Join([]string{ "${config.Arm64Ldflags}", diff --git a/cc/config/arm_device.go b/cc/config/arm_device.go index 0fe5e6883..4466632ea 100644 --- a/cc/config/arm_device.go +++ b/cc/config/arm_device.go @@ -39,6 +39,10 @@ var ( armLldflags = armLdflags + armFixCortexA8LdFlags = []string{"-Wl,--fix-cortex-a8"} + + armNoFixCortexA8LdFlags = []string{"-Wl,--no-fix-cortex-a8"} + armArmCflags = []string{ "-fstrict-aliasing", } @@ -179,6 +183,9 @@ func init() { exportStringListStaticVariable("ArmLdflags", armLdflags) exportStringListStaticVariable("ArmLldflags", armLldflags) + exportStringListStaticVariable("ArmFixCortexA8LdFlags", armFixCortexA8LdFlags) + exportStringListStaticVariable("ArmNoFixCortexA8LdFlags", armNoFixCortexA8LdFlags) + // Clang cflags exportStringListStaticVariable("ArmToolchainCflags", armToolchainCflags) exportStringListStaticVariable("ArmCflags", armCflags) @@ -188,8 +195,8 @@ func init() { exportStringListStaticVariable("ArmArmCflags", armArmCflags) exportStringListStaticVariable("ArmThumbCflags", armThumbCflags) - exportedStringListDictVars.Set("ArmArchVariantCflags", armArchVariantCflags) - exportedStringListDictVars.Set("ArmCpuVariantCflags", armCpuVariantCflags) + exportedVariableReferenceDictVars.Set("ArmArchVariantCflags", armArchVariantCflagsVar) + exportedVariableReferenceDictVars.Set("ArmCpuVariantCflags", armCpuVariantCflagsVar) // Clang arch variant cflags exportStringListStaticVariable("ArmArmv7ACflags", armArchVariantCflags["armv7-a"]) @@ -324,12 +331,12 @@ func armToolchainFactory(arch android.Arch) Toolchain { switch arch.CpuVariant { case "cortex-a8", "": // Generic ARM might be a Cortex A8 -- better safe than sorry - fixCortexA8 = "-Wl,--fix-cortex-a8" + fixCortexA8 = "${config.ArmFixCortexA8LdFlags}" default: - fixCortexA8 = "-Wl,--no-fix-cortex-a8" + fixCortexA8 = "${config.ArmNoFixCortexA8LdFlags}" } case "armv7-a": - fixCortexA8 = "-Wl,--fix-cortex-a8" + fixCortexA8 = "${config.ArmFixCortexA8LdFlags}" case "armv8-a", "armv8-2a": // Nothing extra for armv8-a/armv8-2a default: diff --git a/cc/config/bp2build.go b/cc/config/bp2build.go index eca516107..73f65f539 100644 --- a/cc/config/bp2build.go +++ b/cc/config/bp2build.go @@ -38,6 +38,8 @@ var ( exportedStringListVars = exportedStringListVariables{} exportedStringVars = exportedStringVariables{} exportedStringListDictVars = exportedStringListDictVariables{} + // Note: these can only contain references to other variables and must be printed last + exportedVariableReferenceDictVars = exportedVariableReferenceDictVariables{} /// Maps containing variables that are dependent on the build config. exportedConfigDependingVars = exportedConfigDependingVariables{} @@ -62,6 +64,7 @@ func validateCharacters(s string) string { type bazelConstant struct { variableName string internalDefinition string + sortLast bool } type exportedStringVariables map[string]string @@ -168,6 +171,36 @@ func (m exportedStringListDictVariables) asBazel(_ android.Config, _ exportedStr return ret } +type exportedVariableReferenceDictVariables map[string]map[string]string + +func (m exportedVariableReferenceDictVariables) Set(k string, v map[string]string) { + m[k] = v +} + +func (m exportedVariableReferenceDictVariables) asBazel(_ android.Config, _ exportedStringVariables, + _ exportedStringListVariables, _ exportedConfigDependingVariables) []bazelConstant { + ret := make([]bazelConstant, 0, len(m)) + for n, dict := range m { + for k, v := range dict { + matches, err := variableReference(v) + if err != nil { + panic(err) + } else if !matches.matches { + panic(fmt.Errorf("Expected a variable reference, got %q", v)) + } else if len(matches.fullVariableReference) != len(v) { + panic(fmt.Errorf("Expected only a variable reference, got %q", v)) + } + dict[k] = "_" + matches.variable + } + ret = append(ret, bazelConstant{ + variableName: n, + internalDefinition: starlark_fmt.PrintDict(dict, 0), + sortLast: true, + }) + } + return ret +} + // BazelCcToolchainVars generates bzl file content containing variables for // Bazel's cc_toolchain configuration. func BazelCcToolchainVars(config android.Config) string { @@ -175,7 +208,8 @@ func BazelCcToolchainVars(config android.Config) string { config, exportedStringListDictVars, exportedStringListVars, - exportedStringVars) + exportedStringVars, + exportedVariableReferenceDictVars) } func bazelToolchainVars(config android.Config, vars ...bazelVarExporter) string { @@ -186,7 +220,12 @@ func bazelToolchainVars(config android.Config, vars ...bazelVarExporter) string results = append(results, v.asBazel(config, exportedStringVars, exportedStringListVars, exportedConfigDependingVars)...) } - sort.Slice(results, func(i, j int) bool { return results[i].variableName < results[j].variableName }) + sort.Slice(results, func(i, j int) bool { + if results[i].sortLast != results[j].sortLast { + return !results[i].sortLast + } + return results[i].variableName < results[j].variableName + }) definitions := make([]string, 0, len(results)) constants := make([]string, 0, len(results)) @@ -207,6 +246,32 @@ func bazelToolchainVars(config android.Config, vars ...bazelVarExporter) string return ret } +type match struct { + matches bool + fullVariableReference string + variable string +} + +func variableReference(input string) (match, error) { + // e.g. "${ExternalCflags}" + r := regexp.MustCompile(`\${(?:config\.)?([a-zA-Z0-9_]+)}`) + + matches := r.FindStringSubmatch(input) + if len(matches) == 0 { + return match{}, nil + } + if len(matches) != 2 { + return match{}, fmt.Errorf("Expected to only match 1 subexpression in %s, got %d", input, len(matches)-1) + } + return match{ + matches: true, + fullVariableReference: matches[0], + // Index 1 of FindStringSubmatch contains the subexpression match + // (variable name) of the capture group. + variable: matches[1], + }, nil +} + // expandVar recursively expand interpolated variables in the exportedVars scope. // // We're using a string slice to track the seen variables to avoid @@ -216,8 +281,6 @@ func bazelToolchainVars(config android.Config, vars ...bazelVarExporter) string // interpolation stacks are deep (n > 1). func expandVar(config android.Config, toExpand string, stringScope exportedStringVariables, stringListScope exportedStringListVariables, exportedVars exportedConfigDependingVariables) ([]string, error) { - // e.g. "${ExternalCflags}" - r := regexp.MustCompile(`\${([a-zA-Z0-9_]+)}`) // Internal recursive function. var expandVarInternal func(string, map[string]bool) (string, error) @@ -225,20 +288,18 @@ func expandVar(config android.Config, toExpand string, stringScope exportedStrin var ret string remainingString := toExpand for len(remainingString) > 0 { - matches := r.FindStringSubmatch(remainingString) - if len(matches) == 0 { + matches, err := variableReference(remainingString) + if err != nil { + panic(err) + } + if !matches.matches { return ret + remainingString, nil } - if len(matches) != 2 { - panic(fmt.Errorf("Expected to only match 1 subexpression in %s, got %d", remainingString, len(matches)-1)) - } - matchIndex := strings.Index(remainingString, matches[0]) + matchIndex := strings.Index(remainingString, matches.fullVariableReference) ret += remainingString[:matchIndex] - remainingString = remainingString[matchIndex+len(matches[0]):] + remainingString = remainingString[matchIndex+len(matches.fullVariableReference):] - // Index 1 of FindStringSubmatch contains the subexpression match - // (variable name) of the capture group. - variable := matches[1] + variable := matches.variable // toExpand contains a variable. if _, ok := seenVars[variable]; ok { return ret, fmt.Errorf( diff --git a/cc/config/bp2build_test.go b/cc/config/bp2build_test.go index 4cbf0c6f3..9a8178af6 100644 --- a/cc/config/bp2build_test.go +++ b/cc/config/bp2build_test.go @@ -47,6 +47,14 @@ func TestExpandVars(t *testing.T) { toExpand: "${foo}", expectedValues: []string{"bar"}, }, + { + description: "single level expansion with short-name for string var", + stringScope: exportedStringVariables{ + "foo": "bar", + }, + toExpand: "${config.foo}", + expectedValues: []string{"bar"}, + }, { description: "single level expansion string list var", stringListScope: exportedStringListVariables{ @@ -224,7 +232,30 @@ constants = struct( )`, }, { - name: "sorts across types", + name: "exports dict with var refs", + vars: []bazelVarExporter{ + exportedVariableReferenceDictVariables{ + "a": map[string]string{"b1": "${b2}"}, + "c": map[string]string{"d1": "${config.d2}"}, + }, + }, + expectedOut: `# GENERATED FOR BAZEL FROM SOONG. DO NOT EDIT. + +_a = { + "b1": _b2, +} + +_c = { + "d1": _d2, +} + +constants = struct( + a = _a, + c = _c, +)`, + }, + { + name: "sorts across types with variable references last", vars: []bazelVarExporter{ exportedStringVariables{ "b": "b-val", @@ -238,6 +269,10 @@ constants = struct( "a": map[string][]string{"a1": []string{"a2"}}, "f": map[string][]string{"f1": []string{"f2"}}, }, + exportedVariableReferenceDictVariables{ + "aa": map[string]string{"b1": "${b}"}, + "cc": map[string]string{"d1": "${config.d}"}, + }, }, expectedOut: `# GENERATED FOR BAZEL FROM SOONG. DO NOT EDIT. @@ -257,6 +292,14 @@ _f = { "f1": ["f2"], } +_aa = { + "b1": _b, +} + +_cc = { + "d1": _d, +} + constants = struct( a = _a, b = _b, @@ -264,6 +307,8 @@ constants = struct( d = _d, e = _e, f = _f, + aa = _aa, + cc = _cc, )`, }, } diff --git a/cc/config/x86_64_device.go b/cc/config/x86_64_device.go index 0da51cb4e..164e7a67f 100644 --- a/cc/config/x86_64_device.go +++ b/cc/config/x86_64_device.go @@ -78,14 +78,6 @@ var ( "popcnt": []string{"-mpopcnt"}, "aes_ni": []string{"-maes"}, } - - x86_64DefaultArchVariantFeatures = []string{ - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "popcnt", - } ) const ( @@ -93,8 +85,6 @@ const ( ) func init() { - android.RegisterDefaultArchVariantFeatures(android.Android, android.X86_64, x86_64DefaultArchVariantFeatures...) - exportedStringListVars.Set("X86_64DefaultArchVariantFeatures", x86_64DefaultArchVariantFeatures) pctx.StaticVariable("x86_64GccVersion", x86_64GccVersion)