From d6fd013394892f810ec40ce92809d2df9ee948ce Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 6 Nov 2023 13:54:06 -0800 Subject: [PATCH] Support generating module_info.json in Soong Generate module_info.json for some Soong modules in Soong in order to pass fewer properties to Kati, which can prevent Kati reanalysis when some Android.bp changes are made. Soong modules can export a ModuleInfoJSONProvider containing the data that should be included in module-info.json. During the androidmk singleton the providers are collected and written to a single JSON file. Make then merges the Soong modules into its own modules. For now, to keep the result as similar as possible to the module-info.json currently being generated by Make, only modules that are exported to Make are written to the Soong module-info.json. Bug: 309006256 Test: Compare module-info.json Change-Id: I996520eb48e04743d43ac11c9aba0f3ada7745de --- android/Android.bp | 1 + android/androidmk.go | 99 ++++++-- android/module.go | 84 +++++++ android/module_context.go | 16 ++ android/module_info_json.go | 103 ++++++++ cmd/merge_module_info_json/Android.bp | 30 +++ .../merge_module_info_json.go | 223 ++++++++++++++++++ .../merge_module_info_json_test.go | 58 +++++ 8 files changed, 590 insertions(+), 24 deletions(-) create mode 100644 android/module_info_json.go create mode 100644 cmd/merge_module_info_json/Android.bp create mode 100644 cmd/merge_module_info_json/merge_module_info_json.go create mode 100644 cmd/merge_module_info_json/merge_module_info_json_test.go diff --git a/android/Android.bp b/android/Android.bp index 26317b8e4..3a7ffd09c 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -62,6 +62,7 @@ bootstrap_go_package { "metrics.go", "module.go", "module_context.go", + "module_info_json.go", "mutator.go", "namespace.go", "neverallow.go", diff --git a/android/androidmk.go b/android/androidmk.go index a0ed1e449..a3334dc2e 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -30,6 +30,7 @@ import ( "reflect" "runtime" "sort" + "strconv" "strings" "github.com/google/blueprint" @@ -626,6 +627,10 @@ func (a *AndroidMkEntries) fillInEntries(ctx fillInEntriesContext, mod blueprint a.SetPath("LOCAL_SOONG_LICENSE_METADATA", licenseMetadata.LicenseMetadataPath) } + if _, ok := SingletonModuleProvider(ctx, mod, ModuleInfoJSONProvider); ok { + a.SetBool("LOCAL_SOONG_MODULE_INFO_JSON", true) + } + extraCtx := &androidMkExtraEntriesContext{ ctx: ctx, mod: mod, @@ -643,14 +648,14 @@ func (a *AndroidMkEntries) fillInEntries(ctx fillInEntriesContext, mod blueprint } } +func (a *AndroidMkEntries) disabled() bool { + return a.Disabled || !a.OutputFile.Valid() +} + // write flushes the AndroidMkEntries's in-struct data populated by AndroidMkEntries into the // given Writer object. func (a *AndroidMkEntries) write(w io.Writer) { - if a.Disabled { - return - } - - if !a.OutputFile.Valid() { + if a.disabled() { return } @@ -696,7 +701,9 @@ func (c *androidMkSingleton) GenerateBuildActions(ctx SingletonContext) { return } - err := translateAndroidMk(ctx, absolutePath(transMk.String()), androidMkModulesList) + moduleInfoJSON := PathForOutput(ctx, "module-info"+String(ctx.Config().productVariables.Make_suffix)+".json") + + err := translateAndroidMk(ctx, absolutePath(transMk.String()), moduleInfoJSON, androidMkModulesList) if err != nil { ctx.Errorf(err.Error()) } @@ -707,14 +714,16 @@ func (c *androidMkSingleton) GenerateBuildActions(ctx SingletonContext) { }) } -func translateAndroidMk(ctx SingletonContext, absMkFile string, mods []blueprint.Module) error { +func translateAndroidMk(ctx SingletonContext, absMkFile string, moduleInfoJSONPath WritablePath, mods []blueprint.Module) error { buf := &bytes.Buffer{} + var moduleInfoJSONs []*ModuleInfoJSON + fmt.Fprintln(buf, "LOCAL_MODULE_MAKEFILE := $(lastword $(MAKEFILE_LIST))") typeStats := make(map[string]int) for _, mod := range mods { - err := translateAndroidMkModule(ctx, buf, mod) + err := translateAndroidMkModule(ctx, buf, &moduleInfoJSONs, mod) if err != nil { os.Remove(absMkFile) return err @@ -736,10 +745,36 @@ func translateAndroidMk(ctx SingletonContext, absMkFile string, mods []blueprint fmt.Fprintf(buf, "STATS.SOONG_MODULE_TYPE.%s := %d\n", mod_type, typeStats[mod_type]) } - return pathtools.WriteFileIfChanged(absMkFile, buf.Bytes(), 0666) + err := pathtools.WriteFileIfChanged(absMkFile, buf.Bytes(), 0666) + if err != nil { + return err + } + + return writeModuleInfoJSON(ctx, moduleInfoJSONs, moduleInfoJSONPath) } -func translateAndroidMkModule(ctx SingletonContext, w io.Writer, mod blueprint.Module) error { +func writeModuleInfoJSON(ctx SingletonContext, moduleInfoJSONs []*ModuleInfoJSON, moduleInfoJSONPath WritablePath) error { + moduleInfoJSONBuf := &strings.Builder{} + moduleInfoJSONBuf.WriteString("[") + for i, moduleInfoJSON := range moduleInfoJSONs { + if i != 0 { + moduleInfoJSONBuf.WriteString(",\n") + } + moduleInfoJSONBuf.WriteString("{") + moduleInfoJSONBuf.WriteString(strconv.Quote(moduleInfoJSON.core.RegisterName)) + moduleInfoJSONBuf.WriteString(":") + err := encodeModuleInfoJSON(moduleInfoJSONBuf, moduleInfoJSON) + moduleInfoJSONBuf.WriteString("}") + if err != nil { + return err + } + } + moduleInfoJSONBuf.WriteString("]") + WriteFileRule(ctx, moduleInfoJSONPath, moduleInfoJSONBuf.String()) + return nil +} + +func translateAndroidMkModule(ctx SingletonContext, w io.Writer, moduleInfoJSONs *[]*ModuleInfoJSON, mod blueprint.Module) error { defer func() { if r := recover(); r != nil { panic(fmt.Errorf("%s in translateAndroidMkModule for module %s variant %s", @@ -748,17 +783,23 @@ func translateAndroidMkModule(ctx SingletonContext, w io.Writer, mod blueprint.M }() // Additional cases here require review for correct license propagation to make. + var err error switch x := mod.(type) { case AndroidMkDataProvider: - return translateAndroidModule(ctx, w, mod, x) + err = translateAndroidModule(ctx, w, moduleInfoJSONs, mod, x) case bootstrap.GoBinaryTool: - return translateGoBinaryModule(ctx, w, mod, x) + err = translateGoBinaryModule(ctx, w, mod, x) case AndroidMkEntriesProvider: - return translateAndroidMkEntriesModule(ctx, w, mod, x) + err = translateAndroidMkEntriesModule(ctx, w, moduleInfoJSONs, mod, x) default: // Not exported to make so no make variables to set. - return nil } + + if err != nil { + return err + } + + return err } // A simple, special Android.mk entry output func to make it possible to build blueprint tools using @@ -801,8 +842,8 @@ func (data *AndroidMkData) fillInData(ctx fillInEntriesContext, mod blueprint.Mo // A support func for the deprecated AndroidMkDataProvider interface. Use AndroidMkEntryProvider // instead. -func translateAndroidModule(ctx SingletonContext, w io.Writer, mod blueprint.Module, - provider AndroidMkDataProvider) error { +func translateAndroidModule(ctx SingletonContext, w io.Writer, moduleInfoJSONs *[]*ModuleInfoJSON, + mod blueprint.Module, provider AndroidMkDataProvider) error { amod := mod.(Module).base() if shouldSkipAndroidMkProcessing(amod) { @@ -864,17 +905,19 @@ func translateAndroidModule(ctx SingletonContext, w io.Writer, mod blueprint.Mod WriteAndroidMkData(w, data) } + if !data.Entries.disabled() { + if moduleInfoJSON, ok := SingletonModuleProvider(ctx, mod, ModuleInfoJSONProvider); ok { + *moduleInfoJSONs = append(*moduleInfoJSONs, moduleInfoJSON) + } + } + return nil } // A support func for the deprecated AndroidMkDataProvider interface. Use AndroidMkEntryProvider // instead. func WriteAndroidMkData(w io.Writer, data AndroidMkData) { - if data.Disabled { - return - } - - if !data.OutputFile.Valid() { + if data.Entries.disabled() { return } @@ -889,18 +932,26 @@ func WriteAndroidMkData(w io.Writer, data AndroidMkData) { fmt.Fprintln(w, "include "+data.Include) } -func translateAndroidMkEntriesModule(ctx SingletonContext, w io.Writer, mod blueprint.Module, - provider AndroidMkEntriesProvider) error { +func translateAndroidMkEntriesModule(ctx SingletonContext, w io.Writer, moduleInfoJSONs *[]*ModuleInfoJSON, + mod blueprint.Module, provider AndroidMkEntriesProvider) error { if shouldSkipAndroidMkProcessing(mod.(Module).base()) { return nil } + entriesList := provider.AndroidMkEntries() + // Any new or special cases here need review to verify correct propagation of license information. - for _, entries := range provider.AndroidMkEntries() { + for _, entries := range entriesList { entries.fillInEntries(ctx, mod) entries.write(w) } + if len(entriesList) > 0 && !entriesList[0].disabled() { + if moduleInfoJSON, ok := SingletonModuleProvider(ctx, mod, ModuleInfoJSONProvider); ok { + *moduleInfoJSONs = append(*moduleInfoJSONs, moduleInfoJSON) + } + } + return nil } diff --git a/android/module.go b/android/module.go index 328b38373..1a428e52d 100644 --- a/android/module.go +++ b/android/module.go @@ -23,6 +23,7 @@ import ( "net/url" "path/filepath" "reflect" + "slices" "sort" "strings" @@ -876,6 +877,10 @@ type ModuleBase struct { // The path to the generated license metadata file for the module. licenseMetadataFile WritablePath + + // moduleInfoJSON can be filled out by GenerateAndroidBuildActions to write a JSON file that will + // be included in the final module-info.json produced by Make. + moduleInfoJSON *ModuleInfoJSON } func (m *ModuleBase) AddJSONData(d *map[string]interface{}) { @@ -1771,11 +1776,90 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) buildLicenseMetadata(ctx, m.licenseMetadataFile) + if m.moduleInfoJSON != nil { + var installed InstallPaths + installed = append(installed, m.katiInstalls.InstallPaths()...) + installed = append(installed, m.katiSymlinks.InstallPaths()...) + installed = append(installed, m.katiInitRcInstalls.InstallPaths()...) + installed = append(installed, m.katiVintfInstalls.InstallPaths()...) + installedStrings := installed.Strings() + + var targetRequired, hostRequired []string + if ctx.Host() { + targetRequired = m.commonProperties.Target_required + } else { + hostRequired = m.commonProperties.Host_required + } + + var data []string + for _, d := range m.testData { + data = append(data, d.ToRelativeInstallPath()) + } + + if m.moduleInfoJSON.Uninstallable { + installedStrings = nil + if len(m.moduleInfoJSON.CompatibilitySuites) == 1 && m.moduleInfoJSON.CompatibilitySuites[0] == "null-suite" { + m.moduleInfoJSON.CompatibilitySuites = nil + m.moduleInfoJSON.TestConfig = nil + m.moduleInfoJSON.AutoTestConfig = nil + data = nil + } + } + + m.moduleInfoJSON.core = CoreModuleInfoJSON{ + RegisterName: m.moduleInfoRegisterName(ctx, m.moduleInfoJSON.SubName), + Path: []string{ctx.ModuleDir()}, + Installed: installedStrings, + ModuleName: m.BaseModuleName() + m.moduleInfoJSON.SubName, + SupportedVariants: []string{m.moduleInfoVariant(ctx)}, + TargetDependencies: targetRequired, + HostDependencies: hostRequired, + Data: data, + } + SetProvider(ctx, ModuleInfoJSONProvider, m.moduleInfoJSON) + } + m.buildParams = ctx.buildParams m.ruleParams = ctx.ruleParams m.variables = ctx.variables } +func (m *ModuleBase) moduleInfoRegisterName(ctx ModuleContext, subName string) string { + name := m.BaseModuleName() + + prefix := "" + if ctx.Host() { + if ctx.Os() != ctx.Config().BuildOS { + prefix = "host_cross_" + } + } + suffix := "" + arches := slices.Clone(ctx.Config().Targets[ctx.Os()]) + arches = slices.DeleteFunc(arches, func(target Target) bool { + return target.NativeBridge != ctx.Target().NativeBridge + }) + if len(arches) > 0 && ctx.Arch().ArchType != arches[0].Arch.ArchType { + if ctx.Arch().ArchType.Multilib == "lib32" { + suffix = "_32" + } else { + suffix = "_64" + } + } + return prefix + name + subName + suffix +} + +func (m *ModuleBase) moduleInfoVariant(ctx ModuleContext) string { + variant := "DEVICE" + if ctx.Host() { + if ctx.Os() != ctx.Config().BuildOS { + variant = "HOST_CROSS" + } else { + variant = "HOST" + } + } + return variant +} + // Check the supplied dist structure to make sure that it is valid. // // property - the base property, e.g. dist or dists[1], which is combined with the diff --git a/android/module_context.go b/android/module_context.go index 81692d5a2..e772f8bc4 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -210,6 +210,11 @@ type ModuleContext interface { // LicenseMetadataFile returns the path where the license metadata for this module will be // generated. LicenseMetadataFile() Path + + // ModuleInfoJSON returns a pointer to the ModuleInfoJSON struct that can be filled out by + // GenerateAndroidBuildActions. If it is called then the struct will be written out and included in + // the module-info.json generated by Make, and Make will not generate its own data for this module. + ModuleInfoJSON() *ModuleInfoJSON } type moduleContext struct { @@ -518,6 +523,8 @@ func (m *moduleContext) installFile(installPath InstallPath, name string, srcPat if !m.skipInstall() { deps = append(deps, InstallPaths(m.module.base().installFilesDepSet.ToList())...) + deps = append(deps, m.module.base().installedInitRcPaths...) + deps = append(deps, m.module.base().installedVintfFragmentsPaths...) var implicitDeps, orderOnlyDeps Paths @@ -695,6 +702,15 @@ func (m *moduleContext) LicenseMetadataFile() Path { return m.module.base().licenseMetadataFile } +func (m *moduleContext) ModuleInfoJSON() *ModuleInfoJSON { + if moduleInfoJSON := m.module.base().moduleInfoJSON; moduleInfoJSON != nil { + return moduleInfoJSON + } + moduleInfoJSON := &ModuleInfoJSON{} + m.module.base().moduleInfoJSON = moduleInfoJSON + return moduleInfoJSON +} + // Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must // be tagged with `android:"path" to support automatic source module dependency resolution. // diff --git a/android/module_info_json.go b/android/module_info_json.go new file mode 100644 index 000000000..1c0a38e5a --- /dev/null +++ b/android/module_info_json.go @@ -0,0 +1,103 @@ +package android + +import ( + "encoding/json" + "io" + "slices" + + "github.com/google/blueprint" +) + +type CoreModuleInfoJSON struct { + RegisterName string `json:"-"` + Path []string `json:"path,omitempty"` // $(sort $(ALL_MODULES.$(m).PATH)) + Installed []string `json:"installed,omitempty"` // $(sort $(ALL_MODULES.$(m).INSTALLED)) + ModuleName string `json:"module_name,omitempty"` // $(ALL_MODULES.$(m).MODULE_NAME) + SupportedVariants []string `json:"supported_variants,omitempty"` // $(sort $(ALL_MODULES.$(m).SUPPORTED_VARIANTS)) + HostDependencies []string `json:"host_dependencies,omitempty"` // $(sort $(ALL_MODULES.$(m).HOST_REQUIRED_FROM_TARGET)) + TargetDependencies []string `json:"target_dependencies,omitempty"` // $(sort $(ALL_MODULES.$(m).TARGET_REQUIRED_FROM_HOST)) + Data []string `json:"data,omitempty"` // $(sort $(ALL_MODULES.$(m).TEST_DATA)) +} + +type ModuleInfoJSON struct { + core CoreModuleInfoJSON + SubName string `json:"-"` + Uninstallable bool `json:"-"` + Class []string `json:"class,omitempty"` // $(sort $(ALL_MODULES.$(m).CLASS)) + Tags []string `json:"tags,omitempty"` // $(sort $(ALL_MODULES.$(m).TAGS)) + Dependencies []string `json:"dependencies,omitempty"` // $(sort $(ALL_DEPS.$(m).ALL_DEPS)) + SharedLibs []string `json:"shared_libs,omitempty"` // $(sort $(ALL_MODULES.$(m).SHARED_LIBS)) + StaticLibs []string `json:"static_libs,omitempty"` // $(sort $(ALL_MODULES.$(m).STATIC_LIBS)) + SystemSharedLibs []string `json:"system_shared_libs,omitempty"` // $(sort $(ALL_MODULES.$(m).SYSTEM_SHARED_LIBS)) + Srcs []string `json:"srcs,omitempty"` // $(sort $(ALL_MODULES.$(m).SRCS)) + SrcJars []string `json:"srcjars,omitempty"` // $(sort $(ALL_MODULES.$(m).SRCJARS)) + ClassesJar []string `json:"classes_jar,omitempty"` // $(sort $(ALL_MODULES.$(m).CLASSES_JAR)) + TestMainlineModules []string `json:"test_mainline_modules,omitempty"` // $(sort $(ALL_MODULES.$(m).TEST_MAINLINE_MODULES)) + IsUnitTest bool `json:"is_unit_test,omitempty"` // $(ALL_MODULES.$(m).IS_UNIT_TEST) + TestOptionsTags []string `json:"test_options_tags,omitempty"` // $(sort $(ALL_MODULES.$(m).TEST_OPTIONS_TAGS)) + RuntimeDependencies []string `json:"runtime_dependencies,omitempty"` // $(sort $(ALL_MODULES.$(m).LOCAL_RUNTIME_LIBRARIES)) + StaticDependencies []string `json:"static_dependencies,omitempty"` // $(sort $(ALL_MODULES.$(m).LOCAL_STATIC_LIBRARIES)) + DataDependencies []string `json:"data_dependencies,omitempty"` // $(sort $(ALL_MODULES.$(m).TEST_DATA_BINS)) + + CompatibilitySuites []string `json:"compatibility_suites,omitempty"` // $(sort $(ALL_MODULES.$(m).COMPATIBILITY_SUITES)) + AutoTestConfig []string `json:"auto_test_config,omitempty"` // $(ALL_MODULES.$(m).auto_test_config) + TestConfig []string `json:"test_config,omitempty"` // $(strip $(ALL_MODULES.$(m).TEST_CONFIG) $(ALL_MODULES.$(m).EXTRA_TEST_CONFIGS) +} + +//ALL_DEPS.$(LOCAL_MODULE).ALL_DEPS := $(sort \ +//$(ALL_DEPS.$(LOCAL_MODULE).ALL_DEPS) \ +//$(LOCAL_STATIC_LIBRARIES) \ +//$(LOCAL_WHOLE_STATIC_LIBRARIES) \ +//$(LOCAL_SHARED_LIBRARIES) \ +//$(LOCAL_DYLIB_LIBRARIES) \ +//$(LOCAL_RLIB_LIBRARIES) \ +//$(LOCAL_PROC_MACRO_LIBRARIES) \ +//$(LOCAL_HEADER_LIBRARIES) \ +//$(LOCAL_STATIC_JAVA_LIBRARIES) \ +//$(LOCAL_JAVA_LIBRARIES) \ +//$(LOCAL_JNI_SHARED_LIBRARIES)) + +type combinedModuleInfoJSON struct { + *CoreModuleInfoJSON + *ModuleInfoJSON +} + +func encodeModuleInfoJSON(w io.Writer, moduleInfoJSON *ModuleInfoJSON) error { + moduleInfoJSONCopy := *moduleInfoJSON + + sortAndUnique := func(s *[]string) { + *s = slices.Clone(*s) + slices.Sort(*s) + *s = slices.Compact(*s) + } + + sortAndUnique(&moduleInfoJSONCopy.core.Path) + sortAndUnique(&moduleInfoJSONCopy.core.Installed) + sortAndUnique(&moduleInfoJSONCopy.core.SupportedVariants) + sortAndUnique(&moduleInfoJSONCopy.core.HostDependencies) + sortAndUnique(&moduleInfoJSONCopy.core.TargetDependencies) + sortAndUnique(&moduleInfoJSONCopy.core.Data) + + sortAndUnique(&moduleInfoJSONCopy.Class) + sortAndUnique(&moduleInfoJSONCopy.Tags) + sortAndUnique(&moduleInfoJSONCopy.Dependencies) + sortAndUnique(&moduleInfoJSONCopy.SharedLibs) + sortAndUnique(&moduleInfoJSONCopy.StaticLibs) + sortAndUnique(&moduleInfoJSONCopy.SystemSharedLibs) + sortAndUnique(&moduleInfoJSONCopy.Srcs) + sortAndUnique(&moduleInfoJSONCopy.SrcJars) + sortAndUnique(&moduleInfoJSONCopy.ClassesJar) + sortAndUnique(&moduleInfoJSONCopy.TestMainlineModules) + sortAndUnique(&moduleInfoJSONCopy.TestOptionsTags) + sortAndUnique(&moduleInfoJSONCopy.RuntimeDependencies) + sortAndUnique(&moduleInfoJSONCopy.StaticDependencies) + sortAndUnique(&moduleInfoJSONCopy.DataDependencies) + sortAndUnique(&moduleInfoJSONCopy.CompatibilitySuites) + sortAndUnique(&moduleInfoJSONCopy.AutoTestConfig) + sortAndUnique(&moduleInfoJSONCopy.TestConfig) + + encoder := json.NewEncoder(w) + return encoder.Encode(combinedModuleInfoJSON{&moduleInfoJSONCopy.core, &moduleInfoJSONCopy}) +} + +var ModuleInfoJSONProvider = blueprint.NewProvider[*ModuleInfoJSON]() diff --git a/cmd/merge_module_info_json/Android.bp b/cmd/merge_module_info_json/Android.bp new file mode 100644 index 000000000..1ae6a4756 --- /dev/null +++ b/cmd/merge_module_info_json/Android.bp @@ -0,0 +1,30 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +blueprint_go_binary { + name: "merge_module_info_json", + srcs: [ + "merge_module_info_json.go", + ], + deps: [ + "soong-response", + ], + testSrcs: [ + "merge_module_info_json_test.go", + ], +} diff --git a/cmd/merge_module_info_json/merge_module_info_json.go b/cmd/merge_module_info_json/merge_module_info_json.go new file mode 100644 index 000000000..0143984a8 --- /dev/null +++ b/cmd/merge_module_info_json/merge_module_info_json.go @@ -0,0 +1,223 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// merge_module_info_json is a utility that merges module_info.json files generated by +// Soong and Make. + +package main + +import ( + "android/soong/response" + "cmp" + "encoding/json" + "flag" + "fmt" + "os" + "slices" +) + +var ( + out = flag.String("o", "", "output file") + listFile = flag.String("l", "", "input file list file") +) + +func usage() { + fmt.Fprintf(os.Stderr, "usage: %s -o \n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprintln(os.Stderr, "merge_module_info_json reads input files that each contain an array of json objects") + fmt.Fprintln(os.Stderr, "and writes them out as a single json array to the output file.") + + os.Exit(2) +} + +func main() { + flag.Usage = usage + flag.Parse() + + if *out == "" { + fmt.Fprintf(os.Stderr, "%s: error: -o is required\n", os.Args[0]) + usage() + } + + inputs := flag.Args() + if *listFile != "" { + listFileInputs, err := readListFile(*listFile) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to read list file %s: %s", *listFile, err) + os.Exit(1) + } + inputs = append(inputs, listFileInputs...) + } + + err := mergeJsonObjects(*out, inputs) + + if err != nil { + fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err.Error()) + os.Exit(1) + } +} + +func readListFile(file string) ([]string, error) { + f, err := os.Open(*listFile) + if err != nil { + return nil, err + } + return response.ReadRspFile(f) +} + +func mergeJsonObjects(output string, inputs []string) error { + combined := make(map[string]any) + for _, input := range inputs { + objects, err := decodeObjectFromJson(input) + if err != nil { + return err + } + + for _, object := range objects { + for k, v := range object { + if old, exists := combined[k]; exists { + v = combine(old, v) + } + combined[k] = v + } + } + } + + f, err := os.Create(output) + if err != nil { + return fmt.Errorf("failed to open output file: %w", err) + } + encoder := json.NewEncoder(f) + encoder.SetIndent("", " ") + err = encoder.Encode(combined) + if err != nil { + return fmt.Errorf("failed to encode to output file: %w", err) + } + + return nil +} + +func decodeObjectFromJson(input string) ([]map[string]any, error) { + f, err := os.Open(input) + if err != nil { + return nil, fmt.Errorf("failed to open input file: %w", err) + } + + decoder := json.NewDecoder(f) + var object any + err = decoder.Decode(&object) + if err != nil { + return nil, fmt.Errorf("failed to parse input file %q: %w", input, err) + } + + switch o := object.(type) { + case []any: + var ret []map[string]any + for _, arrayElement := range o { + if m, ok := arrayElement.(map[string]any); ok { + ret = append(ret, m) + } else { + return nil, fmt.Errorf("unknown JSON type in array %T", arrayElement) + } + } + return ret, nil + + case map[string]any: + return []map[string]any{o}, nil + } + + return nil, fmt.Errorf("unknown JSON type %T", object) +} + +func combine(old, new any) any { + // fmt.Printf("%#v %#v\n", old, new) + switch oldTyped := old.(type) { + case map[string]any: + if newObject, ok := new.(map[string]any); ok { + return combineObjects(oldTyped, newObject) + } else { + panic(fmt.Errorf("expected map[string]any, got %#v", new)) + } + case []any: + if newArray, ok := new.([]any); ok { + return combineArrays(oldTyped, newArray) + } else { + panic(fmt.Errorf("expected []any, got %#v", new)) + } + case string: + if newString, ok := new.(string); ok { + if oldTyped != newString { + panic(fmt.Errorf("strings %q and %q don't match", oldTyped, newString)) + } + return oldTyped + } else { + panic(fmt.Errorf("expected []any, got %#v", new)) + } + default: + panic(fmt.Errorf("can't combine type %T", old)) + } +} + +func combineObjects(old, new map[string]any) map[string]any { + for k, newField := range new { + // HACK: Don't merge "test_config" field. This matches the behavior in base_rules.mk that overwrites + // instead of appending ALL_MODULES.$(my_register_name).TEST_CONFIG, keeping the + if k == "test_config" { + old[k] = newField + continue + } + if oldField, exists := old[k]; exists { + oldField = combine(oldField, newField) + old[k] = oldField + } else { + old[k] = newField + } + } + + return old +} + +func combineArrays(old, new []any) []any { + containsNonStrings := false + for _, oldElement := range old { + switch oldElement.(type) { + case string: + default: + containsNonStrings = true + } + } + for _, newElement := range new { + found := false + for _, oldElement := range old { + if oldElement == newElement { + found = true + break + } + } + if !found { + switch newElement.(type) { + case string: + default: + containsNonStrings = true + } + old = append(old, newElement) + } + } + if !containsNonStrings { + slices.SortFunc(old, func(a, b any) int { + return cmp.Compare(a.(string), b.(string)) + }) + } + return old +} diff --git a/cmd/merge_module_info_json/merge_module_info_json_test.go b/cmd/merge_module_info_json/merge_module_info_json_test.go new file mode 100644 index 000000000..dbf1aa15a --- /dev/null +++ b/cmd/merge_module_info_json/merge_module_info_json_test.go @@ -0,0 +1,58 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "reflect" + "testing" +) + +func Test_combine(t *testing.T) { + tests := []struct { + name string + old any + new any + want any + }{ + { + name: "objects", + old: map[string]any{ + "foo": "bar", + "baz": []any{"a"}, + }, + new: map[string]any{ + "foo": "bar", + "baz": []any{"b"}, + }, + want: map[string]any{ + "foo": "bar", + "baz": []any{"a", "b"}, + }, + }, + { + name: "arrays", + old: []any{"foo", "bar"}, + new: []any{"foo", "baz"}, + want: []any{"bar", "baz", "foo"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := combine(tt.old, tt.new); !reflect.DeepEqual(got, tt.want) { + t.Errorf("combine() = %v, want %v", got, tt.want) + } + }) + } +}