From b051c4ede37e33bfb7c56e8c6fcf0ab010915fc4 Mon Sep 17 00:00:00 2001 From: Sasha Smundak Date: Thu, 5 Nov 2020 20:45:07 -0800 Subject: [PATCH] Product config makefiles to Starlark converter Test: treehugger; internal tests in mk2rbc_test.go Bug: 172923994 Change-Id: I43120b9c181ef2b8d9453e743233811b0fec268b --- androidmk/parser/make_strings.go | 9 + androidmk/parser/parser.go | 3 +- mk2rbc/Android.bp | 39 + mk2rbc/TODO | 14 + mk2rbc/cmd/mk2rbc.go | 498 +++++++++++ mk2rbc/expr.go | 580 +++++++++++++ mk2rbc/mk2rbc.go | 1344 ++++++++++++++++++++++++++++++ mk2rbc/mk2rbc_test.go | 857 +++++++++++++++++++ mk2rbc/node.go | 237 ++++++ mk2rbc/types.go | 5 + mk2rbc/variable.go | 300 +++++++ 11 files changed, 3885 insertions(+), 1 deletion(-) create mode 100644 mk2rbc/Android.bp create mode 100644 mk2rbc/TODO create mode 100644 mk2rbc/cmd/mk2rbc.go create mode 100644 mk2rbc/expr.go create mode 100644 mk2rbc/mk2rbc.go create mode 100644 mk2rbc/mk2rbc_test.go create mode 100644 mk2rbc/node.go create mode 100644 mk2rbc/variable.go diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go index 3c4815ea5..aac4c4efe 100644 --- a/androidmk/parser/make_strings.go +++ b/androidmk/parser/make_strings.go @@ -278,6 +278,15 @@ func (ms *MakeString) ReplaceLiteral(input string, output string) { } } +// If MakeString is $(var) after trimming, returns var +func (ms *MakeString) SingleVariable() (*MakeString, bool) { + if len(ms.Strings) != 2 || strings.TrimSpace(ms.Strings[0]) != "" || + strings.TrimSpace(ms.Strings[1]) != "" { + return nil, false + } + return ms.Variables[0].Name, true +} + func splitAnyN(s, sep string, n int) []string { ret := []string{} for n == -1 || n > 1 { diff --git a/androidmk/parser/parser.go b/androidmk/parser/parser.go index 5afef652a..d24efc10d 100644 --- a/androidmk/parser/parser.go +++ b/androidmk/parser/parser.go @@ -216,13 +216,14 @@ func (p *parser) parseDirective() bool { // Nothing case "else": p.ignoreSpaces() - if p.tok != '\n' { + if p.tok != '\n' && p.tok != '#' { d = p.scanner.TokenText() p.accept(scanner.Ident) if d == "ifdef" || d == "ifndef" || d == "ifeq" || d == "ifneq" { d = "el" + d p.ignoreSpaces() expression = p.parseExpression() + expression.TrimRightSpaces() } else { p.errorf("expected ifdef/ifndef/ifeq/ifneq, found %s", d) } diff --git a/mk2rbc/Android.bp b/mk2rbc/Android.bp new file mode 100644 index 000000000..3ea3f7fd0 --- /dev/null +++ b/mk2rbc/Android.bp @@ -0,0 +1,39 @@ +// +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +blueprint_go_binary { + name: "mk2rbc", + srcs: ["cmd/mk2rbc.go"], + deps: [ + "mk2rbc-lib", + "androidmk-parser", + ], +} + +bootstrap_go_package { + name: "mk2rbc-lib", + pkgPath: "android/soong/mk2rbc", + srcs: [ + "android_products.go", + "config_variables.go", + "expr.go", + "mk2rbc.go", + "node.go", + "soong_variables.go", + "types.go", + "variable.go", + ], + deps: ["androidmk-parser"], +} diff --git a/mk2rbc/TODO b/mk2rbc/TODO new file mode 100644 index 000000000..731deb6a5 --- /dev/null +++ b/mk2rbc/TODO @@ -0,0 +1,14 @@ +* Checking filter/filter-out results is incorrect if pattern contains '%' +* Need heuristics to recognize that a variable is local. Propose to use lowercase. +* Need heuristics for the local variable type. Propose '_list' suffix +* Internal source tree has variables in the inherit-product macro argument. Handle it +* Enumerate all environment variables that configuration files use. +* Break mk2rbc.go into multiple files. +* If variable's type is not yet known, try to divine it from the value assigned to it + (it may be a variable of the known type, or a function result) +* ifneq (,$(VAR)) should translate to + if getattr(<>, "VAR", ): +* Launcher file needs to have same suffix as the rest of the generated files +* Implement $(shell) function +* Write execution tests +* Review all TODOs in mk2rbc.go \ No newline at end of file diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go new file mode 100644 index 000000000..aa01e3bdc --- /dev/null +++ b/mk2rbc/cmd/mk2rbc.go @@ -0,0 +1,498 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The application to convert product configuration makefiles to Starlark. +// Converts either given list of files (and optionally the dependent files +// of the same kind), or all all product configuration makefiles in the +// given source tree. +// Previous version of a converted file can be backed up. +// Optionally prints detailed statistics at the end. +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "runtime/debug" + "sort" + "strings" + "time" + + "android/soong/androidmk/parser" + "android/soong/mk2rbc" +) + +var ( + rootDir = flag.String("root", ".", "the value of // for load paths") + // TODO(asmundak): remove this option once there is a consensus on suffix + suffix = flag.String("suffix", ".rbc", "generated files' suffix") + dryRun = flag.Bool("dry_run", false, "dry run") + recurse = flag.Bool("convert_dependents", false, "convert all dependent files") + mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`) + warn = flag.Bool("warnings", false, "warn about partially failed conversions") + verbose = flag.Bool("v", false, "print summary") + errstat = flag.Bool("error_stat", false, "print error statistics") + traceVar = flag.String("trace", "", "comma-separated list of variables to trace") + // TODO(asmundak): this option is for debugging + allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //") + outputTop = flag.String("outdir", "", "write output files into this directory hierarchy") + launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_") + printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit") + traceCalls = flag.Bool("trace_calls", false, "trace function calls") +) + +func init() { + // Poor man's flag aliasing: works, but the usage string is ugly and + // both flag and its alias can be present on the command line + flagAlias := func(target string, alias string) { + if f := flag.Lookup(target); f != nil { + flag.Var(f.Value, alias, "alias for --"+f.Name) + return + } + quit("cannot alias unknown flag " + target) + } + flagAlias("suffix", "s") + flagAlias("root", "d") + flagAlias("dry_run", "n") + flagAlias("convert_dependents", "r") + flagAlias("warnings", "w") + flagAlias("error_stat", "e") +} + +var backupSuffix string +var tracedVariables []string +var errorLogger = errorsByType{data: make(map[string]datum)} + +func main() { + flag.Usage = func() { + cmd := filepath.Base(os.Args[0]) + fmt.Fprintf(flag.CommandLine.Output(), + "Usage: %[1]s flags file...\n"+ + "or: %[1]s flags --launcher=PATH PRODUCT\n", cmd) + flag.PrintDefaults() + } + flag.Parse() + + // Delouse + if *suffix == ".mk" { + quit("cannot use .mk as generated file suffix") + } + if *suffix == "" { + quit("suffix cannot be empty") + } + if *outputTop != "" { + if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil { + quit(err) + } + s, err := filepath.Abs(*outputTop) + if err != nil { + quit(err) + } + *outputTop = s + } + if *allInSource && len(flag.Args()) > 0 { + quit("file list cannot be specified when -all is present") + } + if *allInSource && *launcher != "" { + quit("--all and --launcher are mutually exclusive") + } + + // Flag-driven adjustments + if (*suffix)[0] != '.' { + *suffix = "." + *suffix + } + if *mode == "backup" { + backupSuffix = time.Now().Format("20060102150405") + } + if *traceVar != "" { + tracedVariables = strings.Split(*traceVar, ",") + } + + // Find out global variables + getConfigVariables() + getSoongVariables() + + if *printProductConfigMap { + productConfigMap := buildProductConfigMap() + var products []string + for p := range productConfigMap { + products = append(products, p) + } + sort.Strings(products) + for _, p := range products { + fmt.Println(p, productConfigMap[p]) + } + os.Exit(0) + } + if len(flag.Args()) == 0 { + flag.Usage() + } + // Convert! + ok := true + if *launcher != "" { + if len(flag.Args()) != 1 { + quit(fmt.Errorf("a launcher can be generated only for a single product")) + } + product := flag.Args()[0] + productConfigMap := buildProductConfigMap() + path, found := productConfigMap[product] + if !found { + quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product", + product)) + } + ok = convertOne(path) && ok + err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path))) + if err != nil { + fmt.Fprintf(os.Stderr, "%s:%s", path, err) + ok = false + } + + } else { + files := flag.Args() + if *allInSource { + productConfigMap := buildProductConfigMap() + for _, path := range productConfigMap { + files = append(files, path) + } + } + for _, mkFile := range files { + ok = convertOne(mkFile) && ok + } + } + + printStats() + if *errstat { + errorLogger.printStatistics() + } + if !ok { + os.Exit(1) + } +} + +func quit(s interface{}) { + fmt.Fprintln(os.Stderr, s) + os.Exit(2) +} + +func buildProductConfigMap() map[string]string { + const androidProductsMk = "AndroidProducts.mk" + // Build the list of AndroidProducts.mk files: it's + // build/make/target/product/AndroidProducts.mk plus + // device/**/AndroidProducts.mk + targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk) + if _, err := os.Stat(targetAndroidProductsFile); err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n", + targetAndroidProductsFile, err, *rootDir) + } + productConfigMap := make(map[string]string) + if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err) + } + _ = filepath.Walk(filepath.Join(*rootDir, "device"), + func(path string, info os.FileInfo, err error) error { + if info.IsDir() || filepath.Base(path) != androidProductsMk { + return nil + } + if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", path, err) + // Keep going, we want to find all such errors in a single run + } + return nil + }) + return productConfigMap +} + +func getConfigVariables() { + path := filepath.Join(*rootDir, "build", "make", "core", "product.mk") + if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil { + quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)", + err, *rootDir)) + } +} + +// Implements mkparser.Scope, to be used by mkparser.Value.Value() +type fileNameScope struct { + mk2rbc.ScopeBase +} + +func (s fileNameScope) Get(name string) string { + if name != "BUILD_SYSTEM" { + return fmt.Sprintf("$(%s)", name) + } + return filepath.Join(*rootDir, "build", "make", "core") +} + +func getSoongVariables() { + path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk") + err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables) + if err != nil { + quit(err) + } +} + +var converted = make(map[string]*mk2rbc.StarlarkScript) + +//goland:noinspection RegExpRepeatedSpace +var cpNormalizer = regexp.MustCompile( + "# Copyright \\(C\\) 20.. The Android Open Source Project") + +const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project" +const copyright = `# +# Copyright (C) 20xx The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +` + +// Convert a single file. +// Write the result either to the same directory, to the same place in +// the output hierarchy, or to the stdout. +// Optionally, recursively convert the files this one includes by +// $(call inherit-product) or an include statement. +func convertOne(mkFile string) (ok bool) { + if v, ok := converted[mkFile]; ok { + return v != nil + } + converted[mkFile] = nil + defer func() { + if r := recover(); r != nil { + ok = false + fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack()) + } + }() + + mk2starRequest := mk2rbc.Request{ + MkFile: mkFile, + Reader: nil, + RootDir: *rootDir, + OutputDir: *outputTop, + OutputSuffix: *suffix, + TracedVariables: tracedVariables, + TraceCalls: *traceCalls, + WarnPartialSuccess: *warn, + } + if *errstat { + mk2starRequest.ErrorLogger = errorLogger + } + ss, err := mk2rbc.Convert(mk2starRequest) + if err != nil { + fmt.Fprintln(os.Stderr, mkFile, ": ", err) + return false + } + script := ss.String() + outputPath := outputFilePath(mkFile) + + if *dryRun { + fmt.Printf("==== %s ====\n", outputPath) + // Print generated script after removing the copyright header + outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright) + fmt.Println(strings.TrimPrefix(outText, copyright)) + } else { + if err := maybeBackup(outputPath); err != nil { + fmt.Fprintln(os.Stderr, err) + return false + } + if err := writeGenerated(outputPath, script); err != nil { + fmt.Fprintln(os.Stderr, err) + return false + } + } + ok = true + if *recurse { + for _, sub := range ss.SubConfigFiles() { + // File may be absent if it is a conditional load + if _, err := os.Stat(sub); os.IsNotExist(err) { + continue + } + ok = convertOne(sub) && ok + } + } + converted[mkFile] = ss + return ok +} + +// Optionally saves the previous version of the generated file +func maybeBackup(filename string) error { + stat, err := os.Stat(filename) + if os.IsNotExist(err) { + return nil + } + if !stat.Mode().IsRegular() { + return fmt.Errorf("%s exists and is not a regular file", filename) + } + switch *mode { + case "backup": + return os.Rename(filename, filename+backupSuffix) + case "write": + return os.Remove(filename) + default: + return fmt.Errorf("%s already exists, use --mode option", filename) + } +} + +func outputFilePath(mkFile string) string { + path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix + if *outputTop != "" { + path = filepath.Join(*outputTop, path) + } + return path +} + +func writeGenerated(path string, contents string) error { + if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil { + return err + } + if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil { + return err + } + return nil +} + +func printStats() { + var sortedFiles []string + if !*warn && !*verbose { + return + } + for p := range converted { + sortedFiles = append(sortedFiles, p) + } + sort.Strings(sortedFiles) + + nOk, nPartial, nFailed := 0, 0, 0 + for _, f := range sortedFiles { + if converted[f] == nil { + nFailed++ + } else if converted[f].HasErrors() { + nPartial++ + } else { + nOk++ + } + } + if *warn { + if nPartial > 0 { + fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n") + for _, f := range sortedFiles { + if ss := converted[f]; ss != nil && ss.HasErrors() { + fmt.Fprintln(os.Stderr, " ", f) + } + } + } + + if nFailed > 0 { + fmt.Fprintf(os.Stderr, "Conversion failed for files:\n") + for _, f := range sortedFiles { + if converted[f] == nil { + fmt.Fprintln(os.Stderr, " ", f) + } + } + } + } + if *verbose { + fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk) + fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial) + fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed) + } +} + +type datum struct { + count int + formattingArgs []string +} + +type errorsByType struct { + data map[string]datum +} + +func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) { + v, exists := ebt.data[message] + if exists { + v.count++ + } else { + v = datum{1, nil} + } + if strings.Contains(message, "%s") { + var newArg1 string + if len(args) == 0 { + panic(fmt.Errorf(`%s has %%s but args are missing`, message)) + } + newArg1 = fmt.Sprint(args[0]) + if message == "unsupported line" { + newArg1 = node.Dump() + } else if message == "unsupported directive %s" { + if newArg1 == "include" || newArg1 == "-include" { + newArg1 = node.Dump() + } + } + v.formattingArgs = append(v.formattingArgs, newArg1) + } + ebt.data[message] = v +} + +func (ebt errorsByType) printStatistics() { + if len(ebt.data) > 0 { + fmt.Fprintln(os.Stderr, "Error counts:") + } + for message, data := range ebt.data { + if len(data.formattingArgs) == 0 { + fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message) + continue + } + itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30) + fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count) + fmt.Fprintln(os.Stderr, " ", itemsByFreq) + } +} + +func stringsWithFreq(items []string, topN int) (string, int) { + freq := make(map[string]int) + for _, item := range items { + freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++ + } + var sorted []string + for item := range freq { + sorted = append(sorted, item) + } + sort.Slice(sorted, func(i int, j int) bool { + return freq[sorted[i]] > freq[sorted[j]] + }) + sep := "" + res := "" + for i, item := range sorted { + if i >= topN { + res += " ..." + break + } + count := freq[item] + if count > 1 { + res += fmt.Sprintf("%s%s(%d)", sep, item, count) + } else { + res += fmt.Sprintf("%s%s", sep, item) + } + sep = ", " + } + return res, len(sorted) +} diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go new file mode 100644 index 000000000..b06ed90e3 --- /dev/null +++ b/mk2rbc/expr.go @@ -0,0 +1,580 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mk2rbc + +import ( + "fmt" + "strconv" + "strings" + + mkparser "android/soong/androidmk/parser" +) + +// Represents an expression in the Starlark code. An expression has +// a type, and it can be evaluated. +type starlarkExpr interface { + starlarkNode + typ() starlarkType + // Try to substitute variable values. Return substitution result + // and whether it is the same as the original expression. + eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) + // Emit the code to copy the expression, otherwise we will end up + // with source and target pointing to the same list. + emitListVarCopy(gctx *generationContext) +} + +func maybeString(expr starlarkExpr) (string, bool) { + if x, ok := expr.(*stringLiteralExpr); ok { + return x.literal, true + } + return "", false +} + +type stringLiteralExpr struct { + literal string +} + +func (s *stringLiteralExpr) eval(_ map[string]starlarkExpr) (res starlarkExpr, same bool) { + res = s + same = true + return +} + +func (s *stringLiteralExpr) emit(gctx *generationContext) { + gctx.writef("%q", s.literal) +} + +func (_ *stringLiteralExpr) typ() starlarkType { + return starlarkTypeString +} + +func (s *stringLiteralExpr) emitListVarCopy(gctx *generationContext) { + s.emit(gctx) +} + +// Integer literal +type intLiteralExpr struct { + literal int +} + +func (s *intLiteralExpr) eval(_ map[string]starlarkExpr) (res starlarkExpr, same bool) { + res = s + same = true + return +} + +func (s *intLiteralExpr) emit(gctx *generationContext) { + gctx.writef("%d", s.literal) +} + +func (_ *intLiteralExpr) typ() starlarkType { + return starlarkTypeInt +} + +func (s *intLiteralExpr) emitListVarCopy(gctx *generationContext) { + s.emit(gctx) +} + +// interpolateExpr represents Starlark's interpolation operator % list +// we break into a list of chunks, i.e., "first%second%third" % (X, Y) +// will have chunks = ["first", "second", "third"] and args = [X, Y] +type interpolateExpr struct { + chunks []string // string chunks, separated by '%' + args []starlarkExpr +} + +func (xi *interpolateExpr) emit(gctx *generationContext) { + if len(xi.chunks) != len(xi.args)+1 { + panic(fmt.Errorf("malformed interpolateExpr: #chunks(%d) != #args(%d)+1", + len(xi.chunks), len(xi.args))) + } + // Generate format as join of chunks, but first escape '%' in them + format := strings.ReplaceAll(xi.chunks[0], "%", "%%") + for _, chunk := range xi.chunks[1:] { + format += "%s" + strings.ReplaceAll(chunk, "%", "%%") + } + gctx.writef("%q %% ", format) + emitarg := func(arg starlarkExpr) { + if arg.typ() == starlarkTypeList { + gctx.write(`" ".join(`) + arg.emit(gctx) + gctx.write(`)`) + } else { + arg.emit(gctx) + } + } + if len(xi.args) == 1 { + emitarg(xi.args[0]) + } else { + sep := "(" + for _, arg := range xi.args { + gctx.write(sep) + emitarg(arg) + sep = ", " + } + gctx.write(")") + } +} + +func (xi *interpolateExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + same = true + newChunks := []string{xi.chunks[0]} + var newArgs []starlarkExpr + for i, arg := range xi.args { + newArg, sameArg := arg.eval(valueMap) + same = same && sameArg + switch x := newArg.(type) { + case *stringLiteralExpr: + newChunks[len(newChunks)-1] += x.literal + xi.chunks[i+1] + same = false + continue + case *intLiteralExpr: + newChunks[len(newChunks)-1] += strconv.Itoa(x.literal) + xi.chunks[i+1] + same = false + continue + default: + newChunks = append(newChunks, xi.chunks[i+1]) + newArgs = append(newArgs, newArg) + } + } + if same { + res = xi + } else if len(newChunks) == 1 { + res = &stringLiteralExpr{newChunks[0]} + } else { + res = &interpolateExpr{chunks: newChunks, args: newArgs} + } + return +} + +func (_ *interpolateExpr) typ() starlarkType { + return starlarkTypeString +} + +func (xi *interpolateExpr) emitListVarCopy(gctx *generationContext) { + xi.emit(gctx) +} + +type variableRefExpr struct { + ref variable + isDefined bool +} + +func (v *variableRefExpr) eval(map[string]starlarkExpr) (res starlarkExpr, same bool) { + predefined, ok := v.ref.(*predefinedVariable) + if same = !ok; same { + res = v + } else { + res = predefined.value + } + return +} + +func (v *variableRefExpr) emit(gctx *generationContext) { + v.ref.emitGet(gctx, v.isDefined) +} + +func (v *variableRefExpr) typ() starlarkType { + return v.ref.valueType() +} + +func (v *variableRefExpr) emitListVarCopy(gctx *generationContext) { + v.emit(gctx) + if v.typ() == starlarkTypeList { + gctx.write("[:]") // this will copy the list + } +} + +type notExpr struct { + expr starlarkExpr +} + +func (n *notExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + if x, same := n.expr.eval(valueMap); same { + res = n + } else { + res = ¬Expr{expr: x} + } + return +} + +func (n *notExpr) emit(ctx *generationContext) { + ctx.write("not ") + n.expr.emit(ctx) +} + +func (_ *notExpr) typ() starlarkType { + return starlarkTypeBool +} + +func (n *notExpr) emitListVarCopy(gctx *generationContext) { + n.emit(gctx) +} + +type eqExpr struct { + left, right starlarkExpr + isEq bool // if false, it's != +} + +func (eq *eqExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + xLeft, sameLeft := eq.left.eval(valueMap) + xRight, sameRight := eq.right.eval(valueMap) + if same = sameLeft && sameRight; same { + res = eq + } else { + res = &eqExpr{left: xLeft, right: xRight, isEq: eq.isEq} + } + return +} + +func (eq *eqExpr) emit(gctx *generationContext) { + // Are we checking that a variable is empty? + var varRef *variableRefExpr + if s, ok := maybeString(eq.left); ok && s == "" { + varRef, ok = eq.right.(*variableRefExpr) + } else if s, ok := maybeString(eq.right); ok && s == "" { + varRef, ok = eq.left.(*variableRefExpr) + } + if varRef != nil { + // Yes. + if eq.isEq { + gctx.write("not ") + } + varRef.emit(gctx) + return + } + + // General case + eq.left.emit(gctx) + if eq.isEq { + gctx.write(" == ") + } else { + gctx.write(" != ") + } + eq.right.emit(gctx) +} + +func (_ *eqExpr) typ() starlarkType { + return starlarkTypeBool +} + +func (eq *eqExpr) emitListVarCopy(gctx *generationContext) { + eq.emit(gctx) +} + +// variableDefinedExpr corresponds to Make's ifdef VAR +type variableDefinedExpr struct { + v variable +} + +func (v *variableDefinedExpr) eval(_ map[string]starlarkExpr) (res starlarkExpr, same bool) { + res = v + same = true + return + +} + +func (v *variableDefinedExpr) emit(gctx *generationContext) { + if v.v != nil { + v.v.emitDefined(gctx) + return + } + gctx.writef("%s(%q)", cfnWarning, "TODO(VAR)") +} + +func (_ *variableDefinedExpr) typ() starlarkType { + return starlarkTypeBool +} + +func (v *variableDefinedExpr) emitListVarCopy(gctx *generationContext) { + v.emit(gctx) +} + +type listExpr struct { + items []starlarkExpr +} + +func (l *listExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + newItems := make([]starlarkExpr, len(l.items)) + same = true + for i, item := range l.items { + var sameItem bool + newItems[i], sameItem = item.eval(valueMap) + same = same && sameItem + } + if same { + res = l + } else { + res = &listExpr{newItems} + } + return +} + +func (l *listExpr) emit(gctx *generationContext) { + if !gctx.inAssignment || len(l.items) < 2 { + gctx.write("[") + sep := "" + for _, item := range l.items { + gctx.write(sep) + item.emit(gctx) + sep = ", " + } + gctx.write("]") + return + } + + gctx.write("[") + gctx.indentLevel += 2 + + for _, item := range l.items { + gctx.newLine() + item.emit(gctx) + gctx.write(",") + } + gctx.indentLevel -= 2 + gctx.newLine() + gctx.write("]") +} + +func (_ *listExpr) typ() starlarkType { + return starlarkTypeList +} + +func (l *listExpr) emitListVarCopy(gctx *generationContext) { + l.emit(gctx) +} + +func newStringListExpr(items []string) *listExpr { + v := listExpr{} + for _, item := range items { + v.items = append(v.items, &stringLiteralExpr{item}) + } + return &v +} + +// concatExpr generates epxr1 + expr2 + ... + exprN in Starlark. +type concatExpr struct { + items []starlarkExpr +} + +func (c *concatExpr) emit(gctx *generationContext) { + if len(c.items) == 1 { + c.items[0].emit(gctx) + return + } + + if !gctx.inAssignment { + c.items[0].emit(gctx) + for _, item := range c.items[1:] { + gctx.write(" + ") + item.emit(gctx) + } + return + } + gctx.write("(") + c.items[0].emit(gctx) + gctx.indentLevel += 2 + for _, item := range c.items[1:] { + gctx.write(" +") + gctx.newLine() + item.emit(gctx) + } + gctx.write(")") + gctx.indentLevel -= 2 +} + +func (c *concatExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + same = true + xConcat := &concatExpr{items: make([]starlarkExpr, len(c.items))} + for i, item := range c.items { + var sameItem bool + xConcat.items[i], sameItem = item.eval(valueMap) + same = same && sameItem + } + if same { + res = c + } else { + res = xConcat + } + return +} + +func (_ *concatExpr) typ() starlarkType { + return starlarkTypeList +} + +func (c *concatExpr) emitListVarCopy(gctx *generationContext) { + c.emit(gctx) +} + +// inExpr generates [not] in +type inExpr struct { + expr starlarkExpr + list starlarkExpr + isNot bool +} + +func (i *inExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + x := &inExpr{isNot: i.isNot} + var sameExpr, sameList bool + x.expr, sameExpr = i.expr.eval(valueMap) + x.list, sameList = i.list.eval(valueMap) + if same = sameExpr && sameList; same { + res = i + } else { + res = x + } + return +} + +func (i *inExpr) emit(gctx *generationContext) { + i.expr.emit(gctx) + if i.isNot { + gctx.write(" not in ") + } else { + gctx.write(" in ") + } + i.list.emit(gctx) +} + +func (_ *inExpr) typ() starlarkType { + return starlarkTypeBool +} + +func (i *inExpr) emitListVarCopy(gctx *generationContext) { + i.emit(gctx) +} + +type indexExpr struct { + array starlarkExpr + index starlarkExpr +} + +func (ix indexExpr) emit(gctx *generationContext) { + ix.array.emit(gctx) + gctx.write("[") + ix.index.emit(gctx) + gctx.write("]") +} + +func (ix indexExpr) typ() starlarkType { + return starlarkTypeString +} + +func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + newArray, isSameArray := ix.array.eval(valueMap) + newIndex, isSameIndex := ix.index.eval(valueMap) + if same = isSameArray && isSameIndex; same { + res = ix + } else { + res = &indexExpr{newArray, newIndex} + } + return +} + +func (ix indexExpr) emitListVarCopy(gctx *generationContext) { + ix.emit(gctx) +} + +type callExpr struct { + object starlarkExpr // nil if static call + name string + args []starlarkExpr + returnType starlarkType +} + +func (cx *callExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + newCallExpr := &callExpr{name: cx.name, args: make([]starlarkExpr, len(cx.args)), + returnType: cx.returnType} + if cx.object != nil { + newCallExpr.object, same = cx.object.eval(valueMap) + } else { + same = true + } + for i, args := range cx.args { + var s bool + newCallExpr.args[i], s = args.eval(valueMap) + same = same && s + } + if same { + res = cx + } else { + res = newCallExpr + } + return +} + +func (cx *callExpr) emit(gctx *generationContext) { + if cx.object != nil { + gctx.write("(") + cx.object.emit(gctx) + gctx.write(")") + gctx.write(".", cx.name, "(") + } else { + kf, found := knownFunctions[cx.name] + if !found { + panic(fmt.Errorf("callExpr with unknown function %q", cx.name)) + } + if kf.runtimeName[0] == '!' { + panic(fmt.Errorf("callExpr for %q should not be there", cx.name)) + } + gctx.write(kf.runtimeName, "(") + } + sep := "" + for _, arg := range cx.args { + gctx.write(sep) + arg.emit(gctx) + sep = ", " + } + gctx.write(")") +} + +func (cx *callExpr) typ() starlarkType { + return cx.returnType +} + +func (cx *callExpr) emitListVarCopy(gctx *generationContext) { + cx.emit(gctx) +} + +type badExpr struct { + node mkparser.Node + message string +} + +func (b *badExpr) eval(_ map[string]starlarkExpr) (res starlarkExpr, same bool) { + res = b + same = true + return +} + +func (b *badExpr) emit(_ *generationContext) { + panic("implement me") +} + +func (_ *badExpr) typ() starlarkType { + return starlarkTypeUnknown +} + +func (b *badExpr) emitListVarCopy(gctx *generationContext) { + panic("implement me") +} + +func maybeConvertToStringList(expr starlarkExpr) starlarkExpr { + if xString, ok := expr.(*stringLiteralExpr); ok { + return newStringListExpr(strings.Fields(xString.literal)) + } + return expr +} diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go new file mode 100644 index 000000000..55a35e9c2 --- /dev/null +++ b/mk2rbc/mk2rbc.go @@ -0,0 +1,1344 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Convert makefile containing device configuration to Starlark file +// The conversion can handle the following constructs in a makefile: +// * comments +// * simple variable assignments +// * $(call init-product,) +// * $(call inherit-product-if-exists +// * if directives +// All other constructs are carried over to the output starlark file as comments. +// +package mk2rbc + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "text/scanner" + + mkparser "android/soong/androidmk/parser" +) + +const ( + baseUri = "//build/make/core:product_config.rbc" + // The name of the struct exported by the product_config.rbc + // that contains the functions and variables available to + // product configuration Starlark files. + baseName = "rblf" + + // And here are the functions and variables: + cfnGetCfg = baseName + ".cfg" + cfnMain = baseName + ".product_configuration" + cfnPrintVars = baseName + ".printvars" + cfnWarning = baseName + ".warning" + cfnLocalAppend = baseName + ".local_append" + cfnLocalSetDefault = baseName + ".local_set_default" + cfnInherit = baseName + ".inherit" + cfnSetListDefault = baseName + ".setdefault" +) + +const ( + // Phony makefile functions, they are eventually rewritten + // according to knownFunctions map + fileExistsPhony = "$file_exists" + wildcardExistsPhony = "$wildcard_exists" +) + +const ( + callLoadAlways = "inherit-product" + callLoadIf = "inherit-product-if-exists" +) + +var knownFunctions = map[string]struct { + // The name of the runtime function this function call in makefiles maps to. + // If it starts with !, then this makefile function call is rewritten to + // something else. + runtimeName string + returnType starlarkType +}{ + fileExistsPhony: {baseName + ".file_exists", starlarkTypeBool}, + wildcardExistsPhony: {baseName + ".file_wildcard_exists", starlarkTypeBool}, + "add-to-product-copy-files-if-exists": {baseName + ".copy_if_exists", starlarkTypeList}, + "addprefix": {baseName + ".addprefix", starlarkTypeList}, + "addsuffix": {baseName + ".addsuffix", starlarkTypeList}, + "enforce-product-packages-exist": {baseName + ".enforce_product_packages_exist", starlarkTypeVoid}, + "error": {baseName + ".mkerror", starlarkTypeVoid}, + "findstring": {"!findstring", starlarkTypeInt}, + "find-copy-subdir-files": {baseName + ".find_and_copy", starlarkTypeList}, + "filter": {baseName + ".filter", starlarkTypeList}, + "filter-out": {baseName + ".filter_out", starlarkTypeList}, + "info": {baseName + ".mkinfo", starlarkTypeVoid}, + "is-board-platform": {"!is-board-platform", starlarkTypeBool}, + "is-board-platform-in-list": {"!is-board-platform-in-list", starlarkTypeBool}, + "is-product-in-list": {"!is-product-in-list", starlarkTypeBool}, + "is-vendor-board-platform": {"!is-vendor-board-platform", starlarkTypeBool}, + callLoadAlways: {"!inherit-product", starlarkTypeVoid}, + callLoadIf: {"!inherit-product-if-exists", starlarkTypeVoid}, + "produce_copy_files": {baseName + ".produce_copy_files", starlarkTypeList}, + "require-artifacts-in-path": {baseName + ".require_artifacts_in_path", starlarkTypeVoid}, + "require-artifacts-in-path-relaxed": {baseName + ".require_artifacts_in_path_relaxed", starlarkTypeVoid}, + // TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002 + "shell": {baseName + ".shell", starlarkTypeString}, + "strip": {baseName + ".mkstrip", starlarkTypeString}, + "subst": {baseName + ".subst", starlarkTypeString}, + "warning": {baseName + ".mkwarning", starlarkTypeVoid}, + "word": {baseName + "!word", starlarkTypeString}, + "wildcard": {baseName + ".expand_wildcard", starlarkTypeList}, +} + +var builtinFuncRex = regexp.MustCompile( + "^(addprefix|addsuffix|abspath|and|basename|call|dir|error|eval" + + "|flavor|foreach|file|filter|filter-out|findstring|firstword|guile" + + "|if|info|join|lastword|notdir|or|origin|patsubst|realpath" + + "|shell|sort|strip|subst|suffix|value|warning|word|wordlist|words" + + "|wildcard)") + +// Conversion request parameters +type Request struct { + MkFile string // file to convert + Reader io.Reader // if set, read input from this stream instead + RootDir string // root directory path used to resolve included files + OutputSuffix string // generated Starlark files suffix + OutputDir string // if set, root of the output hierarchy + ErrorLogger ErrorMonitorCB + TracedVariables []string // trace assignment to these variables + TraceCalls bool + WarnPartialSuccess bool +} + +// An error sink allowing to gather error statistics. +// NewError is called on every error encountered during processing. +type ErrorMonitorCB interface { + NewError(s string, node mkparser.Node, args ...interface{}) +} + +// Derives module name for a given file. It is base name +// (file name without suffix), with some characters replaced to make it a Starlark identifier +func moduleNameForFile(mkFile string) string { + base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile)) + // TODO(asmundak): what else can be in the product file names? + return strings.ReplaceAll(base, "-", "_") +} + +func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString { + r := &mkparser.MakeString{StringPos: mkString.StringPos} + r.Strings = append(r.Strings, mkString.Strings...) + r.Variables = append(r.Variables, mkString.Variables...) + return r +} + +func isMakeControlFunc(s string) bool { + return s == "error" || s == "warning" || s == "info" +} + +// Starlark output generation context +type generationContext struct { + buf strings.Builder + starScript *StarlarkScript + indentLevel int + inAssignment bool + tracedCount int +} + +func NewGenerateContext(ss *StarlarkScript) *generationContext { + return &generationContext{starScript: ss} +} + +// emit returns generated script +func (gctx *generationContext) emit() string { + ss := gctx.starScript + + // The emitted code has the following layout: + // + // preamble, i.e., + // load statement for the runtime support + // load statement for each unique submodule pulled in by this one + // def init(g, handle): + // cfg = rblf.cfg(handle) + // + // + + iNode := len(ss.nodes) + for i, node := range ss.nodes { + if _, ok := node.(*commentNode); !ok { + iNode = i + break + } + node.emit(gctx) + } + + gctx.emitPreamble() + + gctx.newLine() + // The arguments passed to the init function are the global dictionary + // ('g') and the product configuration dictionary ('cfg') + gctx.write("def init(g, handle):") + gctx.indentLevel++ + if gctx.starScript.traceCalls { + gctx.newLine() + gctx.writef(`print(">%s")`, gctx.starScript.mkFile) + } + gctx.newLine() + gctx.writef("cfg = %s(handle)", cfnGetCfg) + for _, node := range ss.nodes[iNode:] { + node.emit(gctx) + } + + if ss.hasErrors && ss.warnPartialSuccess { + gctx.newLine() + gctx.writef("%s(%q, %q)", cfnWarning, filepath.Base(ss.mkFile), "partially successful conversion") + } + if gctx.starScript.traceCalls { + gctx.newLine() + gctx.writef(`print("<%s")`, gctx.starScript.mkFile) + } + gctx.indentLevel-- + gctx.write("\n") + return gctx.buf.String() +} + +func (gctx *generationContext) emitPreamble() { + gctx.newLine() + gctx.writef("load(%q, %q)", baseUri, baseName) + // Emit exactly one load statement for each URI. + loadedSubConfigs := make(map[string]string) + for _, sc := range gctx.starScript.inherited { + uri := sc.path + if m, ok := loadedSubConfigs[uri]; ok { + // No need to emit load statement, but fix module name. + sc.moduleLocalName = m + continue + } + if !sc.loadAlways { + uri += "|init" + } + gctx.newLine() + gctx.writef("load(%q, %s = \"init\")", uri, sc.entryName()) + loadedSubConfigs[uri] = sc.moduleLocalName + } + gctx.write("\n") +} + +func (gctx *generationContext) emitPass() { + gctx.newLine() + gctx.write("pass") +} + +func (gctx *generationContext) write(ss ...string) { + for _, s := range ss { + gctx.buf.WriteString(s) + } +} + +func (gctx *generationContext) writef(format string, args ...interface{}) { + gctx.write(fmt.Sprintf(format, args...)) +} + +func (gctx *generationContext) newLine() { + if gctx.buf.Len() == 0 { + return + } + gctx.write("\n") + gctx.writef("%*s", 2*gctx.indentLevel, "") +} + +type knownVariable struct { + name string + class varClass + valueType starlarkType +} + +type knownVariables map[string]knownVariable + +func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) { + v, exists := pcv[name] + if !exists { + pcv[name] = knownVariable{name, varClass, valueType} + return + } + // Conflict resolution: + // * config class trumps everything + // * any type trumps unknown type + match := varClass == v.class + if !match { + if varClass == VarClassConfig { + v.class = VarClassConfig + match = true + } else if v.class == VarClassConfig { + match = true + } + } + if valueType != v.valueType { + if valueType != starlarkTypeUnknown { + if v.valueType == starlarkTypeUnknown { + v.valueType = valueType + } else { + match = false + } + } + } + if !match { + fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n", + name, varClass, valueType, v.class, v.valueType) + } +} + +// All known product variables. +var KnownVariables = make(knownVariables) + +func init() { + for _, kv := range []string{ + // Kernel-related variables that we know are lists. + "BOARD_VENDOR_KERNEL_MODULES", + "BOARD_VENDOR_RAMDISK_KERNEL_MODULES", + "BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD", + "BOARD_RECOVERY_KERNEL_MODULES", + // Other variables we knwo are lists + "ART_APEX_JARS", + } { + KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList) + } +} + +type nodeReceiver interface { + newNode(node starlarkNode) +} + +// Information about the generated Starlark script. +type StarlarkScript struct { + mkFile string + moduleName string + mkPos scanner.Position + nodes []starlarkNode + inherited []*inheritedModule + hasErrors bool + topDir string + traceCalls bool // print enter/exit each init function + warnPartialSuccess bool +} + +func (ss *StarlarkScript) newNode(node starlarkNode) { + ss.nodes = append(ss.nodes, node) +} + +// varAssignmentScope points to the last assignment for each variable +// in the current block. It is used during the parsing to chain +// the assignments to a variable together. +type varAssignmentScope struct { + outer *varAssignmentScope + vars map[string]*assignmentNode +} + +// parseContext holds the script we are generating and all the ephemeral data +// needed during the parsing. +type parseContext struct { + script *StarlarkScript + nodes []mkparser.Node // Makefile as parsed by mkparser + currentNodeIndex int // Node in it we are processing + ifNestLevel int + moduleNameCount map[string]int // count of imported modules with given basename + fatalError error + builtinMakeVars map[string]starlarkExpr + outputSuffix string + errorLogger ErrorMonitorCB + tracedVariables map[string]bool // variables to be traced in the generated script + variables map[string]variable + varAssignments *varAssignmentScope + receiver nodeReceiver // receptacle for the generated starlarkNode's + receiverStack []nodeReceiver + outputDir string +} + +func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext { + predefined := []struct{ name, value string }{ + {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")}, + {"LOCAL_PATH", filepath.Dir(ss.mkFile)}, + {"TOPDIR", ss.topDir}, + // TODO(asmundak): maybe read it from build/make/core/envsetup.mk? + {"TARGET_COPY_OUT_SYSTEM", "system"}, + {"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"}, + {"TARGET_COPY_OUT_DATA", "data"}, + {"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")}, + {"TARGET_COPY_OUT_OEM", "oem"}, + {"TARGET_COPY_OUT_RAMDISK", "ramdisk"}, + {"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"}, + {"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"}, + {"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"}, + {"TARGET_COPY_OUT_ROOT", "root"}, + {"TARGET_COPY_OUT_RECOVERY", "recovery"}, + {"TARGET_COPY_OUT_VENDOR", "||VENDOR-PATH-PH||"}, + {"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"}, + {"TARGET_COPY_OUT_PRODUCT", "||PRODUCT-PATH-PH||"}, + {"TARGET_COPY_OUT_PRODUCT_SERVICES", "||PRODUCT-PATH-PH||"}, + {"TARGET_COPY_OUT_SYSTEM_EXT", "||SYSTEM_EXT-PATH-PH||"}, + {"TARGET_COPY_OUT_ODM", "||ODM-PATH-PH||"}, + {"TARGET_COPY_OUT_VENDOR_DLKM", "||VENDOR_DLKM-PATH-PH||"}, + {"TARGET_COPY_OUT_ODM_DLKM", "||ODM_DLKM-PATH-PH||"}, + // TODO(asmundak): to process internal config files, we need the following variables: + // BOARD_CONFIG_VENDOR_PATH + // TARGET_VENDOR + // target_base_product + // + + // the following utility variables are set in build/make/common/core.mk: + {"empty", ""}, + {"space", " "}, + {"comma", ","}, + {"newline", "\n"}, + {"pound", "#"}, + {"backslash", "\\"}, + } + ctx := &parseContext{ + script: ss, + nodes: nodes, + currentNodeIndex: 0, + ifNestLevel: 0, + moduleNameCount: make(map[string]int), + builtinMakeVars: map[string]starlarkExpr{}, + variables: make(map[string]variable), + } + ctx.pushVarAssignments() + for _, item := range predefined { + ctx.variables[item.name] = &predefinedVariable{ + baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString}, + value: &stringLiteralExpr{item.value}, + } + } + + return ctx +} + +func (ctx *parseContext) lastAssignment(name string) *assignmentNode { + for va := ctx.varAssignments; va != nil; va = va.outer { + if v, ok := va.vars[name]; ok { + return v + } + } + return nil +} + +func (ctx *parseContext) setLastAssignment(name string, asgn *assignmentNode) { + ctx.varAssignments.vars[name] = asgn +} + +func (ctx *parseContext) pushVarAssignments() { + va := &varAssignmentScope{ + outer: ctx.varAssignments, + vars: make(map[string]*assignmentNode), + } + ctx.varAssignments = va +} + +func (ctx *parseContext) popVarAssignments() { + ctx.varAssignments = ctx.varAssignments.outer +} + +func (ctx *parseContext) pushReceiver(rcv nodeReceiver) { + ctx.receiverStack = append(ctx.receiverStack, ctx.receiver) + ctx.receiver = rcv +} + +func (ctx *parseContext) popReceiver() { + last := len(ctx.receiverStack) - 1 + if last < 0 { + panic(fmt.Errorf("popReceiver: receiver stack empty")) + } + ctx.receiver = ctx.receiverStack[last] + ctx.receiverStack = ctx.receiverStack[0:last] +} + +func (ctx *parseContext) hasNodes() bool { + return ctx.currentNodeIndex < len(ctx.nodes) +} + +func (ctx *parseContext) getNode() mkparser.Node { + if !ctx.hasNodes() { + return nil + } + node := ctx.nodes[ctx.currentNodeIndex] + ctx.currentNodeIndex++ + return node +} + +func (ctx *parseContext) backNode() { + if ctx.currentNodeIndex <= 0 { + panic("Cannot back off") + } + ctx.currentNodeIndex-- +} + +func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) { + // Handle only simple variables + if !a.Name.Const() { + ctx.errorf(a, "Only simple variables are handled") + return + } + name := a.Name.Strings[0] + lhs := ctx.addVariable(name) + if lhs == nil { + ctx.errorf(a, "unknown variable %s", name) + return + } + _, isTraced := ctx.tracedVariables[name] + asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced} + if lhs.valueType() == starlarkTypeUnknown { + // Try to divine variable type from the RHS + asgn.value = ctx.parseMakeString(a, a.Value) + if xBad, ok := asgn.value.(*badExpr); ok { + ctx.wrapBadExpr(xBad) + return + } + inferred_type := asgn.value.typ() + if inferred_type != starlarkTypeUnknown { + if ogv, ok := lhs.(*otherGlobalVariable); ok { + ogv.typ = inferred_type + } else if pcv, ok := lhs.(*productConfigVariable); ok { + pcv.typ = inferred_type + } else { + panic(fmt.Errorf("cannot assign new type to a variable %s, its flavor is %T", lhs.name(), lhs)) + } + } + } + if lhs.valueType() == starlarkTypeList { + xConcat := ctx.buildConcatExpr(a) + if xConcat == nil { + return + } + switch len(xConcat.items) { + case 0: + asgn.value = &listExpr{} + case 1: + asgn.value = xConcat.items[0] + default: + asgn.value = xConcat + } + } else { + asgn.value = ctx.parseMakeString(a, a.Value) + if xBad, ok := asgn.value.(*badExpr); ok { + ctx.wrapBadExpr(xBad) + return + } + } + + // TODO(asmundak): move evaluation to a separate pass + asgn.value, _ = asgn.value.eval(ctx.builtinMakeVars) + + asgn.previous = ctx.lastAssignment(name) + ctx.setLastAssignment(name, asgn) + switch a.Type { + case "=", ":=": + asgn.flavor = asgnSet + case "+=": + if asgn.previous == nil && !asgn.lhs.isPreset() { + asgn.flavor = asgnMaybeAppend + } else { + asgn.flavor = asgnAppend + } + case "?=": + asgn.flavor = asgnMaybeSet + default: + panic(fmt.Errorf("unexpected assignment type %s", a.Type)) + } + + ctx.receiver.newNode(asgn) +} + +func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) *concatExpr { + xConcat := &concatExpr{} + var xItemList *listExpr + addToItemList := func(x ...starlarkExpr) { + if xItemList == nil { + xItemList = &listExpr{[]starlarkExpr{}} + } + xItemList.items = append(xItemList.items, x...) + } + finishItemList := func() { + if xItemList != nil { + xConcat.items = append(xConcat.items, xItemList) + xItemList = nil + } + } + + items := a.Value.Words() + for _, item := range items { + // A function call in RHS is supposed to return a list, all other item + // expressions return individual elements. + switch x := ctx.parseMakeString(a, item).(type) { + case *badExpr: + ctx.wrapBadExpr(x) + return nil + case *stringLiteralExpr: + addToItemList(maybeConvertToStringList(x).(*listExpr).items...) + default: + switch x.typ() { + case starlarkTypeList: + finishItemList() + xConcat.items = append(xConcat.items, x) + case starlarkTypeString: + finishItemList() + xConcat.items = append(xConcat.items, &callExpr{ + object: x, + name: "split", + args: nil, + returnType: starlarkTypeList, + }) + default: + addToItemList(x) + } + } + } + if xItemList != nil { + xConcat.items = append(xConcat.items, xItemList) + } + return xConcat +} + +func (ctx *parseContext) newInheritedModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) *inheritedModule { + var path string + x, _ := pathExpr.eval(ctx.builtinMakeVars) + s, ok := x.(*stringLiteralExpr) + if !ok { + ctx.errorf(v, "inherit-product/include argument is too complex") + return nil + } + + path = s.literal + moduleName := moduleNameForFile(path) + moduleLocalName := "_" + moduleName + n, found := ctx.moduleNameCount[moduleName] + if found { + moduleLocalName += fmt.Sprintf("%d", n) + } + ctx.moduleNameCount[moduleName] = n + 1 + ln := &inheritedModule{ + path: ctx.loadedModulePath(path), + originalPath: path, + moduleName: moduleName, + moduleLocalName: moduleLocalName, + loadAlways: loadAlways, + } + ctx.script.inherited = append(ctx.script.inherited, ln) + return ln +} + +func (ctx *parseContext) handleInheritModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) { + if im := ctx.newInheritedModule(v, pathExpr, loadAlways); im != nil { + ctx.receiver.newNode(&inheritNode{im}) + } +} + +func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) { + if ln := ctx.newInheritedModule(v, pathExpr, loadAlways); ln != nil { + ctx.receiver.newNode(&includeNode{ln}) + } +} + +func (ctx *parseContext) handleVariable(v *mkparser.Variable) { + // Handle: + // $(call inherit-product,...) + // $(call inherit-product-if-exists,...) + // $(info xxx) + // $(warning xxx) + // $(error xxx) + expr := ctx.parseReference(v, v.Name) + switch x := expr.(type) { + case *callExpr: + if x.name == callLoadAlways || x.name == callLoadIf { + ctx.handleInheritModule(v, x.args[0], x.name == callLoadAlways) + } else if isMakeControlFunc(x.name) { + // File name is the first argument + args := []starlarkExpr{ + &stringLiteralExpr{ctx.script.mkFile}, + x.args[0], + } + ctx.receiver.newNode(&exprNode{ + &callExpr{name: x.name, args: args, returnType: starlarkTypeUnknown}, + }) + } else { + ctx.receiver.newNode(&exprNode{expr}) + } + case *badExpr: + ctx.wrapBadExpr(x) + return + default: + ctx.errorf(v, "cannot handle %s", v.Dump()) + return + } +} + +func (ctx *parseContext) handleDefine(directive *mkparser.Directive) { + tokens := strings.Fields(directive.Args.Strings[0]) + ctx.errorf(directive, "define is not supported: %s", tokens[0]) +} + +func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) { + ssSwitch := &switchNode{} + ctx.pushReceiver(ssSwitch) + for ctx.processBranch(ifDirective); ctx.hasNodes() && ctx.fatalError == nil; { + node := ctx.getNode() + switch x := node.(type) { + case *mkparser.Directive: + switch x.Name { + case "else", "elifdef", "elifndef", "elifeq", "elifneq": + ctx.processBranch(x) + case "endif": + ctx.popReceiver() + ctx.receiver.newNode(ssSwitch) + return + default: + ctx.errorf(node, "unexpected directive %s", x.Name) + } + default: + ctx.errorf(ifDirective, "unexpected statement") + } + } + if ctx.fatalError == nil { + ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump()) + } + ctx.popReceiver() +} + +// processBranch processes a single branch (if/elseif/else) until the next directive +// on the same level. +func (ctx *parseContext) processBranch(check *mkparser.Directive) { + block := switchCase{gate: ctx.parseCondition(check)} + defer func() { + ctx.popVarAssignments() + ctx.ifNestLevel-- + + }() + ctx.pushVarAssignments() + ctx.ifNestLevel++ + + ctx.pushReceiver(&block) + for ctx.hasNodes() { + node := ctx.getNode() + if ctx.handleSimpleStatement(node) { + continue + } + switch d := node.(type) { + case *mkparser.Directive: + switch d.Name { + case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif": + ctx.popReceiver() + ctx.receiver.newNode(&block) + ctx.backNode() + return + case "ifdef", "ifndef", "ifeq", "ifneq": + ctx.handleIfBlock(d) + default: + ctx.errorf(d, "unexpected directive %s", d.Name) + } + default: + ctx.errorf(node, "unexpected statement") + } + } + ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump()) + ctx.popReceiver() +} + +func (ctx *parseContext) newIfDefinedNode(check *mkparser.Directive) (starlarkExpr, bool) { + if !check.Args.Const() { + return ctx.newBadExpr(check, "ifdef variable ref too complex: %s", check.Args.Dump()), false + } + v := ctx.addVariable(check.Args.Strings[0]) + return &variableDefinedExpr{v}, true +} + +func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode { + switch check.Name { + case "ifdef", "ifndef", "elifdef", "elifndef": + v, ok := ctx.newIfDefinedNode(check) + if ok && strings.HasSuffix(check.Name, "ndef") { + v = ¬Expr{v} + } + return &ifNode{ + isElif: strings.HasPrefix(check.Name, "elif"), + expr: v, + } + case "ifeq", "ifneq", "elifeq", "elifneq": + return &ifNode{ + isElif: strings.HasPrefix(check.Name, "elif"), + expr: ctx.parseCompare(check), + } + case "else": + return &elseNode{} + default: + panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump())) + } +} + +func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr { + message := fmt.Sprintf(text, args...) + if ctx.errorLogger != nil { + ctx.errorLogger.NewError(text, node, args) + } + ctx.script.hasErrors = true + return &badExpr{node, message} +} + +func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr { + // Strip outer parentheses + mkArg := cloneMakeString(cond.Args) + mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ") + n := len(mkArg.Strings) + mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ") + args := mkArg.Split(",") + // TODO(asmundak): handle the case where the arguments are in quotes and space-separated + if len(args) != 2 { + return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump()) + } + args[0].TrimRightSpaces() + args[1].TrimLeftSpaces() + + isEq := !strings.HasSuffix(cond.Name, "neq") + switch xLeft := ctx.parseMakeString(cond, args[0]).(type) { + case *stringLiteralExpr, *variableRefExpr: + switch xRight := ctx.parseMakeString(cond, args[1]).(type) { + case *stringLiteralExpr, *variableRefExpr: + return &eqExpr{left: xLeft, right: xRight, isEq: isEq} + case *badExpr: + return xRight + default: + expr, ok := ctx.parseCheckFunctionCallResult(cond, xLeft, args[1]) + if ok { + return expr + } + return ctx.newBadExpr(cond, "right operand is too complex: %s", args[1].Dump()) + } + case *badExpr: + return xLeft + default: + switch xRight := ctx.parseMakeString(cond, args[1]).(type) { + case *stringLiteralExpr, *variableRefExpr: + expr, ok := ctx.parseCheckFunctionCallResult(cond, xRight, args[0]) + if ok { + return expr + } + return ctx.newBadExpr(cond, "left operand is too complex: %s", args[0].Dump()) + case *badExpr: + return xRight + default: + return ctx.newBadExpr(cond, "operands are too complex: (%s,%s)", args[0].Dump(), args[1].Dump()) + } + } +} + +func (ctx *parseContext) parseCheckFunctionCallResult(directive *mkparser.Directive, xValue starlarkExpr, + varArg *mkparser.MakeString) (starlarkExpr, bool) { + mkSingleVar, ok := varArg.SingleVariable() + if !ok { + return nil, false + } + expr := ctx.parseReference(directive, mkSingleVar) + negate := strings.HasSuffix(directive.Name, "neq") + checkIsSomethingFunction := func(xCall *callExpr) starlarkExpr { + s, ok := maybeString(xValue) + if !ok || s != "true" { + return ctx.newBadExpr(directive, + fmt.Sprintf("the result of %s can be compared only to 'true'", xCall.name)) + } + if len(xCall.args) < 1 { + return ctx.newBadExpr(directive, "%s requires an argument", xCall.name) + } + return nil + } + switch x := expr.(type) { + case *callExpr: + switch x.name { + case "filter": + return ctx.parseCompareFilterFuncResult(directive, x, xValue, !negate), true + case "filter-out": + return ctx.parseCompareFilterFuncResult(directive, x, xValue, negate), true + case "wildcard": + return ctx.parseCompareWildcardFuncResult(directive, x, xValue, negate), true + case "findstring": + return ctx.parseCheckFindstringFuncResult(directive, x, xValue, negate), true + case "strip": + return ctx.parseCompareStripFuncResult(directive, x, xValue, negate), true + case "is-board-platform": + if xBad := checkIsSomethingFunction(x); xBad != nil { + return xBad, true + } + return &eqExpr{ + left: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false}, + right: x.args[0], + isEq: !negate, + }, true + case "is-board-platform-in-list": + if xBad := checkIsSomethingFunction(x); xBad != nil { + return xBad, true + } + return &inExpr{ + expr: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false}, + list: maybeConvertToStringList(x.args[0]), + isNot: negate, + }, true + case "is-product-in-list": + if xBad := checkIsSomethingFunction(x); xBad != nil { + return xBad, true + } + return &inExpr{ + expr: &variableRefExpr{ctx.addVariable("TARGET_PRODUCT"), true}, + list: maybeConvertToStringList(x.args[0]), + isNot: negate, + }, true + case "is-vendor-board-platform": + if xBad := checkIsSomethingFunction(x); xBad != nil { + return xBad, true + } + s, ok := maybeString(x.args[0]) + if !ok { + return ctx.newBadExpr(directive, "cannot handle non-constant argument to is-vendor-board-platform"), true + } + return &inExpr{ + expr: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false}, + list: &variableRefExpr{ctx.addVariable(s + "_BOARD_PLATFORMS"), true}, + isNot: negate, + }, true + default: + return ctx.newBadExpr(directive, "Unknown function in ifeq: %s", x.name), true + } + case *badExpr: + return x, true + default: + return nil, false + } +} + +func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive, + filterFuncCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { + // We handle: + // * ifeq/ifneq (,$(filter v1 v2 ..., $(VAR)) becomes if VAR not in/in ["v1", "v2", ...] + // * ifeq/ifneq (,$(filter $(VAR), v1 v2 ...) becomes if VAR not in/in ["v1", "v2", ...] + // * ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...) becomes if VAR in/not in ["v1", "v2"] + // TODO(Asmundak): check the last case works for filter-out, too. + xPattern := filterFuncCall.args[0] + xText := filterFuncCall.args[1] + var xInList *stringLiteralExpr + var xVar starlarkExpr + var ok bool + switch x := xValue.(type) { + case *stringLiteralExpr: + if x.literal != "" { + return ctx.newBadExpr(cond, "filter comparison to non-empty value: %s", xValue) + } + // Either pattern or text should be const, and the + // non-const one should be varRefExpr + if xInList, ok = xPattern.(*stringLiteralExpr); ok { + xVar = xText + } else if xInList, ok = xText.(*stringLiteralExpr); ok { + xVar = xPattern + } + case *variableRefExpr: + if v, ok := xPattern.(*variableRefExpr); ok { + if xInList, ok = xText.(*stringLiteralExpr); ok && v.ref.name() == x.ref.name() { + // ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...), flip negate, + // it's the opposite to what is done when comparing to empty. + xVar = xPattern + negate = !negate + } + } + } + if xVar != nil && xInList != nil { + if _, ok := xVar.(*variableRefExpr); ok { + slExpr := newStringListExpr(strings.Fields(xInList.literal)) + // Generate simpler code for the common cases: + if xVar.typ() == starlarkTypeList { + if len(slExpr.items) == 1 { + // Checking that a string belongs to list + return &inExpr{isNot: negate, list: xVar, expr: slExpr.items[0]} + } else { + // TODO(asmundak): + panic("TBD") + } + } + return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: xVar} + } + } + return ctx.newBadExpr(cond, "filter arguments are too complex: %s", cond.Dump()) +} + +func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive, + xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { + if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" { + return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue) + } + callFunc := wildcardExistsPhony + if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") { + callFunc = fileExistsPhony + } + var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool} + if !negate { + cc = ¬Expr{cc} + } + return cc +} + +func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive, + xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { + if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" { + return ctx.newBadExpr(directive, "findstring result can be compared only to empty: %s", xValue) + } + return &eqExpr{ + left: &callExpr{ + object: xCall.args[1], + name: "find", + args: []starlarkExpr{xCall.args[0]}, + returnType: starlarkTypeInt, + }, + right: &intLiteralExpr{-1}, + isEq: !negate, + } +} + +func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive, + xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { + if _, ok := xValue.(*stringLiteralExpr); !ok { + return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue) + } + return &eqExpr{ + left: &callExpr{ + name: "strip", + args: xCall.args, + returnType: starlarkTypeString, + }, + right: xValue, isEq: !negate} +} + +// parses $(...), returning an expression +func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr { + ref.TrimLeftSpaces() + ref.TrimRightSpaces() + refDump := ref.Dump() + + // Handle only the case where the first (or only) word is constant + words := ref.SplitN(" ", 2) + if !words[0].Const() { + return ctx.newBadExpr(node, "reference is too complex: %s", refDump) + } + + // If it is a single word, it can be a simple variable + // reference or a function call + if len(words) == 1 { + if isMakeControlFunc(refDump) || refDump == "shell" { + return &callExpr{ + name: refDump, + args: []starlarkExpr{&stringLiteralExpr{""}}, + returnType: starlarkTypeUnknown, + } + } + if v := ctx.addVariable(refDump); v != nil { + return &variableRefExpr{v, ctx.lastAssignment(v.name()) != nil} + } + return ctx.newBadExpr(node, "unknown variable %s", refDump) + } + + expr := &callExpr{name: words[0].Dump(), returnType: starlarkTypeUnknown} + args := words[1] + args.TrimLeftSpaces() + // Make control functions and shell need special treatment as everything + // after the name is a single text argument + if isMakeControlFunc(expr.name) || expr.name == "shell" { + x := ctx.parseMakeString(node, args) + if xBad, ok := x.(*badExpr); ok { + return xBad + } + expr.args = []starlarkExpr{x} + return expr + } + if expr.name == "call" { + words = args.SplitN(",", 2) + if words[0].Empty() || !words[0].Const() { + return ctx.newBadExpr(nil, "cannot handle %s", refDump) + } + expr.name = words[0].Dump() + if len(words) < 2 { + return expr + } + args = words[1] + } + if kf, found := knownFunctions[expr.name]; found { + expr.returnType = kf.returnType + } else { + return ctx.newBadExpr(node, "cannot handle invoking %s", expr.name) + } + switch expr.name { + case "word": + return ctx.parseWordFunc(node, args) + case "subst": + return ctx.parseSubstFunc(node, args) + default: + for _, arg := range args.Split(",") { + arg.TrimLeftSpaces() + arg.TrimRightSpaces() + x := ctx.parseMakeString(node, arg) + if xBad, ok := x.(*badExpr); ok { + return xBad + } + expr.args = append(expr.args, x) + } + } + return expr +} + +func (ctx *parseContext) parseSubstFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr { + words := args.Split(",") + if len(words) != 3 { + return ctx.newBadExpr(node, "subst function should have 3 arguments") + } + if !words[0].Const() || !words[1].Const() { + return ctx.newBadExpr(node, "subst function's from and to arguments should be constant") + } + from := words[0].Strings[0] + to := words[1].Strings[0] + words[2].TrimLeftSpaces() + words[2].TrimRightSpaces() + obj := ctx.parseMakeString(node, words[2]) + return &callExpr{ + object: obj, + name: "replace", + args: []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}}, + returnType: starlarkTypeString, + } +} + +func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr { + words := args.Split(",") + if len(words) != 2 { + return ctx.newBadExpr(node, "word function should have 2 arguments") + } + var index uint64 = 0 + if words[0].Const() { + index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64) + } + if index < 1 { + return ctx.newBadExpr(node, "word index should be constant positive integer") + } + words[1].TrimLeftSpaces() + words[1].TrimRightSpaces() + array := ctx.parseMakeString(node, words[1]) + if xBad, ok := array.(*badExpr); ok { + return xBad + } + if array.typ() != starlarkTypeList { + array = &callExpr{object: array, name: "split", returnType: starlarkTypeList} + } + return indexExpr{array, &intLiteralExpr{int(index - 1)}} +} + +func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr { + if mk.Const() { + return &stringLiteralExpr{mk.Dump()} + } + if mkRef, ok := mk.SingleVariable(); ok { + return ctx.parseReference(node, mkRef) + } + // If we reached here, it's neither string literal nor a simple variable, + // we need a full-blown interpolation node that will generate + // "a%b%c" % (X, Y) for a$(X)b$(Y)c + xInterp := &interpolateExpr{args: make([]starlarkExpr, len(mk.Variables))} + for i, ref := range mk.Variables { + arg := ctx.parseReference(node, ref.Name) + if x, ok := arg.(*badExpr); ok { + return x + } + xInterp.args[i] = arg + } + xInterp.chunks = append(xInterp.chunks, mk.Strings...) + return xInterp +} + +// Handles the statements whose treatment is the same in all contexts: comment, +// assignment, variable (which is a macro call in reality) and all constructs that +// do not handle in any context ('define directive and any unrecognized stuff). +// Return true if we handled it. +func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) bool { + handled := true + switch x := node.(type) { + case *mkparser.Comment: + ctx.insertComment("#" + x.Comment) + case *mkparser.Assignment: + ctx.handleAssignment(x) + case *mkparser.Variable: + ctx.handleVariable(x) + case *mkparser.Directive: + switch x.Name { + case "define": + ctx.handleDefine(x) + case "include", "-include": + ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-') + default: + handled = false + } + default: + ctx.errorf(x, "unsupported line %s", x.Dump()) + } + return handled +} + +func (ctx *parseContext) insertComment(s string) { + ctx.receiver.newNode(&commentNode{strings.TrimSpace(s)}) +} + +func (ctx *parseContext) carryAsComment(failedNode mkparser.Node) { + for _, line := range strings.Split(failedNode.Dump(), "\n") { + ctx.insertComment("# " + line) + } +} + +// records that the given node failed to be converted and includes an explanatory message +func (ctx *parseContext) errorf(failedNode mkparser.Node, message string, args ...interface{}) { + if ctx.errorLogger != nil { + ctx.errorLogger.NewError(message, failedNode, args...) + } + message = fmt.Sprintf(message, args...) + ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", message)) + ctx.carryAsComment(failedNode) + ctx.script.hasErrors = true +} + +func (ctx *parseContext) wrapBadExpr(xBad *badExpr) { + ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", xBad.message)) + ctx.carryAsComment(xBad.node) +} + +func (ctx *parseContext) loadedModulePath(path string) string { + // During the transition to Roboleaf some of the product configuration files + // will be converted and checked in while the others will be generated on the fly + // and run. The runner (rbcrun application) accommodates this by allowing three + // different ways to specify the loaded file location: + // 1) load(":",...) loads from the same directory + // 2) load("//path/relative/to/source/root:", ...) loads source tree + // 3) load("/absolute/path/to/ absolute path + // If the file being generated and the file it wants to load are in the same directory, + // generate option 1. + // Otherwise, if output directory is not specified, generate 2) + // Finally, if output directory has been specified and the file being generated and + // the file it wants to load from are in the different directories, generate 2) or 3): + // * if the file being loaded exists in the source tree, generate 2) + // * otherwise, generate 3) + // Finally, figure out the loaded module path and name and create a node for it + loadedModuleDir := filepath.Dir(path) + base := filepath.Base(path) + loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix + if loadedModuleDir == filepath.Dir(ctx.script.mkFile) { + return ":" + loadedModuleName + } + if ctx.outputDir == "" { + return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName) + } + if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil { + return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName) + } + return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName) +} + +func (ss *StarlarkScript) String() string { + return NewGenerateContext(ss).emit() +} + +func (ss *StarlarkScript) SubConfigFiles() []string { + var subs []string + for _, src := range ss.inherited { + subs = append(subs, src.originalPath) + } + return subs +} + +func (ss *StarlarkScript) HasErrors() bool { + return ss.hasErrors +} + +// Convert reads and parses a makefile. If successful, parsed tree +// is returned and then can be passed to String() to get the generated +// Starlark file. +func Convert(req Request) (*StarlarkScript, error) { + reader := req.Reader + if reader == nil { + mkContents, err := ioutil.ReadFile(req.MkFile) + if err != nil { + return nil, err + } + reader = bytes.NewBuffer(mkContents) + } + parser := mkparser.NewParser(req.MkFile, reader) + nodes, errs := parser.Parse() + if len(errs) > 0 { + for _, e := range errs { + fmt.Fprintln(os.Stderr, "ERROR:", e) + } + return nil, fmt.Errorf("bad makefile %s", req.MkFile) + } + starScript := &StarlarkScript{ + moduleName: moduleNameForFile(req.MkFile), + mkFile: req.MkFile, + topDir: req.RootDir, + traceCalls: req.TraceCalls, + warnPartialSuccess: req.WarnPartialSuccess, + } + ctx := newParseContext(starScript, nodes) + ctx.outputSuffix = req.OutputSuffix + ctx.outputDir = req.OutputDir + ctx.errorLogger = req.ErrorLogger + if len(req.TracedVariables) > 0 { + ctx.tracedVariables = make(map[string]bool) + for _, v := range req.TracedVariables { + ctx.tracedVariables[v] = true + } + } + ctx.pushReceiver(starScript) + for ctx.hasNodes() && ctx.fatalError == nil { + node := ctx.getNode() + if ctx.handleSimpleStatement(node) { + continue + } + switch x := node.(type) { + case *mkparser.Directive: + switch x.Name { + case "ifeq", "ifneq", "ifdef", "ifndef": + ctx.handleIfBlock(x) + default: + ctx.errorf(x, "unexpected directive %s", x.Name) + } + default: + ctx.errorf(x, "unsupported line") + } + } + if ctx.fatalError != nil { + return nil, ctx.fatalError + } + return starScript, nil +} + +func Launcher(path, name string) string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName) + fmt.Fprintf(&buf, "load(%q, \"init\")\n", path) + fmt.Fprintf(&buf, "g, config = %s(%q, init)\n", cfnMain, name) + fmt.Fprintf(&buf, "%s(g, config)\n", cfnPrintVars) + return buf.String() +} + +func MakePath2ModuleName(mkPath string) string { + return strings.TrimSuffix(mkPath, filepath.Ext(mkPath)) +} diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go new file mode 100644 index 000000000..54263b8d4 --- /dev/null +++ b/mk2rbc/mk2rbc_test.go @@ -0,0 +1,857 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mk2rbc + +import ( + "bytes" + "strings" + "testing" +) + +var testCases = []struct { + desc string + mkname string + in string + expected string +}{ + { + desc: "Comment", + mkname: "product.mk", + in: ` +# Comment +# FOO= a\ + b +`, + expected: `# Comment +# FOO= a +# b +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) +`, + }, + { + desc: "Name conversion", + mkname: "path/bar-baz.mk", + in: ` +# Comment +`, + expected: `# Comment +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) +`, + }, + { + desc: "Item variable", + mkname: "pixel3.mk", + in: ` +PRODUCT_NAME := Pixel 3 +PRODUCT_MODEL := +local_var = foo +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_NAME"] = "Pixel 3" + cfg["PRODUCT_MODEL"] = "" + _local_var = "foo" +`, + }, + { + desc: "List variable", + mkname: "pixel4.mk", + in: ` +PRODUCT_PACKAGES = package1 package2 +PRODUCT_COPY_FILES += file2:target +PRODUCT_PACKAGES += package3 +PRODUCT_COPY_FILES = +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_PACKAGES"] = [ + "package1", + "package2", + ] + rblf.setdefault(handle, "PRODUCT_COPY_FILES") + cfg["PRODUCT_COPY_FILES"] += ["file2:target"] + cfg["PRODUCT_PACKAGES"] += ["package3"] + cfg["PRODUCT_COPY_FILES"] = [] +`, + }, + { + desc: "Unknown function", + mkname: "product.mk", + in: ` +PRODUCT_NAME := $(call foo, bar) +`, + expected: `# MK2RBC TRANSLATION ERROR: cannot handle invoking foo +# PRODUCT_NAME := $(call foo, bar) +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.warning("product.mk", "partially successful conversion") +`, + }, + { + desc: "Inherit configuration always", + mkname: "product.mk", + in: ` +ifdef PRODUCT_NAME +$(call inherit-product, part.mk) +else # Comment +$(call inherit-product, $(LOCAL_PATH)/part.mk) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") +load(":part.star", _part_init = "init") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g.get("PRODUCT_NAME") != None: + rblf.inherit(handle, "part", _part_init) + else: + # Comment + rblf.inherit(handle, "./part", _part_init) +`, + }, + { + desc: "Inherit configuration if it exists", + mkname: "product.mk", + in: ` +$(call inherit-product-if-exists, part.mk) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") +load(":part.star|init", _part_init = "init") + +def init(g, handle): + cfg = rblf.cfg(handle) + if _part_init != None: + rblf.inherit(handle, "part", _part_init) +`, + }, + + { + desc: "Include configuration", + mkname: "product.mk", + in: ` +ifdef PRODUCT_NAME +include part.mk +else +-include $(LOCAL_PATH)/part.mk) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") +load(":part.star", _part_init = "init") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g.get("PRODUCT_NAME") != None: + _part_init(g, handle) + else: + if _part_init != None: + _part_init(g, handle) +`, + }, + + { + desc: "Synonymous inherited configurations", + mkname: "path/product.mk", + in: ` +$(call inherit-product, foo/font.mk) +$(call inherit-product, bar/font.mk) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") +load("//foo:font.star", _font_init = "init") +load("//bar:font.star", _font1_init = "init") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.inherit(handle, "foo/font", _font_init) + rblf.inherit(handle, "bar/font", _font1_init) +`, + }, + { + desc: "Directive define", + mkname: "product.mk", + in: ` +define some-macro + $(info foo) +endef +`, + expected: `# MK2RBC TRANSLATION ERROR: define is not supported: some-macro +# define some-macro +# $(info foo) +# endef +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.warning("product.mk", "partially successful conversion") +`, + }, + { + desc: "Ifdef", + mkname: "product.mk", + in: ` +ifdef PRODUCT_NAME + PRODUCT_NAME = gizmo +else +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g.get("PRODUCT_NAME") != None: + cfg["PRODUCT_NAME"] = "gizmo" + else: + pass +`, + }, + { + desc: "Simple functions", + mkname: "product.mk", + in: ` +$(warning this is the warning) +$(warning) +$(info this is the info) +$(error this is the error) +PRODUCT_NAME:=$(shell echo *) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.mkwarning("product.mk", "this is the warning") + rblf.mkwarning("product.mk", "") + rblf.mkinfo("product.mk", "this is the info") + rblf.mkerror("product.mk", "this is the error") + cfg["PRODUCT_NAME"] = rblf.shell("echo *") +`, + }, + { + desc: "Empty if", + mkname: "product.mk", + in: ` +ifdef PRODUCT_NAME +# Comment +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g.get("PRODUCT_NAME") != None: + # Comment + pass +`, + }, + { + desc: "if/else/endif", + mkname: "product.mk", + in: ` +ifndef PRODUCT_NAME + PRODUCT_NAME=gizmo1 +else + PRODUCT_NAME=gizmo2 +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if not g.get("PRODUCT_NAME") != None: + cfg["PRODUCT_NAME"] = "gizmo1" + else: + cfg["PRODUCT_NAME"] = "gizmo2" +`, + }, + { + desc: "else if", + mkname: "product.mk", + in: ` +ifdef PRODUCT_NAME + PRODUCT_NAME = gizmo +else ifndef PRODUCT_PACKAGES # Comment +endif + `, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g.get("PRODUCT_NAME") != None: + cfg["PRODUCT_NAME"] = "gizmo" + elif not g.get("PRODUCT_PACKAGES") != None: + # Comment + pass +`, + }, + { + desc: "ifeq / ifneq", + mkname: "product.mk", + in: ` +ifeq (aosp_arm, $(TARGET_PRODUCT)) + PRODUCT_MODEL = pix2 +else + PRODUCT_MODEL = pix21 +endif +ifneq (aosp_x86, $(TARGET_PRODUCT)) + PRODUCT_MODEL = pix3 +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if "aosp_arm" == g["TARGET_PRODUCT"]: + cfg["PRODUCT_MODEL"] = "pix2" + else: + cfg["PRODUCT_MODEL"] = "pix21" + if "aosp_x86" != g["TARGET_PRODUCT"]: + cfg["PRODUCT_MODEL"] = "pix3" +`, + }, + { + desc: "Check filter result", + mkname: "product.mk", + in: ` +ifeq (,$(filter userdebug eng, $(TARGET_BUILD_VARIANT))) +endif +ifneq (,$(filter userdebug,$(TARGET_BUILD_VARIANT)) +endif +ifneq (,$(filter plaf,$(PLATFORM_LIST))) +endif +ifeq ($(TARGET_BUILD_VARIANT), $(filter $(TARGET_BUILD_VARIANT), userdebug eng)) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g["TARGET_BUILD_VARIANT"] not in ["userdebug", "eng"]: + pass + if g["TARGET_BUILD_VARIANT"] in ["userdebug"]: + pass + if "plaf" in g.get("PLATFORM_LIST", []): + pass + if g["TARGET_BUILD_VARIANT"] in ["userdebug", "eng"]: + pass +`, + }, + { + desc: "Get filter result", + mkname: "product.mk", + in: ` +PRODUCT_LIST2=$(filter-out %/foo.ko,$(wildcard path/*.ko)) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_LIST2"] = rblf.filter_out("%/foo.ko", rblf.expand_wildcard("path/*.ko")) +`, + }, + { + desc: "filter $(VAR), values", + mkname: "product.mk", + in: ` +ifeq (,$(filter $(TARGET_PRODUCT), yukawa_gms yukawa_sei510_gms) + ifneq (,$(filter $(TARGET_PRODUCT), yukawa_gms) + endif +endif + +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g["TARGET_PRODUCT"] not in ["yukawa_gms", "yukawa_sei510_gms"]: + if g["TARGET_PRODUCT"] in ["yukawa_gms"]: + pass +`, + }, + { + desc: "ifeq", + mkname: "product.mk", + in: ` +ifeq (aosp, $(TARGET_PRODUCT)) # Comment +else ifneq (, $(TARGET_PRODUCT)) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if "aosp" == g["TARGET_PRODUCT"]: + # Comment + pass + elif g["TARGET_PRODUCT"]: + pass +`, + }, + { + desc: "Nested if", + mkname: "product.mk", + in: ` +ifdef PRODUCT_NAME + PRODUCT_PACKAGES = pack-if0 + ifdef PRODUCT_MODEL + PRODUCT_PACKAGES = pack-if-if + else ifdef PRODUCT_NAME + PRODUCT_PACKAGES = pack-if-elif + else + PRODUCT_PACKAGES = pack-if-else + endif + PRODUCT_PACKAGES = pack-if +else ifneq (,$(TARGET_PRODUCT)) + PRODUCT_PACKAGES = pack-elif +else + PRODUCT_PACKAGES = pack-else +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g.get("PRODUCT_NAME") != None: + cfg["PRODUCT_PACKAGES"] = ["pack-if0"] + if g.get("PRODUCT_MODEL") != None: + cfg["PRODUCT_PACKAGES"] = ["pack-if-if"] + elif g.get("PRODUCT_NAME") != None: + cfg["PRODUCT_PACKAGES"] = ["pack-if-elif"] + else: + cfg["PRODUCT_PACKAGES"] = ["pack-if-else"] + cfg["PRODUCT_PACKAGES"] = ["pack-if"] + elif g["TARGET_PRODUCT"]: + cfg["PRODUCT_PACKAGES"] = ["pack-elif"] + else: + cfg["PRODUCT_PACKAGES"] = ["pack-else"] +`, + }, + { + desc: "Wildcard", + mkname: "product.mk", + in: ` +ifeq (,$(wildcard foo.mk)) +endif +ifneq (,$(wildcard foo*.mk)) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if not rblf.file_exists("foo.mk"): + pass + if rblf.file_wildcard_exists("foo*.mk"): + pass +`, + }, + { + desc: "ifneq $(X),true", + mkname: "product.mk", + in: ` +ifneq ($(VARIABLE),true) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g.get("VARIABLE", "") != "true": + pass +`, + }, + { + desc: "Const neq", + mkname: "product.mk", + in: ` +ifneq (1,0) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if "1" != "0": + pass +`, + }, + { + desc: "is-board calls", + mkname: "product.mk", + in: ` +ifeq ($(call is-board-platform-in-list,msm8998), true) +else ifneq ($(call is-board-platform,copper),true) +else ifneq ($(call is-vendor-board-platform,QCOM),true) +else ifeq ($(call is-product-in-list, $(PLATFORM_LIST)), true) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if g.get("TARGET_BOARD_PLATFORM", "") in ["msm8998"]: + pass + elif g.get("TARGET_BOARD_PLATFORM", "") != "copper": + pass + elif g.get("TARGET_BOARD_PLATFORM", "") not in g["QCOM_BOARD_PLATFORMS"]: + pass + elif g["TARGET_PRODUCT"] in g.get("PLATFORM_LIST", []): + pass +`, + }, + { + desc: "findstring call", + mkname: "product.mk", + in: ` +ifneq ($(findstring foo,$(PRODUCT_PACKAGES)),) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if (cfg.get("PRODUCT_PACKAGES", [])).find("foo") != -1: + pass +`, + }, + { + desc: "rhs call", + mkname: "product.mk", + in: ` +PRODUCT_COPY_FILES = $(call add-to-product-copy-files-if-exists, path:distpath) \ + $(call find-copy-subdir-files, *, fromdir, todir) $(wildcard foo.*) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_COPY_FILES"] = (rblf.copy_if_exists("path:distpath") + + rblf.find_and_copy("*", "fromdir", "todir") + + rblf.expand_wildcard("foo.*")) +`, + }, + { + desc: "inferred type", + mkname: "product.mk", + in: ` +HIKEY_MODS := $(wildcard foo/*.ko) +BOARD_VENDOR_KERNEL_MODULES += $(HIKEY_MODS) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["HIKEY_MODS"] = rblf.expand_wildcard("foo/*.ko") + g.setdefault("BOARD_VENDOR_KERNEL_MODULES", []) + g["BOARD_VENDOR_KERNEL_MODULES"] += g["HIKEY_MODS"] +`, + }, + { + desc: "list with vars", + mkname: "product.mk", + in: ` +PRODUCT_COPY_FILES += path1:$(TARGET_PRODUCT)/path1 $(PRODUCT_MODEL)/path2:$(TARGET_PRODUCT)/path2 +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.setdefault(handle, "PRODUCT_COPY_FILES") + cfg["PRODUCT_COPY_FILES"] += (("path1:%s/path1" % g["TARGET_PRODUCT"]).split() + + ("%s/path2:%s/path2" % (cfg.get("PRODUCT_MODEL", ""), g["TARGET_PRODUCT"])).split()) +`, + }, + { + desc: "misc calls", + mkname: "product.mk", + in: ` +$(call enforce-product-packages-exist,) +$(call enforce-product-packages-exist, foo) +$(call require-artifacts-in-path, foo, bar) +$(call require-artifacts-in-path-relaxed, foo, bar) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.enforce_product_packages_exist("") + rblf.enforce_product_packages_exist("foo") + rblf.require_artifacts_in_path("foo", "bar") + rblf.require_artifacts_in_path_relaxed("foo", "bar") +`, + }, + { + desc: "list with functions", + mkname: "product.mk", + in: ` +PRODUCT_COPY_FILES := $(call find-copy-subdir-files,*.kl,from1,to1) \ + $(call find-copy-subdir-files,*.kc,from2,to2) \ + foo bar +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_COPY_FILES"] = (rblf.find_and_copy("*.kl", "from1", "to1") + + rblf.find_and_copy("*.kc", "from2", "to2") + + [ + "foo", + "bar", + ]) +`, + }, + { + desc: "Text functions", + mkname: "product.mk", + in: ` +PRODUCT_COPY_FILES := $(addprefix pfx-,a b c) +PRODUCT_COPY_FILES := $(addsuffix .sff, a b c) +PRODUCT_NAME := $(word 1, $(subst ., ,$(TARGET_BOARD_PLATFORM))) + +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_COPY_FILES"] = rblf.addprefix("pfx-", "a b c") + cfg["PRODUCT_COPY_FILES"] = rblf.addsuffix(".sff", "a b c") + cfg["PRODUCT_NAME"] = ((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " ")).split()[0] +`, + }, + { + desc: "assignment flavors", + mkname: "product.mk", + in: ` +PRODUCT_LIST1 := a +PRODUCT_LIST2 += a +PRODUCT_LIST1 += b +PRODUCT_LIST2 += b +PRODUCT_LIST3 ?= a +PRODUCT_LIST1 = c +PLATFORM_LIST += x +PRODUCT_PACKAGES := $(PLATFORM_LIST) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_LIST1"] = ["a"] + rblf.setdefault(handle, "PRODUCT_LIST2") + cfg["PRODUCT_LIST2"] += ["a"] + cfg["PRODUCT_LIST1"] += ["b"] + cfg["PRODUCT_LIST2"] += ["b"] + if cfg.get("PRODUCT_LIST3") == None: + cfg["PRODUCT_LIST3"] = ["a"] + cfg["PRODUCT_LIST1"] = ["c"] + g.setdefault("PLATFORM_LIST", []) + g["PLATFORM_LIST"] += ["x"] + cfg["PRODUCT_PACKAGES"] = g["PLATFORM_LIST"][:] +`, + }, + { + desc: "assigment flavors2", + mkname: "product.mk", + in: ` +PRODUCT_LIST1 = a +ifeq (0,1) + PRODUCT_LIST1 += b + PRODUCT_LIST2 += b +endif +PRODUCT_LIST1 += c +PRODUCT_LIST2 += c +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_LIST1"] = ["a"] + if "0" == "1": + cfg["PRODUCT_LIST1"] += ["b"] + rblf.setdefault(handle, "PRODUCT_LIST2") + cfg["PRODUCT_LIST2"] += ["b"] + cfg["PRODUCT_LIST1"] += ["c"] + rblf.setdefault(handle, "PRODUCT_LIST2") + cfg["PRODUCT_LIST2"] += ["c"] +`, + }, + { + desc: "string split", + mkname: "product.mk", + in: ` +PRODUCT_LIST1 = a +local = b +local += c +FOO = d +FOO += e +PRODUCT_LIST1 += $(local) +PRODUCT_LIST1 += $(FOO) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_LIST1"] = ["a"] + _local = "b" + _local += " " + "c" + g["FOO"] = "d" + g["FOO"] += " " + "e" + cfg["PRODUCT_LIST1"] += (_local).split() + cfg["PRODUCT_LIST1"] += (g["FOO"]).split() +`, + }, + { + desc: "apex_jars", + mkname: "product.mk", + in: ` +PRODUCT_BOOT_JARS := $(ART_APEX_JARS) framework-minus-apex +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + cfg["PRODUCT_BOOT_JARS"] = (g.get("ART_APEX_JARS", []) + + ["framework-minus-apex"]) +`, + }, + { + desc: "strip function", + mkname: "product.mk", + in: ` +ifeq ($(filter hwaddress,$(PRODUCT_PACKAGES)),) + PRODUCT_PACKAGES := $(strip $(PRODUCT_PACKAGES) hwaddress) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if "hwaddress" not in cfg.get("PRODUCT_PACKAGES", []): + cfg["PRODUCT_PACKAGES"] = (rblf.mkstrip("%s hwaddress" % " ".join(cfg.get("PRODUCT_PACKAGES", [])))).split() +`, + }, + { + desc: "strip func in condition", + mkname: "product.mk", + in: ` +ifneq ($(strip $(TARGET_VENDOR)),) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + if rblf.mkstrip(g.get("TARGET_VENDOR", "")) != "": + pass +`, + }, + { + desc: "ref after set", + mkname: "product.mk", + in: ` +PRODUCT_ADB_KEYS:=value +FOO := $(PRODUCT_ADB_KEYS) +ifneq (,$(PRODUCT_ADB_KEYS)) +endif +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["PRODUCT_ADB_KEYS"] = "value" + g["FOO"] = g["PRODUCT_ADB_KEYS"] + if g["PRODUCT_ADB_KEYS"]: + pass +`, + }, + { + desc: "ref before set", + mkname: "product.mk", + in: ` +V1 := $(PRODUCT_ADB_KEYS) +ifeq (,$(PRODUCT_ADB_KEYS)) + V2 := $(PRODUCT_ADB_KEYS) + PRODUCT_ADB_KEYS:=foo + V3 := $(PRODUCT_ADB_KEYS) +endif`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["V1"] = g.get("PRODUCT_ADB_KEYS", "") + if not g.get("PRODUCT_ADB_KEYS", ""): + g["V2"] = g.get("PRODUCT_ADB_KEYS", "") + g["PRODUCT_ADB_KEYS"] = "foo" + g["V3"] = g["PRODUCT_ADB_KEYS"] +`, + }, +} + +var known_variables = []struct { + name string + class varClass + starlarkType +}{ + {"PRODUCT_NAME", VarClassConfig, starlarkTypeString}, + {"PRODUCT_MODEL", VarClassConfig, starlarkTypeString}, + {"PRODUCT_PACKAGES", VarClassConfig, starlarkTypeList}, + {"PRODUCT_BOOT_JARS", VarClassConfig, starlarkTypeList}, + {"PRODUCT_COPY_FILES", VarClassConfig, starlarkTypeList}, + {"PRODUCT_IS_64BIT", VarClassConfig, starlarkTypeString}, + {"PRODUCT_LIST1", VarClassConfig, starlarkTypeList}, + {"PRODUCT_LIST2", VarClassConfig, starlarkTypeList}, + {"PRODUCT_LIST3", VarClassConfig, starlarkTypeList}, + {"TARGET_PRODUCT", VarClassSoong, starlarkTypeString}, + {"TARGET_BUILD_VARIANT", VarClassSoong, starlarkTypeString}, + {"TARGET_BOARD_PLATFORM", VarClassSoong, starlarkTypeString}, + {"QCOM_BOARD_PLATFORMS", VarClassSoong, starlarkTypeString}, + {"PLATFORM_LIST", VarClassSoong, starlarkTypeList}, // TODO(asmundak): make it local instead of soong +} + +func TestGood(t *testing.T) { + for _, v := range known_variables { + KnownVariables.NewVariable(v.name, v.class, v.starlarkType) + } + for _, test := range testCases { + t.Run(test.desc, + func(t *testing.T) { + ss, err := Convert(Request{ + MkFile: test.mkname, + Reader: bytes.NewBufferString(test.in), + RootDir: ".", + OutputSuffix: ".star", + WarnPartialSuccess: true, + }) + if err != nil { + t.Error(err) + return + } + got := ss.String() + if got != test.expected { + t.Errorf("%q failed\nExpected:\n%s\nActual:\n%s\n", test.desc, + strings.ReplaceAll(test.expected, "\n", "␤\n"), + strings.ReplaceAll(got, "\n", "␤\n")) + } + }) + } +} diff --git a/mk2rbc/node.go b/mk2rbc/node.go new file mode 100644 index 000000000..d4b42226f --- /dev/null +++ b/mk2rbc/node.go @@ -0,0 +1,237 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mk2rbc + +import ( + "fmt" + "strings" + + mkparser "android/soong/androidmk/parser" +) + +// A parsed node for which starlark code will be generated +// by calling emit(). +type starlarkNode interface { + emit(ctx *generationContext) +} + +// Types used to keep processed makefile data: +type commentNode struct { + text string +} + +func (c *commentNode) emit(gctx *generationContext) { + chunks := strings.Split(c.text, "\\\n") + gctx.newLine() + gctx.write(chunks[0]) // It has '#' at the beginning already. + for _, chunk := range chunks[1:] { + gctx.newLine() + gctx.write("#", chunk) + } +} + +type inheritedModule struct { + path string // Converted Starlark file path + originalPath string // Makefile file path + moduleName string + moduleLocalName string + loadAlways bool +} + +func (im inheritedModule) name() string { + return MakePath2ModuleName(im.originalPath) +} + +func (im inheritedModule) entryName() string { + return im.moduleLocalName + "_init" +} + +type inheritNode struct { + *inheritedModule +} + +func (inn *inheritNode) emit(gctx *generationContext) { + // Unconditional case: + // rblf.inherit(handle, , module_init) + // Conditional case: + // if _init != None: + // same as above + gctx.newLine() + if inn.loadAlways { + gctx.writef("%s(handle, %q, %s)", cfnInherit, inn.name(), inn.entryName()) + return + } + gctx.writef("if %s != None:", inn.entryName()) + gctx.indentLevel++ + gctx.newLine() + gctx.writef("%s(handle, %q, %s)", cfnInherit, inn.name(), inn.entryName()) + gctx.indentLevel-- +} + +type includeNode struct { + *inheritedModule +} + +func (inn *includeNode) emit(gctx *generationContext) { + gctx.newLine() + if inn.loadAlways { + gctx.writef("%s(g, handle)", inn.entryName()) + return + } + gctx.writef("if %s != None:", inn.entryName()) + gctx.indentLevel++ + gctx.newLine() + gctx.writef("%s(g, handle)", inn.entryName()) + gctx.indentLevel-- +} + +type assignmentFlavor int + +const ( + // Assignment flavors + asgnSet assignmentFlavor = iota // := or = + asgnMaybeSet assignmentFlavor = iota // ?= and variable may be unset + asgnAppend assignmentFlavor = iota // += and variable has been set before + asgnMaybeAppend assignmentFlavor = iota // += and variable may be unset +) + +type assignmentNode struct { + lhs variable + value starlarkExpr + mkValue *mkparser.MakeString + flavor assignmentFlavor + isTraced bool + previous *assignmentNode +} + +func (asgn *assignmentNode) emit(gctx *generationContext) { + gctx.newLine() + gctx.inAssignment = true + asgn.lhs.emitSet(gctx, asgn) + gctx.inAssignment = false + + if asgn.isTraced { + gctx.newLine() + gctx.tracedCount++ + gctx.writef(`print("%s.%d: %s := ", `, gctx.starScript.mkFile, gctx.tracedCount, asgn.lhs.name()) + asgn.lhs.emitGet(gctx, true) + gctx.writef(")") + } +} + +type exprNode struct { + expr starlarkExpr +} + +func (exn *exprNode) emit(gctx *generationContext) { + gctx.newLine() + exn.expr.emit(gctx) +} + +type ifNode struct { + isElif bool // true if this is 'elif' statement + expr starlarkExpr +} + +func (in *ifNode) emit(gctx *generationContext) { + ifElif := "if " + if in.isElif { + ifElif = "elif " + } + + gctx.newLine() + if bad, ok := in.expr.(*badExpr); ok { + gctx.write("# MK2STAR ERROR converting:") + gctx.newLine() + gctx.writef("# %s", bad.node.Dump()) + gctx.newLine() + gctx.writef("# %s", bad.message) + gctx.newLine() + // The init function emits a warning if the conversion was not + // fullly successful, so here we (arbitrarily) take the false path. + gctx.writef("%sFalse:", ifElif) + return + } + gctx.write(ifElif) + in.expr.emit(gctx) + gctx.write(":") +} + +type elseNode struct{} + +func (br *elseNode) emit(gctx *generationContext) { + gctx.newLine() + gctx.write("else:") +} + +// switchCase represents as single if/elseif/else branch. All the necessary +// info about flavor (if/elseif/else) is supposed to be kept in `gate`. +type switchCase struct { + gate starlarkNode + nodes []starlarkNode +} + +func (cb *switchCase) newNode(node starlarkNode) { + cb.nodes = append(cb.nodes, node) +} + +func (cb *switchCase) emit(gctx *generationContext) { + cb.gate.emit(gctx) + gctx.indentLevel++ + hasStatements := false + emitNode := func(node starlarkNode) { + if _, ok := node.(*commentNode); !ok { + hasStatements = true + } + node.emit(gctx) + } + if len(cb.nodes) > 0 { + emitNode(cb.nodes[0]) + for _, node := range cb.nodes[1:] { + emitNode(node) + } + if !hasStatements { + gctx.emitPass() + } + } else { + gctx.emitPass() + } + gctx.indentLevel-- +} + +// A single complete if ... elseif ... else ... endif sequences +type switchNode struct { + ssCases []*switchCase +} + +func (ssw *switchNode) newNode(node starlarkNode) { + switch br := node.(type) { + case *switchCase: + ssw.ssCases = append(ssw.ssCases, br) + default: + panic(fmt.Errorf("expected switchCase node, got %t", br)) + } +} + +func (ssw *switchNode) emit(gctx *generationContext) { + if len(ssw.ssCases) == 0 { + gctx.emitPass() + } else { + ssw.ssCases[0].emit(gctx) + for _, ssCase := range ssw.ssCases[1:] { + ssCase.emit(gctx) + } + } +} diff --git a/mk2rbc/types.go b/mk2rbc/types.go index 22c8b58f7..16254648b 100644 --- a/mk2rbc/types.go +++ b/mk2rbc/types.go @@ -18,6 +18,11 @@ package mk2rbc type starlarkType int const ( + // Variable types. Initially we only know the types of the product + // configuration variables that are lists, and the types of some + // hardwired variables. The remaining variables are first entered as + // having an unknown type and treated as strings, but sometimes we + // can infer variable's type from the value assigned to it. starlarkTypeUnknown starlarkType = iota starlarkTypeList starlarkType = iota starlarkTypeString starlarkType = iota diff --git a/mk2rbc/variable.go b/mk2rbc/variable.go new file mode 100644 index 000000000..56db192d8 --- /dev/null +++ b/mk2rbc/variable.go @@ -0,0 +1,300 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mk2rbc + +import ( + "fmt" + "os" + "strings" +) + +type variable interface { + name() string + emitGet(gctx *generationContext, isDefined bool) + emitSet(gctx *generationContext, asgn *assignmentNode) + emitDefined(gctx *generationContext) + valueType() starlarkType + defaultValueString() string + isPreset() bool +} + +type baseVariable struct { + nam string + typ starlarkType + preset bool // true if it has been initialized at startup +} + +func (v baseVariable) name() string { + return v.nam +} + +func (v baseVariable) valueType() starlarkType { + return v.typ +} + +func (v baseVariable) isPreset() bool { + return v.preset +} + +var defaultValuesByType = map[starlarkType]string{ + starlarkTypeUnknown: `""`, + starlarkTypeList: "[]", + starlarkTypeString: `""`, + starlarkTypeInt: "0", + starlarkTypeBool: "False", + starlarkTypeVoid: "None", +} + +func (v baseVariable) defaultValueString() string { + if v, ok := defaultValuesByType[v.valueType()]; ok { + return v + } + panic(fmt.Errorf("%s has unknown type %q", v.name(), v.valueType())) +} + +type productConfigVariable struct { + baseVariable +} + +func (pcv productConfigVariable) emitSet(gctx *generationContext, asgn *assignmentNode) { + emitAssignment := func() { + pcv.emitGet(gctx, true) + gctx.write(" = ") + asgn.value.emitListVarCopy(gctx) + } + emitAppend := func() { + pcv.emitGet(gctx, true) + gctx.write(" += ") + if pcv.valueType() == starlarkTypeString { + gctx.writef(`" " + `) + } + asgn.value.emit(gctx) + } + + switch asgn.flavor { + case asgnSet: + emitAssignment() + case asgnAppend: + emitAppend() + case asgnMaybeAppend: + // If we are not sure variable has been assigned before, emit setdefault + if pcv.typ == starlarkTypeList { + gctx.writef("%s(handle, %q)", cfnSetListDefault, pcv.name()) + } else { + gctx.writef("cfg.setdefault(%q, %s)", pcv.name(), pcv.defaultValueString()) + } + gctx.newLine() + emitAppend() + case asgnMaybeSet: + gctx.writef("if cfg.get(%q) == None:", pcv.nam) + gctx.indentLevel++ + gctx.newLine() + emitAssignment() + gctx.indentLevel-- + } +} + +func (pcv productConfigVariable) emitGet(gctx *generationContext, isDefined bool) { + if isDefined || pcv.isPreset() { + gctx.writef("cfg[%q]", pcv.nam) + } else { + gctx.writef("cfg.get(%q, %s)", pcv.nam, pcv.defaultValueString()) + } +} + +func (pcv productConfigVariable) emitDefined(gctx *generationContext) { + gctx.writef("g.get(%q) != None", pcv.name()) +} + +type otherGlobalVariable struct { + baseVariable +} + +func (scv otherGlobalVariable) emitSet(gctx *generationContext, asgn *assignmentNode) { + emitAssignment := func() { + scv.emitGet(gctx, true) + gctx.write(" = ") + asgn.value.emitListVarCopy(gctx) + } + + emitAppend := func() { + scv.emitGet(gctx, true) + gctx.write(" += ") + if scv.valueType() == starlarkTypeString { + gctx.writef(`" " + `) + } + asgn.value.emit(gctx) + } + + switch asgn.flavor { + case asgnSet: + emitAssignment() + case asgnAppend: + emitAppend() + case asgnMaybeAppend: + // If we are not sure variable has been assigned before, emit setdefault + gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) + gctx.newLine() + emitAppend() + case asgnMaybeSet: + gctx.writef("if g.get(%q) == None:", scv.nam) + gctx.indentLevel++ + gctx.newLine() + emitAssignment() + gctx.indentLevel-- + } +} + +func (scv otherGlobalVariable) emitGet(gctx *generationContext, isDefined bool) { + if isDefined || scv.isPreset() { + gctx.writef("g[%q]", scv.nam) + } else { + gctx.writef("g.get(%q, %s)", scv.nam, scv.defaultValueString()) + } +} + +func (scv otherGlobalVariable) emitDefined(gctx *generationContext) { + gctx.writef("g.get(%q) != None", scv.name()) +} + +type localVariable struct { + baseVariable +} + +func (lv localVariable) emitDefined(_ *generationContext) { + panic("implement me") +} + +func (lv localVariable) String() string { + return "_" + lv.nam +} + +func (lv localVariable) emitSet(gctx *generationContext, asgn *assignmentNode) { + switch asgn.flavor { + case asgnSet: + gctx.writef("%s = ", lv) + asgn.value.emitListVarCopy(gctx) + case asgnAppend: + lv.emitGet(gctx, false) + gctx.write(" += ") + if lv.valueType() == starlarkTypeString { + gctx.writef(`" " + `) + } + asgn.value.emit(gctx) + case asgnMaybeAppend: + gctx.writef("%s(%q, ", cfnLocalAppend, lv) + asgn.value.emit(gctx) + gctx.write(")") + case asgnMaybeSet: + gctx.writef("%s(%q, ", cfnLocalSetDefault, lv) + asgn.value.emit(gctx) + gctx.write(")") + } +} + +func (lv localVariable) emitGet(gctx *generationContext, _ bool) { + gctx.writef("%s", lv) +} + +type predefinedVariable struct { + baseVariable + value starlarkExpr +} + +func (pv predefinedVariable) emitGet(gctx *generationContext, _ bool) { + pv.value.emit(gctx) +} + +func (pv predefinedVariable) emitSet(_ *generationContext, asgn *assignmentNode) { + if expectedValue, ok1 := maybeString(pv.value); ok1 { + actualValue, ok2 := maybeString(asgn.value) + if ok2 { + if actualValue == expectedValue { + return + } + fmt.Fprintf(os.Stderr, "cannot set predefined variable %s to %q, its value should be %q", + pv.name(), actualValue, expectedValue) + return + } + } + panic(fmt.Errorf("cannot set predefined variable %s to %q", pv.name(), asgn.mkValue.Dump())) +} + +func (pv predefinedVariable) emitDefined(gctx *generationContext) { + gctx.write("True") +} + +var localProductConfigVariables = map[string]string{ + "LOCAL_AUDIO_PRODUCT_PACKAGE": "PRODUCT_PACKAGES", + "LOCAL_AUDIO_PRODUCT_COPY_FILES": "PRODUCT_COPY_FILES", + "LOCAL_AUDIO_DEVICE_PACKAGE_OVERLAYS": "DEVICE_PACKAGE_OVERLAYS", + "LOCAL_DUMPSTATE_PRODUCT_PACKAGE": "PRODUCT_PACKAGES", + "LOCAL_GATEKEEPER_PRODUCT_PACKAGE": "PRODUCT_PACKAGES", + "LOCAL_HEALTH_PRODUCT_PACKAGE": "PRODUCT_PACKAGES", + "LOCAL_SENSOR_PRODUCT_PACKAGE": "PRODUCT_PACKAGES", + "LOCAL_KEYMASTER_PRODUCT_PACKAGE": "PRODUCT_PACKAGES", + "LOCAL_KEYMINT_PRODUCT_PACKAGE": "PRODUCT_PACKAGES", +} + +var presetVariables = map[string]bool{ + "BUILD_ID": true, + "HOST_ARCH": true, + "HOST_OS": true, + "HOST_BUILD_TYPE": true, + "OUT_DIR": true, + "PLATFORM_VERSION_CODENAME": true, + "PLATFORM_VERSION": true, + "TARGET_ARCH": true, + "TARGET_ARCH_VARIANT": true, + "TARGET_BUILD_TYPE": true, + "TARGET_BUILD_VARIANT": true, + "TARGET_PRODUCT": true, +} + +// addVariable returns a variable with a given name. A variable is +// added if it does not exist yet. +func (ctx *parseContext) addVariable(name string) variable { + v, found := ctx.variables[name] + if !found { + _, preset := presetVariables[name] + if vi, found := KnownVariables[name]; found { + switch vi.class { + case VarClassConfig: + v = &productConfigVariable{baseVariable{nam: name, typ: vi.valueType, preset: preset}} + case VarClassSoong: + v = &otherGlobalVariable{baseVariable{nam: name, typ: vi.valueType, preset: preset}} + } + } else if name == strings.ToLower(name) { + // Heuristics: if variable's name is all lowercase, consider it local + // string variable. + v = &localVariable{baseVariable{nam: name, typ: starlarkTypeString}} + } else { + vt := starlarkTypeUnknown + if strings.HasPrefix(name, "LOCAL_") { + // Heuristics: local variables that contribute to corresponding config variables + if cfgVarName, found := localProductConfigVariables[name]; found { + vi, found2 := KnownVariables[cfgVarName] + if !found2 { + panic(fmt.Errorf("unknown config variable %s for %s", cfgVarName, name)) + } + vt = vi.valueType + } + } + v = &otherGlobalVariable{baseVariable{nam: name, typ: vt}} + } + ctx.variables[name] = v + } + return v +}