diff --git a/core/dumpconfig.mk b/core/dumpconfig.mk index dd3ef43429..9b1f2c29e5 100644 --- a/core/dumpconfig.mk +++ b/core/dumpconfig.mk @@ -36,6 +36,10 @@ ifeq (,$(DUMPCONFIG_FILE)) $(error stopping) endif +# Skip the second inclusion of all of the product config files, because +# we will do these checks in the product_config tool. +SKIP_ARTIFACT_PATH_REQUIREMENT_PRODUCTS_CHECK := true + # Before we do anything else output the format version. $(file > $(DUMPCONFIG_FILE),dumpconfig_version,1) $(file >> $(DUMPCONFIG_FILE),dumpconfig_file,$(DUMPCONFIG_FILE)) @@ -75,7 +79,7 @@ $(eval $(file >> $(DUMPCONFIG_FILE),inherit,$(strip $(1)),$(strip $(2)))) endef # Args: -# $(1): Config phase (PRODUCT or DEVICE) +# $(1): Config phase (PRODUCT, EXPAND, or DEVICE) # $(2): Root nodes to import # $(3): All variable names # $(4): Single-value variables @@ -104,10 +108,21 @@ DUMPCONFIG_SKIP_VARS := \ .KATI_SYMBOLS \ 1 \ 2 \ + 3 \ + 4 \ + 5 \ + 6 \ + 7 \ + 8 \ + 9 \ LOCAL_PATH \ MAKEFILE_LIST \ PARENT_PRODUCT_FILES \ current_mk \ + _eiv_ev \ + _eiv_i \ + _eiv_sv \ + _eiv_tv \ inherit_var \ np \ _node_import_context \ diff --git a/core/product.mk b/core/product.mk index 170402a6f7..8976dd9174 100644 --- a/core/product.mk +++ b/core/product.mk @@ -606,6 +606,8 @@ get-product-var = $(PRODUCTS.$(strip $(1)).$(2)) # to a shorthand that is more convenient to read from elsewhere. # define strip-product-vars +$(call dump-phase-start,PRODUCT-EXPAND,,$(_product_var_list),$(_product_single_value_vars), \ + build/make/core/product.mk) \ $(foreach v,\ $(_product_var_list) \ PRODUCT_ENFORCE_PACKAGES_EXIST \ @@ -613,7 +615,8 @@ $(foreach v,\ $(eval $(v) := $(strip $(PRODUCTS.$(INTERNAL_PRODUCT).$(v)))) \ $(eval get-product-var = $$(if $$(filter $$(1),$$(INTERNAL_PRODUCT)),$$($$(2)),$$(PRODUCTS.$$(strip $$(1)).$$(2)))) \ $(KATI_obsolete_var PRODUCTS.$(INTERNAL_PRODUCT).$(v),Use $(v) instead) \ -) +) \ +$(call dump-phase-end,build/make/core/product.mk) endef define add-to-product-copy-files-if-exists diff --git a/core/product_config.mk b/core/product_config.mk index 6d886ec8d6..c1c08d1491 100644 --- a/core/product_config.mk +++ b/core/product_config.mk @@ -163,12 +163,14 @@ endif # Import all or just the current product makefile # Quick check $(check-all-products) +ifeq ($(SKIP_ARTIFACT_PATH_REQUIREMENT_PRODUCTS_CHECK),) # Import all the products that have made artifact path requirements, so that we can verify # the artifacts they produce. # These are imported after check-all-products because some of them might not be real products. $(foreach makefile,$(ARTIFACT_PATH_REQUIREMENT_PRODUCTS),\ $(if $(filter-out $(makefile),$(PRODUCTS)),$(eval $(call import-products,$(makefile))))\ ) +endif ifneq ($(filter dump-products, $(MAKECMDGOALS)),) $(dump-products) @@ -181,14 +183,16 @@ INTERNAL_PRODUCT := $(call resolve-short-product-name, $(TARGET_PRODUCT)) ifneq ($(current_product_makefile),$(INTERNAL_PRODUCT)) $(error PRODUCT_NAME inconsistent in $(current_product_makefile) and $(INTERNAL_PRODUCT)) endif -current_product_makefile := -all_product_makefiles := -all_product_configs := + ############################################################################ # Strip and assign the PRODUCT_ variables. $(call strip-product-vars) +current_product_makefile := +all_product_makefiles := +all_product_configs := + ############################################################################# # Quick check and assign default values diff --git a/tools/product_config/inherit_tree.py b/tools/product_config/inherit_tree.py new file mode 100755 index 0000000000..ae8a2753e4 --- /dev/null +++ b/tools/product_config/inherit_tree.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# +# Run from the root of the tree, after product-config has been run to see +# the product inheritance hierarchy for the current lunch target. +# + +import csv +import sys + +def PrintNodes(graph, node, prefix): + sys.stdout.write("%s%s" % (prefix, node)) + children = graph.get(node, []) + if children: + sys.stdout.write(" {\n") + for child in sorted(graph.get(node, [])): + PrintNodes(graph, child, prefix + " ") + sys.stdout.write("%s}\n" % prefix); + else: + sys.stdout.write("\n") + +def main(argv): + if len(argv) != 2: + print("usage: inherit_tree.py out/$TARGET_PRODUCT-$TARGET_BUILD_VARIANT/dumpconfig.csv") + sys.exit(1) + + root = None + graph = {} + with open(argv[1], newline='') as csvfile: + for line in csv.reader(csvfile): + if not root: + # Look for PRODUCTS + if len(line) < 3 or line[0] != "phase" or line[1] != "PRODUCTS": + continue + root = line[2] + else: + # Everything else + if len(line) < 3 or line[0] != "inherit": + continue + graph.setdefault(line[1], list()).append(line[2]) + + PrintNodes(graph, root, "") + + +if __name__ == "__main__": + main(sys.argv) + +# vim: set expandtab ts=2 sw=2 sts=2: + diff --git a/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java b/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java index ca31cd520b..39bd5dfe7a 100644 --- a/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java +++ b/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java @@ -31,14 +31,20 @@ public class ConvertMakeToGenericConfig { mErrors = errors; } - public GenericConfig convert(MakeConfig make) { + public GenericConfig convert(Map make) { final GenericConfig result = new GenericConfig(); + final MakeConfig products = make.get("PRODUCTS"); + if (products == null) { + mErrors.ERROR_DUMPCONFIG.add("Could not find PRODUCTS phase in dumpconfig output."); + return null; + } + // Base class fields - result.copyFrom(make); + result.copyFrom(products); // Each file - for (MakeConfig.ConfigFile f: make.getConfigFiles()) { + for (MakeConfig.ConfigFile f: products.getConfigFiles()) { final GenericConfig.ConfigFile genericFile = new GenericConfig.ConfigFile(f.getFilename()); result.addConfigFile(genericFile); @@ -77,7 +83,7 @@ public class ConvertMakeToGenericConfig { for (final Map.Entry entry: block.getVars().entrySet()) { final String varName = entry.getKey(); final GenericConfig.Assign assign = convertAssignment(block.getBlockType(), - block.getInheritedFile(), make.getVarType(varName), varName, + block.getInheritedFile(), products.getVarType(varName), varName, entry.getValue(), prevBlock.getVar(varName)); if (assign != null) { genericFile.addStatement(assign); @@ -100,6 +106,29 @@ public class ConvertMakeToGenericConfig { prevBlock = block; } } + + // Overwrite the final variables with the ones that come from the PRODUCTS-EXPAND phase. + // Drop the ones that were newly defined between the two phases, but leave values + // that were modified between. We do need to reproduce that logic in this tool. + final MakeConfig expand = make.get("PRODUCT-EXPAND"); + if (expand == null) { + mErrors.ERROR_DUMPCONFIG.add("Could not find PRODUCT-EXPAND phase in dumpconfig" + + " output."); + return null; + } + final Map productsFinal = products.getFinalVariables(); + final Map expandInitial = expand.getInitialVariables(); + final Map expandFinal = expand.getFinalVariables(); + final Map finalFinal = result.getFinalVariables(); + finalFinal.clear(); + for (Map.Entry var: expandFinal.entrySet()) { + final String varName = var.getKey(); + if (expandInitial.containsKey(varName) && !productsFinal.containsKey(varName)) { + continue; + } + finalFinal.put(varName, var.getValue()); + } + return result; } @@ -113,7 +142,7 @@ public class ConvertMakeToGenericConfig { return new GenericConfig.Assign(varName, varVal); } else if (!varVal.equals(prevVal)) { // The value changed from the last block. - if (varVal.equals("")) { + if (varVal.length() == 0) { // It was set to empty return new GenericConfig.Assign(varName, varVal); } else { diff --git a/tools/product_config/src/com/android/build/config/DumpConfigParser.java b/tools/product_config/src/com/android/build/config/DumpConfigParser.java index 6da96c1208..c4cd9630ae 100644 --- a/tools/product_config/src/com/android/build/config/DumpConfigParser.java +++ b/tools/product_config/src/com/android/build/config/DumpConfigParser.java @@ -44,13 +44,13 @@ import java.util.regex.Pattern; * 4 The location of the variable, as best tracked by kati */ public class DumpConfigParser { - private static final boolean DEBUG = true; + private static final boolean DEBUG = false; private final Errors mErrors; private final String mFilename; private final Reader mReader; - private final ArrayList mResults = new ArrayList(); + private final Map mResults = new HashMap(); private static final Pattern LIST_SEPARATOR = Pattern.compile("\\s+"); @@ -64,9 +64,9 @@ public class DumpConfigParser { } /** - * Parse the text into a list of MakeConfig objects. + * Parse the text into a map of the phase names to MakeConfig objects. */ - public static ArrayList parse(Errors errors, String filename, Reader reader) + public static Map parse(Errors errors, String filename, Reader reader) throws CsvParser.ParseException, IOException { DumpConfigParser parser = new DumpConfigParser(errors, filename, reader); parser.parseImpl(); @@ -130,7 +130,16 @@ public class DumpConfigParser { makeConfig = new MakeConfig(); makeConfig.setPhase(fields.get(1)); makeConfig.setRootNodes(splitList(fields.get(2))); - mResults.add(makeConfig); + // If there is a duplicate phase of the same name, continue parsing, but + // don't add it. Emit a warning. + if (!mResults.containsKey(makeConfig.getPhase())) { + mResults.put(makeConfig.getPhase(), makeConfig); + } else { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "Duplicate phase: " + makeConfig.getPhase() + + ". This one will be dropped."); + } initialVariables = makeConfig.getInitialVariables(); finalVariables = makeConfig.getFinalVariables(); diff --git a/tools/product_config/src/com/android/build/config/ErrorReporter.java b/tools/product_config/src/com/android/build/config/ErrorReporter.java index 5d87636d14..0a0c9f4a84 100644 --- a/tools/product_config/src/com/android/build/config/ErrorReporter.java +++ b/tools/product_config/src/com/android/build/config/ErrorReporter.java @@ -171,7 +171,7 @@ public class ErrorReporter { /** * An instance of an error happening. */ - public class Entry { + public static class Entry { private final Category mCategory; private final Position mPosition; private final String mMessage; diff --git a/tools/product_config/src/com/android/build/config/Errors.java b/tools/product_config/src/com/android/build/config/Errors.java index 92a4b30544..b333e78d96 100644 --- a/tools/product_config/src/com/android/build/config/Errors.java +++ b/tools/product_config/src/com/android/build/config/Errors.java @@ -59,4 +59,16 @@ public class Errors extends ErrorReporter { // if we're seeing this. public final Category ERROR_IMPROPER_PRODUCT_VAR_MARKER = new Category(7, true, Level.ERROR, "Bad input from dumpvars causing corrupted product variables."); + + public final Category ERROR_MISSING_CONFIG_FILE = new Category(8, true, Level.ERROR, + "Unable to find config file."); + + public final Category ERROR_INFINITE_RECURSION = new Category(9, true, Level.ERROR, + "A file tries to inherit-product from itself or its own inherited products."); + + // TODO: This will become obsolete when it is possible to have starlark-based product + // config files. + public final Category WARNING_DIFFERENT_FROM_KATI = new Category(1000, true, Level.WARNING, + "The cross-check with the original kati implementation failed."); + } diff --git a/tools/product_config/src/com/android/build/config/FlatConfig.java b/tools/product_config/src/com/android/build/config/FlatConfig.java new file mode 100644 index 0000000000..6f277fef18 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/FlatConfig.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Flattened configuration -- set of variables after all assignments and inherits have + * been executed. + */ +public class FlatConfig extends ConfigBase { + + private final TreeMap mValues = new TreeMap(); + + public TreeMap getValues() { + return mValues; + } +} diff --git a/tools/product_config/src/com/android/build/config/FlattenConfig.java b/tools/product_config/src/com/android/build/config/FlattenConfig.java new file mode 100644 index 0000000000..a19802b88e --- /dev/null +++ b/tools/product_config/src/com/android/build/config/FlattenConfig.java @@ -0,0 +1,474 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.regex.Pattern; + +public class FlattenConfig { + private static final Pattern RE_SPACE = Pattern.compile("\\p{Space}+"); + private static final String PRODUCTS_PREFIX = "PRODUCTS"; + + private final Errors mErrors; + private final GenericConfig mGenericConfig; + private final Map mGenericConfigs; + private final FlatConfig mResult = new FlatConfig(); + private final Map mVariables; + /** + * Files that have been visited, to prevent infinite recursion. There are no + * conditionals at this point in the processing, so we don't need a stack, just + * a single set. + */ + private final Set mStack = new HashSet(); + + + private FlattenConfig(Errors errors, GenericConfig genericConfig) { + mErrors = errors; + mGenericConfig = genericConfig; + mGenericConfigs = genericConfig.getFiles(); + mVariables = mResult.getValues(); + + // Base class fields + mResult.copyFrom(genericConfig); + } + + /** + * Flatten a GenericConfig to a FlatConfig. + * + * Makes three passes through the genericConfig, one to flatten the single variables, + * one to flatten the list variables, and one to flatten the unknown variables. Each + * has a slightly different algorithm. + */ + public static FlatConfig flatten(Errors errors, GenericConfig genericConfig) { + final FlattenConfig flattener = new FlattenConfig(errors, genericConfig); + return flattener.flattenImpl(); + } + + private FlatConfig flattenImpl() { + final List rootNodes = mGenericConfig.getRootNodes(); + if (rootNodes.size() == 0) { + mErrors.ERROR_DUMPCONFIG.add("No root nodes in PRODUCTS phase."); + return null; + } else if (rootNodes.size() != 1) { + final StringBuilder msg = new StringBuilder( + "Ignoring extra root nodes in PRODUCTS phase. All nodes are:"); + for (final String rn: rootNodes) { + msg.append(' '); + msg.append(rn); + } + mErrors.WARNING_DUMPCONFIG.add(msg.toString()); + } + final String root = rootNodes.get(0); + + // TODO: Do we need to worry about the initial state of variables? Anything + // that from the product config + + flattenListVars(root); + flattenSingleVars(root); + flattenUnknownVars(root); + flattenInheritsFrom(root); + + setDefaultKnownVars(); + + // TODO: This only supports the single product mode of import-nodes, which is all the + // real build does. m product-graph and friends will have to be rewritten. + mVariables.put("PRODUCTS", new Value(VarType.UNKNOWN, new Str(root))); + + return mResult; + } + + interface AssignCallback { + void onAssignStatement(GenericConfig.Assign assign); + } + + interface InheritCallback { + void onInheritStatement(GenericConfig.Inherit assign); + } + + /** + * Do a bunch of validity checks, and then iterate through each of the statements + * in the given file. For Assignments, the callback is only called for variables + * matching varType. + * + * Adds makefiles which have been traversed to the 'seen' set, and will not traverse + * into an inherit statement if its makefile has already been seen. + */ + private void forEachStatement(Str filename, VarType varType, Set seen, + AssignCallback assigner, InheritCallback inheriter) { + if (mStack.contains(filename)) { + mErrors.ERROR_INFINITE_RECURSION.add(filename.getPosition(), + "File is already in the inherit-product stack: " + filename); + return; + } + + mStack.add(filename); + try { + final GenericConfig.ConfigFile genericFile = mGenericConfigs.get(filename.toString()); + + if (genericFile == null) { + mErrors.ERROR_MISSING_CONFIG_FILE.add(filename.getPosition(), + "Unable to find config file: " + filename); + return; + } + + for (final GenericConfig.Statement statement: genericFile.getStatements()) { + if (statement instanceof GenericConfig.Assign) { + if (assigner != null) { + final GenericConfig.Assign assign = (GenericConfig.Assign)statement; + final String varName = assign.getName(); + + // Assert that we're not stomping on another variable, which + // really should be impossible at this point. + assertVarType(filename, varName); + + if (mGenericConfig.getVarType(varName) == varType) { + assigner.onAssignStatement(assign); + } + } + } else if (statement instanceof GenericConfig.Inherit) { + if (inheriter != null) { + final GenericConfig.Inherit inherit = (GenericConfig.Inherit)statement; + if (seen != null) { + if (seen.contains(inherit.getFilename().toString())) { + continue; + } + seen.add(inherit.getFilename().toString()); + } + inheriter.onInheritStatement(inherit); + } + } + } + } finally { + // Also executes after return statements, so we always remove this. + mStack.remove(filename); + } + } + + /** + * Call 'inheriter' for each child of 'filename' in alphabetical order. + */ + private void forEachInheritAlpha(final Str filename, VarType varType, Set seen, + InheritCallback inheriter) { + final TreeMap alpha = new TreeMap(); + forEachStatement(filename, varType, null, null, + (inherit) -> { + alpha.put(inherit.getFilename(), inherit); + }); + for (final GenericConfig.Inherit inherit: alpha.values()) { + // Handle 'seen' here where we actaully call back, not before, so that + // the proper traversal order is preserved. + if (seen != null) { + if (seen.contains(inherit.getFilename().toString())) { + continue; + } + seen.add(inherit.getFilename().toString()); + } + inheriter.onInheritStatement(inherit); + } + } + + /** + * Traverse the inheritance hierarchy, setting list-value product config variables. + */ + private void flattenListVars(final String filename) { + Map vars = flattenListVars(new Str(filename), new HashSet()); + // Add the result of the recursion to mVariables. We know there will be + // no collisions because this function only handles list variables. + for (Map.Entry entry: vars.entrySet()) { + mVariables.put(entry.getKey(), entry.getValue()); + } + } + + /** + * Return the variables defined, recursively, by 'filename.' The 'seen' set + * accumulates which nodes have been visited, as each is only done once. + * + * This convoluted algorithm isn't ideal, but it matches what is in node_fns.mk. + */ + private Map flattenListVars(final Str filename, Set seen) { + Map result = new HashMap(); + + // Recurse into our children first in alphabetical order, building a map of + // that filename to its flattened values. The order matters here because + // we will only look at each child once, and when a file appears multiple + // times, its variables must have the right set, based on whether it's been + // seen before. This preserves the order from node_fns.mk. + + // Child filename --> { varname --> value } + final Map> children = new HashMap(); + forEachInheritAlpha(filename, VarType.LIST, seen, + (inherit) -> { + final Str child = inherit.getFilename(); + children.put(child, flattenListVars(child, seen)); + }); + + // Now, traverse the values again in the original source order to concatenate the values. + // Note that the contcatenation order is *different* from the inherit order above. + forEachStatement(filename, VarType.LIST, null, + (assign) -> { + assignToListVar(result, assign.getName(), assign.getValue()); + }, + (inherit) -> { + final Map child = children.get(inherit.getFilename()); + // child == null happens if this node has been visited before. + if (child != null) { + for (Map.Entry entry: child.entrySet()) { + final String varName = entry.getKey(); + final Value varVal = entry.getValue(); + appendToListVar(result, varName, varVal.getList()); + } + } + }); + + return result; + } + + /** + * Traverse the inheritance hierarchy, setting single-value product config variables. + */ + private void flattenSingleVars(final String filename) { + flattenSingleVars(new Str(filename), new HashSet(), new HashSet()); + } + + private void flattenSingleVars(final Str filename, Set seen1, Set seen2) { + // flattenSingleVars has two loops. The first sets all variables that are + // defined for *this* file. The second traverses through the inheritance, + // to fill in values that weren't defined in this file. The first appearance of + // the variable is the one that wins. + + forEachStatement(filename, VarType.SINGLE, seen1, + (assign) -> { + final String varName = assign.getName(); + Value v = mVariables.get(varName); + // Only take the first value that we see for single variables. + Value value = mVariables.get(varName); + if (!mVariables.containsKey(varName)) { + final List valueList = assign.getValue(); + // There should never be more than one item in this list, because + // SINGLE values should never be appended to. + if (valueList.size() != 1) { + final StringBuilder positions = new StringBuilder("["); + for (Str s: valueList) { + positions.append(s.getPosition()); + } + positions.append(" ]"); + throw new RuntimeException("Value list found for SINGLE variable " + + varName + " size=" + valueList.size() + + "positions=" + positions.toString()); + } + mVariables.put(varName, + new Value(VarType.SINGLE, + valueList.get(0))); + } + }, null); + + forEachInheritAlpha(filename, VarType.SINGLE, seen2, + (inherit) -> { + flattenSingleVars(inherit.getFilename(), seen1, seen2); + }); + } + + /** + * Traverse the inheritance hierarchy and flatten the values + */ + private void flattenUnknownVars(String filename) { + flattenUnknownVars(new Str(filename), new HashSet()); + } + + private void flattenUnknownVars(final Str filename, Set seen) { + // flattenUnknownVars has two loops: First to attempt to set the variable from + // this file, and then a second loop to handle the inheritance. This is odd + // but it matches the order the files are included in node_fns.mk. The last appearance + // of the value is the one that wins. + + forEachStatement(filename, VarType.UNKNOWN, null, + (assign) -> { + // Overwrite the current value with whatever is now in the file. + mVariables.put(assign.getName(), + new Value(VarType.UNKNOWN, + flattenAssignList(assign, new Str("")))); + }, null); + + forEachInheritAlpha(filename, VarType.UNKNOWN, seen, + (inherit) -> { + flattenUnknownVars(inherit.getFilename(), seen); + }); + } + + String prefix = ""; + + /** + * Sets the PRODUCTS..INHERITS_FROM variables. + */ + private void flattenInheritsFrom(final String filename) { + flattenInheritsFrom(new Str(filename)); + } + + /** + * This flatten function, unlike the others visits all of the nodes regardless + * of whether they have been seen before, because that's what the make code does. + */ + private void flattenInheritsFrom(final Str filename) { + // Recurse, and gather the list our chlidren + final TreeSet children = new TreeSet(); + forEachStatement(filename, VarType.LIST, null, null, + (inherit) -> { + children.add(inherit.getFilename()); + flattenInheritsFrom(inherit.getFilename()); + }); + + final String varName = "PRODUCTS." + filename + ".INHERITS_FROM"; + if (children.size() > 0) { + // Build the space separated list. + boolean first = true; + final StringBuilder val = new StringBuilder(); + for (Str child: children) { + if (first) { + first = false; + } else { + val.append(' '); + } + val.append(child); + } + mVariables.put(varName, new Value(VarType.UNKNOWN, new Str(val.toString()))); + } else { + // Clear whatever flattenUnknownVars happened to have put in. + mVariables.remove(varName); + } + } + + /** + * Throw an exception if there's an existing variable with a different type. + */ + private void assertVarType(Str filename, String varName) { + if (mGenericConfig.getVarType(varName) == VarType.UNKNOWN) { + final Value prevValue = mVariables.get(varName); + if (prevValue != null + && prevValue.getVarType() != VarType.UNKNOWN) { + throw new RuntimeException("Mismatched var types:" + + " filename=" + filename + + " varType=" + mGenericConfig.getVarType(varName) + + " varName=" + varName + + " prevValue=" + Value.debugString(prevValue)); + } + } + } + + /** + * Depending on whether the assignment is prepending, appending, setting, etc., + * update the value. We can infer which of those operations it is by the length + * and contents of the values. Each value in the list was originally separated + * by the previous value. + */ + private void assignToListVar(Map vars, String varName, List items) { + final Value value = vars.get(varName); + final List orig = value == null ? new ArrayList() : value.getList(); + final List result = new ArrayList(); + if (items.size() > 0) { + for (int i = 0; i < items.size(); i++) { + if (i != 0) { + result.addAll(orig); + } + final Str item = items.get(i); + addWords(result, item); + } + } + vars.put(varName, new Value(result)); + } + + /** + * Appends all of the words in in 'items' to an entry in vars keyed by 'varName', + * creating one if necessary. + */ + private static void appendToListVar(Map vars, String varName, List items) { + Value value = vars.get(varName); + if (value == null) { + value = new Value(new ArrayList()); + vars.put(varName, value); + } + final List out = value.getList(); + for (Str item: items) { + addWords(out, item); + } + } + + /** + * Split 'item' on spaces, and add each of them as a word to 'out'. + */ + private static void addWords(List out, Str item) { + for (String word: RE_SPACE.split(item.toString().trim())) { + if (word.length() > 0) { + out.add(new Str(item.getPosition(), word)); + } + } + } + + /** + * Flatten the list of strings in an Assign statement, using the previous value + * as a separator. + */ + private Str flattenAssignList(GenericConfig.Assign assign, Str previous) { + final StringBuilder result = new StringBuilder(); + Position position = previous.getPosition(); + final List list = assign.getValue(); + final int size = list.size(); + for (int i = 0; i < size; i++) { + final Str item = list.get(i); + result.append(item.toString()); + if (i != size - 1) { + result.append(previous); + } + final Position pos = item.getPosition(); + if (pos != null && pos.getFile() != null) { + position = pos; + } + } + return new Str(position, result.toString()); + } + + /** + * Make sure that each of the product config variables has a default value. + */ + private void setDefaultKnownVars() { + for (Map.Entry entry: mGenericConfig.getProductVars().entrySet()) { + final String varName = entry.getKey(); + final VarType varType = entry.getValue(); + + final Value val = mVariables.get(varName); + if (val == null) { + mVariables.put(varName, new Value(varType)); + } + } + + + // TODO: These two for now as well, until we can rewrite the enforce packages exist + // handling. + if (!mVariables.containsKey("PRODUCT_ENFORCE_PACKAGES_EXIST")) { + mVariables.put("PRODUCT_ENFORCE_PACKAGES_EXIST", new Value(VarType.UNKNOWN)); + } + if (!mVariables.containsKey("PRODUCT_ENFORCE_PACKAGES_EXIST_ALLOW_LIST")) { + mVariables.put("PRODUCT_ENFORCE_PACKAGES_EXIST_ALLOW_LIST", new Value(VarType.UNKNOWN)); + } + } +} diff --git a/tools/product_config/src/com/android/build/config/Kati.java b/tools/product_config/src/com/android/build/config/Kati.java index 026ddb5a59..4fa229753e 100644 --- a/tools/product_config/src/com/android/build/config/Kati.java +++ b/tools/product_config/src/com/android/build/config/Kati.java @@ -16,11 +16,11 @@ package com.android.build.config; -import java.util.List; +import java.util.Map; /** * Wrapper for invoking kati. */ public interface Kati { - public MakeConfig loadProductConfig(); + public Map loadProductConfig(); } diff --git a/tools/product_config/src/com/android/build/config/KatiImpl.java b/tools/product_config/src/com/android/build/config/KatiImpl.java index feb374cb74..de11f36685 100644 --- a/tools/product_config/src/com/android/build/config/KatiImpl.java +++ b/tools/product_config/src/com/android/build/config/KatiImpl.java @@ -56,17 +56,16 @@ public class KatiImpl implements Kati { } @Override - public MakeConfig loadProductConfig() { + public Map loadProductConfig() { final String csvPath = getDumpConfigCsvPath(); try { File workDir = new File(getWorkDirPath()); - if (!workDir.mkdirs()) { + if ((workDir.exists() && !workDir.isDirectory()) || !workDir.mkdirs()) { mErrors.ERROR_KATI.add("Unable to create directory: " + workDir); return null; // TODO: throw exception? } - System.out.println("running kati"); String out = mCommand.run(new String[] { "-f", "build/make/core/dumpconfig.mk", "DUMPCONFIG_FILE=" + csvPath @@ -89,17 +88,14 @@ public class KatiImpl implements Kati { } try (FileReader reader = new FileReader(csvPath)) { - System.out.println("csvPath=" + csvPath); - List makeConfigs = DumpConfigParser.parse(mErrors, csvPath, reader); + Map makeConfigs = DumpConfigParser.parse(mErrors, csvPath, reader); if (makeConfigs.size() == 0) { // TODO: Issue error? return null; } - // TODO: There are multiple passes. That should be cleaned up in the make - // build system, but for now, the first one is the one we want. - return makeConfigs.get(0); + return makeConfigs; } catch (CsvParser.ParseException ex) { mErrors.ERROR_KATI.add(new Position(csvPath, ex.getLine()), "Unable to parse output of dumpconfig.mk: " + ex.getMessage()); diff --git a/tools/product_config/src/com/android/build/config/Main.java b/tools/product_config/src/com/android/build/config/Main.java index 7417fc7dcc..5cec55ede7 100644 --- a/tools/product_config/src/com/android/build/config/Main.java +++ b/tools/product_config/src/com/android/build/config/Main.java @@ -18,6 +18,7 @@ package com.android.build.config; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.TreeSet; public class Main { @@ -30,30 +31,44 @@ public class Main { } void run() { - System.out.println("Hello World"); - // TODO: Check the build environment to make sure we're running in a real // build environment, e.g. actually inside a source tree, with TARGET_PRODUCT // and TARGET_BUILD_VARIANT defined, etc. Kati kati = new KatiImpl(mErrors, mOptions); - MakeConfig makeConfig = kati.loadProductConfig(); - if (makeConfig == null || mErrors.hadError()) { + Map makeConfigs = kati.loadProductConfig(); + if (makeConfigs == null || mErrors.hadError()) { return; } - - System.out.println(); - System.out.println("===================="); - System.out.println("PRODUCT CONFIG FILES"); - System.out.println("===================="); - makeConfig.printToStream(System.out); + if (false) { + for (MakeConfig makeConfig: (new TreeMap(makeConfigs)).values()) { + System.out.println(); + System.out.println("======================================="); + System.out.println("PRODUCT CONFIG FILES : " + makeConfig.getPhase()); + System.out.println("======================================="); + makeConfig.printToStream(System.out); + } + } ConvertMakeToGenericConfig m2g = new ConvertMakeToGenericConfig(mErrors); - GenericConfig generic = m2g.convert(makeConfig); + GenericConfig generic = m2g.convert(makeConfigs); + if (false) { + System.out.println("======================"); + System.out.println("REGENERATED MAKE FILES"); + System.out.println("======================"); + MakeWriter.write(System.out, generic, 0); + } - System.out.println("======================"); - System.out.println("REGENERATED MAKE FILES"); - System.out.println("======================"); - MakeWriter.write(System.out, generic, 0); + // TODO: Lookup shortened name as used in PRODUCT_NAME / TARGET_PRODUCT + FlatConfig flat = FlattenConfig.flatten(mErrors, generic); + if (false) { + System.out.println("======================="); + System.out.println("FLATTENED VARIABLE LIST"); + System.out.println("======================="); + MakeWriter.write(System.out, flat, 0); + } + + OutputChecker checker = new OutputChecker(flat); + checker.reportErrors(mErrors); // TODO: Run kati and extract the variables and convert all that into starlark files. @@ -97,7 +112,10 @@ public class Main { } finally { // Print errors and warnings errors.printErrors(System.err); + if (errors.hadError()) { + exitCode = 1; + } + System.exit(exitCode); } - System.exit(exitCode); } } diff --git a/tools/product_config/src/com/android/build/config/MakeWriter.java b/tools/product_config/src/com/android/build/config/MakeWriter.java index 58dfcc090e..15fd095f49 100644 --- a/tools/product_config/src/com/android/build/config/MakeWriter.java +++ b/tools/product_config/src/com/android/build/config/MakeWriter.java @@ -30,15 +30,20 @@ public class MakeWriter { private final boolean mWriteAnnotations; public static void write(PrintStream out, GenericConfig config, int flags) { - (new MakeWriter(flags)).write(out, config); + (new MakeWriter(flags)).writeGeneric(out, config); } + public static void write(PrintStream out, FlatConfig config, int flags) { + (new MakeWriter(flags)).writeFlat(out, config); + } + + private MakeWriter(int flags) { mWriteHeader = (flags & FLAG_WRITE_HEADER) != 0; mWriteAnnotations = (flags & FLAG_WRITE_ANNOTATIONS) != 0; } - private void write(PrintStream out, GenericConfig config) { + private void writeGeneric(PrintStream out, GenericConfig config) { for (GenericConfig.ConfigFile file: config.getFiles().values()) { out.println("---------------------------------------------------------"); out.println("FILE: " + file.getFilename()); @@ -49,7 +54,7 @@ public class MakeWriter { out.println("---------------------------------------------------------"); out.println("VARIABLES TOUCHED BY MAKE BASED CONFIG:"); out.println("---------------------------------------------------------"); - writeStrVars(out, getModifiedVars(config.getInitialVariables(), + writeStrVars(out, OutputChecker.getModifiedVars(config.getInitialVariables(), config.getFinalVariables()), config); } @@ -109,28 +114,6 @@ public class MakeWriter { out.println(); } - private static Map getModifiedVars(Map before, - Map after) { - final HashMap result = new HashMap(); - // Entries that were added or changed. - for (Map.Entry afterEntry: after.entrySet()) { - final String varName = afterEntry.getKey(); - final Str afterValue = afterEntry.getValue(); - final Str beforeValue = before.get(varName); - if (beforeValue == null || !beforeValue.equals(afterValue)) { - result.put(varName, afterValue); - } - } - // removed Entries that were removed, we just treat them as - for (Map.Entry beforeEntry: before.entrySet()) { - final String varName = beforeEntry.getKey(); - if (!after.containsKey(varName)) { - result.put(varName, new Str("")); - } - } - return result; - } - private static class Var { Var(String name, Str val) { this.name = name; @@ -152,4 +135,27 @@ public class MakeWriter { out.println(var.val.getPosition() + var.name + " := " + var.val); } } + + private void writeFlat(PrintStream out, FlatConfig config) { + // TODO: Print positions. + for (Map.Entry entry: config.getValues().entrySet()) { + out.print(entry.getKey()); + out.print(" := "); + + final Value value = entry.getValue(); + if (value.getVarType() == VarType.LIST) { + final List list = value.getList(); + final int size = list.size(); + for (int i = 0; i < size; i++) { + out.print(list.get(i).toString()); + if (i != size - 1) { + out.print(" \\\n "); + } + } + } else { + out.print(value.getStr().toString()); + } + out.println(); + } + } } diff --git a/tools/product_config/src/com/android/build/config/Options.java b/tools/product_config/src/com/android/build/config/Options.java index 4e6048425e..ed544dc2ab 100644 --- a/tools/product_config/src/com/android/build/config/Options.java +++ b/tools/product_config/src/com/android/build/config/Options.java @@ -87,7 +87,7 @@ public class Options { } static class Parser { - private class ParseException extends Exception { + private static class ParseException extends Exception { public ParseException(String message) { super(message); } diff --git a/tools/product_config/src/com/android/build/config/OutputChecker.java b/tools/product_config/src/com/android/build/config/OutputChecker.java new file mode 100644 index 0000000000..228f9f13a5 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/OutputChecker.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.util.Arrays; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Compares the make-based configuration as reported by dumpconfig.mk + * with what was computed from the new tool. + */ +public class OutputChecker { + // Differences that we know about, either know issues to be fixed or intentional. + private static final RegexSet IGNORED_VARIABLES = new RegexSet( + // TODO: Rewrite the enforce packages exist logic into this tool. + "PRODUCT_ENFORCE_PACKAGES_EXIST", + "PRODUCT_ENFORCE_PACKAGES_EXIST_ALLOW_LIST", + "PRODUCTS\\..*\\.PRODUCT_ENFORCE_PACKAGES_EXIST", + "PRODUCTS\\..*\\.PRODUCT_ENFORCE_PACKAGES_EXIST_ALLOW_LIST", + + // This is generated by this tool, but comes later in the make build system. + "INTERNAL_PRODUCT"); + + private final FlatConfig mConfig; + private final TreeMap mVariables; + + /** + * Represents the before and after state of a variable. + */ + public static class Variable { + public final String name; + public final VarType type; + public final Str original; + public final Value updated; + + public Variable(String name, VarType type, Str original) { + this(name, type, original, null); + } + + public Variable(String name, VarType type, Str original, Value updated) { + this.name = name; + this.type = type; + this.original = original; + this.updated = updated; + } + + /** + * Return copy of this Variable with the updated field also set. + */ + public Variable addUpdated(Value updated) { + return new Variable(name, type, original, updated); + } + + /** + * Return whether normalizedOriginal and normalizedUpdate are equal. + */ + public boolean isSame() { + final Str normalizedOriginal = Value.normalize(original); + final Str normalizedUpdated = Value.normalize(updated); + if (normalizedOriginal == null && normalizedUpdated == null) { + return true; + } else if (normalizedOriginal != null) { + return normalizedOriginal.equals(normalizedUpdated); + } else { + return false; + } + } + } + + /** + * Construct OutputChecker with the config it will check. + */ + public OutputChecker(FlatConfig config) { + mConfig = config; + mVariables = getVariables(config); + } + + /** + * Add a WARNING_DIFFERENT_FROM_KATI for each of the variables which have changed. + */ + public void reportErrors(Errors errors) { + for (Variable var: getDifferences()) { + if (IGNORED_VARIABLES.matches(var.name)) { + continue; + } + errors.WARNING_DIFFERENT_FROM_KATI.add("product_config processing differs from" + + " kati processing for " + var.type + " variable " + var.name + ".\n" + + "original: " + + Value.oneLinePerWord(var.original, "") + "\n" + + "updated: " + + Value.oneLinePerWord(var.updated, "")); + } + } + + /** + * Get the Variables that are different between the normalized form of the original + * and updated. If one is null and the other is not, even if one is an empty string, + * the values are considered different. + */ + public List getDifferences() { + final ArrayList result = new ArrayList(); + for (Variable var: mVariables.values()) { + if (!var.isSame()) { + result.add(var); + } + } + return result; + } + + /** + * Get all of the variables for this config. + * + * VisibleForTesting + */ + static TreeMap getVariables(FlatConfig config) { + final TreeMap result = new TreeMap(); + + // Add the original values to mAll + for (Map.Entry entry: getModifiedVars(config.getInitialVariables(), + config.getFinalVariables()).entrySet()) { + final String name = entry.getKey(); + result.put(name, new Variable(name, config.getVarType(name), entry.getValue())); + } + + // Add the updated values to mAll + for (Map.Entry entry: config.getValues().entrySet()) { + final String name = entry.getKey(); + final Value value = entry.getValue(); + Variable var = result.get(name); + if (var == null) { + result.put(name, new Variable(name, config.getVarType(name), null, value)); + } else { + result.put(name, var.addUpdated(value)); + } + } + + return result; + } + + /** + * Get the entries that are different in the two maps. + */ + public static Map getModifiedVars(Map before, + Map after) { + final HashMap result = new HashMap(); + + // Entries that were added or changed. + for (Map.Entry afterEntry: after.entrySet()) { + final String varName = afterEntry.getKey(); + final Str afterValue = afterEntry.getValue(); + final Str beforeValue = before.get(varName); + if (beforeValue == null || !beforeValue.equals(afterValue)) { + result.put(varName, afterValue); + } + } + + // removed Entries that were removed, we just treat them as empty string + for (Map.Entry beforeEntry: before.entrySet()) { + final String varName = beforeEntry.getKey(); + if (!after.containsKey(varName)) { + result.put(varName, new Str("")); + } + } + + return result; + } +} diff --git a/tools/product_config/src/com/android/build/config/RegexSet.java b/tools/product_config/src/com/android/build/config/RegexSet.java new file mode 100644 index 0000000000..70fcd294ab --- /dev/null +++ b/tools/product_config/src/com/android/build/config/RegexSet.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.util.regex.Pattern; + +/** + * Returns whether a string matches one of a set of presupplied regexes. + */ +public class RegexSet { + private final Pattern[] mPatterns; + + public RegexSet(String... patterns) { + mPatterns = new Pattern[patterns.length]; + for (int i = 0; i < patterns.length; i++) { + mPatterns[i] = Pattern.compile(patterns[i]); + } + } + + public boolean matches(String s) { + for (Pattern p: mPatterns) { + if (p.matcher(s).matches()) { + return true; + } + } + return false; + } +} + diff --git a/tools/product_config/src/com/android/build/config/Str.java b/tools/product_config/src/com/android/build/config/Str.java index 9c345a6329..2516b76cf8 100644 --- a/tools/product_config/src/com/android/build/config/Str.java +++ b/tools/product_config/src/com/android/build/config/Str.java @@ -22,7 +22,7 @@ import java.util.List; /** * A String and a Position, where it came from in source code. */ -public class Str { +public class Str implements Comparable { private String mValue; private Position mPosition; @@ -36,6 +36,10 @@ public class Str { mPosition = pos; } + public int length() { + return mValue.length(); + } + @Override public String toString() { return mValue; @@ -51,16 +55,11 @@ public class Str { */ @Override public boolean equals(Object o) { - if (o == null) { - return false; - } else if (o instanceof String) { - return mValue.equals(o); - } else if (o instanceof Str) { - final Str that = (Str)o; - return mValue.equals(that.mValue); - } else { + if (!(o instanceof Str)) { return false; } + final Str that = (Str)o; + return mValue.equals(that.mValue); } @Override @@ -68,6 +67,11 @@ public class Str { return mValue.hashCode(); } + @Override + public int compareTo(Str that) { + return this.mValue.compareTo(that.mValue); + } + public static ArrayList toList(Position pos, List list) { final ArrayList result = new ArrayList(list.size()); for (String s: list) { diff --git a/tools/product_config/src/com/android/build/config/Value.java b/tools/product_config/src/com/android/build/config/Value.java new file mode 100644 index 0000000000..9bd6401ae2 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/Value.java @@ -0,0 +1,218 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; +import java.util.regex.Matcher; + +/** + * Class to hold the two types of variables we support, strings and lists of strings. + */ +public class Value { + private static final Pattern SPACES = Pattern.compile("\\s+"); + + private final VarType mVarType; + private final Str mStr; + private final ArrayList mList; + + /** + * Construct an appropriately typed empty value. + */ + public Value(VarType varType) { + mVarType = varType; + if (varType == VarType.LIST) { + mStr = null; + mList = new ArrayList(); + mList.add(new Str("")); + } else { + mStr = new Str(""); + mList = null; + } + } + + public Value(VarType varType, Str str) { + mVarType = varType; + mStr = str; + mList = null; + } + + public Value(List list) { + mVarType = VarType.LIST; + mStr = null; + mList = new ArrayList(list); + } + + public VarType getVarType() { + return mVarType; + } + + public Str getStr() { + return mStr; + } + + public List getList() { + return mList; + } + + /** + * Normalize a string that is behaving as a list. + */ + public static String normalize(String str) { + if (str == null) { + return null; + } + return SPACES.matcher(str.trim()).replaceAll(" ").trim(); + } + + /** + * Normalize a string that is behaving as a list. + */ + public static Str normalize(Str str) { + if (str == null) { + return null; + } + return new Str(str.getPosition(), normalize(str.toString())); + } + + /** + * Normalize a this Value into the same format as normalize(Str). + */ + public static Str normalize(Value val) { + if (val == null) { + return null; + } + if (val.mStr != null) { + return normalize(val.mStr); + } + + if (val.mList.size() == 0) { + return new Str(""); + } + + StringBuilder result = new StringBuilder(); + final int size = val.mList.size(); + boolean first = true; + for (int i = 0; i < size; i++) { + String s = val.mList.get(i).toString().trim(); + if (s.length() > 0) { + if (!first) { + result.append(" "); + } else { + first = false; + } + result.append(s); + } + } + + // Just use the first item's position. + return new Str(val.mList.get(0).getPosition(), result.toString()); + } + + /** + * Put each word in 'str' on its own line in make format. If 'val' is null, + * 'nullValue' is returned. + */ + public static String oneLinePerWord(Value val, String nullValue) { + if (val == null) { + return nullValue; + } + final String s = normalize(val).toString(); + final Matcher m = SPACES.matcher(s); + final StringBuilder result = new StringBuilder(); + if (s.length() > 0 && (val.mVarType == VarType.LIST || m.find())) { + result.append("\\\n "); + } + result.append(m.replaceAll(" \\\\\n ")); + return result.toString(); + } + + /** + * Put each word in 'str' on its own line in make format. If 'str' is null, + * nullValue is returned. + */ + public static String oneLinePerWord(Str str, String nullValue) { + if (str == null) { + return nullValue; + } + final Matcher m = SPACES.matcher(normalize(str.toString())); + final StringBuilder result = new StringBuilder(); + if (m.find()) { + result.append("\\\n "); + } + result.append(m.replaceAll(" \\\\\n ")); + return result.toString(); + } + + /** + * Return a string representing this value with detailed debugging information. + */ + public static String debugString(Value val) { + if (val == null) { + return "null"; + } + + final StringBuilder str = new StringBuilder("Value("); + if (val.mStr != null) { + str.append("mStr="); + str.append("\""); + str.append(val.mStr.toString()); + str.append("\""); + if (false) { + str.append(" ("); + str.append(val.mStr.getPosition().toString()); + str.append(")"); + } + } + if (val.mList != null) { + str.append("mList="); + str.append("["); + for (Str s: val.mList) { + str.append(" \""); + str.append(s.toString()); + if (false) { + str.append("\" ("); + str.append(s.getPosition().toString()); + str.append(")"); + } else { + str.append("\""); + } + } + str.append(" ]"); + } + str.append(")"); + return str.toString(); + } + + /** + * Get the Positions of all of the parts of this Value. + */ + public List getPositions() { + List result = new ArrayList(); + if (mStr != null) { + result.add(mStr.getPosition()); + } + if (mList != null) { + for (Str str: mList) { + result.add(str.getPosition()); + } + } + return result; + } +} + diff --git a/tools/product_config/test.sh b/tools/product_config/test.sh new file mode 100755 index 0000000000..a910df8016 --- /dev/null +++ b/tools/product_config/test.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# +# This script runs the full set of tests for product config: +# 1. Build the product-config tool. +# 2. Run the unit tests. +# 3. Run the product config for every product available in the current +# source tree, for each of user, userdebug and eng. +# - To restrict which products or variants are run, set the +# PRODUCTS or VARIANTS environment variables. +# - Products for which the make based product config fails are +# skipped. +# + +# The PRODUCTS variable is used by the build, and setting it in the environment +# interferes with that, so unset it. (That should probably be fixed) +products=$PRODUCTS +variants=$VARIANTS +unset PRODUCTS +unset VARIANTS + +# Don't use lunch from the user's shell +unset TARGET_PRODUCT +unset TARGET_BUILD_VARIANT + +function die() { + format=$1 + shift + printf "$format\nStopping...\n" $@ >&2 + exit 1; +} + +[[ -f build/make/envsetup.sh ]] || die "Run this script from the root of the tree." +: ${products:=$(build/soong/soong_ui.bash --dumpvar-mode all_named_products | sed -e "s/ /\n/g" | sort -u )} +: ${variants:="user userdebug eng"} +: ${CKATI_BIN:=prebuilts/build-tools/$(build/soong/soong_ui.bash --dumpvar-mode HOST_PREBUILT_TAG)/bin/ckati} + +function if_signal_exit() { + [[ $1 -lt 128 ]] || exit $1 +} + +build/soong/soong_ui.bash --build-mode --all-modules --dir="$(pwd)" product-config-test product-config \ + || die "Build failed." + +echo +echo Running unit tests +java -jar out/host/linux-x86/testcases/product-config-test/product-config-test.jar +unit_tests=$? +if_signal_exit $unit_tests + +failed_baseline_checks= +for product in $products ; do + for variant in $variants ; do + echo + echo Checking to see if $product-$variant works with make + TARGET_PRODUCT=$product TARGET_BUILD_VARIANT=$variant build/soong/soong_ui.bash --dumpvar-mode TARGET_PRODUCT &> /dev/null + exit_status=$? + if_signal_exit $exit_status + if [ $exit_status -ne 0 ] ; then + echo Combo fails with make, skipping product-config test run for $product-$variant + else + echo Running product-config for $product-$variant + rm -rf out/config/$product-$variant + TARGET_PRODUCT=$product TARGET_BUILD_VARIANT=$variant product-config \ + --ckati_bin $CKATI_BIN \ + --error 1000 + exit_status=$? + if_signal_exit $exit_status + if [ $exit_status -ne 0 ] ; then + failed_baseline_checks="$failed_baseline_checks $product-$variant" + fi + fi + done +done + +echo +echo +echo "------------------------------" +echo SUMMARY +echo "------------------------------" + +echo -n "Unit tests " +if [ $unit_tests -eq 0 ] ; then echo PASSED ; else echo FAILED ; fi + +echo -n "Baseline checks " +if [ "$failed_baseline_checks" = "" ] ; then echo PASSED ; else echo FAILED ; fi +for combo in $failed_baseline_checks ; do + echo " ... $combo" +done +