Generate FlatConfig objects from GenericConfig objects.

Doesn't include tests. More of those will come later.

Test: build/make/tools/product_config/test.sh
Change-Id: Icd2b455ac5f7b4773ba332fc40e994dc6f024f1b

core/dumpconfig.mk

@@ -36,6 +36,10 @@ ifeq (,$(DUMPCONFIG_FILE))
     $(error stopping)
 endif
 
+# Skip the second inclusion of all of the product config files, because
+# we will do these checks in the product_config tool.
+SKIP_ARTIFACT_PATH_REQUIREMENT_PRODUCTS_CHECK := true
+
 # Before we do anything else output the format version.
 $(file > $(DUMPCONFIG_FILE),dumpconfig_version,1)
 $(file >> $(DUMPCONFIG_FILE),dumpconfig_file,$(DUMPCONFIG_FILE))
@@ -75,7 +79,7 @@ $(eval $(file >> $(DUMPCONFIG_FILE),inherit,$(strip $(1)),$(strip $(2))))
 endef
 
 # Args:
-#   $(1): Config phase (PRODUCT or DEVICE)
+#   $(1): Config phase (PRODUCT, EXPAND, or DEVICE)
 #   $(2): Root nodes to import
 #   $(3): All variable names
 #   $(4): Single-value variables
@@ -104,10 +108,21 @@ DUMPCONFIG_SKIP_VARS := \
 	.KATI_SYMBOLS \
 	1 \
 	2 \
+	3 \
+	4 \
+	5 \
+	6 \
+	7 \
+	8 \
+	9 \
 	LOCAL_PATH \
 	MAKEFILE_LIST \
 	PARENT_PRODUCT_FILES \
 	current_mk \
+	_eiv_ev \
+	_eiv_i \
+	_eiv_sv \
+	_eiv_tv \
 	inherit_var \
 	np \
 	_node_import_context \

core/product.mk

@@ -606,6 +606,8 @@ get-product-var = $(PRODUCTS.$(strip $(1)).$(2))
 # to a shorthand that is more convenient to read from elsewhere.
 #
 define strip-product-vars
+$(call dump-phase-start,PRODUCT-EXPAND,,$(_product_var_list),$(_product_single_value_vars), \
+		build/make/core/product.mk) \
 $(foreach v,\
   $(_product_var_list) \
     PRODUCT_ENFORCE_PACKAGES_EXIST \
@@ -613,7 +615,8 @@ $(foreach v,\
   $(eval $(v) := $(strip $(PRODUCTS.$(INTERNAL_PRODUCT).$(v)))) \
   $(eval get-product-var = $$(if $$(filter $$(1),$$(INTERNAL_PRODUCT)),$$($$(2)),$$(PRODUCTS.$$(strip $$(1)).$$(2)))) \
   $(KATI_obsolete_var PRODUCTS.$(INTERNAL_PRODUCT).$(v),Use $(v) instead) \
-)
+) \
+$(call dump-phase-end,build/make/core/product.mk)
 endef
 
 define add-to-product-copy-files-if-exists

core/product_config.mk

@@ -163,12 +163,14 @@ endif  # Import all or just the current product makefile
 # Quick check
 $(check-all-products)
 
+ifeq ($(SKIP_ARTIFACT_PATH_REQUIREMENT_PRODUCTS_CHECK),)
 # Import all the products that have made artifact path requirements, so that we can verify
 # the artifacts they produce.
 # These are imported after check-all-products because some of them might not be real products.
 $(foreach makefile,$(ARTIFACT_PATH_REQUIREMENT_PRODUCTS),\
   $(if $(filter-out $(makefile),$(PRODUCTS)),$(eval $(call import-products,$(makefile))))\
 )
+endif
 
 ifneq ($(filter dump-products, $(MAKECMDGOALS)),)
 $(dump-products)
@@ -181,14 +183,16 @@ INTERNAL_PRODUCT := $(call resolve-short-product-name, $(TARGET_PRODUCT))
 ifneq ($(current_product_makefile),$(INTERNAL_PRODUCT))
 $(error PRODUCT_NAME inconsistent in $(current_product_makefile) and $(INTERNAL_PRODUCT))
 endif
-current_product_makefile :=
+
-all_product_makefiles :=
-all_product_configs :=
 
 ############################################################################
 # Strip and assign the PRODUCT_ variables.
 $(call strip-product-vars)
 
+current_product_makefile :=
+all_product_makefiles :=
+all_product_configs :=
+
 #############################################################################
 # Quick check and assign default values
 

tools/product_config/inherit_tree.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+#
+# Run from the root of the tree, after product-config has been run to see
+# the product inheritance hierarchy for the current lunch target.
+#
+
+import csv
+import sys
+
+def PrintNodes(graph, node, prefix):
+  sys.stdout.write("%s%s" % (prefix, node))
+  children = graph.get(node, [])
+  if children:
+    sys.stdout.write(" {\n")
+    for child in sorted(graph.get(node, [])):
+      PrintNodes(graph, child, prefix + "  ")
+    sys.stdout.write("%s}\n" % prefix);
+  else:
+    sys.stdout.write("\n")
+
+def main(argv):
+  if len(argv) != 2:
+    print("usage: inherit_tree.py out/$TARGET_PRODUCT-$TARGET_BUILD_VARIANT/dumpconfig.csv")
+    sys.exit(1)
+
+  root = None
+  graph = {}
+  with open(argv[1], newline='') as csvfile:
+    for line in csv.reader(csvfile):
+      if not root:
+        # Look for PRODUCTS
+        if len(line) < 3 or line[0] != "phase" or line[1] != "PRODUCTS":
+          continue
+        root = line[2]
+      else:
+        # Everything else
+        if len(line) < 3 or line[0] != "inherit":
+          continue
+        graph.setdefault(line[1], list()).append(line[2])
+
+  PrintNodes(graph, root, "")
+
+
+if __name__ == "__main__":
+  main(sys.argv)
+
+# vim: set expandtab ts=2 sw=2 sts=2:
+

tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java

@@ -31,14 +31,20 @@ public class ConvertMakeToGenericConfig {
         mErrors = errors;
     }
 
-    public GenericConfig convert(MakeConfig make) {
+    public GenericConfig convert(Map<String, MakeConfig> make) {
         final GenericConfig result = new GenericConfig();
 
+        final MakeConfig products = make.get("PRODUCTS");
+        if (products == null) {
+            mErrors.ERROR_DUMPCONFIG.add("Could not find PRODUCTS phase in dumpconfig output.");
+            return null;
+        }
+
         // Base class fields
-        result.copyFrom(make);
+        result.copyFrom(products);
 
         // Each file
-        for (MakeConfig.ConfigFile f: make.getConfigFiles()) {
+        for (MakeConfig.ConfigFile f: products.getConfigFiles()) {
             final GenericConfig.ConfigFile genericFile
                     = new GenericConfig.ConfigFile(f.getFilename());
             result.addConfigFile(genericFile);
@@ -77,7 +83,7 @@ public class ConvertMakeToGenericConfig {
                 for (final Map.Entry<String, Str> entry: block.getVars().entrySet()) {
                     final String varName = entry.getKey();
                     final GenericConfig.Assign assign = convertAssignment(block.getBlockType(),
-                            block.getInheritedFile(), make.getVarType(varName), varName,
+                            block.getInheritedFile(), products.getVarType(varName), varName,
                             entry.getValue(), prevBlock.getVar(varName));
                     if (assign != null) {
                         genericFile.addStatement(assign);
@@ -100,6 +106,29 @@ public class ConvertMakeToGenericConfig {
                 prevBlock = block;
             }
         }
+
+        // Overwrite the final variables with the ones that come from the PRODUCTS-EXPAND phase.
+        // Drop the ones that were newly defined between the two phases, but leave values
+        // that were modified between.  We do need to reproduce that logic in this tool.
+        final MakeConfig expand = make.get("PRODUCT-EXPAND");
+        if (expand == null) {
+            mErrors.ERROR_DUMPCONFIG.add("Could not find PRODUCT-EXPAND phase in dumpconfig"
+                    + " output.");
+            return null;
+        }
+        final Map<String, Str> productsFinal = products.getFinalVariables();
+        final Map<String, Str> expandInitial = expand.getInitialVariables();
+        final Map<String, Str> expandFinal = expand.getFinalVariables();
+        final Map<String, Str> finalFinal = result.getFinalVariables();
+        finalFinal.clear();
+        for (Map.Entry<String, Str> var: expandFinal.entrySet()) {
+            final String varName = var.getKey();
+            if (expandInitial.containsKey(varName) && !productsFinal.containsKey(varName)) {
+                continue;
+            }
+            finalFinal.put(varName, var.getValue());
+        }
+
         return result;
     }
 
@@ -113,7 +142,7 @@ public class ConvertMakeToGenericConfig {
             return new GenericConfig.Assign(varName, varVal);
         } else if (!varVal.equals(prevVal)) {
             // The value changed from the last block.
-            if (varVal.equals("")) {
+            if (varVal.length() == 0) {
                 // It was set to empty
                 return new GenericConfig.Assign(varName, varVal);
             } else {

tools/product_config/src/com/android/build/config/DumpConfigParser.java

@@ -44,13 +44,13 @@ import java.util.regex.Pattern;
  *          4       The location of the variable, as best tracked by kati
  */
 public class DumpConfigParser {
-    private static final boolean DEBUG = true;
+    private static final boolean DEBUG = false;
 
     private final Errors mErrors;
     private final String mFilename;
     private final Reader mReader;
 
-    private final ArrayList<MakeConfig> mResults = new ArrayList();
+    private final Map<String,MakeConfig> mResults = new HashMap();
 
     private static final Pattern LIST_SEPARATOR = Pattern.compile("\\s+");
 
@@ -64,9 +64,9 @@ public class DumpConfigParser {
     }
 
     /**
-     * Parse the text into a list of MakeConfig objects.
+     * Parse the text into a map of the phase names to MakeConfig objects.
      */
-    public static ArrayList<MakeConfig> parse(Errors errors, String filename, Reader reader)
+    public static Map<String,MakeConfig> parse(Errors errors, String filename, Reader reader)
             throws CsvParser.ParseException, IOException {
         DumpConfigParser parser = new DumpConfigParser(errors, filename, reader);
         parser.parseImpl();
@@ -130,7 +130,16 @@ public class DumpConfigParser {
                 makeConfig = new MakeConfig();
                 makeConfig.setPhase(fields.get(1));
                 makeConfig.setRootNodes(splitList(fields.get(2)));
-                mResults.add(makeConfig);
+                // If there is a duplicate phase of the same name, continue parsing, but
+                // don't add it.  Emit a warning.
+                if (!mResults.containsKey(makeConfig.getPhase())) {
+                    mResults.put(makeConfig.getPhase(), makeConfig);
+                } else {
+                    mErrors.WARNING_DUMPCONFIG.add(
+                            new Position(mFilename, line.getLine()),
+                            "Duplicate phase: " + makeConfig.getPhase()
+                                + ". This one will be dropped.");
+                }
                 initialVariables = makeConfig.getInitialVariables();
                 finalVariables = makeConfig.getFinalVariables();
 

tools/product_config/src/com/android/build/config/ErrorReporter.java

@@ -171,7 +171,7 @@ public class ErrorReporter {
     /**
      * An instance of an error happening.
      */
-    public class Entry {
+    public static class Entry {
         private final Category mCategory;
         private final Position mPosition;
         private final String mMessage;

tools/product_config/src/com/android/build/config/Errors.java

@@ -59,4 +59,16 @@ public class Errors extends ErrorReporter {
     // if we're seeing this.
     public final Category ERROR_IMPROPER_PRODUCT_VAR_MARKER = new Category(7, true, Level.ERROR,
             "Bad input from dumpvars causing corrupted product variables.");
+
+    public final Category ERROR_MISSING_CONFIG_FILE = new Category(8, true, Level.ERROR,
+            "Unable to find config file.");
+
+    public final Category ERROR_INFINITE_RECURSION = new Category(9, true, Level.ERROR,
+            "A file tries to inherit-product from itself or its own inherited products.");
+
+    // TODO: This will become obsolete when it is possible to have starlark-based product
+    // config files.
+    public final Category WARNING_DIFFERENT_FROM_KATI = new Category(1000, true, Level.WARNING,
+            "The cross-check with the original kati implementation failed.");
+
 }

tools/product_config/src/com/android/build/config/FlatConfig.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Flattened configuration -- set of variables after all assignments and inherits have
+ * been executed.
+ */
+public class FlatConfig extends ConfigBase {
+
+    private final TreeMap<String, Value> mValues = new TreeMap();
+
+    public TreeMap<String, Value> getValues() {
+        return mValues;
+    }
+}

tools/product_config/src/com/android/build/config/FlattenConfig.java

@@ -0,0 +1,474 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+
+public class FlattenConfig {
+    private static final Pattern RE_SPACE = Pattern.compile("\\p{Space}+");
+    private static final String PRODUCTS_PREFIX = "PRODUCTS";
+
+    private final Errors mErrors;
+    private final GenericConfig mGenericConfig;
+    private final Map<String, GenericConfig.ConfigFile> mGenericConfigs;
+    private final FlatConfig mResult = new FlatConfig();
+    private final Map<String, Value> mVariables;
+    /**
+     * Files that have been visited, to prevent infinite recursion. There are no
+     * conditionals at this point in the processing, so we don't need a stack, just
+     * a single set.
+     */
+    private final Set<Str> mStack = new HashSet();
+
+
+    private FlattenConfig(Errors errors, GenericConfig genericConfig) {
+        mErrors = errors;
+        mGenericConfig = genericConfig;
+        mGenericConfigs = genericConfig.getFiles();
+        mVariables = mResult.getValues();
+
+        // Base class fields
+        mResult.copyFrom(genericConfig);
+    }
+
+    /**
+     * Flatten a GenericConfig to a FlatConfig.
+     *
+     * Makes three passes through the genericConfig, one to flatten the single variables,
+     * one to flatten the list variables, and one to flatten the unknown variables. Each
+     * has a slightly different algorithm.
+     */
+    public static FlatConfig flatten(Errors errors, GenericConfig genericConfig) {
+        final FlattenConfig flattener = new FlattenConfig(errors, genericConfig);
+        return flattener.flattenImpl();
+    }
+
+    private FlatConfig flattenImpl() {
+        final List<String> rootNodes = mGenericConfig.getRootNodes();
+        if (rootNodes.size() == 0) {
+            mErrors.ERROR_DUMPCONFIG.add("No root nodes in PRODUCTS phase.");
+            return null;
+        } else if (rootNodes.size() != 1) {
+            final StringBuilder msg = new StringBuilder(
+                    "Ignoring extra root nodes in PRODUCTS phase. All nodes are:");
+            for (final String rn: rootNodes) {
+                msg.append(' ');
+                msg.append(rn);
+            }
+            mErrors.WARNING_DUMPCONFIG.add(msg.toString());
+        }
+        final String root = rootNodes.get(0);
+
+        // TODO: Do we need to worry about the initial state of variables? Anything
+        // that from the product config
+
+        flattenListVars(root);
+        flattenSingleVars(root);
+        flattenUnknownVars(root);
+        flattenInheritsFrom(root);
+
+        setDefaultKnownVars();
+
+        // TODO: This only supports the single product mode of import-nodes, which is all the
+        // real build does. m product-graph and friends will have to be rewritten.
+        mVariables.put("PRODUCTS", new Value(VarType.UNKNOWN, new Str(root)));
+
+        return mResult;
+    }
+
+    interface AssignCallback {
+        void onAssignStatement(GenericConfig.Assign assign);
+    }
+
+    interface InheritCallback {
+        void onInheritStatement(GenericConfig.Inherit assign);
+    }
+
+    /**
+     * Do a bunch of validity checks, and then iterate through each of the statements
+     * in the given file.  For Assignments, the callback is only called for variables
+     * matching varType.
+     *
+     * Adds makefiles which have been traversed to the 'seen' set, and will not traverse
+     * into an inherit statement if its makefile has already been seen.
+     */
+    private void forEachStatement(Str filename, VarType varType, Set<String> seen,
+            AssignCallback assigner, InheritCallback inheriter) {
+        if (mStack.contains(filename)) {
+            mErrors.ERROR_INFINITE_RECURSION.add(filename.getPosition(),
+                    "File is already in the inherit-product stack: " + filename);
+            return;
+        }
+
+        mStack.add(filename);
+        try {
+            final GenericConfig.ConfigFile genericFile = mGenericConfigs.get(filename.toString());
+
+            if (genericFile == null) {
+                mErrors.ERROR_MISSING_CONFIG_FILE.add(filename.getPosition(),
+                        "Unable to find config file: " + filename);
+                return;
+            }
+
+            for (final GenericConfig.Statement statement: genericFile.getStatements()) {
+                if (statement instanceof GenericConfig.Assign) {
+                    if (assigner != null) {
+                        final GenericConfig.Assign assign = (GenericConfig.Assign)statement;
+                        final String varName = assign.getName();
+
+                        // Assert that we're not stomping on another variable, which
+                        // really should be impossible at this point.
+                        assertVarType(filename, varName);
+
+                        if (mGenericConfig.getVarType(varName) == varType) {
+                            assigner.onAssignStatement(assign);
+                        }
+                    }
+                } else if (statement instanceof GenericConfig.Inherit) {
+                    if (inheriter != null) {
+                        final GenericConfig.Inherit inherit = (GenericConfig.Inherit)statement;
+                        if (seen != null) {
+                            if (seen.contains(inherit.getFilename().toString())) {
+                                continue;
+                            }
+                            seen.add(inherit.getFilename().toString());
+                        }
+                        inheriter.onInheritStatement(inherit);
+                    }
+                }
+            }
+        } finally {
+            // Also executes after return statements, so we always remove this.
+            mStack.remove(filename);
+        }
+    }
+
+    /**
+     * Call 'inheriter' for each child of 'filename' in alphabetical order.
+     */
+    private void forEachInheritAlpha(final Str filename, VarType varType, Set<String> seen,
+            InheritCallback inheriter) {
+        final TreeMap<Str, GenericConfig.Inherit> alpha = new TreeMap();
+        forEachStatement(filename, varType, null, null,
+                (inherit) -> {
+                    alpha.put(inherit.getFilename(), inherit);
+                });
+        for (final GenericConfig.Inherit inherit: alpha.values()) {
+            // Handle 'seen' here where we actaully call back, not before, so that
+            // the proper traversal order is preserved.
+            if (seen != null) {
+                if (seen.contains(inherit.getFilename().toString())) {
+                    continue;
+                }
+                seen.add(inherit.getFilename().toString());
+            }
+            inheriter.onInheritStatement(inherit);
+        }
+    }
+
+    /**
+     * Traverse the inheritance hierarchy, setting list-value product config variables.
+     */
+    private void flattenListVars(final String filename) {
+        Map<String, Value> vars = flattenListVars(new Str(filename), new HashSet());
+        // Add the result of the recursion to mVariables. We know there will be
+        // no collisions because this function only handles list variables.
+        for (Map.Entry<String, Value> entry: vars.entrySet()) {
+            mVariables.put(entry.getKey(), entry.getValue());
+        }
+    }
+
+    /**
+     * Return the variables defined, recursively, by 'filename.' The 'seen' set
+     * accumulates which nodes have been visited, as each is only done once.
+     *
+     * This convoluted algorithm isn't ideal, but it matches what is in node_fns.mk.
+     */
+    private Map<String, Value> flattenListVars(final Str filename, Set<String> seen) {
+        Map<String, Value> result = new HashMap();
+
+        // Recurse into our children first in alphabetical order, building a map of
+        // that filename to its flattened values.  The order matters here because
+        // we will only look at each child once, and when a file appears multiple
+        // times, its variables must have the right set, based on whether it's been
+        // seen before. This preserves the order from node_fns.mk.
+
+        // Child filename --> { varname --> value }
+        final Map<Str, Map<String, Value>> children = new HashMap();
+        forEachInheritAlpha(filename, VarType.LIST, seen,
+                (inherit) -> {
+                    final Str child = inherit.getFilename();
+                    children.put(child, flattenListVars(child, seen));
+                });
+
+        // Now, traverse the values again in the original source order to concatenate the values.
+        // Note that the contcatenation order is *different* from the inherit order above.
+        forEachStatement(filename, VarType.LIST, null,
+                (assign) -> {
+                    assignToListVar(result, assign.getName(), assign.getValue());
+                },
+                (inherit) -> {
+                    final Map<String, Value> child = children.get(inherit.getFilename());
+                    // child == null happens if this node has been visited before.
+                    if (child != null) {
+                        for (Map.Entry<String, Value> entry: child.entrySet()) {
+                            final String varName = entry.getKey();
+                            final Value varVal = entry.getValue();
+                            appendToListVar(result, varName, varVal.getList());
+                        }
+                    }
+                });
+
+        return result;
+    }
+
+    /**
+     * Traverse the inheritance hierarchy, setting single-value product config variables.
+     */
+    private void flattenSingleVars(final String filename) {
+        flattenSingleVars(new Str(filename), new HashSet(), new HashSet());
+    }
+
+    private void flattenSingleVars(final Str filename, Set<String> seen1, Set<String> seen2) {
+        // flattenSingleVars has two loops.  The first sets all variables that are
+        // defined for *this* file.  The second traverses through the inheritance,
+        // to fill in values that weren't defined in this file.  The first appearance of
+        // the variable is the one that wins.
+
+        forEachStatement(filename, VarType.SINGLE, seen1,
+                (assign) -> {
+                    final String varName = assign.getName();
+                    Value v = mVariables.get(varName);
+                    // Only take the first value that we see for single variables.
+                    Value value = mVariables.get(varName);
+                    if (!mVariables.containsKey(varName)) {
+                        final List<Str> valueList = assign.getValue();
+                        // There should never be more than one item in this list, because
+                        // SINGLE values should never be appended to.
+                        if (valueList.size() != 1) {
+                            final StringBuilder positions = new StringBuilder("[");
+                            for (Str s: valueList) {
+                                positions.append(s.getPosition());
+                            }
+                            positions.append(" ]");
+                            throw new RuntimeException("Value list found for SINGLE variable "
+                                    + varName + " size=" + valueList.size()
+                                    + "positions=" + positions.toString());
+                        }
+                        mVariables.put(varName,
+                                new Value(VarType.SINGLE,
+                                    valueList.get(0)));
+                    }
+                }, null);
+
+        forEachInheritAlpha(filename, VarType.SINGLE, seen2,
+                (inherit) -> {
+                    flattenSingleVars(inherit.getFilename(), seen1, seen2);
+                });
+    }
+
+    /**
+     * Traverse the inheritance hierarchy and flatten the values
+     */
+    private void flattenUnknownVars(String filename) {
+        flattenUnknownVars(new Str(filename), new HashSet());
+    }
+
+    private void flattenUnknownVars(final Str filename, Set<String> seen) {
+        // flattenUnknownVars has two loops: First to attempt to set the variable from
+        // this file, and then a second loop to handle the inheritance.  This is odd
+        // but it matches the order the files are included in node_fns.mk. The last appearance
+        // of the value is the one that wins.
+
+        forEachStatement(filename, VarType.UNKNOWN, null,
+                (assign) -> {
+                    // Overwrite the current value with whatever is now in the file.
+                    mVariables.put(assign.getName(),
+                            new Value(VarType.UNKNOWN,
+                                flattenAssignList(assign, new Str(""))));
+                }, null);
+
+        forEachInheritAlpha(filename, VarType.UNKNOWN, seen,
+                (inherit) -> {
+                    flattenUnknownVars(inherit.getFilename(), seen);
+                });
+    }
+
+    String prefix = "";
+
+    /**
+     * Sets the PRODUCTS.<filename>.INHERITS_FROM variables.
+     */
+    private void flattenInheritsFrom(final String filename) {
+        flattenInheritsFrom(new Str(filename));
+    }
+
+    /**
+     * This flatten function, unlike the others visits all of the nodes regardless
+     * of whether they have been seen before, because that's what the make code does.
+     */
+    private void flattenInheritsFrom(final Str filename) {
+        // Recurse, and gather the list our chlidren
+        final TreeSet<Str> children = new TreeSet();
+        forEachStatement(filename, VarType.LIST, null, null,
+                (inherit) -> {
+                    children.add(inherit.getFilename());
+                    flattenInheritsFrom(inherit.getFilename());
+                });
+
+        final String varName = "PRODUCTS." + filename + ".INHERITS_FROM";
+        if (children.size() > 0) {
+            // Build the space separated list.
+            boolean first = true;
+            final StringBuilder val = new StringBuilder();
+            for (Str child: children) {
+                if (first) {
+                    first = false;
+                } else {
+                    val.append(' ');
+                }
+                val.append(child);
+            }
+            mVariables.put(varName, new Value(VarType.UNKNOWN, new Str(val.toString())));
+        } else {
+            // Clear whatever flattenUnknownVars happened to have put in.
+            mVariables.remove(varName);
+        }
+    }
+
+    /**
+     * Throw an exception if there's an existing variable with a different type.
+     */
+    private void assertVarType(Str filename, String varName) {
+        if (mGenericConfig.getVarType(varName) == VarType.UNKNOWN) {
+            final Value prevValue = mVariables.get(varName);
+            if (prevValue != null
+                    && prevValue.getVarType() != VarType.UNKNOWN) {
+                throw new RuntimeException("Mismatched var types:"
+                        + " filename=" + filename
+                        + " varType=" + mGenericConfig.getVarType(varName)
+                        + " varName=" + varName
+                        + " prevValue=" + Value.debugString(prevValue));
+            }
+        }
+    }
+
+    /**
+     * Depending on whether the assignment is prepending, appending, setting, etc.,
+     * update the value.  We can infer which of those operations it is by the length
+     * and contents of the values. Each value in the list was originally separated
+     * by the previous value.
+     */
+    private void assignToListVar(Map<String, Value> vars, String varName, List<Str> items) {
+        final Value value = vars.get(varName);
+        final List<Str> orig = value == null ? new ArrayList() : value.getList();
+        final List<Str> result = new ArrayList();
+        if (items.size() > 0) {
+            for (int i = 0; i < items.size(); i++) {
+                if (i != 0) {
+                    result.addAll(orig);
+                }
+                final Str item = items.get(i);
+                addWords(result, item);
+            }
+        }
+        vars.put(varName, new Value(result));
+    }
+
+    /**
+     * Appends all of the words in in 'items' to an entry in vars keyed by 'varName',
+     * creating one if necessary.
+     */
+    private static void appendToListVar(Map<String, Value> vars, String varName, List<Str> items) {
+        Value value = vars.get(varName);
+        if (value == null) {
+            value = new Value(new ArrayList());
+            vars.put(varName, value);
+        }
+        final List<Str> out = value.getList();
+        for (Str item: items) {
+            addWords(out, item);
+        }
+    }
+
+    /**
+     * Split 'item' on spaces, and add each of them as a word to 'out'.
+     */
+    private static void addWords(List<Str> out, Str item) {
+        for (String word: RE_SPACE.split(item.toString().trim())) {
+            if (word.length() > 0) {
+                out.add(new Str(item.getPosition(), word));
+            }
+        }
+    }
+
+    /**
+     * Flatten the list of strings in an Assign statement, using the previous value
+     * as a separator.
+     */
+    private Str flattenAssignList(GenericConfig.Assign assign, Str previous) {
+        final StringBuilder result = new StringBuilder();
+        Position position = previous.getPosition();
+        final List<Str> list = assign.getValue();
+        final int size = list.size();
+        for (int i = 0; i < size; i++) {
+            final Str item = list.get(i);
+            result.append(item.toString());
+            if (i != size - 1) {
+                result.append(previous);
+            }
+            final Position pos = item.getPosition();
+            if (pos != null && pos.getFile() != null) {
+                position = pos;
+            }
+        }
+        return new Str(position, result.toString());
+    }
+
+    /**
+     * Make sure that each of the product config variables has a default value.
+     */
+    private void setDefaultKnownVars() {
+        for (Map.Entry<String, VarType> entry: mGenericConfig.getProductVars().entrySet()) {
+            final String varName = entry.getKey();
+            final VarType varType = entry.getValue();
+
+            final Value val = mVariables.get(varName);
+            if (val == null) {
+                mVariables.put(varName, new Value(varType));
+            }
+        }
+
+
+        // TODO: These two for now as well, until we can rewrite the enforce packages exist
+        // handling.
+        if (!mVariables.containsKey("PRODUCT_ENFORCE_PACKAGES_EXIST")) {
+            mVariables.put("PRODUCT_ENFORCE_PACKAGES_EXIST", new Value(VarType.UNKNOWN));
+        }
+        if (!mVariables.containsKey("PRODUCT_ENFORCE_PACKAGES_EXIST_ALLOW_LIST")) {
+            mVariables.put("PRODUCT_ENFORCE_PACKAGES_EXIST_ALLOW_LIST", new Value(VarType.UNKNOWN));
+        }
+    }
+}

tools/product_config/src/com/android/build/config/Kati.java

@@ -16,11 +16,11 @@
 
 package com.android.build.config;
 
-import java.util.List;
+import java.util.Map;
 
 /**
  * Wrapper for invoking kati.
  */
 public interface Kati {
-    public MakeConfig loadProductConfig();
+    public Map<String, MakeConfig> loadProductConfig();
 }

tools/product_config/src/com/android/build/config/KatiImpl.java

@@ -56,17 +56,16 @@ public class KatiImpl implements Kati {
     }
 
     @Override
-    public MakeConfig loadProductConfig() {
+    public Map<String, MakeConfig> loadProductConfig() {
         final String csvPath = getDumpConfigCsvPath();
         try {
             File workDir = new File(getWorkDirPath());
 
-            if (!workDir.mkdirs()) {
+            if ((workDir.exists() && !workDir.isDirectory()) || !workDir.mkdirs()) {
                 mErrors.ERROR_KATI.add("Unable to create directory: " + workDir);
                 return null; // TODO: throw exception?
             }
 
-            System.out.println("running kati");
             String out = mCommand.run(new String[] {
                     "-f", "build/make/core/dumpconfig.mk",
                     "DUMPCONFIG_FILE=" + csvPath
@@ -89,17 +88,14 @@ public class KatiImpl implements Kati {
         }
 
         try (FileReader reader = new FileReader(csvPath)) {
-            System.out.println("csvPath=" + csvPath);
+            Map<String, MakeConfig> makeConfigs = DumpConfigParser.parse(mErrors, csvPath, reader);
-            List<MakeConfig> makeConfigs = DumpConfigParser.parse(mErrors, csvPath, reader);
 
             if (makeConfigs.size() == 0) {
                 // TODO: Issue error?
                 return null;
             }
 
-            // TODO: There are multiple passes. That should be cleaned up in the make
+            return makeConfigs;
-            // build system, but for now, the first one is the one we want.
-            return makeConfigs.get(0);
         } catch (CsvParser.ParseException ex) {
             mErrors.ERROR_KATI.add(new Position(csvPath, ex.getLine()),
                     "Unable to parse output of dumpconfig.mk: " + ex.getMessage());

tools/product_config/src/com/android/build/config/Main.java

@@ -18,6 +18,7 @@ package com.android.build.config;
 
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.TreeSet;
 
 public class Main {
@@ -30,30 +31,44 @@ public class Main {
     }
 
     void run() {
-        System.out.println("Hello World");
-
         // TODO: Check the build environment to make sure we're running in a real
         // build environment, e.g. actually inside a source tree, with TARGET_PRODUCT
         // and TARGET_BUILD_VARIANT defined, etc.
         Kati kati = new KatiImpl(mErrors, mOptions);
-        MakeConfig makeConfig = kati.loadProductConfig();
+        Map<String, MakeConfig> makeConfigs = kati.loadProductConfig();
-        if (makeConfig == null || mErrors.hadError()) {
+        if (makeConfigs == null || mErrors.hadError()) {
             return;
         }
-
+        if (false) {
-        System.out.println();
+            for (MakeConfig makeConfig: (new TreeMap<String, MakeConfig>(makeConfigs)).values()) {
-        System.out.println("====================");
+                System.out.println();
-        System.out.println("PRODUCT CONFIG FILES");
+                System.out.println("=======================================");
-        System.out.println("====================");
+                System.out.println("PRODUCT CONFIG FILES : " + makeConfig.getPhase());
-        makeConfig.printToStream(System.out);
+                System.out.println("=======================================");
+                makeConfig.printToStream(System.out);
+            }
+        }
 
         ConvertMakeToGenericConfig m2g = new ConvertMakeToGenericConfig(mErrors);
-        GenericConfig generic = m2g.convert(makeConfig);
+        GenericConfig generic = m2g.convert(makeConfigs);
+        if (false) {
+            System.out.println("======================");
+            System.out.println("REGENERATED MAKE FILES");
+            System.out.println("======================");
+            MakeWriter.write(System.out, generic, 0);
+        }
 
-        System.out.println("======================");
+        // TODO: Lookup shortened name as used in PRODUCT_NAME / TARGET_PRODUCT
-        System.out.println("REGENERATED MAKE FILES");
+        FlatConfig flat = FlattenConfig.flatten(mErrors, generic);
-        System.out.println("======================");
+        if (false) {
-        MakeWriter.write(System.out, generic, 0);
+            System.out.println("=======================");
+            System.out.println("FLATTENED VARIABLE LIST");
+            System.out.println("=======================");
+            MakeWriter.write(System.out, flat, 0);
+        }
+
+        OutputChecker checker = new OutputChecker(flat);
+        checker.reportErrors(mErrors);
 
         // TODO: Run kati and extract the variables and convert all that into starlark files.
 
@@ -97,7 +112,10 @@ public class Main {
         } finally {
             // Print errors and warnings
             errors.printErrors(System.err);
+            if (errors.hadError()) {
+                exitCode = 1;
+            }
+            System.exit(exitCode);
         }
-        System.exit(exitCode);
     }
 }

tools/product_config/src/com/android/build/config/MakeWriter.java

@@ -30,15 +30,20 @@ public class MakeWriter {
     private final boolean mWriteAnnotations;
 
     public static void write(PrintStream out, GenericConfig config, int flags) {
-        (new MakeWriter(flags)).write(out, config);
+        (new MakeWriter(flags)).writeGeneric(out, config);
     }
 
+    public static void write(PrintStream out, FlatConfig config, int flags) {
+        (new MakeWriter(flags)).writeFlat(out, config);
+    }
+
+
     private MakeWriter(int flags) {
         mWriteHeader = (flags & FLAG_WRITE_HEADER) != 0;
         mWriteAnnotations = (flags & FLAG_WRITE_ANNOTATIONS) != 0;
     }
 
-    private void write(PrintStream out, GenericConfig config) {
+    private void writeGeneric(PrintStream out, GenericConfig config) {
         for (GenericConfig.ConfigFile file: config.getFiles().values()) {
             out.println("---------------------------------------------------------");
             out.println("FILE: " + file.getFilename());
@@ -49,7 +54,7 @@ public class MakeWriter {
         out.println("---------------------------------------------------------");
         out.println("VARIABLES TOUCHED BY MAKE BASED CONFIG:");
         out.println("---------------------------------------------------------");
-        writeStrVars(out, getModifiedVars(config.getInitialVariables(),
+        writeStrVars(out, OutputChecker.getModifiedVars(config.getInitialVariables(),
                                           config.getFinalVariables()), config);
     }
 
@@ -109,28 +114,6 @@ public class MakeWriter {
         out.println();
     }
 
-    private static Map<String, Str> getModifiedVars(Map<String, Str> before,
-            Map<String, Str> after) {
-        final HashMap<String, Str> result = new HashMap();
-        // Entries that were added or changed.
-        for (Map.Entry<String, Str> afterEntry: after.entrySet()) {
-            final String varName = afterEntry.getKey();
-            final Str afterValue = afterEntry.getValue();
-            final Str beforeValue = before.get(varName);
-            if (beforeValue == null || !beforeValue.equals(afterValue)) {
-                result.put(varName, afterValue);
-            }
-        }
-        // removed Entries that were removed, we just treat them as  
-        for (Map.Entry<String, Str> beforeEntry: before.entrySet()) {
-            final String varName = beforeEntry.getKey();
-            if (!after.containsKey(varName)) {
-                result.put(varName, new Str(""));
-            }
-        }
-        return result;
-    }
-
     private static class Var {
         Var(String name, Str val) {
             this.name = name;
@@ -152,4 +135,27 @@ public class MakeWriter {
             out.println(var.val.getPosition() + var.name + " := " + var.val);
         }
     }
+
+    private void writeFlat(PrintStream out, FlatConfig config) {
+        // TODO: Print positions.
+        for (Map.Entry<String, Value> entry: config.getValues().entrySet()) {
+            out.print(entry.getKey());
+            out.print(" := ");
+
+            final Value value = entry.getValue();
+            if (value.getVarType() == VarType.LIST) {
+                final List<Str> list = value.getList();
+                final int size = list.size();
+                for (int i = 0; i < size; i++) {
+                    out.print(list.get(i).toString());
+                    if (i != size - 1) {
+                        out.print(" \\\n        ");
+                    }
+                }
+            } else {
+                out.print(value.getStr().toString());
+            }
+            out.println();
+        }
+    }
 }

tools/product_config/src/com/android/build/config/Options.java

@@ -87,7 +87,7 @@ public class Options {
     }
 
     static class Parser {
-        private class ParseException extends Exception {
+        private static class ParseException extends Exception {
             public ParseException(String message) {
                 super(message);
             }

tools/product_config/src/com/android/build/config/OutputChecker.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Compares the make-based configuration as reported by dumpconfig.mk
+ * with what was computed from the new tool.
+ */
+public class OutputChecker {
+    // Differences that we know about, either know issues to be fixed or intentional.
+    private static final RegexSet IGNORED_VARIABLES = new RegexSet(
+            // TODO: Rewrite the enforce packages exist logic into this tool.
+            "PRODUCT_ENFORCE_PACKAGES_EXIST",
+            "PRODUCT_ENFORCE_PACKAGES_EXIST_ALLOW_LIST",
+            "PRODUCTS\\..*\\.PRODUCT_ENFORCE_PACKAGES_EXIST",
+            "PRODUCTS\\..*\\.PRODUCT_ENFORCE_PACKAGES_EXIST_ALLOW_LIST",
+
+            // This is generated by this tool, but comes later in the make build system.
+            "INTERNAL_PRODUCT");
+
+    private final FlatConfig mConfig;
+    private final TreeMap<String, Variable> mVariables;
+
+    /**
+     * Represents the before and after state of a variable.
+     */
+    public static class Variable {
+        public final String name;
+        public final VarType type;
+        public final Str original;
+        public final Value updated;
+
+        public Variable(String name, VarType type, Str original) {
+            this(name, type, original, null);
+        }
+
+        public Variable(String name, VarType type, Str original, Value updated) {
+            this.name = name;
+            this.type = type;
+            this.original = original;
+            this.updated = updated;
+        }
+
+        /**
+         * Return copy of this Variable with the updated field also set.
+         */
+        public Variable addUpdated(Value updated) {
+            return new Variable(name, type, original, updated);
+        }
+
+        /**
+         * Return whether normalizedOriginal and normalizedUpdate are equal.
+         */
+        public boolean isSame() {
+            final Str normalizedOriginal = Value.normalize(original);
+            final Str normalizedUpdated = Value.normalize(updated);
+            if (normalizedOriginal == null && normalizedUpdated == null) {
+                return true;
+            } else if (normalizedOriginal != null) {
+                return normalizedOriginal.equals(normalizedUpdated);
+            } else {
+                return false;
+            }
+        }
+    }
+
+    /**
+     * Construct OutputChecker with the config it will check.
+     */
+    public OutputChecker(FlatConfig config) {
+        mConfig = config;
+        mVariables = getVariables(config);
+    }
+
+    /**
+     * Add a WARNING_DIFFERENT_FROM_KATI for each of the variables which have changed.
+     */
+    public void reportErrors(Errors errors) {
+        for (Variable var: getDifferences()) {
+            if (IGNORED_VARIABLES.matches(var.name)) {
+                continue;
+            }
+            errors.WARNING_DIFFERENT_FROM_KATI.add("product_config processing differs from"
+                    + " kati processing for " + var.type + " variable " + var.name + ".\n"
+                    + "original: "
+                    + Value.oneLinePerWord(var.original, "<null>") + "\n"
+                    + "updated: "
+                    + Value.oneLinePerWord(var.updated, "<null>"));
+        }
+    }
+
+    /**
+     * Get the Variables that are different between the normalized form of the original
+     * and updated.  If one is null and the other is not, even if one is an empty string,
+     * the values are considered different.
+     */
+    public List<Variable> getDifferences() {
+        final ArrayList<Variable> result = new ArrayList();
+        for (Variable var: mVariables.values()) {
+            if (!var.isSame()) {
+                result.add(var);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Get all of the variables for this config.
+     *
+     * VisibleForTesting
+     */
+    static TreeMap<String, Variable> getVariables(FlatConfig config) {
+        final TreeMap<String, Variable> result = new TreeMap();
+
+        // Add the original values to mAll
+        for (Map.Entry<String, Str> entry: getModifiedVars(config.getInitialVariables(),
+                    config.getFinalVariables()).entrySet()) {
+            final String name = entry.getKey();
+            result.put(name, new Variable(name, config.getVarType(name), entry.getValue()));
+        }
+
+        // Add the updated values to mAll
+        for (Map.Entry<String, Value> entry: config.getValues().entrySet()) {
+            final String name = entry.getKey();
+            final Value value = entry.getValue();
+            Variable var = result.get(name);
+            if (var == null) {
+                result.put(name, new Variable(name, config.getVarType(name), null, value));
+            } else {
+                result.put(name, var.addUpdated(value));
+            }
+        }
+
+        return result;
+    }
+
+    /**
+     * Get the entries that are different in the two maps.
+     */
+    public static Map<String, Str> getModifiedVars(Map<String, Str> before,
+            Map<String, Str> after) {
+        final HashMap<String, Str> result = new HashMap();
+
+        // Entries that were added or changed.
+        for (Map.Entry<String, Str> afterEntry: after.entrySet()) {
+            final String varName = afterEntry.getKey();
+            final Str afterValue = afterEntry.getValue();
+            final Str beforeValue = before.get(varName);
+            if (beforeValue == null || !beforeValue.equals(afterValue)) {
+                result.put(varName, afterValue);
+            }
+        }
+
+        // removed Entries that were removed, we just treat them as empty string
+        for (Map.Entry<String, Str> beforeEntry: before.entrySet()) {
+            final String varName = beforeEntry.getKey();
+            if (!after.containsKey(varName)) {
+                result.put(varName, new Str(""));
+            }
+        }
+
+        return result;
+    }
+}

tools/product_config/src/com/android/build/config/RegexSet.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.util.regex.Pattern;
+
+/**
+ * Returns whether a string matches one of a set of presupplied regexes.
+ */
+public class RegexSet {
+    private final Pattern[] mPatterns;
+
+    public RegexSet(String... patterns) {
+        mPatterns = new Pattern[patterns.length];
+        for (int i = 0; i < patterns.length; i++) {
+            mPatterns[i] = Pattern.compile(patterns[i]);
+        }
+    }
+
+    public boolean matches(String s) {
+        for (Pattern p: mPatterns) {
+            if (p.matcher(s).matches()) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+

tools/product_config/src/com/android/build/config/Str.java

@@ -22,7 +22,7 @@ import java.util.List;
 /**
  * A String and a Position, where it came from in source code.
  */
-public class Str {
+public class Str implements Comparable<Str> {
     private String mValue;
     private Position mPosition;
 
@@ -36,6 +36,10 @@ public class Str {
         mPosition = pos;
     }
 
+    public int length() {
+        return mValue.length();
+    }
+
     @Override
     public String toString() {
         return mValue;
@@ -51,16 +55,11 @@ public class Str {
      */
     @Override
     public boolean equals(Object o) {
-        if (o == null) {
+        if (!(o instanceof Str)) {
-            return false;
-        } else if (o instanceof String) {
-            return mValue.equals(o);
-        } else if (o instanceof Str) {
-            final Str that = (Str)o;
-            return mValue.equals(that.mValue);
-        } else {
             return false;
         }
+        final Str that = (Str)o;
+        return mValue.equals(that.mValue);
     }
 
     @Override
@@ -68,6 +67,11 @@ public class Str {
         return mValue.hashCode();
     }
 
+    @Override
+    public int compareTo(Str that) {
+        return this.mValue.compareTo(that.mValue);
+    }
+
     public static ArrayList<Str> toList(Position pos, List<String> list) {
         final ArrayList<Str> result = new ArrayList(list.size());
         for (String s: list) {

tools/product_config/src/com/android/build/config/Value.java

@@ -0,0 +1,218 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
+/**
+ * Class to hold the two types of variables we support, strings and lists of strings.
+ */
+public class Value {
+    private static final Pattern SPACES = Pattern.compile("\\s+");
+
+    private final VarType mVarType;
+    private final Str mStr;
+    private final ArrayList<Str> mList;
+
+    /**
+     * Construct an appropriately typed empty value.
+     */
+    public Value(VarType varType) {
+        mVarType = varType;
+        if (varType == VarType.LIST) {
+            mStr = null;
+            mList = new ArrayList();
+            mList.add(new Str(""));
+        } else {
+            mStr = new Str("");
+            mList = null;
+        }
+    }
+
+    public Value(VarType varType, Str str) {
+        mVarType = varType;
+        mStr = str;
+        mList = null;
+    }
+
+    public Value(List<Str> list) {
+        mVarType = VarType.LIST;
+        mStr = null;
+        mList = new ArrayList(list);
+    }
+
+    public VarType getVarType() {
+        return mVarType;
+    }
+
+    public Str getStr() {
+        return mStr;
+    }
+
+    public List<Str> getList() {
+        return mList;
+    }
+
+    /**
+     * Normalize a string that is behaving as a list.
+     */
+    public static String normalize(String str) {
+        if (str == null) {
+            return null;
+        }
+        return SPACES.matcher(str.trim()).replaceAll(" ").trim();
+    }
+
+    /**
+     * Normalize a string that is behaving as a list.
+     */
+    public static Str normalize(Str str) {
+        if (str == null) {
+            return null;
+        }
+        return new Str(str.getPosition(), normalize(str.toString()));
+    }
+
+    /**
+     * Normalize a this Value into the same format as normalize(Str).
+     */
+    public static Str normalize(Value val) {
+        if (val == null) {
+            return null;
+        }
+        if (val.mStr != null) {
+            return normalize(val.mStr);
+        }
+
+        if (val.mList.size() == 0) {
+            return new Str("");
+        }
+
+        StringBuilder result = new StringBuilder();
+        final int size = val.mList.size();
+        boolean first = true;
+        for (int i = 0; i < size; i++) {
+            String s = val.mList.get(i).toString().trim();
+            if (s.length() > 0) {
+                if (!first) {
+                    result.append(" ");
+                } else {
+                    first = false;
+                }
+                result.append(s);
+            }
+        }
+
+        // Just use the first item's position.
+        return new Str(val.mList.get(0).getPosition(), result.toString());
+    }
+
+    /**
+     * Put each word in 'str' on its own line in make format. If 'val' is null,
+     * 'nullValue' is returned.
+     */
+    public static String oneLinePerWord(Value val, String nullValue) {
+        if (val == null) {
+            return nullValue;
+        }
+        final String s = normalize(val).toString();
+        final Matcher m = SPACES.matcher(s);
+        final StringBuilder result = new StringBuilder();
+        if (s.length() > 0 && (val.mVarType == VarType.LIST || m.find())) {
+            result.append("\\\n  ");
+        }
+        result.append(m.replaceAll(" \\\\\n  "));
+        return result.toString();
+    }
+
+    /**
+     * Put each word in 'str' on its own line in make format. If 'str' is null,
+     * nullValue is returned.
+     */
+    public static String oneLinePerWord(Str str, String nullValue) {
+        if (str == null) {
+            return nullValue;
+        }
+        final Matcher m = SPACES.matcher(normalize(str.toString()));
+        final StringBuilder result = new StringBuilder();
+        if (m.find()) {
+            result.append("\\\n  ");
+        }
+        result.append(m.replaceAll(" \\\\\n  "));
+        return result.toString();
+    }
+
+    /**
+     * Return a string representing this value with detailed debugging information.
+     */
+    public static String debugString(Value val) {
+        if (val == null) {
+            return "null";
+        }
+
+        final StringBuilder str = new StringBuilder("Value(");
+        if (val.mStr != null) {
+            str.append("mStr=");
+            str.append("\"");
+            str.append(val.mStr.toString());
+            str.append("\"");
+            if (false) {
+                str.append(" (");
+                str.append(val.mStr.getPosition().toString());
+                str.append(")");
+            }
+        }
+        if (val.mList != null) {
+            str.append("mList=");
+            str.append("[");
+            for (Str s: val.mList) {
+                str.append(" \"");
+                str.append(s.toString());
+                if (false) {
+                    str.append("\" (");
+                    str.append(s.getPosition().toString());
+                    str.append(")");
+                } else {
+                    str.append("\"");
+                }
+            }
+            str.append(" ]");
+        }
+        str.append(")");
+        return str.toString();
+    }
+
+    /**
+     * Get the Positions of all of the parts of this Value.
+     */
+    public List<Position> getPositions() {
+        List<Position> result = new ArrayList();
+        if (mStr != null) {
+            result.add(mStr.getPosition());
+        }
+        if (mList != null) {
+            for (Str str: mList) {
+                result.add(str.getPosition());
+            }
+        }
+        return result;
+    }
+}
+

tools/product_config/test.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+#
+# This script runs the full set of tests for product config:
+# 1. Build the product-config tool.
+# 2. Run the unit tests.
+# 3. Run the product config for every product available in the current
+#    source tree, for each of user, userdebug and eng.
+#       - To restrict which products or variants are run, set the
+#         PRODUCTS or VARIANTS environment variables.
+#       - Products for which the make based product config fails are
+#         skipped.
+#
+
+# The PRODUCTS variable is used by the build, and setting it in the environment
+# interferes with that, so unset it.  (That should probably be fixed)
+products=$PRODUCTS
+variants=$VARIANTS
+unset PRODUCTS
+unset VARIANTS
+
+# Don't use lunch from the user's shell
+unset TARGET_PRODUCT
+unset TARGET_BUILD_VARIANT
+
+function die() {
+    format=$1
+    shift
+    printf "$format\nStopping...\n" $@ >&2
+    exit 1;
+}
+
+[[ -f build/make/envsetup.sh ]] || die "Run this script from the root of the tree."
+: ${products:=$(build/soong/soong_ui.bash --dumpvar-mode all_named_products | sed -e "s/ /\n/g" | sort -u )}
+: ${variants:="user userdebug eng"}
+: ${CKATI_BIN:=prebuilts/build-tools/$(build/soong/soong_ui.bash --dumpvar-mode HOST_PREBUILT_TAG)/bin/ckati}
+
+function if_signal_exit() {
+    [[ $1 -lt 128 ]] || exit $1
+}
+
+build/soong/soong_ui.bash --build-mode --all-modules --dir="$(pwd)" product-config-test product-config \
+    || die "Build failed."
+
+echo
+echo Running unit tests
+java -jar out/host/linux-x86/testcases/product-config-test/product-config-test.jar
+unit_tests=$?
+if_signal_exit $unit_tests
+
+failed_baseline_checks=
+for product in $products ; do
+    for variant in $variants ; do
+        echo
+        echo Checking to see if $product-$variant works with make
+        TARGET_PRODUCT=$product TARGET_BUILD_VARIANT=$variant build/soong/soong_ui.bash --dumpvar-mode TARGET_PRODUCT &> /dev/null
+        exit_status=$?
+        if_signal_exit $exit_status
+        if [ $exit_status -ne 0 ] ; then
+            echo Combo fails with make, skipping product-config test run for $product-$variant
+        else
+            echo Running product-config for $product-$variant
+            rm -rf out/config/$product-$variant
+            TARGET_PRODUCT=$product TARGET_BUILD_VARIANT=$variant product-config \
+                            --ckati_bin $CKATI_BIN \
+                            --error 1000
+            exit_status=$?
+            if_signal_exit $exit_status
+            if [ $exit_status -ne 0 ] ; then
+                failed_baseline_checks="$failed_baseline_checks $product-$variant"
+            fi
+        fi
+    done
+done
+
+echo
+echo
+echo "------------------------------"
+echo SUMMARY
+echo "------------------------------"
+
+echo -n "Unit tests        "
+if [ $unit_tests -eq 0 ] ; then echo PASSED ; else echo FAILED ; fi
+
+echo -n "Baseline checks   "
+if [ "$failed_baseline_checks" = "" ] ; then echo PASSED ; else echo FAILED ; fi
+for combo in $failed_baseline_checks ; do
+    echo "                   ... $combo"
+done
+