diff --git a/core/dumpconfig.mk b/core/dumpconfig.mk new file mode 100644 index 0000000000..cb7fbcb3ca --- /dev/null +++ b/core/dumpconfig.mk @@ -0,0 +1,121 @@ +# Read and dump the product configuration. + +# Called from the product-config tool, not from the main build system. + +# +# Ensure we are being called correctly +# +ifndef KATI + $(warning Kati must be used to call dumpconfig.mk, not make.) + $(error stopping) +endif + +ifdef DEFAULT_GOAL + $(warning Calling dumpconfig.mk from inside the make build system is not) + $(warning supported. It is only meant to be called via kati by product-confing.) + $(error stopping) +endif + +ifndef TARGET_PRODUCT + $(warning dumpconfig.mk requires TARGET_PRODUCT to be set) + $(error stopping) +endif + +ifndef TARGET_BUILD_VARIANT + $(warning dumpconfig.mk requires TARGET_BUILD_VARIANT to be set) + $(error stopping) +endif + +ifneq (build/make/core/config.mk,$(wildcard build/make/core/config.mk)) + $(warning dumpconfig must be called from the root of the source tree) + $(error stopping) +endif + +ifeq (,$(DUMPCONFIG_FILE)) + $(warning dumpconfig requires DUMPCONFIG_FILE to be set) + $(error stopping) +endif + +# Before we do anything else output the format version. +$(file > $(DUMPCONFIG_FILE),dumpconfig_version,1) +$(file >> $(DUMPCONFIG_FILE),dumpconfig_file,$(DUMPCONFIG_FILE)) + +# Default goal for dumpconfig +dumpconfig: + $(file >> $(DUMPCONFIG_FILE),***DONE***) + @echo ***DONE*** + +# TODO(Remove): These need to be set externally +OUT_DIR := out +TMPDIR = /tmp/build-temp +BUILD_DATETIME_FILE := $(OUT_DIR)/build_date.txt + +# Escape quotation marks for CSV, and wraps in quotation marks. +define escape-for-csv +"$(subst ","",$1)" +endef + +# Args: +# $(1): include stack +define dump-import-start +$(eval $(file >> $(DUMPCONFIG_FILE),import,$(strip $(1)))) +endef + +# Args: +# $(1): include stack +define dump-import-done +$(eval $(file >> $(DUMPCONFIG_FILE),imported,$(strip $(1)))) +endef + +# Args: +# $(1): Current file +# $(2): Inherited file +define dump-inherit +$(eval $(file >> $(DUMPCONFIG_FILE),inherit,$(strip $(1)),$(strip $(2)))) +endef + +# Args: +# $(1): Config phase (PRODUCT or DEVICE) +# $(2): Root nodes to import +# $(3): All variable names +# $(4): Single-value variables +define dump-product-var-names +$(eval $(file >> $(DUMPCONFIG_FILE),phase,$(strip $(1)),$(strip $(2)))) \ +$(foreach var,$(3), \ + $(eval $(file >> $(DUMPCONFIG_FILE),var,$(if $(filter $(4),$(var)),single,list),$(var))) \ +) +endef + +define dump-debug +$(eval $(file >> $(DUMPCONFIG_FILE),debug,$(1))) +endef + +# Skip these when dumping. They're not used and they cause a lot of noise in the dump. +DUMPCONFIG_SKIP_VARS := \ + .VARIABLES \ + .KATI_SYMBOLS \ + 1 \ + 2 \ + LOCAL_PATH \ + MAKEFILE_LIST \ + PARENT_PRODUCT_FILES \ + current_mk \ + inherit_var \ + np \ + _node_import_context \ + _included \ + _include_stack \ + _in \ + _nic.% + +# Args: +# $(1): Makefile that was included +# $(2): block (before,import,after) +define dump-config-vals +$(foreach var,$(filter-out $(DUMPCONFIG_SKIP_VARS),$(.KATI_SYMBOLS)),\ + $(eval $(file >> $(DUMPCONFIG_FILE),val,$(call escape-for-csv,$(1)),$(2),$(call escape-for-csv,$(var)),$(call escape-for-csv,$($(var))),$(call escape-for-csv,$(KATI_variable_location $(var))))) \ +) +endef + +include build/make/core/config.mk + diff --git a/core/node_fns.mk b/core/node_fns.mk index b81d60c0f5..878a4dd43e 100644 --- a/core/node_fns.mk +++ b/core/node_fns.mk @@ -195,7 +195,11 @@ define _import-node $(call clear-var-list, $(3)) $(eval LOCAL_PATH := $(patsubst %/,%,$(dir $(2)))) $(eval MAKEFILE_LIST :=) + $(call dump-import-start,$(_include_stack)) + $(call dump-config-vals,$(2),before) $(eval include $(2)) + $(call dump-import-done,$(_include_stack)) + $(call dump-config-vals,$(2),after) $(eval _included := $(filter-out $(2),$(MAKEFILE_LIST))) $(eval MAKEFILE_LIST :=) $(eval LOCAL_PATH :=) @@ -256,6 +260,7 @@ $(if \ $(if $(_include_stack),$(eval $(error ASSERTION FAILED: _include_stack \ should be empty here: $(_include_stack))),) \ $(eval _include_stack := ) \ + $(call dump-product-var-names,$(1),$(2),$(3),$(4)) \ $(call _import-nodes-inner,$(_node_import_context),$(_in),$(3),$(4)) \ $(call move-var-list,$(_node_import_context).$(_in),$(1).$(_in),$(3)) \ $(eval _node_import_context :=) \ diff --git a/core/product.mk b/core/product.mk index ce9bacff04..58902025bf 100644 --- a/core/product.mk +++ b/core/product.mk @@ -460,7 +460,9 @@ define inherit-product $(eval current_mk := $(strip $(word 1,$(_include_stack)))) \ $(eval inherit_var := PRODUCTS.$(current_mk).INHERITS_FROM) \ $(eval $(inherit_var) := $(sort $($(inherit_var)) $(np))) \ - $(eval PARENT_PRODUCT_FILES := $(sort $(PARENT_PRODUCT_FILES) $(current_mk))) + $(eval PARENT_PRODUCT_FILES := $(sort $(PARENT_PRODUCT_FILES) $(current_mk))) \ + $(call dump-inherit,$(strip $(word 1,$(_include_stack))),$(1)) \ + $(call dump-config-vals,$(current_mk),inherit) endef # Specifies a number of path prefixes, relative to PRODUCT_OUT, where the diff --git a/tools/product_config/src/com/android/build/config/ConfigBase.java b/tools/product_config/src/com/android/build/config/ConfigBase.java new file mode 100644 index 0000000000..5ac1fc2bf7 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/ConfigBase.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Common parts between MakeConfig and the to-be-added GenericConfig, BazelConfig and SoongConfig. + */ +public class ConfigBase { + protected String mPhase; + protected List mRootNodes; + + /** + * The variables that are handled specially. + */ + protected final TreeMap mProductVars = new TreeMap(); + + /** + * Whether a product config variable is a list or single-value variable. + */ + public enum VarType { + LIST, + SINGLE, + UNKNOWN // For non-product vars + } + + public void setPhase(String phase) { + mPhase = phase; + } + + public String getPhase() { + return mPhase; + } + + public void setRootNodes(List filenames) { + mRootNodes = new ArrayList(filenames); + } + + public List getRootNodes() { + return mRootNodes; + } + + public void addProductVar(String name, VarType type) { + mProductVars.put(name, type); + } + + public VarType getVarType(String name) { + final VarType t = mProductVars.get(name); + if (t != null) { + return t; + } else { + return VarType.UNKNOWN; + } + } + + public boolean isProductVar(String name) { + return mProductVars.get(name) != null; + } +} diff --git a/tools/product_config/src/com/android/build/config/DumpConfigParser.java b/tools/product_config/src/com/android/build/config/DumpConfigParser.java new file mode 100644 index 0000000000..b954f32758 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/DumpConfigParser.java @@ -0,0 +1,304 @@ + +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.android.build.config; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Parses the output of ckati building build/make/core/dumpconfig.mk. + * + * The format is as follows: + * - All processed lines are colon (':') separated fields. + * - Lines before the dumpconfig_version line are dropped for forward compatibility + * - Lines where the first field is config_var describe variables declared in makefiles + * (implemented by the dump-config-vals macro) + * Field Description + * 0 "config_var" row type + * 1 Product makefile being processed + * 2 The variable name + * 3 The value of the variable + * 4 The location of the variable, as best tracked by kati + */ +public class DumpConfigParser { + private static final boolean DEBUG = true; + + private final Errors mErrors; + private final String mFilename; + private final Reader mReader; + + private final ArrayList mResults = new ArrayList(); + + private static final Pattern LIST_SEPARATOR = Pattern.compile("\\s+"); + + public class BuildPhase { + + } + + /** + * Constructor. + */ + private DumpConfigParser(Errors errors, String filename, Reader reader) { + mErrors = errors; + mFilename = filename; + mReader = reader; + } + + /** + * Parse the text into a list of MakeConfig objects. + */ + public static ArrayList parse(Errors errors, String filename, Reader reader) + throws CsvParser.ParseException, IOException { + DumpConfigParser parser = new DumpConfigParser(errors, filename, reader); + parser.parseImpl(); + return parser.mResults; + } + + /** + * Parse the input. + */ + private void parseImpl() throws CsvParser.ParseException, IOException { + final List lines = CsvParser.parse(mReader); + final int lineCount = lines.size(); + int index = 0; + + int dumpconfigVersion = 0; + + // Ignore lines until until we get a dumpconfig_version line for forward compatibility. + // In a previous life, this loop parsed from all of kati's stdout, not just the file + // that dumpconfig.mk writes, but it's harmless to leave this loop in. It gives us a + // little bit of flexibility which we probably won't need anyway, this tool probably + // won't diverge from dumpconfig.mk anyway. + for (; index < lineCount; index++) { + final CsvParser.Line line = lines.get(index); + final List fields = line.getFields(); + + if (matchLineType(line, "dumpconfig_version", 1)) { + try { + dumpconfigVersion = Integer.parseInt(fields.get(1)); + } catch (NumberFormatException ex) { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "Couldn't parse dumpconfig_version: " + fields.get(1)); + } + break; + } + } + + // If we never saw dumpconfig_version, there's a problem with the command, so stop. + if (dumpconfigVersion == 0) { + mErrors.ERROR_DUMPCONFIG.fatal( + new Position(mFilename), + "Never saw a valid dumpconfig_version line."); + } + + // Any lines before the start signal will be dropped. We create garbage objects + // here to avoid having to check for null everywhere. + MakeConfig makeConfig = new MakeConfig(); + MakeConfig.ConfigFile configFile = new MakeConfig.ConfigFile(""); + MakeConfig.Block block = new MakeConfig.Block(MakeConfig.BlockType.UNSET); + + // Number of "phases" we've seen so far. + for (; index < lineCount; index++) { + final CsvParser.Line line = lines.get(index); + final List fields = line.getFields(); + final String lineType = fields.get(0); + + if (matchLineType(line, "phase", 2)) { + makeConfig = new MakeConfig(); + makeConfig.setPhase(fields.get(1)); + makeConfig.setRootNodes(splitList(fields.get(2))); + mResults.add(makeConfig); + + if (DEBUG) { + System.out.println("PHASE:"); + System.out.println(" " + makeConfig.getPhase()); + System.out.println(" " + makeConfig.getRootNodes()); + } + } else if (matchLineType(line, "var", 2)) { + final MakeConfig.VarType type = "list".equals(fields.get(1)) + ? MakeConfig.VarType.LIST : MakeConfig.VarType.SINGLE; + makeConfig.addProductVar(fields.get(2), type); + + if (DEBUG) { + System.out.println(" VAR: " + type + " " + fields.get(2)); + } + } else if (matchLineType(line, "import", 1)) { + final List importStack = splitList(fields.get(1)); + if (importStack.size() == 0) { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "'import' line with empty include stack."); + continue; + } + + // The beginning of importing a new file. + configFile = new MakeConfig.ConfigFile(importStack.get(0)); + if (makeConfig.addConfigFile(configFile) != null) { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "Duplicate file imported in section: " + configFile.getFilename()); + } + // We expect a Variable block next. + block = new MakeConfig.Block(MakeConfig.BlockType.BEFORE); + configFile.addBlock(block); + + if (DEBUG) { + System.out.println(" IMPORT: " + configFile.getFilename()); + } + } else if (matchLineType(line, "inherit", 2)) { + final String currentFile = fields.get(1); + final String inheritedFile = fields.get(2); + if (!configFile.getFilename().equals(currentFile)) { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "Unexpected current file in 'inherit' line '" + currentFile + + "' while processing '" + configFile.getFilename() + "'"); + continue; + } + + // There is already a file in progress, so add another var block to that. + block = new MakeConfig.Block(MakeConfig.BlockType.INHERIT); + block.setInheritedFile(inheritedFile); + configFile.addBlock(block); + + if (DEBUG) { + System.out.println(" INHERIT: " + inheritedFile); + } + } else if (matchLineType(line, "imported", 1)) { + final List importStack = splitList(fields.get(1)); + if (importStack.size() == 0) { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "'imported' line with empty include stack."); + continue; + } + final String currentFile = importStack.get(0); + if (!configFile.getFilename().equals(currentFile)) { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "Unexpected current file in 'imported' line '" + currentFile + + "' while processing '" + configFile.getFilename() + "'"); + continue; + } + + // There is already a file in progress, so add another var block to that. + // This will be the last one, but will check that after parsing. + block = new MakeConfig.Block(MakeConfig.BlockType.AFTER); + configFile.addBlock(block); + + if (DEBUG) { + System.out.println(" AFTER: " + currentFile); + } + } else if (matchLineType(line, "val", 5)) { + final String productMakefile = fields.get(1); + final MakeConfig.BlockType blockType = parseBlockType(line, fields.get(2)); + final String varName = fields.get(3); + final String varValue = fields.get(4); + final Position pos = Position.parse(fields.get(5)); + + if (!productMakefile.equals(configFile.getFilename())) { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "Mismatched 'val' product makefile." + + " Expected: " + configFile.getFilename() + + " Saw: " + productMakefile); + continue; + } + if (blockType == null) { + continue; + } + if (blockType != block.getBlockType()) { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "Mismatched 'val' block type." + + " Expected: " + block.getBlockType() + + " Saw: " + blockType); + } + + // Add the value to the block in progress + block.addValue(varName, new Str(pos, varValue)); + } else { + if (DEBUG) { + System.out.print("# "); + for (int d = 0; d < fields.size(); d++) { + System.out.print(fields.get(d)); + if (d != fields.size() - 1) { + System.out.print(","); + } + } + System.out.println(); + } + } + } + } + + /** + * Return true if the line type matches 'lineType' and there are at least 'fieldCount' + * fields (not including the first field which is the line type). + */ + private boolean matchLineType(CsvParser.Line line, String lineType, int fieldCount) { + final List fields = line.getFields(); + if (!lineType.equals(fields.get(0))) { + return false; + } + if (fields.size() < (fieldCount + 1)) { + mErrors.WARNING_DUMPCONFIG.add(new Position(mFilename, line.getLine()), + fields.get(0) + " line has " + fields.size() + " fields. Expected at least " + + (fieldCount + 1) + " fields."); + return false; + } + return true; + } + + /** + * Split a string with space separated items (i.e. the make list format) into a List. + */ + private static List splitList(String text) { + // Arrays.asList returns a fixed-length List, so we copy it into an ArrayList to not + // propagate that surprise detail downstream. + return new ArrayList(Arrays.asList(LIST_SEPARATOR.split(text.trim()))); + } + + /** + * Parse a BockType or issue a warning if it can't be parsed. + */ + private MakeConfig.BlockType parseBlockType(CsvParser.Line line, String text) { + if ("before".equals(text)) { + return MakeConfig.BlockType.BEFORE; + } else if ("inherit".equals(text)) { + return MakeConfig.BlockType.INHERIT; + } else if ("after".equals(text)) { + return MakeConfig.BlockType.AFTER; + } else { + mErrors.WARNING_DUMPCONFIG.add( + new Position(mFilename, line.getLine()), + "Invalid block type: " + text); + return null; + } + } +} diff --git a/tools/product_config/src/com/android/build/config/Errors.java b/tools/product_config/src/com/android/build/config/Errors.java index 5424338fba..9290b72d7d 100644 --- a/tools/product_config/src/com/android/build/config/Errors.java +++ b/tools/product_config/src/com/android/build/config/Errors.java @@ -45,4 +45,11 @@ public class Errors extends ErrorReporter { public final Category ERROR_KATI = new Category(3, false, Level.ERROR, "Error executing or reading from Kati."); + + public final Category WARNING_DUMPCONFIG = new Category(4, true, Level.WARNING, + "Anomaly parsing the output of kati and dumpconfig.mk."); + + public final Category ERROR_DUMPCONFIG = new Category(5, false, Level.ERROR, + "Error parsing the output of kati and dumpconfig.mk."); + } diff --git a/tools/product_config/src/com/android/build/config/Kati.java b/tools/product_config/src/com/android/build/config/Kati.java new file mode 100644 index 0000000000..026ddb5a59 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/Kati.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.util.List; + +/** + * Wrapper for invoking kati. + */ +public interface Kati { + public MakeConfig loadProductConfig(); +} diff --git a/tools/product_config/src/com/android/build/config/KatiImpl.java b/tools/product_config/src/com/android/build/config/KatiImpl.java new file mode 100644 index 0000000000..feb374cb74 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/KatiImpl.java @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class KatiImpl implements Kati { + // Subdirectory inside out for config stuff. + private static final String CONFIG_SUBDIR = "config"; + + private final Errors mErrors; + private final Options mOptions; + private final KatiCommand mCommand; + + // TODO: Do we need to consider the whole or a greater subset of the + // environment (or a hash of it?). In theory product-variant is enough, but we know + // people use stuff from the environment, even though we're trying to get rid of that. + private String getWorkDirPath() { + return Paths.get(mOptions.getOutDir(), CONFIG_SUBDIR, + mOptions.getProduct() + '-' + mOptions.getVariant()).toString(); + } + + private String getDumpConfigCsvPath() { + return Paths.get(getWorkDirPath(), "dumpconfig.csv").toString(); + } + + public KatiImpl(Errors errors, Options options) { + this(errors, options, new KatiCommandImpl(errors, options)); + } + + // VisibleForTesting + public KatiImpl(Errors errors, Options options, KatiCommand command) { + mErrors = errors; + mOptions = options; + mCommand = command; + } + + @Override + public MakeConfig loadProductConfig() { + final String csvPath = getDumpConfigCsvPath(); + try { + File workDir = new File(getWorkDirPath()); + + if (!workDir.mkdirs()) { + mErrors.ERROR_KATI.add("Unable to create directory: " + workDir); + return null; // TODO: throw exception? + } + + System.out.println("running kati"); + String out = mCommand.run(new String[] { + "-f", "build/make/core/dumpconfig.mk", + "DUMPCONFIG_FILE=" + csvPath + }); + + if (!out.contains("***DONE***")) { + mErrors.ERROR_KATI.add( + "Unknown error with kati, but it didn't print ***DONE*** message"); + return null; // TODO: throw exception? + } + // TODO: Check that output was good. + } catch (KatiCommand.KatiException ex) { + mErrors.ERROR_KATI.add("Error running kati:\n" + ex.getStderr()); + return null; + } + + if (!(new File(csvPath)).canRead()) { + mErrors.ERROR_KATI.add("Kati ran but did not create " + csvPath); + return null; + } + + try (FileReader reader = new FileReader(csvPath)) { + System.out.println("csvPath=" + csvPath); + List makeConfigs = DumpConfigParser.parse(mErrors, csvPath, reader); + + if (makeConfigs.size() == 0) { + // TODO: Issue error? + return null; + } + + // TODO: There are multiple passes. That should be cleaned up in the make + // build system, but for now, the first one is the one we want. + return makeConfigs.get(0); + } catch (CsvParser.ParseException ex) { + mErrors.ERROR_KATI.add(new Position(csvPath, ex.getLine()), + "Unable to parse output of dumpconfig.mk: " + ex.getMessage()); + return null; // TODO: throw exception? + } catch (IOException ex) { + System.out.println(ex); + mErrors.ERROR_KATI.add("Unable to read " + csvPath + ": " + ex.getMessage()); + return null; // TODO: throw exception? + } + } +} diff --git a/tools/product_config/src/com/android/build/config/Main.java b/tools/product_config/src/com/android/build/config/Main.java index e577dfef26..81d9e7b4fb 100644 --- a/tools/product_config/src/com/android/build/config/Main.java +++ b/tools/product_config/src/com/android/build/config/Main.java @@ -16,6 +16,9 @@ package com.android.build.config; +import java.util.List; +import java.util.Map; + public class Main { private final Errors mErrors; private final Options mOptions; @@ -31,6 +34,17 @@ public class Main { // TODO: Check the build environment to make sure we're running in a real // build environment, e.g. actually inside a source tree, with TARGET_PRODUCT // and TARGET_BUILD_VARIANT defined, etc. + Kati kati = new KatiImpl(mErrors, mOptions); + MakeConfig makeConfig = kati.loadProductConfig(); + if (makeConfig == null || mErrors.hadError()) { + return; + } + + System.out.println(); + System.out.println("===================="); + System.out.println("PRODUCT CONFIG FILES"); + System.out.println("===================="); + makeConfig.printToStream(System.out); // TODO: Run kati and extract the variables and convert all that into starlark files. @@ -38,8 +52,6 @@ public class Main { // TODO: Get the variables that were defined in starlark and use that to write // out the make, soong and bazel input files. - mErrors.ERROR_COMMAND_LINE.add("asdf"); - throw new RuntimeException("poop"); } public static void main(String[] args) { diff --git a/tools/product_config/src/com/android/build/config/MakeConfig.java b/tools/product_config/src/com/android/build/config/MakeConfig.java new file mode 100644 index 0000000000..300b655035 --- /dev/null +++ b/tools/product_config/src/com/android/build/config/MakeConfig.java @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class MakeConfig extends ConfigBase { + /** + * The config files that were imported in this config pass. + */ + protected final ArrayList mConfigFiles = new ArrayList(); + + public enum BlockType { + UNSET, + BEFORE, + INHERIT, + AFTER + } + + public static class ConfigFile { + /** + * The name of the file, relative to the tree root. + */ + private final String mFilename; + + /** + * Sections of variable definitions and import statements. Product config + * files will always have at least one block. + */ + private final ArrayList mBlocks = new ArrayList(); + + public ConfigFile(String filename) { + mFilename = filename; + } + + public String getFilename() { + return mFilename; + } + + public void addBlock(Block block) { + mBlocks.add(block); + } + + public ArrayList getBlocks() { + return mBlocks; + } + } + + /** + * A set of variables that were defined. + */ + public static class Block { + private final BlockType mBlockType; + private final TreeMap mValues = new TreeMap(); + private String mInheritedFile; + + public Block(BlockType blockType) { + mBlockType = blockType; + } + + public BlockType getBlockType() { + return mBlockType; + } + + public void addValue(String varName, Str varValue) { + mValues.put(varName, varValue); + } + + public TreeMap getValues() { + return mValues; + } + + public void setInheritedFile(String filename) { + mInheritedFile = filename; + } + + public String getInheritedFile() { + return mInheritedFile; + } + } + + /** + * Adds the given config file. Returns any one previously added, or null. + */ + public ConfigFile addConfigFile(ConfigFile file) { + ConfigFile prev = null; + for (ConfigFile f: mConfigFiles) { + if (f.getFilename().equals(file.getFilename())) { + prev = f; + break; + } + } + mConfigFiles.add(file); + return prev; + } + + public List getConfigFiles() { + return mConfigFiles; + } + + public void printToStream(PrintStream out) { + out.println("MakeConfig {"); + out.println(" phase: " + mPhase); + out.println(" rootNodes: " + mRootNodes); + out.print(" singleVars: [ "); + for (Map.Entry entry: mProductVars.entrySet()) { + if (entry.getValue() == VarType.SINGLE) { + out.print(entry.getKey()); + out.print(" "); + } + } + out.println("]"); + out.print(" listVars: [ "); + for (Map.Entry entry: mProductVars.entrySet()) { + if (entry.getValue() == VarType.LIST) { + out.print(entry.getKey()); + out.print(" "); + } + } + out.println("]"); + out.println(" configFiles: ["); + for (final ConfigFile configFile: mConfigFiles) { + out.println(" ConfigFile {"); + out.println(" filename: " + configFile.getFilename()); + out.println(" blocks: ["); + for (Block block: configFile.getBlocks()) { + out.println(" Block {"); + out.println(" type: " + block.getBlockType()); + if (block.getBlockType() == BlockType.INHERIT) { + out.println(" inherited: " + block.getInheritedFile()); + } + out.println(" values: {"); + for (Map.Entry var: block.getValues().entrySet()) { + if (!var.getKey().equals("PRODUCT_PACKAGES")) { + continue; + } + out.println(" " + var.getKey() + ": " + var.getValue()); + } + out.println(" }"); + out.println(" }"); + } + out.println(" ]"); + out.println(" }"); + } + out.println(" ] // configFiles"); + out.println("} // MakeConfig"); + } +} diff --git a/tools/product_config/src/com/android/build/config/Position.java b/tools/product_config/src/com/android/build/config/Position.java index 795394271e..266021d8bf 100644 --- a/tools/product_config/src/com/android/build/config/Position.java +++ b/tools/product_config/src/com/android/build/config/Position.java @@ -16,6 +16,9 @@ package com.android.build.config; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * Position in a source file. */ @@ -25,6 +28,9 @@ public class Position implements Comparable { */ public static final int NO_LINE = -1; + private static final Pattern REGEX = Pattern.compile("([^:]*)(?::(\\d)*)?:?\\s*"); + public static final String UNKNOWN = ""; + private final String mFile; private final int mLine; @@ -63,12 +69,39 @@ public class Position implements Comparable { return mLine; } + /** + * Return a Position object from a string containing :, or the default + * Position(null, NO_LINE) if the string can't be parsed. + */ + public static Position parse(String str) { + final Matcher m = REGEX.matcher(str); + if (!m.matches()) { + return new Position(); + } + String filename = m.group(1); + if (filename.length() == 0 || UNKNOWN.equals(filename)) { + filename = null; + } + String lineString = m.group(2); + int line; + if (lineString == null || lineString.length() == 0) { + line = NO_LINE; + } else { + try { + line = Integer.parseInt(lineString); + } catch (NumberFormatException ex) { + line = NO_LINE; + } + } + return new Position(filename, line); + } + @Override public String toString() { if (mFile == null && mLine == NO_LINE) { return ""; } else if (mFile == null && mLine != NO_LINE) { - return ":" + mLine + ": "; + return UNKNOWN + ":" + mLine + ": "; } else if (mFile != null && mLine == NO_LINE) { return mFile + ": "; } else { // if (mFile != null && mLine != NO_LINE) diff --git a/tools/product_config/src/com/android/build/config/Str.java b/tools/product_config/src/com/android/build/config/Str.java new file mode 100644 index 0000000000..7dbe2e5d6b --- /dev/null +++ b/tools/product_config/src/com/android/build/config/Str.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +/** + * A String and a Position, where it came from in source code. + */ +public class Str { + private String mValue; + private Position mPosition; + + public Str(String s) { + mValue = s; + mPosition = new Position(); + } + + public Str(Position pos, String s) { + mValue = s; + mPosition = pos; + } + + @Override + public String toString() { + return mValue; + } + + public Position getPosition() { + return mPosition; + } + + /** + * Str is equal if the string value is equal, regardless of whether the position + * is the same. + */ + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } else if (o instanceof String) { + return mValue.equals(o); + } else if (o instanceof Str) { + final Str that = (Str)o; + return mValue.equals(that.mValue); + } else { + return false; + } + } + + @Override + public int hashCode() { + return mValue.hashCode(); + } +} diff --git a/tools/product_config/test/com/android/build/config/PositionTest.java b/tools/product_config/test/com/android/build/config/PositionTest.java new file mode 100644 index 0000000000..82b5dd4604 --- /dev/null +++ b/tools/product_config/test/com/android/build/config/PositionTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.build.config; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; + +public class PositionTest { + + @Test + public void testParseEmpty() { + final Position pos = Position.parse(""); + + Assert.assertEquals(null, pos.getFile()); + Assert.assertEquals(Position.NO_LINE, pos.getLine()); + } + + @Test + public void testParseOnlyFile() { + final Position pos = Position.parse("asdf"); + + Assert.assertEquals("asdf", pos.getFile()); + Assert.assertEquals(Position.NO_LINE, pos.getLine()); + } + + @Test + public void testParseBoth() { + final Position pos = Position.parse("asdf:1"); + + Assert.assertEquals("asdf", pos.getFile()); + Assert.assertEquals(1, pos.getLine()); + } + + @Test + public void testParseEndsWithColon() { + final Position pos = Position.parse("asdf:"); + + Assert.assertEquals("asdf", pos.getFile()); + Assert.assertEquals(Position.NO_LINE, pos.getLine()); + } + + @Test + public void testParseEndsWithSpace() { + final Position pos = Position.parse("asdf: "); + + Assert.assertEquals("asdf", pos.getFile()); + Assert.assertEquals(Position.NO_LINE, pos.getLine()); + } + + +} + diff --git a/tools/product_config/test/com/android/build/config/TestRunner.java b/tools/product_config/test/com/android/build/config/TestRunner.java index 79d87d50c7..546518fbe2 100644 --- a/tools/product_config/test/com/android/build/config/TestRunner.java +++ b/tools/product_config/test/com/android/build/config/TestRunner.java @@ -41,7 +41,8 @@ public class TestRunner { }); Result result = junit.run(CsvParserTest.class, ErrorReporterTest.class, - OptionsTest.class); + OptionsTest.class, + PositionTest.class); if (!result.wasSuccessful()) { System.out.println("\n*** FAILED ***"); }