Add script that dumps all product config varaibles.

Test: build/make/ci/dump_product_config
Change-Id: If6b48fa69b6836ab2c06d6ebe22487001bcd6e77

ci/buildbot.py

@@ -0,0 +1,43 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for interacting with buildbot, with a simulation in a local environment"""
+
+import os
+import sys
+
+# Check that the script is running from the root of the tree. Prevents subtle
+# errors later, and CI always runs from the root of the tree.
+if not os.path.exists("build/make/ci/buildbot.py"):
+    raise Exception("CI script must be run from the root of the tree instead of: "
+                    + os.getcwd())
+
+# Check that we are using the hermetic interpreter
+if "prebuilts/build-tools/" not in sys.executable:
+    raise Exception("CI script must be run using the hermetic interpreter from "
+                    + "prebuilts/build-tools instead of: " + sys.executable)
+
+
+def OutDir():
+    "Get the out directory. Will create it if needed."
+    result = os.environ.get("OUT_DIR", "out")
+    os.makedirs(result, exist_ok=True)
+    return result
+
+def DistDir():
+    "Get the dist directory. Will create it if needed."
+    result = os.environ.get("DIST_DIR", os.path.join(OutDir(), "dist"))
+    os.makedirs(result, exist_ok=True)
+    return result
+

ci/dump_product_config

@@ -0,0 +1,353 @@
+#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B
+
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script to collect all of the make variables from all product config combos.
+
+This script must be run from the root of the source tree.
+
+See GetArgs() below or run dump_product_config for more information.
+"""
+
+import argparse
+import asyncio
+import contextlib
+import csv
+import dataclasses
+import json
+import multiprocessing
+import os
+import subprocess
+import sys
+import time
+from typing import List, Dict, Tuple, Optional
+
+import buildbot
+
+# We have some BIG variables
+csv.field_size_limit(sys.maxsize)
+
+
+class DataclassJSONEncoder(json.JSONEncoder):
+    """JSONEncoder for our custom types."""
+    def default(self, o):
+        if dataclasses.is_dataclass(o):
+            return dataclasses.asdict(o)
+        return super().default(o)
+
+
+def GetProducts():
+    """Get the all of the available TARGET_PRODUCT values."""
+    try:
+        stdout = subprocess.check_output(["build/soong/bin/list_products"], text=True)
+    except subprocess.CalledProcessError:
+        sys.exit(1)
+    return [s.strip() for s in stdout.splitlines() if s.strip()]
+
+
+def GetReleases(product):
+    """For a given product, get the release configs available to it."""
+    if True:
+        # Hard code the list
+        mainline_products = [
+            "module_arm",
+            "module_x86",
+            "module_arm64",
+            "module_riscv64",
+            "module_x86_64",
+            "module_arm64only",
+            "module_x86_64only",
+        ]
+        if product in mainline_products:
+            return ["trunk_staging", "trunk", "mainline"]
+        else:
+            return ["trunk_staging", "trunk", "next"]
+    else:
+        # Get it from the build system
+        try:
+            stdout = subprocess.check_output(["build/soong/bin/list_releases", product], text=True)
+        except subprocess.CalledProcessError:
+            sys.exit(1)
+        return [s.strip() for s in stdout.splitlines() if s.strip()]
+
+
+def GenerateAllLunchTargets():
+    """Generate the full list of lunch targets."""
+    for product in GetProducts():
+        for release in GetReleases(product):
+            for variant in ["user", "userdebug", "eng"]:
+                yield (product, release, variant)
+
+
+async def ParallelExec(parallelism, tasks):
+    '''
+    ParallelExec takes a parallelism number, and an iterator of tasks to run.
+    Then it will run all the tasks, but a maximum of parallelism will be run at
+    any given time. The tasks must be async functions that accept one argument,
+    which will be an integer id of the worker that they're running on.
+    '''
+    tasks = iter(tasks)
+
+    overall_start = time.monotonic()
+    # lists so they can be modified from the inner function
+    total_duration = [0]
+    count = [0]
+    async def dispatch(worker):
+        while True:
+            try:
+                task = next(tasks)
+                item_start = time.monotonic()
+                await task(worker)
+                now = time.monotonic()
+                item_duration = now - item_start
+                count[0] += 1
+                total_duration[0] += item_duration
+                sys.stderr.write(f"Timing: Items processed: {count[0]}, Wall time: {now-overall_start:0.1f} sec, Throughput: {(now-overall_start)/count[0]:0.3f} sec per item, Average duration: {total_duration[0]/count[0]:0.1f} sec\n")
+            except StopIteration:
+                return
+
+    await asyncio.gather(*[dispatch(worker) for worker in range(parallelism)])
+
+
+async def DumpProductConfigs(out, generator, out_dir):
+    """Collects all of the product config data and store it in file."""
+    # Write the outer json list by hand so we can stream it
+    out.write("[")
+    try:
+        first_result = [True] # a list so it can be modified from the inner function
+        def run(lunch):
+            async def curried(worker):
+                sys.stderr.write(f"running: {'-'.join(lunch)}\n")
+                result = await DumpOneProductConfig(lunch, os.path.join(out_dir, f"lunchable_{worker}"))
+                if first_result[0]:
+                    out.write("\n")
+                    first_result[0] = False
+                else:
+                    out.write(",\n")
+                result.dumpToFile(out)
+                sys.stderr.write(f"finished: {'-'.join(lunch)}\n")
+            return curried
+
+        await ParallelExec(multiprocessing.cpu_count(), (run(lunch) for lunch in generator))
+    finally:
+        # Close the json regardless of how we exit
+        out.write("\n]\n")
+
+
+@dataclasses.dataclass(frozen=True)
+class Variable:
+    """A variable name, value and where it was set."""
+    name: str
+    value: str
+    location: str
+
+
+@dataclasses.dataclass(frozen=True)
+class ProductResult:
+    product: str
+    release: str
+    variant: str
+    board_includes: List[str]
+    product_includes: Dict[str, List[str]]
+    product_graph: List[Tuple[str, str]]
+    board_vars: List[Variable]
+    product_vars: List[Variable]
+
+    def dumpToFile(self, f):
+        json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder)
+
+
+@dataclasses.dataclass(frozen=True)
+class ProductError:
+    product: str
+    release: str
+    variant: str
+    error: str
+
+    def dumpToFile(self, f):
+        json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder)
+
+
+def NormalizeInheritGraph(lists):
+    """Flatten the inheritance graph to a simple list for easier querying."""
+    result = set()
+    for item in lists:
+        for i in range(len(item)):
+            result.add((item[i+1] if i < len(item)-1 else "", item[i]))
+    return sorted(list(result))
+
+
+def ParseDump(lunch, filename) -> ProductResult:
+    """Parses the csv and returns a tuple of the data."""
+    def diff(initial, final):
+        return [after for after in final.values() if
+                initial.get(after.name, Variable(after.name, "", "<unset>")).value != after.value]
+    product_initial = {}
+    product_final = {}
+    board_initial = {}
+    board_final = {}
+    inherit_product = [] # The stack of inherit-product calls
+    product_includes = {} # Other files included by each of the properly imported files
+    board_includes = [] # Files included by boardconfig
+    with open(filename) as f:
+        phase = ""
+        for line in csv.reader(f):
+            if line[0] == "phase":
+                phase = line[1]
+            elif line[0] == "val":
+                # TOOD: We should skip these somewhere else.
+                if line[3].startswith("_ALL_RELEASE_FLAGS"):
+                    continue
+                if line[3].startswith("PRODUCTS."):
+                    continue
+                if phase == "PRODUCTS":
+                    if line[2] == "initial":
+                        product_initial[line[3]] = Variable(line[3], line[4], line[5])
+                if phase == "PRODUCT-EXPAND":
+                    if line[2] == "final":
+                        product_final[line[3]] = Variable(line[3], line[4], line[5])
+                if phase == "BOARD":
+                    if line[2] == "initial":
+                        board_initial[line[3]] = Variable(line[3], line[4], line[5])
+                    if line[2] == "final":
+                        board_final[line[3]] = Variable(line[3], line[4], line[5])
+            elif line[0] == "imported":
+                imports = [s.strip() for s in line[1].split()]
+                if imports:
+                    inherit_product.append(imports)
+                    inc = [s.strip() for s in line[2].split()]
+                    for f in inc:
+                        product_includes.setdefault(imports[0], []).append(f)
+            elif line[0] == "board_config_files":
+                board_includes += [s.strip() for s in line[1].split()]
+    return ProductResult(
+        product = lunch[0],
+        release = lunch[1],
+        variant = lunch[2],
+        product_vars = diff(product_initial, product_final),
+        board_vars = diff(board_initial, board_final),
+        product_graph = NormalizeInheritGraph(inherit_product),
+        product_includes = product_includes,
+        board_includes = board_includes
+    )
+
+
+async def DumpOneProductConfig(lunch, out_dir) -> ProductResult | ProductError:
+    """Print a single config's lunch info to stdout."""
+    product, release, variant = lunch
+
+    dumpconfig_file = os.path.join(out_dir, f"{product}-{release}-{variant}.csv")
+
+    # Run get_build_var to bootstrap soong_ui for this target
+    env = dict(os.environ)
+    env["TARGET_PRODUCT"] = product
+    env["TARGET_RELEASE"] = release
+    env["TARGET_BUILD_VARIANT"] = variant
+    env["OUT_DIR"] = out_dir
+    process = await asyncio.create_subprocess_exec(
+        "build/soong/bin/get_build_var",
+        "TARGET_PRODUCT",
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        env=env
+    )
+    stdout, _ = await process.communicate()
+    stdout = stdout.decode()
+
+    if process.returncode != 0:
+        return ProductError(
+            product = product,
+            release = release,
+            variant = variant,
+            error = stdout
+        )
+    else:
+        # Run kati to extract the data
+        process = await asyncio.create_subprocess_exec(
+            "prebuilts/build-tools/linux-x86/bin/ckati",
+            "-f",
+            "build/make/core/dumpconfig.mk",
+            f"TARGET_PRODUCT={product}",
+            f"TARGET_RELEASE={release}",
+            f"TARGET_BUILD_VARIANT={variant}",
+            f"DUMPCONFIG_FILE={dumpconfig_file}",
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            env=env
+        )
+        stdout, _ = await process.communicate()
+        if process.returncode != 0:
+            stdout = stdout.decode()
+            return ProductError(
+                product = product,
+                release = release,
+                variant = variant,
+                error = stdout
+            )
+        else:
+            # Parse and record the output
+            return ParseDump(lunch, dumpconfig_file)
+
+
+def GetArgs():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+            description="Collect all of the make variables from product config.",
+            epilog="NOTE: This script must be run from the root of the source tree.")
+    parser.add_argument("--lunch", nargs="*")
+    parser.add_argument("--dist", action="store_true")
+
+    return parser.parse_args()
+
+
+async def main():
+    args = GetArgs()
+
+    out_dir = buildbot.OutDir()
+
+    if args.dist:
+        cm = open(os.path.join(buildbot.DistDir(), "all_product_config.json"), "w")
+    else:
+        cm = contextlib.nullcontext(sys.stdout)
+
+
+    with cm as out:
+        if args.lunch:
+            lunches = [lunch.split("-") for lunch in args.lunch]
+            fail = False
+            for i in range(len(lunches)):
+                if len(lunches[i]) != 3:
+                    sys.stderr.write(f"Malformed lunch targets: {args.lunch[i]}\n")
+                    fail = True
+            if fail:
+                sys.exit(1)
+            if len(lunches) == 1:
+                result = await DumpOneProductConfig(lunches[0], out_dir)
+                result.dumpToFile(out)
+                out.write("\n")
+            else:
+                await DumpProductConfigs(out, lunches, out_dir)
+        else:
+            # All configs mode. This will exec single config mode in parallel
+            # for each lunch combo. Write output to $DIST_DIR.
+            await DumpProductConfigs(out, GenerateAllLunchTargets(), out_dir)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+
+
+# vim: set syntax=python ts=4 sw=4 sts=4:
+