diff --git a/ci/buildbot.py b/ci/buildbot.py new file mode 100644 index 0000000000..97097be598 --- /dev/null +++ b/ci/buildbot.py @@ -0,0 +1,43 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for interacting with buildbot, with a simulation in a local environment""" + +import os +import sys + +# Check that the script is running from the root of the tree. Prevents subtle +# errors later, and CI always runs from the root of the tree. +if not os.path.exists("build/make/ci/buildbot.py"): + raise Exception("CI script must be run from the root of the tree instead of: " + + os.getcwd()) + +# Check that we are using the hermetic interpreter +if "prebuilts/build-tools/" not in sys.executable: + raise Exception("CI script must be run using the hermetic interpreter from " + + "prebuilts/build-tools instead of: " + sys.executable) + + +def OutDir(): + "Get the out directory. Will create it if needed." + result = os.environ.get("OUT_DIR", "out") + os.makedirs(result, exist_ok=True) + return result + +def DistDir(): + "Get the dist directory. Will create it if needed." + result = os.environ.get("DIST_DIR", os.path.join(OutDir(), "dist")) + os.makedirs(result, exist_ok=True) + return result + diff --git a/ci/dump_product_config b/ci/dump_product_config new file mode 100755 index 0000000000..77b51dd281 --- /dev/null +++ b/ci/dump_product_config @@ -0,0 +1,353 @@ +#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B + +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to collect all of the make variables from all product config combos. + +This script must be run from the root of the source tree. + +See GetArgs() below or run dump_product_config for more information. +""" + +import argparse +import asyncio +import contextlib +import csv +import dataclasses +import json +import multiprocessing +import os +import subprocess +import sys +import time +from typing import List, Dict, Tuple, Optional + +import buildbot + +# We have some BIG variables +csv.field_size_limit(sys.maxsize) + + +class DataclassJSONEncoder(json.JSONEncoder): + """JSONEncoder for our custom types.""" + def default(self, o): + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return super().default(o) + + +def GetProducts(): + """Get the all of the available TARGET_PRODUCT values.""" + try: + stdout = subprocess.check_output(["build/soong/bin/list_products"], text=True) + except subprocess.CalledProcessError: + sys.exit(1) + return [s.strip() for s in stdout.splitlines() if s.strip()] + + +def GetReleases(product): + """For a given product, get the release configs available to it.""" + if True: + # Hard code the list + mainline_products = [ + "module_arm", + "module_x86", + "module_arm64", + "module_riscv64", + "module_x86_64", + "module_arm64only", + "module_x86_64only", + ] + if product in mainline_products: + return ["trunk_staging", "trunk", "mainline"] + else: + return ["trunk_staging", "trunk", "next"] + else: + # Get it from the build system + try: + stdout = subprocess.check_output(["build/soong/bin/list_releases", product], text=True) + except subprocess.CalledProcessError: + sys.exit(1) + return [s.strip() for s in stdout.splitlines() if s.strip()] + + +def GenerateAllLunchTargets(): + """Generate the full list of lunch targets.""" + for product in GetProducts(): + for release in GetReleases(product): + for variant in ["user", "userdebug", "eng"]: + yield (product, release, variant) + + +async def ParallelExec(parallelism, tasks): + ''' + ParallelExec takes a parallelism number, and an iterator of tasks to run. + Then it will run all the tasks, but a maximum of parallelism will be run at + any given time. The tasks must be async functions that accept one argument, + which will be an integer id of the worker that they're running on. + ''' + tasks = iter(tasks) + + overall_start = time.monotonic() + # lists so they can be modified from the inner function + total_duration = [0] + count = [0] + async def dispatch(worker): + while True: + try: + task = next(tasks) + item_start = time.monotonic() + await task(worker) + now = time.monotonic() + item_duration = now - item_start + count[0] += 1 + total_duration[0] += item_duration + sys.stderr.write(f"Timing: Items processed: {count[0]}, Wall time: {now-overall_start:0.1f} sec, Throughput: {(now-overall_start)/count[0]:0.3f} sec per item, Average duration: {total_duration[0]/count[0]:0.1f} sec\n") + except StopIteration: + return + + await asyncio.gather(*[dispatch(worker) for worker in range(parallelism)]) + + +async def DumpProductConfigs(out, generator, out_dir): + """Collects all of the product config data and store it in file.""" + # Write the outer json list by hand so we can stream it + out.write("[") + try: + first_result = [True] # a list so it can be modified from the inner function + def run(lunch): + async def curried(worker): + sys.stderr.write(f"running: {'-'.join(lunch)}\n") + result = await DumpOneProductConfig(lunch, os.path.join(out_dir, f"lunchable_{worker}")) + if first_result[0]: + out.write("\n") + first_result[0] = False + else: + out.write(",\n") + result.dumpToFile(out) + sys.stderr.write(f"finished: {'-'.join(lunch)}\n") + return curried + + await ParallelExec(multiprocessing.cpu_count(), (run(lunch) for lunch in generator)) + finally: + # Close the json regardless of how we exit + out.write("\n]\n") + + +@dataclasses.dataclass(frozen=True) +class Variable: + """A variable name, value and where it was set.""" + name: str + value: str + location: str + + +@dataclasses.dataclass(frozen=True) +class ProductResult: + product: str + release: str + variant: str + board_includes: List[str] + product_includes: Dict[str, List[str]] + product_graph: List[Tuple[str, str]] + board_vars: List[Variable] + product_vars: List[Variable] + + def dumpToFile(self, f): + json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder) + + +@dataclasses.dataclass(frozen=True) +class ProductError: + product: str + release: str + variant: str + error: str + + def dumpToFile(self, f): + json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder) + + +def NormalizeInheritGraph(lists): + """Flatten the inheritance graph to a simple list for easier querying.""" + result = set() + for item in lists: + for i in range(len(item)): + result.add((item[i+1] if i < len(item)-1 else "", item[i])) + return sorted(list(result)) + + +def ParseDump(lunch, filename) -> ProductResult: + """Parses the csv and returns a tuple of the data.""" + def diff(initial, final): + return [after for after in final.values() if + initial.get(after.name, Variable(after.name, "", "")).value != after.value] + product_initial = {} + product_final = {} + board_initial = {} + board_final = {} + inherit_product = [] # The stack of inherit-product calls + product_includes = {} # Other files included by each of the properly imported files + board_includes = [] # Files included by boardconfig + with open(filename) as f: + phase = "" + for line in csv.reader(f): + if line[0] == "phase": + phase = line[1] + elif line[0] == "val": + # TOOD: We should skip these somewhere else. + if line[3].startswith("_ALL_RELEASE_FLAGS"): + continue + if line[3].startswith("PRODUCTS."): + continue + if phase == "PRODUCTS": + if line[2] == "initial": + product_initial[line[3]] = Variable(line[3], line[4], line[5]) + if phase == "PRODUCT-EXPAND": + if line[2] == "final": + product_final[line[3]] = Variable(line[3], line[4], line[5]) + if phase == "BOARD": + if line[2] == "initial": + board_initial[line[3]] = Variable(line[3], line[4], line[5]) + if line[2] == "final": + board_final[line[3]] = Variable(line[3], line[4], line[5]) + elif line[0] == "imported": + imports = [s.strip() for s in line[1].split()] + if imports: + inherit_product.append(imports) + inc = [s.strip() for s in line[2].split()] + for f in inc: + product_includes.setdefault(imports[0], []).append(f) + elif line[0] == "board_config_files": + board_includes += [s.strip() for s in line[1].split()] + return ProductResult( + product = lunch[0], + release = lunch[1], + variant = lunch[2], + product_vars = diff(product_initial, product_final), + board_vars = diff(board_initial, board_final), + product_graph = NormalizeInheritGraph(inherit_product), + product_includes = product_includes, + board_includes = board_includes + ) + + +async def DumpOneProductConfig(lunch, out_dir) -> ProductResult | ProductError: + """Print a single config's lunch info to stdout.""" + product, release, variant = lunch + + dumpconfig_file = os.path.join(out_dir, f"{product}-{release}-{variant}.csv") + + # Run get_build_var to bootstrap soong_ui for this target + env = dict(os.environ) + env["TARGET_PRODUCT"] = product + env["TARGET_RELEASE"] = release + env["TARGET_BUILD_VARIANT"] = variant + env["OUT_DIR"] = out_dir + process = await asyncio.create_subprocess_exec( + "build/soong/bin/get_build_var", + "TARGET_PRODUCT", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env + ) + stdout, _ = await process.communicate() + stdout = stdout.decode() + + if process.returncode != 0: + return ProductError( + product = product, + release = release, + variant = variant, + error = stdout + ) + else: + # Run kati to extract the data + process = await asyncio.create_subprocess_exec( + "prebuilts/build-tools/linux-x86/bin/ckati", + "-f", + "build/make/core/dumpconfig.mk", + f"TARGET_PRODUCT={product}", + f"TARGET_RELEASE={release}", + f"TARGET_BUILD_VARIANT={variant}", + f"DUMPCONFIG_FILE={dumpconfig_file}", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env + ) + stdout, _ = await process.communicate() + if process.returncode != 0: + stdout = stdout.decode() + return ProductError( + product = product, + release = release, + variant = variant, + error = stdout + ) + else: + # Parse and record the output + return ParseDump(lunch, dumpconfig_file) + + +def GetArgs(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Collect all of the make variables from product config.", + epilog="NOTE: This script must be run from the root of the source tree.") + parser.add_argument("--lunch", nargs="*") + parser.add_argument("--dist", action="store_true") + + return parser.parse_args() + + +async def main(): + args = GetArgs() + + out_dir = buildbot.OutDir() + + if args.dist: + cm = open(os.path.join(buildbot.DistDir(), "all_product_config.json"), "w") + else: + cm = contextlib.nullcontext(sys.stdout) + + + with cm as out: + if args.lunch: + lunches = [lunch.split("-") for lunch in args.lunch] + fail = False + for i in range(len(lunches)): + if len(lunches[i]) != 3: + sys.stderr.write(f"Malformed lunch targets: {args.lunch[i]}\n") + fail = True + if fail: + sys.exit(1) + if len(lunches) == 1: + result = await DumpOneProductConfig(lunches[0], out_dir) + result.dumpToFile(out) + out.write("\n") + else: + await DumpProductConfigs(out, lunches, out_dir) + else: + # All configs mode. This will exec single config mode in parallel + # for each lunch combo. Write output to $DIST_DIR. + await DumpProductConfigs(out, GenerateAllLunchTargets(), out_dir) + + +if __name__ == "__main__": + asyncio.run(main()) + + +# vim: set syntax=python ts=4 sw=4 sts=4: +