From c1f344e980fedd538adc385630d7c0388a441d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A1n=20Ca=C3=B1as?= Date: Wed, 24 Apr 2024 16:23:41 -0400 Subject: [PATCH] Add * support products and modules Passing "*" to --products is equivalent to passing the all_named_producs build variable Passing "*" to --modules passes the contents of PRODUCT_OUT/all_modules.txt The total length of the text of all_modules can easily exceed the maximum argument size for the OS. The proper solution is likely to call getconf ARG_MAX, then concatenate the command to be executed to check against the limit. Instead, the modules are processed in batches of 40k modules. As long as module names don't get extremely long, this should keep us under the ARG_MAX limit. In testing, using --modules "*" gets split into two batches. Test: build/make/tools/whichgit --modules "*" Test: build/make/tools/whichgit --modules "*" --unused Test: build/make/tools/whichgit --modules "*" --products "*" Existing use-cases should remain unchanged: TEST: build/make/tools/whichgit --modules framework Change-Id: Ifa947daea2d439df0145e6def92637b67a8b5d22 --- tools/whichgit | 61 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/tools/whichgit b/tools/whichgit index 8cf84f5629..55c8c6fb5a 100755 --- a/tools/whichgit +++ b/tools/whichgit @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import itertools import os import subprocess import sys @@ -10,15 +11,34 @@ def get_build_var(var): check=True, capture_output=True, text=True).stdout.strip() +def get_all_modules(): + product_out = subprocess.run(["build/soong/soong_ui.bash", "--dumpvar-mode", "--abs", "PRODUCT_OUT"], + check=True, capture_output=True, text=True).stdout.strip() + result = subprocess.run(["cat", product_out + "/all_modules.txt"], check=True, capture_output=True, text=True) + return result.stdout.strip().split("\n") + + +def batched(iterable, n): + # introduced in itertools 3.12, could delete once that's universally available + if n < 1: + raise ValueError('n must be at least one') + it = iter(iterable) + while batch := tuple(itertools.islice(it, n)): + yield batch + + def get_sources(modules): - result = subprocess.run(["./prebuilts/build-tools/linux-x86/bin/ninja", "-f", - "out/combined-" + os.environ["TARGET_PRODUCT"] + ".ninja", - "-t", "inputs", "-d", ] + modules, - stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=False, text=True) - if result.returncode != 0: - sys.stderr.write(result.stdout) - sys.exit(1) - return set([f for f in result.stdout.split("\n") if not f.startswith("out/")]) + sources = set() + for module_group in batched(modules, 40_000): + result = subprocess.run(["./prebuilts/build-tools/linux-x86/bin/ninja", "-f", + "out/combined-" + os.environ["TARGET_PRODUCT"] + ".ninja", + "-t", "inputs", "-d", ] + list(module_group), + stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=False, text=True) + if result.returncode != 0: + sys.stderr.write(result.stdout) + sys.exit(1) + sources.update(set([f for f in result.stdout.split("\n") if not f.startswith("out/")])) + return sources def m_nothing(): @@ -57,13 +77,13 @@ def main(argv): # Argument parsing ap = argparse.ArgumentParser(description="List the required git projects for the given modules") ap.add_argument("--products", nargs="*", - help="The TARGET_PRODUCT to check. If not provided just uses whatever has" - + " already been built") + help="One or more TARGET_PRODUCT to check, or \"*\" for all. If not provided" + + "just uses whatever has already been built") ap.add_argument("--variants", nargs="*", help="The TARGET_BUILD_VARIANTS to check. If not provided just uses whatever has" + " already been built, or eng if --products is supplied") ap.add_argument("--modules", nargs="*", - help="The build modules to check, or droid if not supplied") + help="The build modules to check, or \"*\" for all, or droid if not supplied") ap.add_argument("--why", nargs="*", help="Also print the input files used in these projects, or \"*\" for all") ap.add_argument("--unused", help="List the unused git projects for the given modules rather than" @@ -72,22 +92,33 @@ def main(argv): modules = args.modules if args.modules else ["droid"] + match args.products: + case ["*"]: + products = get_build_var("all_named_products").split(" ") + case _: + products = args.products + # Get the list of sources for all of the requested build combos - if not args.products and not args.variants: + if not products and not args.variants: + m_nothing() + if args.modules == ["*"]: + modules = get_all_modules() sources = get_sources(modules) else: - if not args.products: + if not products: sys.stderr.write("Error: --products must be supplied if --variants is supplied") sys.exit(1) sources = set() build_num = 1 - for product in args.products: + for product in products: os.environ["TARGET_PRODUCT"] = product variants = args.variants if args.variants else ["user", "userdebug", "eng"] for variant in variants: - sys.stderr.write(f"Analyzing build {build_num} of {len(args.products)*len(variants)}\r") + sys.stderr.write(f"Analyzing build {build_num} of {len(products)*len(variants)}\r") os.environ["TARGET_BUILD_VARIANT"] = variant m_nothing() + if args.modules == ["*"]: + modules = get_all_modules() sources.update(get_sources(modules)) build_num += 1 sys.stderr.write("\n\n")