diff --git a/envsetup.sh b/envsetup.sh index b49bb8a21e..b079d4199b 100644 --- a/envsetup.sh +++ b/envsetup.sh @@ -456,7 +456,7 @@ function multitree_lunch() if $(echo "$1" | grep -q '^-') ; then # Calls starting with a -- argument are passed directly and the function # returns with the lunch.py exit code. - build/make/orchestrator/core/lunch.py "$@" + build/build/make/orchestrator/core/lunch.py "$@" code=$? if [[ $code -eq 2 ]] ; then echo 1>&2 @@ -467,7 +467,7 @@ function multitree_lunch() fi else # All other calls go through the --lunch variant of lunch.py - results=($(build/make/orchestrator/core/lunch.py --lunch "$@")) + results=($(build/build/make/orchestrator/core/lunch.py --lunch "$@")) code=$? if [[ $code -eq 2 ]] ; then echo 1>&2 @@ -944,6 +944,34 @@ function gettop fi } +# TODO: Merge into gettop as part of launching multitree +function multitree_gettop +{ + local TOPFILE=build/build/make/core/envsetup.mk + if [ -n "$TOP" -a -f "$TOP/$TOPFILE" ] ; then + # The following circumlocution ensures we remove symlinks from TOP. + (cd "$TOP"; PWD= /bin/pwd) + else + if [ -f $TOPFILE ] ; then + # The following circumlocution (repeated below as well) ensures + # that we record the true directory name and not one that is + # faked up with symlink names. + PWD= /bin/pwd + else + local HERE=$PWD + local T= + while [ \( ! \( -f $TOPFILE \) \) -a \( "$PWD" != "/" \) ]; do + \cd .. + T=`PWD= /bin/pwd -P` + done + \cd "$HERE" + if [ -f "$T/$TOPFILE" ]; then + echo "$T" + fi + fi + fi +} + function croot() { local T=$(gettop) @@ -1826,6 +1854,21 @@ function make() _wrap_build $(get_make_command "$@") "$@" } +function _multitree_lunch_error() +{ + >&2 echo "Couldn't locate the top of the tree. Please run \'source build/envsetup.sh\' and multitree_lunch from the root of your workspace." +} + +function multitree_build() +{ + if T="$(multitree_gettop)"; then + "$T/build/build/orchestrator/core/orchestrator.py" "$@" + else + _multitree_lunch_error + return 1 + fi +} + function provision() { if [ ! "$ANDROID_PRODUCT_OUT" ]; then diff --git a/orchestrator/README b/orchestrator/README new file mode 100644 index 0000000000..ce6f5c3e19 --- /dev/null +++ b/orchestrator/README @@ -0,0 +1,7 @@ +DEMO + +from the root of the workspace + +ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py master/.inner_build +ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py sc-mainline-prod/.inner_build + diff --git a/orchestrator/core/api_assembly.py b/orchestrator/core/api_assembly.py new file mode 100644 index 0000000000..d87a83de3a --- /dev/null +++ b/orchestrator/core/api_assembly.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +def assemble_apis(inner_trees): + + # Find all of the contributions from the inner tree + contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree) + + # Load and validate the contribution files + # TODO: Check timestamps and skip unnecessary work + contributions = [] + for tree_key, filenames in contribution_files_dict.items(): + for filename in filenames: + contribution_data = load_contribution_file(filename) + if not contribution_data: + continue + # TODO: Validate the configs, especially that the domains match what we asked for + # from the lunch config. + contributions.append(contribution_data) + + # Group contributions by language and API surface + stub_libraries = collate_contributions(contributions) + + # Iterate through all of the stub libraries and generate rules to assemble them + # and Android.bp/BUILD files to make those available to inner trees. + # TODO: Parallelize? Skip unnecessary work? + ninja_file = NinjaFile() # TODO: parameters? + build_file = BuildFile() # TODO: parameters? + for stub_library in stub_libraries: + STUB_LANGUAGE_HANDLERS[stub_library.language](ninja_file, build_file, stub_library) + + # TODO: Handle host_executables separately or as a StubLibrary language? + + +def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie): + "Scan an inner_tree's out dir for the api contribution files." + directory = inner_tree.out.api_contributions_dir() + result = [] + with os.scandir(directory) as it: + for dirent in it: + if not dirent.is_file(): + break + if dirent.name.endswith(".json"): + result.append(os.path.join(directory, dirent.name)) + return result + + +def load_contribution_file(filename): + "Load and return the API contribution at filename. On error report error and return None." + with open(filename) as f: + try: + return json.load(f) + except json.decoder.JSONDecodeError as ex: + # TODO: Error reporting + raise ex + + +class StubLibraryContribution(object): + def __init__(self, api_domain, library_contribution): + self.api_domain = api_domain + self.library_contribution = library_contribution + + +class StubLibrary(object): + def __init__(self, language, api_surface, api_surface_version, name): + self.language = language + self.api_surface = api_surface + self.api_surface_version = api_surface_version + self.name = name + self.contributions = [] + + def add_contribution(self, contrib): + self.contributions.append(contrib) + + +def collate_contributions(contributions): + """Take the list of parsed API contribution files, and group targets by API Surface, version, + language and library name, and return a StubLibrary object for each of those. + """ + grouped = {} + for contribution in contributions: + for language in STUB_LANGUAGE_HANDLERS.keys(): + for library in contribution.get(language, []): + key = (language, contribution["name"], contribution["version"], library["name"]) + stub_library = grouped.get(key) + if not stub_library: + stub_library = StubLibrary(language, contribution["name"], + contribution["version"], library["name"]) + grouped[key] = stub_library + stub_library.add_contribution(StubLibraryContribution( + contribution["api_domain"], library)) + return list(grouped.values()) + + +def assemble_cc_api_library(ninja_file, build_file, stub_library): + print("assembling cc_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version, + stub_library.name)) + for contrib in stub_library.contributions: + print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"])) + # TODO: Implement me + + +def assemble_java_api_library(ninja_file, build_file, stub_library): + print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version, + stub_library.name)) + for contrib in stub_library.contributions: + print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"])) + # TODO: Implement me + + +def assemble_resource_api_library(ninja_file, build_file, stub_library): + print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version, + stub_library.name)) + for contrib in stub_library.contributions: + print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"])) + # TODO: Implement me + + +STUB_LANGUAGE_HANDLERS = { + "cc_libraries": assemble_cc_api_library, + "java_libraries": assemble_java_api_library, + "resource_libraries": assemble_resource_api_library, +} + + +class NinjaFile(object): + "Generator for build actions and dependencies." + pass + + +class BuildFile(object): + "Abstract generator for Android.bp files and BUILD files." + pass + + diff --git a/orchestrator/core/api_domain.py b/orchestrator/core/api_domain.py new file mode 100644 index 0000000000..bb7306c34a --- /dev/null +++ b/orchestrator/core/api_domain.py @@ -0,0 +1,28 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ApiDomain(object): + def __init__(self, name, tree, product): + # Product will be null for modules + self.name = name + self.tree = tree + self.product = product + + def __str__(self): + return "ApiDomain(name=\"%s\" tree.root=\"%s\" product=%s)" % ( + self.name, self.tree.root, + "None" if self.product is None else "\"%s\"" % self.product) + diff --git a/orchestrator/core/api_export.py b/orchestrator/core/api_export.py new file mode 100644 index 0000000000..2f26b02b61 --- /dev/null +++ b/orchestrator/core/api_export.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def export_apis_from_tree(tree_key, inner_tree, cookie): + inner_tree.invoke(["export_api_contributions"]) + + diff --git a/orchestrator/core/inner_tree.py b/orchestrator/core/inner_tree.py new file mode 100644 index 0000000000..cdb0d857ac --- /dev/null +++ b/orchestrator/core/inner_tree.py @@ -0,0 +1,155 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +import textwrap + +class InnerTreeKey(object): + """Trees are identified uniquely by their root and the TARGET_PRODUCT they will use to build. + If a single tree uses two different prdoucts, then we won't make assumptions about + them sharing _anything_. + TODO: This is true for soong. It's more likely that bazel could do analysis for two + products at the same time in a single tree, so there's an optimization there to do + eventually.""" + def __init__(self, root, product): + self.root = root + self.product = product + + def __str__(self): + return "TreeKey(root=%s product=%s)" % (enquote(self.root), enquote(self.product)) + + def __hash__(self): + return hash((self.root, self.product)) + + def __eq__(self, other): + return (self.root == other.root and self.product == other.product) + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + return (self.root, self.product) < (other.root, other.product) + + def __le__(self, other): + return (self.root, self.product) <= (other.root, other.product) + + def __gt__(self, other): + return (self.root, self.product) > (other.root, other.product) + + def __ge__(self, other): + return (self.root, self.product) >= (other.root, other.product) + + +class InnerTree(object): + def __init__(self, root, product): + """Initialize with the inner tree root (relative to the workspace root)""" + self.root = root + self.product = product + self.domains = {} + # TODO: Base directory on OUT_DIR + self.out = OutDirLayout(os.path.join("out", "trees", root)) + + def __str__(self): + return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root), + enquote(self.product), + " ".join([enquote(d) for d in sorted(self.domains.keys())])) + + def invoke(self, args): + """Call the inner tree command for this inner tree. Exits on failure.""" + # TODO: Build time tracing + + # Validate that there is a .inner_build command to run at the root of the tree + # so we can print a good error message + inner_build_tool = os.path.join(self.root, ".inner_build") + if not os.access(inner_build_tool, os.X_OK): + sys.stderr.write(("Unable to execute %s. Is there an inner tree or lunch combo" + + " misconfiguration?\n") % inner_build_tool) + sys.exit(1) + + # TODO: This is where we should set up the shared trees + + # Build the command + cmd = [inner_build_tool, "--out_dir", self.out.root()] + for domain_name in sorted(self.domains.keys()): + cmd.append("--api_domain") + cmd.append(domain_name) + cmd += args + + # Run the command + process = subprocess.run(cmd, shell=False) + + # TODO: Probably want better handling of inner tree failures + if process.returncode: + sys.stderr.write("Build error in inner tree: %s\nstopping multitree build.\n" + % self.root) + sys.exit(1) + + +class InnerTrees(object): + def __init__(self, trees, domains): + self.trees = trees + self.domains = domains + + def __str__(self): + "Return a debugging dump of this object" + return textwrap.dedent("""\ + InnerTrees { + trees: [ + %(trees)s + ] + domains: [ + %(domains)s + ] + }""" % { + "trees": "\n ".join(sorted([str(t) for t in self.trees.values()])), + "domains": "\n ".join(sorted([str(d) for d in self.domains.values()])), + }) + + + def for_each_tree(self, func, cookie=None): + """Call func for each of the inner trees once for each product that will be built in it. + + The calls will be in a stable order. + + Return a map of the InnerTreeKey to any results returned from func(). + """ + result = {} + for key in sorted(self.trees.keys()): + result[key] = func(key, self.trees[key], cookie) + return result + + +class OutDirLayout(object): + def __init__(self, root): + "Initialize with the root of the OUT_DIR for the inner tree." + self._root = root + + def root(self): + return self._root + + def tree_info_file(self): + return os.path.join(self._root, "tree_info.json") + + def api_contributions_dir(self): + return os.path.join(self._root, "api_contributions") + + +def enquote(s): + return "None" if s is None else "\"%s\"" % s + + diff --git a/orchestrator/core/interrogate.py b/orchestrator/core/interrogate.py new file mode 100644 index 0000000000..9fe769e5ef --- /dev/null +++ b/orchestrator/core/interrogate.py @@ -0,0 +1,29 @@ +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +def interrogate_tree(tree_key, inner_tree, cookie): + inner_tree.invoke(["describe"]) + + info_json_filename = inner_tree.out.tree_info_file() + + # TODO: Error handling + with open(info_json_filename) as f: + info_json = json.load(f) + + # TODO: Check orchestrator protocol + diff --git a/orchestrator/core/lunch.py b/orchestrator/core/lunch.py index 35dac73341..a6484787e1 100755 --- a/orchestrator/core/lunch.py +++ b/orchestrator/core/lunch.py @@ -24,8 +24,10 @@ EXIT_STATUS_OK = 0 EXIT_STATUS_ERROR = 1 EXIT_STATUS_NEED_HELP = 2 -def FindDirs(path, name, ttl=6): - """Search at most ttl directories deep inside path for a directory called name.""" + +def find_dirs(path, name, ttl=6): + """Search at most ttl directories deep inside path for a directory called name + and yield directories that match.""" # The dance with subdirs is so that we recurse in sorted order. subdirs = [] with os.scandir(path) as it: @@ -40,10 +42,10 @@ def FindDirs(path, name, ttl=6): # Consume filesystem errors, e.g. too many links, permission etc. pass for subdir in subdirs: - yield from FindDirs(os.path.join(path, subdir), name, ttl-1) + yield from find_dirs(os.path.join(path, subdir), name, ttl-1) -def WalkPaths(path, matcher, ttl=10): +def walk_paths(path, matcher, ttl=10): """Do a traversal of all files under path yielding each file that matches matcher.""" # First look for files, then recurse into directories as needed. @@ -62,22 +64,22 @@ def WalkPaths(path, matcher, ttl=10): # Consume filesystem errors, e.g. too many links, permission etc. pass for subdir in sorted(subdirs): - yield from WalkPaths(os.path.join(path, subdir), matcher, ttl-1) + yield from walk_paths(os.path.join(path, subdir), matcher, ttl-1) -def FindFile(path, filename): +def find_file(path, filename): """Return a file called filename inside path, no more than ttl levels deep. Directories are searched alphabetically. """ - for f in WalkPaths(path, lambda x: x == filename): + for f in walk_paths(path, lambda x: x == filename): return f -def FindConfigDirs(workspace_root): +def find_config_dirs(workspace_root): """Find the configuration files in the well known locations inside workspace_root - /build/orchestrator/multitree_combos + /build/build/orchestrator/multitree_combos (AOSP devices, such as cuttlefish) /vendor/**/multitree_combos @@ -89,29 +91,30 @@ def FindConfigDirs(workspace_root): Directories are returned specifically in this order, so that aosp can't be overridden, but vendor overrides device. """ + # TODO: This is not looking in inner trees correctly. # TODO: When orchestrator is in its own git project remove the "make/" here - yield os.path.join(workspace_root, "build/make/orchestrator/multitree_combos") + yield os.path.join(workspace_root, "build/build/make/orchestrator/multitree_combos") dirs = ["vendor", "device"] for d in dirs: - yield from FindDirs(os.path.join(workspace_root, d), "multitree_combos") + yield from find_dirs(os.path.join(workspace_root, d), "multitree_combos") -def FindNamedConfig(workspace_root, shortname): +def find_named_config(workspace_root, shortname): """Find the config with the given shortname inside workspace_root. - Config directories are searched in the order described in FindConfigDirs, + Config directories are searched in the order described in find_config_dirs, and inside those directories, alphabetically.""" filename = shortname + ".mcombo" - for config_dir in FindConfigDirs(workspace_root): - found = FindFile(config_dir, filename) + for config_dir in find_config_dirs(workspace_root): + found = find_file(config_dir, filename) if found: return found return None -def ParseProductVariant(s): +def parse_product_variant(s): """Split a PRODUCT-VARIANT name, or return None if it doesn't match that pattern.""" split = s.split("-") if len(split) != 2: @@ -119,15 +122,15 @@ def ParseProductVariant(s): return split -def ChooseConfigFromArgs(workspace_root, args): +def choose_config_from_args(workspace_root, args): """Return the config file we should use for the given argument, or null if there's no file that matches that.""" if len(args) == 1: # Prefer PRODUCT-VARIANT syntax so if there happens to be a matching # file we don't match that. - pv = ParseProductVariant(args[0]) + pv = parse_product_variant(args[0]) if pv: - config = FindNamedConfig(workspace_root, pv[0]) + config = find_named_config(workspace_root, pv[0]) if config: return (config, pv[1]) return None, None @@ -139,10 +142,12 @@ def ChooseConfigFromArgs(workspace_root, args): class ConfigException(Exception): + ERROR_IDENTIFY = "identify" ERROR_PARSE = "parse" ERROR_CYCLE = "cycle" + ERROR_VALIDATE = "validate" - def __init__(self, kind, message, locations, line=0): + def __init__(self, kind, message, locations=[], line=0): """Error thrown when loading and parsing configurations. Args: @@ -169,13 +174,13 @@ class ConfigException(Exception): self.line = line -def LoadConfig(filename): +def load_config(filename): """Load a config, including processing the inherits fields. Raises: ConfigException on errors """ - def LoadAndMerge(fn, visited): + def load_and_merge(fn, visited): with open(fn) as f: try: contents = json.load(f) @@ -191,34 +196,74 @@ def LoadConfig(filename): if parent in visited: raise ConfigException(ConfigException.ERROR_CYCLE, "Cycle detected in inherits", visited) - DeepMerge(inherited_data, LoadAndMerge(parent, [parent,] + visited)) + deep_merge(inherited_data, load_and_merge(parent, [parent,] + visited)) # Then merge inherited_data into contents, but what's already there will win. - DeepMerge(contents, inherited_data) + deep_merge(contents, inherited_data) contents.pop("inherits", None) return contents - return LoadAndMerge(filename, [filename,]) + return load_and_merge(filename, [filename,]) -def DeepMerge(merged, addition): +def deep_merge(merged, addition): """Merge all fields of addition into merged. Pre-existing fields win.""" for k, v in addition.items(): if k in merged: if isinstance(v, dict) and isinstance(merged[k], dict): - DeepMerge(merged[k], v) + deep_merge(merged[k], v) else: merged[k] = v -def Lunch(args): +def make_config_header(config_file, config, variant): + def make_table(rows): + maxcols = max([len(row) for row in rows]) + widths = [0] * maxcols + for row in rows: + for i in range(len(row)): + widths[i] = max(widths[i], len(row[i])) + text = [] + for row in rows: + rowtext = [] + for i in range(len(row)): + cell = row[i] + rowtext.append(str(cell)) + rowtext.append(" " * (widths[i] - len(cell))) + rowtext.append(" ") + text.append("".join(rowtext)) + return "\n".join(text) + + trees = [("Component", "Path", "Product"), + ("---------", "----", "-------")] + entry = config.get("system", None) + def add_config_tuple(trees, entry, name): + if entry: + trees.append((name, entry.get("tree"), entry.get("product", ""))) + add_config_tuple(trees, config.get("system"), "system") + add_config_tuple(trees, config.get("vendor"), "vendor") + for k, v in config.get("modules", {}).items(): + add_config_tuple(trees, v, k) + + return """======================================== +TARGET_BUILD_COMBO=%(TARGET_BUILD_COMBO)s +TARGET_BUILD_VARIANT=%(TARGET_BUILD_VARIANT)s + +%(trees)s +========================================\n""" % { + "TARGET_BUILD_COMBO": config_file, + "TARGET_BUILD_VARIANT": variant, + "trees": make_table(trees), + } + + +def do_lunch(args): """Handle the lunch command.""" - # Check that we're at the top of a multitree workspace - # TODO: Choose the right sentinel file - if not os.path.exists("build/make/orchestrator"): + # Check that we're at the top of a multitree workspace by seeing if this script exists. + if not os.path.exists("build/build/make/orchestrator/core/lunch.py"): sys.stderr.write("ERROR: lunch.py must be run from the root of a multi-tree workspace\n") return EXIT_STATUS_ERROR # Choose the config file - config_file, variant = ChooseConfigFromArgs(".", args) + config_file, variant = choose_config_from_args(".", args) if config_file == None: sys.stderr.write("Can't find lunch combo file for: %s\n" % " ".join(args)) @@ -229,7 +274,7 @@ def Lunch(args): # Parse the config file try: - config = LoadConfig(config_file) + config = load_config(config_file) except ConfigException as ex: sys.stderr.write(str(ex)) return EXIT_STATUS_ERROR @@ -244,47 +289,81 @@ def Lunch(args): sys.stdout.write("%s\n" % config_file) sys.stdout.write("%s\n" % variant) + # Write confirmation message to stderr + sys.stderr.write(make_config_header(config_file, config, variant)) + return EXIT_STATUS_OK -def FindAllComboFiles(workspace_root): +def find_all_combo_files(workspace_root): """Find all .mcombo files in the prescribed locations in the tree.""" - for dir in FindConfigDirs(workspace_root): - for file in WalkPaths(dir, lambda x: x.endswith(".mcombo")): + for dir in find_config_dirs(workspace_root): + for file in walk_paths(dir, lambda x: x.endswith(".mcombo")): yield file -def IsFileLunchable(config_file): +def is_file_lunchable(config_file): """Parse config_file, flatten the inheritance, and return whether it can be used as a lunch target.""" try: - config = LoadConfig(config_file) + config = load_config(config_file) except ConfigException as ex: sys.stderr.write("%s" % ex) return False return config.get("lunchable", False) -def FindAllLunchable(workspace_root): +def find_all_lunchable(workspace_root): """Find all mcombo files in the tree (rooted at workspace_root) that when parsed (and inheritance is flattened) have lunchable: true.""" - for f in [x for x in FindAllComboFiles(workspace_root) if IsFileLunchable(x)]: + for f in [x for x in find_all_combo_files(workspace_root) if is_file_lunchable(x)]: yield f -def List(): +def load_current_config(): + """Load, validate and return the config as specified in TARGET_BUILD_COMBO. Throws + ConfigException if there is a problem.""" + + # Identify the config file + config_file = os.environ.get("TARGET_BUILD_COMBO") + if not config_file: + raise ConfigException(ConfigException.ERROR_IDENTIFY, + "TARGET_BUILD_COMBO not set. Run lunch or pass a combo file.") + + # Parse the config file + config = load_config(config_file) + + # Validate the config file + if not config.get("lunchable", False): + raise ConfigException(ConfigException.ERROR_VALIDATE, + "Lunch config file (or inherited files) does not have the 'lunchable'" + + " flag set, which means it is probably not a complete lunch spec.", + [config_file,]) + + # TODO: Validate that: + # - there are no modules called system or vendor + # - everything has all the required files + + variant = os.environ.get("TARGET_BUILD_VARIANT") + if not variant: + variant = "eng" # TODO: Is this the right default? + # Validate variant is user, userdebug or eng + + return config_file, config, variant + +def do_list(): """Handle the --list command.""" - for f in sorted(FindAllLunchable(".")): + for f in sorted(find_all_lunchable(".")): print(f) -def Print(args): +def do_print(args): """Handle the --print command.""" # Parse args if len(args) == 0: config_file = os.environ.get("TARGET_BUILD_COMBO") if not config_file: - sys.stderr.write("TARGET_BUILD_COMBO not set. Run lunch or pass a combo file.\n") + sys.stderr.write("TARGET_BUILD_COMBO not set. Run lunch before building.\n") return EXIT_STATUS_NEED_HELP elif len(args) == 1: config_file = args[0] @@ -293,7 +372,7 @@ def Print(args): # Parse the config file try: - config = LoadConfig(config_file) + config = load_config(config_file) except ConfigException as ex: sys.stderr.write(str(ex)) return EXIT_STATUS_ERROR @@ -309,15 +388,15 @@ def main(argv): return EXIT_STATUS_NEED_HELP if len(argv) == 2 and argv[1] == "--list": - List() + do_list() return EXIT_STATUS_OK if len(argv) == 2 and argv[1] == "--print": - return Print(argv[2:]) + return do_print(argv[2:]) return EXIT_STATUS_OK - if (len(argv) == 2 or len(argv) == 3) and argv[1] == "--lunch": - return Lunch(argv[2:]) + if (len(argv) == 3 or len(argv) == 4) and argv[1] == "--lunch": + return do_lunch(argv[2:]) sys.stderr.write("Unknown lunch command: %s\n" % " ".join(argv[1:])) return EXIT_STATUS_NEED_HELP diff --git a/orchestrator/core/orchestrator.py b/orchestrator/core/orchestrator.py new file mode 100755 index 0000000000..e99c9563bd --- /dev/null +++ b/orchestrator/core/orchestrator.py @@ -0,0 +1,123 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +sys.dont_write_bytecode = True +import api_assembly +import api_domain +import api_export +import inner_tree +import interrogate +import lunch + +EXIT_STATUS_OK = 0 +EXIT_STATUS_ERROR = 1 + +API_DOMAIN_SYSTEM = "system" +API_DOMAIN_VENDOR = "vendor" +API_DOMAIN_MODULE = "module" + +def process_config(lunch_config): + """Returns a InnerTrees object based on the configuration requested in the lunch config.""" + def add(domain_name, tree_root, product): + tree_key = inner_tree.InnerTreeKey(tree_root, product) + if tree_key in trees: + tree = trees[tree_key] + else: + tree = inner_tree.InnerTree(tree_root, product) + trees[tree_key] = tree + domain = api_domain.ApiDomain(domain_name, tree, product) + domains[domain_name] = domain + tree.domains[domain_name] = domain + + trees = {} + domains = {} + + system_entry = lunch_config.get("system") + if system_entry: + add(API_DOMAIN_SYSTEM, system_entry["tree"], system_entry["product"]) + + vendor_entry = lunch_config.get("vendor") + if vendor_entry: + add(API_DOMAIN_VENDOR, vendor_entry["tree"], vendor_entry["product"]) + + for module_name, module_entry in lunch_config.get("modules", []).items(): + add(module_name, module_entry["tree"], None) + + return inner_tree.InnerTrees(trees, domains) + + +def build(): + # + # Load lunch combo + # + + # Read the config file + try: + config_file, config, variant = lunch.load_current_config() + except lunch.ConfigException as ex: + sys.stderr.write("%s\n" % ex) + return EXIT_STATUS_ERROR + sys.stdout.write(lunch.make_config_header(config_file, config, variant)) + + # Construct the trees and domains dicts + inner_trees = process_config(config) + + # + # 1. Interrogate the trees + # + inner_trees.for_each_tree(interrogate.interrogate_tree) + # TODO: Detect bazel-only mode + + # + # 2a. API Export + # + inner_trees.for_each_tree(api_export.export_apis_from_tree) + + # + # 2b. API Surface Assembly + # + api_assembly.assemble_apis(inner_trees) + + # + # 3a. API Domain Analysis + # + + # + # 3b. Final Packaging Rules + # + + # + # 4. Build Execution + # + + + # + # Success! + # + return EXIT_STATUS_OK + +def main(argv): + return build() + +if __name__ == "__main__": + sys.exit(main(sys.argv)) + + +# vim: sts=4:ts=4:sw=4 diff --git a/orchestrator/core/test_lunch.py b/orchestrator/core/test_lunch.py index 3c39493ebe..2d85d05958 100755 --- a/orchestrator/core/test_lunch.py +++ b/orchestrator/core/test_lunch.py @@ -23,73 +23,73 @@ import lunch class TestStringMethods(unittest.TestCase): def test_find_dirs(self): - self.assertEqual([x for x in lunch.FindDirs("test/configs", "multitree_combos")], [ + self.assertEqual([x for x in lunch.find_dirs("test/configs", "multitree_combos")], [ "test/configs/build/make/orchestrator/multitree_combos", "test/configs/device/aa/bb/multitree_combos", "test/configs/vendor/aa/bb/multitree_combos"]) def test_find_file(self): # Finds the one in device first because this is searching from the root, - # not using FindNamedConfig. - self.assertEqual(lunch.FindFile("test/configs", "v.mcombo"), + # not using find_named_config. + self.assertEqual(lunch.find_file("test/configs", "v.mcombo"), "test/configs/device/aa/bb/multitree_combos/v.mcombo") def test_find_config_dirs(self): - self.assertEqual([x for x in lunch.FindConfigDirs("test/configs")], [ + self.assertEqual([x for x in lunch.find_config_dirs("test/configs")], [ "test/configs/build/make/orchestrator/multitree_combos", "test/configs/vendor/aa/bb/multitree_combos", "test/configs/device/aa/bb/multitree_combos"]) def test_find_named_config(self): # Inside build/orchestrator, overriding device and vendor - self.assertEqual(lunch.FindNamedConfig("test/configs", "b"), + self.assertEqual(lunch.find_named_config("test/configs", "b"), "test/configs/build/make/orchestrator/multitree_combos/b.mcombo") # Nested dir inside a combo dir - self.assertEqual(lunch.FindNamedConfig("test/configs", "nested"), + self.assertEqual(lunch.find_named_config("test/configs", "nested"), "test/configs/build/make/orchestrator/multitree_combos/nested/nested.mcombo") # Inside vendor, overriding device - self.assertEqual(lunch.FindNamedConfig("test/configs", "v"), + self.assertEqual(lunch.find_named_config("test/configs", "v"), "test/configs/vendor/aa/bb/multitree_combos/v.mcombo") # Inside device - self.assertEqual(lunch.FindNamedConfig("test/configs", "d"), + self.assertEqual(lunch.find_named_config("test/configs", "d"), "test/configs/device/aa/bb/multitree_combos/d.mcombo") # Make sure we don't look too deep (for performance) - self.assertIsNone(lunch.FindNamedConfig("test/configs", "too_deep")) + self.assertIsNone(lunch.find_named_config("test/configs", "too_deep")) def test_choose_config_file(self): # Empty string argument - self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", [""]), + self.assertEqual(lunch.choose_config_from_args("test/configs", [""]), (None, None)) # A PRODUCT-VARIANT name - self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["v-eng"]), + self.assertEqual(lunch.choose_config_from_args("test/configs", ["v-eng"]), ("test/configs/vendor/aa/bb/multitree_combos/v.mcombo", "eng")) # A PRODUCT-VARIANT name that conflicts with a file - self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["b-eng"]), + self.assertEqual(lunch.choose_config_from_args("test/configs", ["b-eng"]), ("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng")) # A PRODUCT-VARIANT that doesn't exist - self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["z-user"]), + self.assertEqual(lunch.choose_config_from_args("test/configs", ["z-user"]), (None, None)) # An explicit file - self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", + self.assertEqual(lunch.choose_config_from_args("test/configs", ["test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng"]), ("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng")) # An explicit file that doesn't exist - self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", + self.assertEqual(lunch.choose_config_from_args("test/configs", ["test/configs/doesnt_exist.mcombo", "eng"]), (None, None)) # An explicit file without a variant should fail - self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", + self.assertEqual(lunch.choose_config_from_args("test/configs", ["test/configs/build/make/orchestrator/multitree_combos/b.mcombo"]), ("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", None)) @@ -97,12 +97,12 @@ class TestStringMethods(unittest.TestCase): def test_config_cycles(self): # Test that we catch cycles with self.assertRaises(lunch.ConfigException) as context: - lunch.LoadConfig("test/configs/parsing/cycles/1.mcombo") + lunch.load_config("test/configs/parsing/cycles/1.mcombo") self.assertEqual(context.exception.kind, lunch.ConfigException.ERROR_CYCLE) def test_config_merge(self): # Test the merge logic - self.assertEqual(lunch.LoadConfig("test/configs/parsing/merge/1.mcombo"), { + self.assertEqual(lunch.load_config("test/configs/parsing/merge/1.mcombo"), { "in_1": "1", "in_1_2": "1", "merged": {"merged_1": "1", @@ -119,7 +119,7 @@ class TestStringMethods(unittest.TestCase): }) def test_list(self): - self.assertEqual(sorted(lunch.FindAllLunchable("test/configs")), + self.assertEqual(sorted(lunch.find_all_lunchable("test/configs")), ["test/configs/build/make/orchestrator/multitree_combos/b.mcombo"]) if __name__ == "__main__": diff --git a/orchestrator/inner_build/common.py b/orchestrator/inner_build/common.py new file mode 100644 index 0000000000..6919e043d6 --- /dev/null +++ b/orchestrator/inner_build/common.py @@ -0,0 +1,56 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +def _parse_arguments(argv): + argv = argv[1:] + """Return an argparse options object.""" + # Top-level parser + parser = argparse.ArgumentParser(prog=".inner_build") + + parser.add_argument("--out_dir", action="store", required=True, + help="root of the output directory for this inner tree's API contributions") + + parser.add_argument("--api_domain", action="append", required=True, + help="which API domains are to be built in this inner tree") + + subparsers = parser.add_subparsers(required=True, dest="command", + help="subcommands") + + # inner_build describe command + describe_parser = subparsers.add_parser("describe", + help="describe the capabilities of this inner tree's build system") + + # create the parser for the "b" command + export_parser = subparsers.add_parser("export_api_contributions", + help="export the API contributions of this inner tree") + + # Parse the arguments + return parser.parse_args(argv) + + +class Commands(object): + def Run(self, argv): + """Parse the command arguments and call the corresponding subcommand method on + this object. + + Throws AttributeError if the method for the command wasn't found. + """ + args = _parse_arguments(argv) + return getattr(self, args.command)(args) + diff --git a/orchestrator/inner_build/inner_build_demo.py b/orchestrator/inner_build/inner_build_demo.py new file mode 100755 index 0000000000..9aafb4df22 --- /dev/null +++ b/orchestrator/inner_build/inner_build_demo.py @@ -0,0 +1,143 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import textwrap + +sys.dont_write_bytecode = True +import common + +def mkdirs(path): + try: + os.makedirs(path) + except FileExistsError: + pass + + +class InnerBuildSoong(common.Commands): + def describe(self, args): + mkdirs(args.out_dir) + + with open(os.path.join(args.out_dir, "tree_info.json"), "w") as f: + f.write(textwrap.dedent("""\ + { + "requires_ninja": true, + "orchestrator_protocol_version": 1 + }""")) + + def export_api_contributions(self, args): + contributions_dir = os.path.join(args.out_dir, "api_contributions") + mkdirs(contributions_dir) + + if "system" in args.api_domain: + with open(os.path.join(contributions_dir, "public_api-1.json"), "w") as f: + # 'name: android' is android.jar + f.write(textwrap.dedent("""\ + { + "name": "public_api", + "version": 1, + "api_domain": "system", + "cc_libraries": [ + { + "name": "libhwui", + "headers": [ + { + "root": "frameworks/base/libs/hwui/apex/include", + "files": [ + "android/graphics/jni_runtime.h", + "android/graphics/paint.h", + "android/graphics/matrix.h", + "android/graphics/canvas.h", + "android/graphics/renderthread.h", + "android/graphics/bitmap.h", + "android/graphics/region.h" + ] + } + ], + "api": [ + "frameworks/base/libs/hwui/libhwui.map.txt" + ] + } + ], + "java_libraries": [ + { + "name": "android", + "api": [ + "frameworks/base/core/api/current.txt" + ] + } + ], + "resource_libraries": [ + { + "name": "android", + "api": "frameworks/base/core/res/res/values/public.xml" + } + ], + "host_executables": [ + { + "name": "aapt2", + "binary": "out/host/bin/aapt2", + "runfiles": [ + "../lib/todo.so" + ] + } + ] + }""")) + elif "com.android.bionic" in args.api_domain: + with open(os.path.join(contributions_dir, "public_api-1.json"), "w") as f: + # 'name: android' is android.jar + f.write(textwrap.dedent("""\ + { + "name": "public_api", + "version": 1, + "api_domain": "system", + "cc_libraries": [ + { + "name": "libc", + "headers": [ + { + "root": "bionic/libc/include", + "files": [ + "stdio.h", + "sys/klog.h" + ] + } + ], + "api": "bionic/libc/libc.map.txt" + } + ], + "java_libraries": [ + { + "name": "android", + "api": [ + "frameworks/base/libs/hwui/api/current.txt" + ] + } + ] + }""")) + + + +def main(argv): + return InnerBuildSoong().Run(argv) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) + + +# vim: sts=4:ts=4:sw=4 diff --git a/orchestrator/inner_build/inner_build_soong.py b/orchestrator/inner_build/inner_build_soong.py new file mode 100755 index 0000000000..a653dcca8c --- /dev/null +++ b/orchestrator/inner_build/inner_build_soong.py @@ -0,0 +1,37 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +sys.dont_write_bytecode = True +import common + +class InnerBuildSoong(common.Commands): + def describe(self, args): + pass + + + def export_api_contributions(self, args): + pass + + +def main(argv): + return InnerBuildSoong().Run(argv) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/orchestrator/multitree_combos/aosp_cf_arm64_phone.mcombo b/orchestrator/multitree_combos/aosp_cf_arm64_phone.mcombo new file mode 100644 index 0000000000..079022611d --- /dev/null +++ b/orchestrator/multitree_combos/aosp_cf_arm64_phone.mcombo @@ -0,0 +1,16 @@ +{ + "lunchable": true, + "system": { + "tree": "master", + "product": "aosp_cf_arm64_phone" + }, + "vendor": { + "tree": "master", + "product": "aosp_cf_arm64_phone" + }, + "modules": { + "com.android.bionic": { + "tree": "sc-mainline-prod" + } + } +}