diff --git a/orchestrator/demo/buffet_helper.py b/orchestrator/demo/buffet_helper.py new file mode 100644 index 0000000000..fa29aeb1ce --- /dev/null +++ b/orchestrator/demo/buffet_helper.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import os +import sys +import yaml + +from hierarchy import parse_hierarchy + + +def main(): + if len(sys.argv) != 2: + print('usage: %s target' % sys.argv[0]) + exit(1) + + args = sys.argv[1].split('-') + if len(args) != 2: + print('target format: {target}-{variant}') + exit(1) + + target, variant = args + + if variant not in ['eng', 'user', 'userdebug']: + print('unknown variant "%s": expected "eng", "user" or "userdebug"' % + variant) + exit(1) + + build_top = os.getenv('BUFFET_BUILD_TOP') + if not build_top: + print('BUFFET_BUILD_TOP is not set; Did you correctly run envsetup.sh?') + exit(1) + + hierarchy_map = parse_hierarchy(build_top) + + if target not in hierarchy_map: + raise RuntimeError( + "unknown target '%s': couldn't find the target. Supported targets are: %s" + % (target, list(hierarchy_map.keys()))) + + hierarchy = [target] + while hierarchy_map[hierarchy[-1]]: + hierarchy.append(hierarchy_map[hierarchy[-1]]) + + print('Target hierarchy for %s: %s' % (target, hierarchy)) + + +if __name__ == '__main__': + main() diff --git a/orchestrator/demo/build_helper.py b/orchestrator/demo/build_helper.py new file mode 100644 index 0000000000..c481f80c9c --- /dev/null +++ b/orchestrator/demo/build_helper.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import copy +import hierarchy +import json +import logging +import filecmp +import os +import shutil +import subprocess +import sys +import tempfile +import collect_metadata +import utils + +BUILD_CMD_TO_ALL = ( + 'clean', + 'installclean', + 'update-meta', +) +BUILD_ALL_EXEMPTION = ( + 'art', +) + +def get_supported_product(ctx, supported_products): + hierarchy_map = hierarchy.parse_hierarchy(ctx.build_top()) + target = ctx.target_product() + + while target not in supported_products: + if target not in hierarchy_map: + return None + target = hierarchy_map[target] + return target + + +def parse_goals(ctx, metadata, goals): + """Parse goals and returns a map from each component to goals. + + e.g. + + "m main art timezone:foo timezone:bar" will return the following dict: { + "main": {"all"}, + "art": {"all"}, + "timezone": {"foo", "bar"}, + } + """ + # for now, goal should look like: + # {component} or {component}:{subgoal} + + ret = collections.defaultdict(set) + + for goal in goals: + # check if the command is for all components + if goal in BUILD_CMD_TO_ALL: + ret['all'].add(goal) + continue + + # should be {component} or {component}:{subgoal} + try: + component, subgoal = goal.split(':') if ':' in goal else (goal, 'all') + except ValueError: + raise RuntimeError( + 'unknown goal: %s: should be {component} or {component}:{subgoal}' % + goal) + if component not in metadata: + raise RuntimeError('unknown goal: %s: component %s not found' % + (goal, component)) + if not get_supported_product(ctx, metadata[component]['lunch_targets']): + raise RuntimeError("can't find matching target. Supported targets are: " + + str(metadata[component]['lunch_targets'])) + + ret[component].add(subgoal) + + return ret + + +def find_cycle(metadata): + """ Finds a cyclic dependency among components. + + This is for debugging. + """ + visited = set() + parent_node = dict() + in_stack = set() + + # Returns a cycle if one is found + def dfs(node): + # visit_order[visit_time[node] - 1] == node + nonlocal visited, parent_node, in_stack + + visited.add(node) + in_stack.add(node) + if 'deps' not in metadata[node]: + in_stack.remove(node) + return None + for next in metadata[node]['deps']: + # We found a cycle (next ~ node) if next is still in the stack + if next in in_stack: + cycle = [node] + while cycle[-1] != next: + cycle.append(parent_node[cycle[-1]]) + return cycle + + # Else, continue searching + if next in visited: + continue + + parent_node[next] = node + result = dfs(next) + if result: + return result + + in_stack.remove(node) + return None + + for component in metadata: + if component in visited: + continue + + result = dfs(component) + if result: + return result + + return None + + +def topological_sort_components(metadata): + """ Performs topological sort on components. + + If A depends on B, B appears first. + """ + # If A depends on B, we want B to appear before A. But the graph in metadata + # is represented as A -> B (B in metadata[A]['deps']). So we sort in the + # reverse order, and then reverse the result again to get the desired order. + indegree = collections.defaultdict(int) + for component in metadata: + if 'deps' not in metadata[component]: + continue + for dep in metadata[component]['deps']: + indegree[dep] += 1 + + component_queue = collections.deque() + for component in metadata: + if indegree[component] == 0: + component_queue.append(component) + + result = [] + while component_queue: + component = component_queue.popleft() + result.append(component) + if 'deps' not in metadata[component]: + continue + for dep in metadata[component]['deps']: + indegree[dep] -= 1 + if indegree[dep] == 0: + component_queue.append(dep) + + # If topological sort fails, there must be a cycle. + if len(result) != len(metadata): + cycle = find_cycle(metadata) + raise RuntimeError('circular dependency found among metadata: %s' % cycle) + + return result[::-1] + + +def add_dependency_goals(ctx, metadata, component, goals): + """ Adds goals that given component depends on.""" + # For now, let's just add "all" + # TODO: add detailed goals (e.g. API build rules, library build rules, etc.) + if 'deps' not in metadata[component]: + return + + for dep in metadata[component]['deps']: + goals[dep].add('all') + + +def sorted_goals_with_dependencies(ctx, metadata, parsed_goals): + """ Analyzes the dependency graph among components, adds build commands for + + dependencies, and then sorts the goals. + + Returns a list of tuples: (component_name, set of subgoals). + Builds should be run in the list's order. + """ + # TODO(inseob@): after topological sort, some components may be built in + # parallel. + + topological_order = topological_sort_components(metadata) + combined_goals = copy.deepcopy(parsed_goals) + + # Add build rules for each component's dependencies + # We do this in reverse order, so it can be transitive. + # e.g. if A depends on B and B depends on C, and we build A, + # C should also be built, in addition to B. + for component in topological_order[::-1]: + if component in combined_goals: + add_dependency_goals(ctx, metadata, component, combined_goals) + + ret = [] + for component in ['all'] + topological_order: + if component in combined_goals: + ret.append((component, combined_goals[component])) + + return ret + + +def run_build(ctx, metadata, component, subgoals): + build_cmd = metadata[component]['build_cmd'] + out_dir = metadata[component]['out_dir'] + default_goals = '' + if 'default_goals' in metadata[component]: + default_goals = metadata[component]['default_goals'] + + if 'all' in subgoals: + goal = default_goals + else: + goal = ' '.join(subgoals) + + build_vars = '' + if 'update-meta' in subgoals: + build_vars = 'TARGET_MULTITREE_UPDATE_META=true' + # TODO(inseob@): shell escape + cmd = [ + '/bin/bash', '-c', + 'source build/envsetup.sh && lunch %s-%s && %s %s %s' % + (get_supported_product(ctx, metadata[component]['lunch_targets']), + ctx.target_build_variant(), build_vars, build_cmd, goal) + ] + logging.debug('cwd: ' + metadata[component]['path']) + logging.debug('running build: ' + str(cmd)) + + subprocess.run(cmd, cwd=metadata[component]['path'], check=True) + + +def run_build_all(ctx, metadata, subgoals): + for component in metadata: + if component in BUILD_ALL_EXEMPTION: + continue + run_build(ctx, metadata, component, subgoals) + + +def find_components(metadata, predicate): + for component in metadata: + if predicate(component): + yield component + + +def import_filegroups(metadata, component, exporting_component, target_file_pairs): + imported_filegroup_dir = os.path.join(metadata[component]['path'], 'imported', exporting_component) + + bp_content = '' + for name, outpaths in target_file_pairs: + bp_content += ('filegroup {{\n' + ' name: "{fname}",\n' + ' srcs: [\n'.format(fname=name)) + for outpath in outpaths: + bp_content += ' "{outfile}",\n'.format(outfile=os.path.basename(outpath)) + bp_content += (' ],\n' + '}\n') + + with tempfile.TemporaryDirectory() as tmp_dir: + with open(os.path.join(tmp_dir, 'Android.bp'), 'w') as fout: + fout.write(bp_content) + for _, outpaths in target_file_pairs: + for outpath in outpaths: + os.symlink(os.path.join(metadata[exporting_component]['path'], outpath), + os.path.join(tmp_dir, os.path.basename(outpath))) + cmp_result = filecmp.dircmp(tmp_dir, imported_filegroup_dir) + if os.path.exists(imported_filegroup_dir) and len( + cmp_result.left_only) + len(cmp_result.right_only) + len( + cmp_result.diff_files) == 0: + # Files are identical, it doesn't need to be written + logging.info( + 'imported files exists and the contents are identical: {} -> {}' + .format(component, exporting_component)) + continue + logging.info('creating symlinks for imported files: {} -> {}'.format( + component, exporting_component)) + os.makedirs(imported_filegroup_dir, exist_ok=True) + shutil.rmtree(imported_filegroup_dir, ignore_errors=True) + shutil.move(tmp_dir, imported_filegroup_dir) + + +def prepare_build(metadata, component): + imported_dir = os.path.join(metadata[component]['path'], 'imported') + if utils.META_DEPS not in metadata[component]: + if os.path.exists(imported_dir): + logging.debug('remove {}'.format(imported_dir)) + shutil.rmtree(imported_dir) + return + + imported_components = set() + for exp_comp in metadata[component][utils.META_DEPS]: + if utils.META_FILEGROUP in metadata[component][utils.META_DEPS][exp_comp]: + filegroups = metadata[component][utils.META_DEPS][exp_comp][utils.META_FILEGROUP] + target_file_pairs = [] + for name in filegroups: + target_file_pairs.append((name, filegroups[name])) + import_filegroups(metadata, component, exp_comp, target_file_pairs) + imported_components.add(exp_comp) + + # Remove directories that are not generated this time. + if os.path.exists(imported_dir): + if len(imported_components) == 0: + shutil.rmtree(imported_dir) + else: + for remove_target in set(os.listdir(imported_dir)) - imported_components: + logging.info('remove unnecessary imported dir: {}'.format(remove_target)) + shutil.rmtree(os.path.join(imported_dir, remove_target)) + + +def main(): + utils.set_logging_config(logging.DEBUG) + ctx = utils.get_build_context() + + logging.info('collecting metadata') + + utils.set_logging_config(True) + + goals = sys.argv[1:] + if not goals: + logging.debug('empty goals. defaults to main') + goals = ['main'] + + logging.debug('goals: ' + str(goals)) + + # Force update the metadata for the 'update-meta' build + metadata_collector = collect_metadata.MetadataCollector( + ctx.components_top(), ctx.out_dir(), + collect_metadata.COMPONENT_METADATA_DIR, + collect_metadata.COMPONENT_METADATA_FILE, + force_update='update-meta' in goals) + metadata_collector.collect() + + metadata = metadata_collector.get_metadata() + logging.debug('metadata: ' + str(metadata)) + + parsed_goals = parse_goals(ctx, metadata, goals) + logging.debug('parsed goals: ' + str(parsed_goals)) + + sorted_goals = sorted_goals_with_dependencies(ctx, metadata, parsed_goals) + logging.debug('sorted goals with deps: ' + str(sorted_goals)) + + for component, subgoals in sorted_goals: + if component == 'all': + run_build_all(ctx, metadata, subgoals) + continue + prepare_build(metadata, component) + run_build(ctx, metadata, component, subgoals) + + +if __name__ == '__main__': + main() diff --git a/orchestrator/demo/collect_metadata.py b/orchestrator/demo/collect_metadata.py new file mode 100755 index 0000000000..148167d3eb --- /dev/null +++ b/orchestrator/demo/collect_metadata.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import copy +import json +import logging +import os +import sys +import yaml +from collections import defaultdict +from typing import ( + List, + Set, +) + +import utils + +# SKIP_COMPONENT_SEARCH = ( +# 'tools', +# ) +COMPONENT_METADATA_DIR = '.repo' +COMPONENT_METADATA_FILE = 'treeinfo.yaml' +GENERATED_METADATA_FILE = 'metadata.json' +COMBINED_METADATA_FILENAME = 'multitree_meta.json' + + +class Dep(object): + def __init__(self, name, component, deps_type): + self.name = name + self.component = component + self.type = deps_type + self.out_paths = list() + + +class ExportedDep(Dep): + def __init__(self, name, component, deps_type): + super().__init__(name, component, deps_type) + + def setOutputPaths(self, output_paths: list): + self.out_paths = output_paths + + +class ImportedDep(Dep): + required_type_map = { + # import type: (required type, get imported module list) + utils.META_FILEGROUP: (utils.META_MODULES, True), + } + + def __init__(self, name, component, deps_type, import_map): + super().__init__(name, component, deps_type) + self.exported_deps: Set[ExportedDep] = set() + self.imported_modules: List[str] = list() + self.required_type = deps_type + get_imported_module = False + if deps_type in ImportedDep.required_type_map: + self.required_type, get_imported_module = ImportedDep.required_type_map[deps_type] + if get_imported_module: + self.imported_modules = import_map[name] + else: + self.imported_modules.append(name) + + def verify_and_add(self, exported: ExportedDep): + if self.required_type != exported.type: + raise RuntimeError( + '{comp} components imports {module} for {imp_type} but it is exported as {exp_type}.' + .format(comp=self.component, module=exported.name, imp_type=self.required_type, exp_type=exported.type)) + self.exported_deps.add(exported) + self.out_paths.extend(exported.out_paths) + # Remove duplicates. We may not use set() which is not JSON serializable + self.out_paths = list(dict.fromkeys(self.out_paths)) + + +class MetadataCollector(object): + """Visit all component directories and collect the metadata from them. + +Example of metadata: +========== +build_cmd: m # build command for this component. 'm' if omitted +out_dir: out # out dir of this component. 'out' if omitted +exports: + libraries: + - name: libopenjdkjvm + - name: libopenjdkjvmd + build_cmd: mma # build command for libopenjdkjvmd if specified + out_dir: out/soong # out dir for libopenjdkjvmd if specified + - name: libctstiagent + APIs: + - api1 + - api2 +imports: + libraries: + - lib1 + - lib2 + APIs: + - import_api1 + - import_api2 +lunch_targets: + - arm64 + - x86_64 +""" + + def __init__(self, component_top, out_dir, meta_dir, meta_file, force_update=False): + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + self.__component_top = component_top + self.__out_dir = out_dir + self.__metadata_path = os.path.join(meta_dir, meta_file) + self.__combined_metadata_path = os.path.join(self.__out_dir, + COMBINED_METADATA_FILENAME) + self.__force_update = force_update + + self.__metadata = dict() + self.__map_exports = dict() + self.__component_set = set() + + def collect(self): + """ Read precomputed combined metadata from the json file. + + If any components have updated their metadata, update the metadata + information and the json file. + """ + timestamp = self.__restore_metadata() + if timestamp and os.path.getmtime(__file__) > timestamp: + logging.info('Update the metadata as the orchestrator has been changed') + self.__force_update = True + self.__collect_from_components(timestamp) + + def get_metadata(self): + """ Returns collected metadata from all components""" + if not self.__metadata: + logging.warning('Metadata is empty') + return copy.deepcopy(self.__metadata) + + def __collect_from_components(self, timestamp): + """ Read metadata from all components + + If any components have newer metadata files or are removed, update the + combined metadata. + """ + metadata_updated = False + for component in os.listdir(self.__component_top): + # if component in SKIP_COMPONENT_SEARCH: + # continue + if self.__read_component_metadata(timestamp, component): + metadata_updated = True + if self.__read_generated_metadata(timestamp, component): + metadata_updated = True + + deleted_components = set() + for meta in self.__metadata: + if meta not in self.__component_set: + logging.info('Component {} is removed'.format(meta)) + deleted_components.add(meta) + metadata_updated = True + for meta in deleted_components: + del self.__metadata[meta] + + if metadata_updated: + self.__update_dependencies() + self.__store_metadata() + logging.info('Metadata updated') + + def __read_component_metadata(self, timestamp, component): + """ Search for the metadata file from a component. + + If the metadata is modified, read the file and update the metadata. + """ + component_path = os.path.join(self.__component_top, component) + metadata_file = os.path.join(component_path, self.__metadata_path) + logging.info( + 'Reading a metadata file from {} component ...'.format(component)) + if not os.path.isfile(metadata_file): + logging.warning('Metadata file {} not found!'.format(metadata_file)) + return False + + self.__component_set.add(component) + if not self.__force_update and timestamp and timestamp > os.path.getmtime(metadata_file): + logging.info('... yaml not changed. Skip') + return False + + with open(metadata_file) as f: + meta = yaml.load(f, Loader=yaml.SafeLoader) + + meta['path'] = component_path + if utils.META_BUILDCMD not in meta: + meta[utils.META_BUILDCMD] = utils.DEFAULT_BUILDCMD + if utils.META_OUTDIR not in meta: + meta[utils.META_OUTDIR] = utils.DEFAULT_OUTDIR + + if utils.META_IMPORTS not in meta: + meta[utils.META_IMPORTS] = defaultdict(dict) + if utils.META_EXPORTS not in meta: + meta[utils.META_EXPORTS] = defaultdict(dict) + + self.__metadata[component] = meta + return True + + def __read_generated_metadata(self, timestamp, component): + """ Read a metadata gerated by 'update-meta' build command from the soong build system + + Soong generate the metadata that has the information of import/export module/files. + Build orchestrator read the generated metadata to collect the dependency information. + + Generated metadata has the following format: + { + "Imported": { + "FileGroups": { + "": [ + "", + ... + ], + ... + } + } + "Exported": { + "": [ + "", + ... + ], + ... + } + } + """ + if component not in self.__component_set: + # skip reading generated metadata if the component metadata file was missing + return False + component_out = os.path.join(self.__component_top, component, self.__metadata[component][utils.META_OUTDIR]) + generated_metadata_file = os.path.join(component_out, 'soong', 'multitree', GENERATED_METADATA_FILE) + if not os.path.isfile(generated_metadata_file): + logging.info('... Soong did not generated the metadata file. Skip') + return False + if not self.__force_update and timestamp and timestamp > os.path.getmtime(generated_metadata_file): + logging.info('... Soong generated metadata not changed. Skip') + return False + + with open(generated_metadata_file, 'r') as gen_meta_json: + try: + gen_metadata = json.load(gen_meta_json) + except json.decoder.JSONDecodeError: + logging.warning('JSONDecodeError!!!: skip reading the {} file'.format( + generated_metadata_file)) + return False + + if utils.SOONG_IMPORTED in gen_metadata: + imported = gen_metadata[utils.SOONG_IMPORTED] + if utils.SOONG_IMPORTED_FILEGROUPS in imported: + self.__metadata[component][utils.META_IMPORTS][utils.META_FILEGROUP] = imported[utils.SOONG_IMPORTED_FILEGROUPS] + if utils.SOONG_EXPORTED in gen_metadata: + self.__metadata[component][utils.META_EXPORTS][utils.META_MODULES] = gen_metadata[utils.SOONG_EXPORTED] + + return True + + def __update_export_map(self): + """ Read metadata of all components and update the export map + + 'libraries' and 'APIs' are special exproted types that are provided manually + from the .yaml metadata files. These need to be replaced with the implementation + in soong gerated metadata. + The export type 'module' is generated from the soong build system from the modules + with 'export: true' property. This export type includes a dictionary with module + names as keys and their output files as values. These output files will be used as + prebuilt sources when generating the imported modules. + """ + self.__map_exports = dict() + for comp in self.__metadata: + if utils.META_EXPORTS not in self.__metadata[comp]: + continue + exports = self.__metadata[comp][utils.META_EXPORTS] + + for export_type in exports: + for module in exports[export_type]: + if export_type == utils.META_LIBS: + name = module[utils.META_LIB_NAME] + else: + name = module + + if name in self.__map_exports: + raise RuntimeError( + 'Exported libs conflict!!!: "{name}" in the {comp} component is already exported by the {prev} component.' + .format(name=name, comp=comp, prev=self.__map_exports[name][utils.EXP_COMPONENT])) + exported_deps = ExportedDep(name, comp, export_type) + if export_type == utils.META_MODULES: + exported_deps.setOutputPaths(exports[export_type][module]) + self.__map_exports[name] = exported_deps + + def __verify_and_add_dependencies(self, component): + """ Search all imported items from the export_map. + + If any imported items are not provided by the other components, report + an error. + Otherwise, add the component dependency and update the exported information to the + import maps. + """ + def verify_and_add_dependencies(imported_dep: ImportedDep): + for module in imported_dep.imported_modules: + if module not in self.__map_exports: + raise RuntimeError( + 'Imported item not found!!!: Imported module "{module}" in the {comp} component is not exported from any other components.' + .format(module=module, comp=imported_dep.component)) + imported_dep.verify_and_add(self.__map_exports[module]) + + deps = self.__metadata[component][utils.META_DEPS] + exp_comp = self.__map_exports[module].component + if exp_comp not in deps: + deps[exp_comp] = defaultdict(defaultdict) + deps[exp_comp][imported_dep.type][imported_dep.name] = imported_dep.out_paths + + self.__metadata[component][utils.META_DEPS] = defaultdict() + imports = self.__metadata[component][utils.META_IMPORTS] + for import_type in imports: + for module in imports[import_type]: + verify_and_add_dependencies(ImportedDep(module, component, import_type, imports[import_type])) + + def __check_imports(self): + """ Search the export map to find the component to import libraries or APIs. + + Update the 'deps' field that includes the dependent components. + """ + for component in self.__metadata: + self.__verify_and_add_dependencies(component) + if utils.META_DEPS in self.__metadata[component]: + logging.debug('{comp} depends on {list} components'.format( + comp=component, list=self.__metadata[component][utils.META_DEPS])) + + def __update_dependencies(self): + """ Generate a dependency graph for the components + + Update __map_exports and the dependency graph with the maps. + """ + self.__update_export_map() + self.__check_imports() + + def __store_metadata(self): + """ Store the __metadata dictionary as json format""" + with open(self.__combined_metadata_path, 'w') as json_file: + json.dump(self.__metadata, json_file, indent=2) + + def __restore_metadata(self): + """ Read the stored json file and return the time stamps of the + + metadata file. + """ + if not os.path.exists(self.__combined_metadata_path): + return None + + with open(self.__combined_metadata_path, 'r') as json_file: + try: + self.__metadata = json.load(json_file) + except json.decoder.JSONDecodeError: + logging.warning('JSONDecodeError!!!: skip reading the {} file'.format( + self.__combined_metadata_path)) + return None + + logging.info('Metadata restored from {}'.format( + self.__combined_metadata_path)) + self.__update_export_map() + return os.path.getmtime(self.__combined_metadata_path) + + +def get_args(): + + def check_dir(path): + if os.path.exists(path) and os.path.isdir(path): + return os.path.normpath(path) + else: + raise argparse.ArgumentTypeError('\"{}\" is not a directory'.format(path)) + + parser = argparse.ArgumentParser() + parser.add_argument( + '--component-top', + help='Scan all components under this directory.', + default=os.path.join(os.path.dirname(__file__), '../../../components'), + type=check_dir) + parser.add_argument( + '--meta-file', + help='Name of the metadata file.', + default=COMPONENT_METADATA_FILE, + type=str) + parser.add_argument( + '--meta-dir', + help='Each component has the metadata in this directory.', + default=COMPONENT_METADATA_DIR, + type=str) + parser.add_argument( + '--out-dir', + help='Out dir for the outer tree. The orchestrator stores the collected metadata in this directory.', + default=os.path.join(os.path.dirname(__file__), '../../../out'), + type=os.path.normpath) + parser.add_argument( + '--force', + '-f', + action='store_true', + help='Force to collect metadata', + ) + parser.add_argument( + '--verbose', + '-v', + help='Increase output verbosity, e.g. "-v", "-vv".', + action='count', + default=0) + return parser.parse_args() + + +def main(): + args = get_args() + utils.set_logging_config(args.verbose) + + metadata_collector = MetadataCollector(args.component_top, args.out_dir, + args.meta_dir, args.meta_file, args.force) + metadata_collector.collect() + + +if __name__ == '__main__': + main() diff --git a/orchestrator/demo/envsetup.sh b/orchestrator/demo/envsetup.sh new file mode 100644 index 0000000000..902a37c2a2 --- /dev/null +++ b/orchestrator/demo/envsetup.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +function buffet() +{ + local product variant selection + if [[ $# -ne 1 ]]; then + echo "usage: buffet [target]" >&2 + return 1 + fi + + selection=$1 + product=${selection%%-*} # Trim everything after first dash + variant=${selection#*-} # Trim everything up to first dash + + if [ -z "$product" ] + then + echo + echo "Invalid lunch combo: $selection" + return 1 + fi + + if [ -z "$variant" ] + then + if [[ "$product" =~ .*_(eng|user|userdebug) ]] + then + echo "Did you mean -${product/*_/}? (dash instead of underscore)" + fi + return 1 + fi + + BUFFET_BUILD_TOP=$(pwd) python3 tools/build/orchestrator/buffet_helper.py $1 || return 1 + + export BUFFET_BUILD_TOP=$(pwd) + export BUFFET_COMPONENTS_TOP=$BUFFET_BUILD_TOP/components + export BUFFET_TARGET_PRODUCT=$product + export BUFFET_TARGET_BUILD_VARIANT=$variant + export BUFFET_TARGET_BUILD_TYPE=release +} + +function m() +{ + if [ -z "$BUFFET_BUILD_TOP" ] + then + echo "Run \"buffet [target]\" first" + return 1 + fi + python3 $BUFFET_BUILD_TOP/tools/build/orchestrator/build_helper.py "$@" +} diff --git a/orchestrator/demo/hierarchy.py b/orchestrator/demo/hierarchy.py new file mode 100644 index 0000000000..ae1825c049 --- /dev/null +++ b/orchestrator/demo/hierarchy.py @@ -0,0 +1,79 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import yaml + + +def parse_hierarchy(build_top): + """Parse build hierarchy file from given build top directory, and returns a dict from child targets to parent targets. + + Example of hierarchy file: + ========== + aosp_arm64: + - armv8 + - aosp_cf_arm64_phone + + armv8: + - aosp_oriole + - aosp_sunfish + + aosp_oriole: + - oriole + + aosp_sunfish: + - sunfish + + oriole: + # leaf + + sunfish: + # leaf + ========== + + If we parse this yaml, we get a dict looking like: + + { + "sunfish": "aosp_sunfish", + "oriole": "aosp_oriole", + "aosp_oriole": "armv8", + "aosp_sunfish": "armv8", + "armv8": "aosp_arm64", + "aosp_cf_arm64_phone": "aosp_arm64", + "aosp_arm64": None, # no parent + } + """ + metadata_path = os.path.join(build_top, 'tools', 'build', 'hierarchy.yaml') + if not os.path.isfile(metadata_path): + raise RuntimeError("target metadata file %s doesn't exist" % metadata_path) + + with open(metadata_path, 'r') as f: + hierarchy_yaml = yaml.load(f, Loader=yaml.SafeLoader) + + hierarchy_map = dict() + + for parent_target, child_targets in hierarchy_yaml.items(): + if not child_targets: + # leaf + continue + for child_target in child_targets: + hierarchy_map[child_target] = parent_target + + for parent_target in hierarchy_yaml: + # targets with no parent + if parent_target not in hierarchy_map: + hierarchy_map[parent_target] = None + + return hierarchy_map diff --git a/orchestrator/demo/hierarchy.yaml b/orchestrator/demo/hierarchy.yaml new file mode 100644 index 0000000000..cc6de4df3d --- /dev/null +++ b/orchestrator/demo/hierarchy.yaml @@ -0,0 +1,37 @@ +# hierarchy of targets + +aosp_arm64: +- armv8 +- aosp_cf_arm64_phone + +armv8: +- mainline_modules_arm64 + +mainline_modules_arm64: +- aosp_oriole +- aosp_sunfish +- aosp_raven + +aosp_oriole: +- oriole + +aosp_sunfish: +- sunfish + +aosp_raven: +- raven + +oriole: +# leaf + +sunfish: +# leaf + +raven: +# leaf + +aosp_cf_arm64_phone: +- cf_arm64_phone + +cf_arm64_phone: +# leaf diff --git a/orchestrator/demo/utils.py b/orchestrator/demo/utils.py new file mode 100644 index 0000000000..5dbbe4aa9f --- /dev/null +++ b/orchestrator/demo/utils.py @@ -0,0 +1,89 @@ +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os + +# default build configuration for each component +DEFAULT_BUILDCMD = 'm' +DEFAULT_OUTDIR = 'out' + +# yaml fields +META_BUILDCMD = 'build_cmd' +META_OUTDIR = 'out_dir' +META_EXPORTS = 'exports' +META_IMPORTS = 'imports' +META_TARGETS = 'lunch_targets' +META_DEPS = 'deps' +# fields under 'exports' and 'imports' +META_LIBS = 'libraries' +META_APIS = 'APIs' +META_FILEGROUP = 'filegroup' +META_MODULES = 'modules' +# fields under 'libraries' +META_LIB_NAME = 'name' + +# fields for generated metadata file +SOONG_IMPORTED = 'Imported' +SOONG_IMPORTED_FILEGROUPS = 'FileGroups' +SOONG_EXPORTED = 'Exported' + +# export map items +EXP_COMPONENT = 'component' +EXP_TYPE = 'type' +EXP_OUTPATHS = 'outpaths' + +class BuildContext: + + def __init__(self): + self._build_top = os.getenv('BUFFET_BUILD_TOP') + self._components_top = os.getenv('BUFFET_COMPONENTS_TOP') + self._target_product = os.getenv('BUFFET_TARGET_PRODUCT') + self._target_build_variant = os.getenv('BUFFET_TARGET_BUILD_VARIANT') + self._target_build_type = os.getenv('BUFFET_TARGET_BUILD_TYPE') + self._out_dir = os.path.join(self._build_top, 'out') + + if not self._build_top: + raise RuntimeError("Can't find root. Did you run buffet?") + + def build_top(self): + return self._build_top + + def components_top(self): + return self._components_top + + def target_product(self): + return self._target_product + + def target_build_variant(self): + return self._target_build_variant + + def target_build_type(self): + return self._target_build_type + + def out_dir(self): + return self._out_dir + + +def get_build_context(): + return BuildContext() + + +def set_logging_config(verbose_level): + verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) + verbosity = min(verbose_level, 2) + logging.basicConfig( + format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', + level=verbose_map[verbosity])