diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py index 6e1f88c36c..dbc468268a 100644 --- a/ci/build_test_suites.py +++ b/ci/build_test_suites.py @@ -20,6 +20,7 @@ import json import logging import os import pathlib +import re import subprocess import sys from typing import Callable @@ -51,6 +52,15 @@ class BuildPlanner: any output zip files needed by the build. """ + _DOWNLOAD_OPTS = { + 'test-config-only-zip', + 'test-zip-file-filter', + 'extra-host-shared-lib-zip', + 'sandbox-tests-zips', + 'additional-files-filter', + 'cts-package-name', + } + def __init__( self, build_context: dict[str, any], @@ -63,12 +73,19 @@ class BuildPlanner: def create_build_plan(self): - if 'optimized_build' not in self.build_context['enabled_build_features']: + if 'optimized_build' not in self.build_context.get( + 'enabledBuildFeatures', [] + ): return BuildPlan(set(self.args.extra_targets), set()) build_targets = set() packaging_functions = set() for target in self.args.extra_targets: + if self._unused_target_exclusion_enabled( + target + ) and not self._build_target_used(target): + continue + target_optimizer_getter = self.target_optimizations.get(target, None) if not target_optimizer_getter: build_targets.add(target) @@ -82,6 +99,60 @@ class BuildPlanner: return BuildPlan(build_targets, packaging_functions) + def _unused_target_exclusion_enabled(self, target: str) -> bool: + return f'{target}_unused_exclusion' in self.build_context.get( + 'enabledBuildFeatures', [] + ) + + def _build_target_used(self, target: str) -> bool: + """Determines whether this target's outputs are used by the test configurations listed in the build context.""" + file_download_regexes = self._aggregate_file_download_regexes() + # For all of a targets' outputs, check if any of the regexes used by tests + # to download artifacts would match it. If any of them do then this target + # is necessary. + for artifact in self._get_target_potential_outputs(target): + for regex in file_download_regexes: + if re.match(regex, artifact): + return True + return False + + def _get_target_potential_outputs(self, target: str) -> set[str]: + tests_suffix = '-tests' + if target.endswith('tests'): + tests_suffix = '' + # This is a list of all the potential zips output by the test suite targets. + # If the test downloads artifacts from any of these zips, we will be + # conservative and avoid skipping the tests. + return { + f'{target}.zip', + f'android-{target}.zip', + f'android-{target}-verifier.zip', + f'{target}{tests_suffix}_list.zip', + f'android-{target}{tests_suffix}_list.zip', + f'{target}{tests_suffix}_host-shared-libs.zip', + f'android-{target}{tests_suffix}_host-shared-libs.zip', + f'{target}{tests_suffix}_configs.zip', + f'android-{target}{tests_suffix}_configs.zip', + } + + def _aggregate_file_download_regexes(self) -> set[re.Pattern]: + """Lists out all test config options to specify targets to download. + + These come in the form of regexes. + """ + all_regexes = set() + for test_info in self._get_test_infos(): + for opt in test_info.get('extraOptions', []): + # check the known list of options for downloading files. + if opt.get('key') in self._DOWNLOAD_OPTS: + all_regexes.update( + re.compile(value) for value in opt.get('values', []) + ) + return all_regexes + + def _get_test_infos(self): + return self.build_context.get('testContext', dict()).get('testInfos', []) + @dataclass(frozen=True) class BuildPlan: @@ -154,7 +225,7 @@ def load_build_context(): def empty_build_context(): - return {'enabled_build_features': []} + return {'enabledBuildFeatures': []} def execute_build_plan(build_plan: BuildPlan): diff --git a/ci/build_test_suites_test.py b/ci/build_test_suites_test.py index a9ff3fbf4d..7efb154487 100644 --- a/ci/build_test_suites_test.py +++ b/ci/build_test_suites_test.py @@ -324,6 +324,55 @@ class BuildPlannerTest(unittest.TestCase): self.run_packaging_functions(build_plan), ) + def test_target_output_used_target_built(self): + build_target = 'test_target' + build_planner = self.create_build_planner( + build_targets={build_target}, + build_context=self.create_build_context( + test_context=self.get_test_context(build_target), + enabled_build_features={'test_target_unused_exclusion'}, + ), + ) + + build_plan = build_planner.create_build_plan() + + self.assertSetEqual(build_plan.build_targets, {build_target}) + + def test_target_regex_used_target_built(self): + build_target = 'test_target' + test_context = self.get_test_context(build_target) + test_context['testInfos'][0]['extraOptions'] = [{ + 'key': 'additional-files-filter', + 'values': [f'.*{build_target}.*\.zip'], + }] + build_planner = self.create_build_planner( + build_targets={build_target}, + build_context=self.create_build_context( + test_context=test_context, + enabled_build_features={'test_target_unused_exclusion'}, + ), + ) + + build_plan = build_planner.create_build_plan() + + self.assertSetEqual(build_plan.build_targets, {build_target}) + + def test_target_output_not_used_target_not_built(self): + build_target = 'test_target' + test_context = self.get_test_context(build_target) + test_context['testInfos'][0]['extraOptions'] = [] + build_planner = self.create_build_planner( + build_targets={build_target}, + build_context=self.create_build_context( + test_context=test_context, + enabled_build_features={'test_target_unused_exclusion'}, + ), + ) + + build_plan = build_planner.create_build_plan() + + self.assertSetEqual(build_plan.build_targets, set()) + def create_build_planner( self, build_targets: set[str], @@ -352,10 +401,10 @@ class BuildPlannerTest(unittest.TestCase): test_context: dict[str, any] = {}, ) -> dict[str, any]: build_context = {} - build_context['enabled_build_features'] = enabled_build_features + build_context['enabledBuildFeatures'] = enabled_build_features if optimized_build_enabled: - build_context['enabled_build_features'].add('optimized_build') - build_context['test_context'] = test_context + build_context['enabledBuildFeatures'].add('optimized_build') + build_context['testContext'] = test_context return build_context def create_args( @@ -398,6 +447,25 @@ class BuildPlannerTest(unittest.TestCase): return output + def get_test_context(self, target: str): + return { + 'testInfos': [ + { + 'name': 'atp_test', + 'target': 'test_target', + 'branch': 'branch', + 'extraOptions': [{ + 'key': 'additional-files-filter', + 'values': [f'{target}.zip'], + }], + 'command': '/tf/command', + 'extraBuildTargets': [ + 'extra_build_target', + ], + }, + ], + } + def wait_until( condition_function: Callable[[], bool], diff --git a/ci/optimized_targets.py b/ci/optimized_targets.py index 224c8c0221..632171906d 100644 --- a/ci/optimized_targets.py +++ b/ci/optimized_targets.py @@ -53,7 +53,7 @@ class NullOptimizer(OptimizedBuildTarget): def get_target_optimizer(target, enabled_flag, build_context, optimizer): - if enabled_flag in build_context['enabled_build_features']: + if enabled_flag in build_context['enabledBuildFeatures']: return optimizer return NullOptimizer(target)