Merge "Don't build targets if they're not used." into main

ci/build_test_suites.py

@@ -20,6 +20,7 @@ import json
 import logging
 import os
 import pathlib
+import re
 import subprocess
 import sys
 from typing import Callable
@@ -52,6 +53,15 @@ class BuildPlanner:
   any output zip files needed by the build.
   """
 
+  _DOWNLOAD_OPTS = {
+      'test-config-only-zip',
+      'test-zip-file-filter',
+      'extra-host-shared-lib-zip',
+      'sandbox-tests-zips',
+      'additional-files-filter',
+      'cts-package-name',
+  }
+
   def __init__(
       self,
       build_context: dict[str, any],
@@ -64,12 +74,19 @@ class BuildPlanner:
 
   def create_build_plan(self):
 
-    if 'optimized_build' not in self.build_context['enabled_build_features']:
+    if 'optimized_build' not in self.build_context.get(
+        'enabledBuildFeatures', []
+    ):
       return BuildPlan(set(self.args.extra_targets), set())
 
     build_targets = set()
     packaging_functions = set()
     for target in self.args.extra_targets:
+      if self._unused_target_exclusion_enabled(
+          target
+      ) and not self._build_target_used(target):
+        continue
+
       target_optimizer_getter = self.target_optimizations.get(target, None)
       if not target_optimizer_getter:
         build_targets.add(target)
@@ -83,6 +100,60 @@ class BuildPlanner:
 
     return BuildPlan(build_targets, packaging_functions)
 
+  def _unused_target_exclusion_enabled(self, target: str) -> bool:
+    return f'{target}_unused_exclusion' in self.build_context.get(
+        'enabledBuildFeatures', []
+    )
+
+  def _build_target_used(self, target: str) -> bool:
+    """Determines whether this target's outputs are used by the test configurations listed in the build context."""
+    file_download_regexes = self._aggregate_file_download_regexes()
+    # For all of a targets' outputs, check if any of the regexes used by tests
+    # to download artifacts would match it. If any of them do then this target
+    # is necessary.
+    for artifact in self._get_target_potential_outputs(target):
+      for regex in file_download_regexes:
+        if re.match(regex, artifact):
+          return True
+    return False
+
+  def _get_target_potential_outputs(self, target: str) -> set[str]:
+    tests_suffix = '-tests'
+    if target.endswith('tests'):
+      tests_suffix = ''
+    # This is a list of all the potential zips output by the test suite targets.
+    # If the test downloads artifacts from any of these zips, we will be
+    # conservative and avoid skipping the tests.
+    return {
+        f'{target}.zip',
+        f'android-{target}.zip',
+        f'android-{target}-verifier.zip',
+        f'{target}{tests_suffix}_list.zip',
+        f'android-{target}{tests_suffix}_list.zip',
+        f'{target}{tests_suffix}_host-shared-libs.zip',
+        f'android-{target}{tests_suffix}_host-shared-libs.zip',
+        f'{target}{tests_suffix}_configs.zip',
+        f'android-{target}{tests_suffix}_configs.zip',
+    }
+
+  def _aggregate_file_download_regexes(self) -> set[re.Pattern]:
+    """Lists out all test config options to specify targets to download.
+
+    These come in the form of regexes.
+    """
+    all_regexes = set()
+    for test_info in self._get_test_infos():
+      for opt in test_info.get('extraOptions', []):
+        # check the known list of options for downloading files.
+        if opt.get('key') in self._DOWNLOAD_OPTS:
+          all_regexes.update(
+              re.compile(value) for value in opt.get('values', [])
+          )
+    return all_regexes
+
+  def _get_test_infos(self):
+    return self.build_context.get('testContext', dict()).get('testInfos', [])
+
 
 @dataclass(frozen=True)
 class BuildPlan:
@@ -155,7 +226,7 @@ def load_build_context():
 
 
 def empty_build_context():
-  return {'enabled_build_features': []}
+  return {'enabledBuildFeatures': []}
 
 
 def execute_build_plan(build_plan: BuildPlan):

ci/build_test_suites_test.py

@@ -324,6 +324,55 @@ class BuildPlannerTest(unittest.TestCase):
         self.run_packaging_functions(build_plan),
     )
 
+  def test_target_output_used_target_built(self):
+    build_target = 'test_target'
+    build_planner = self.create_build_planner(
+        build_targets={build_target},
+        build_context=self.create_build_context(
+            test_context=self.get_test_context(build_target),
+            enabled_build_features={'test_target_unused_exclusion'},
+        ),
+    )
+
+    build_plan = build_planner.create_build_plan()
+
+    self.assertSetEqual(build_plan.build_targets, {build_target})
+
+  def test_target_regex_used_target_built(self):
+    build_target = 'test_target'
+    test_context = self.get_test_context(build_target)
+    test_context['testInfos'][0]['extraOptions'] = [{
+        'key': 'additional-files-filter',
+        'values': [f'.*{build_target}.*\.zip'],
+    }]
+    build_planner = self.create_build_planner(
+        build_targets={build_target},
+        build_context=self.create_build_context(
+            test_context=test_context,
+            enabled_build_features={'test_target_unused_exclusion'},
+        ),
+    )
+
+    build_plan = build_planner.create_build_plan()
+
+    self.assertSetEqual(build_plan.build_targets, {build_target})
+
+  def test_target_output_not_used_target_not_built(self):
+    build_target = 'test_target'
+    test_context = self.get_test_context(build_target)
+    test_context['testInfos'][0]['extraOptions'] = []
+    build_planner = self.create_build_planner(
+        build_targets={build_target},
+        build_context=self.create_build_context(
+            test_context=test_context,
+            enabled_build_features={'test_target_unused_exclusion'},
+        ),
+    )
+
+    build_plan = build_planner.create_build_plan()
+
+    self.assertSetEqual(build_plan.build_targets, set())
+
   def create_build_planner(
       self,
       build_targets: set[str],
@@ -352,10 +401,10 @@ class BuildPlannerTest(unittest.TestCase):
       test_context: dict[str, any] = {},
   ) -> dict[str, any]:
     build_context = {}
-    build_context['enabled_build_features'] = enabled_build_features
+    build_context['enabledBuildFeatures'] = enabled_build_features
     if optimized_build_enabled:
-      build_context['enabled_build_features'].add('optimized_build')
-    build_context['test_context'] = test_context
+      build_context['enabledBuildFeatures'].add('optimized_build')
+    build_context['testContext'] = test_context
     return build_context
 
   def create_args(
@@ -398,6 +447,25 @@ class BuildPlannerTest(unittest.TestCase):
 
     return output
 
+  def get_test_context(self, target: str):
+    return {
+        'testInfos': [
+            {
+                'name': 'atp_test',
+                'target': 'test_target',
+                'branch': 'branch',
+                'extraOptions': [{
+                    'key': 'additional-files-filter',
+                    'values': [f'{target}.zip'],
+                }],
+                'command': '/tf/command',
+                'extraBuildTargets': [
+                    'extra_build_target',
+                ],
+            },
+        ],
+    }
+
 
 def wait_until(
     condition_function: Callable[[], bool],

ci/optimized_targets.py

@@ -53,7 +53,7 @@ class NullOptimizer(OptimizedBuildTarget):
 
 
 def get_target_optimizer(target, enabled_flag, build_context, optimizer):
-  if enabled_flag in build_context['enabled_build_features']:
+  if enabled_flag in build_context['enabledBuildFeatures']:
     return optimizer
 
   return NullOptimizer(target)