Implement package_outputs in GeneralTestsOptimizer
Implement the output packaging step in the GeneralTestsOptimizer. This step takes all the outputs built by the general-tests target (in the case where it was optimized) and packages them into the necessary zips generated by the target normally. Test: atest build_test_suites_test; atest optimized_targets_test Bug: 358215235 Change-Id: I5d27eef4e37137cc9b6e235f52f3856ba0b30460
This commit is contained in:
Luca Farsi
@@ -16,12 +16,13 @@
|
||||
from abc import ABC
|
||||
import argparse
|
||||
import functools
|
||||
from build_context import BuildContext
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Self
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
from build_context import BuildContext
|
||||
import test_mapping_module_retriever
|
||||
|
||||
|
||||
@@ -33,6 +34,9 @@ class OptimizedBuildTarget(ABC):
|
||||
build.
|
||||
"""
|
||||
|
||||
_SOONG_UI_BASH_PATH = 'build/soong/soong_ui.bash'
|
||||
_PREBUILT_SOONG_ZIP_PATH = 'prebuilts/build-tools/linux-x86/bin/soong_zip'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target: str,
|
||||
@@ -75,6 +79,88 @@ class OptimizedBuildTarget(ABC):
|
||||
f'get_build_targets_impl not implemented in {type(self).__name__}'
|
||||
)
|
||||
|
||||
def _generate_zip_options_for_items(
|
||||
self,
|
||||
prefix: str = '',
|
||||
relative_root: str = '',
|
||||
list_files: list[str] | None = None,
|
||||
files: list[str] | None = None,
|
||||
directories: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
if not list_files and not files and not directories:
|
||||
raise RuntimeError(
|
||||
f'No items specified to be added to zip! Prefix: {prefix}, Relative'
|
||||
f' root: {relative_root}'
|
||||
)
|
||||
command_segment = []
|
||||
# These are all soong_zip options so consult soong_zip --help for specifics.
|
||||
if prefix:
|
||||
command_segment.append('-P')
|
||||
command_segment.append(prefix)
|
||||
if relative_root:
|
||||
command_segment.append('-C')
|
||||
command_segment.append(relative_root)
|
||||
if list_files:
|
||||
for list_file in list_files:
|
||||
command_segment.append('-l')
|
||||
command_segment.append(list_file)
|
||||
if files:
|
||||
for file in files:
|
||||
command_segment.append('-f')
|
||||
command_segment.append(file)
|
||||
if directories:
|
||||
for directory in directories:
|
||||
command_segment.append('-D')
|
||||
command_segment.append(directory)
|
||||
|
||||
return command_segment
|
||||
|
||||
def _query_soong_vars(
|
||||
self, src_top: pathlib.Path, soong_vars: list[str]
|
||||
) -> dict[str, str]:
|
||||
process_result = subprocess.run(
|
||||
args=[
|
||||
f'{src_top / self._SOONG_UI_BASH_PATH}',
|
||||
'--dumpvar-mode',
|
||||
'--abs',
|
||||
soong_vars,
|
||||
],
|
||||
env=os.environ,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
)
|
||||
if not process_result.returncode == 0:
|
||||
logging.error('soong dumpvars command failed! stderr:')
|
||||
logging.error(process_result.stderr)
|
||||
raise RuntimeError('Soong dumpvars failed! See log for stderr.')
|
||||
|
||||
if not process_result.stdout:
|
||||
raise RuntimeError(
|
||||
'Necessary soong variables ' + soong_vars + ' not found.'
|
||||
)
|
||||
|
||||
try:
|
||||
return {
|
||||
line.split('=')[0]: line.split('=')[1].strip("'")
|
||||
for line in process_result.stdout.split('\n')
|
||||
}
|
||||
except IndexError as e:
|
||||
raise RuntimeError(
|
||||
'Error parsing soong dumpvars output! See output here:'
|
||||
f' {process_result.stdout}',
|
||||
e,
|
||||
)
|
||||
|
||||
def _base_zip_command(
|
||||
self, src_top: pathlib.Path, dist_dir: pathlib.Path, name: str
|
||||
) -> list[str]:
|
||||
return [
|
||||
f'{src_top / self._PREBUILT_SOONG_ZIP_PATH }',
|
||||
'-d',
|
||||
'-o',
|
||||
f'{dist_dir / name}',
|
||||
]
|
||||
|
||||
|
||||
class NullOptimizer(OptimizedBuildTarget):
|
||||
"""No-op target optimizer.
|
||||
@@ -121,8 +207,6 @@ class ChangeInfo:
|
||||
class GeneralTestsOptimizer(OptimizedBuildTarget):
|
||||
"""general-tests optimizer
|
||||
|
||||
TODO(b/358215235): Implement
|
||||
|
||||
This optimizer reads in the list of changed files from the file located in
|
||||
env[CHANGE_INFO] and uses this list alongside the normal TEST MAPPING logic to
|
||||
determine what test mapping modules will run for the given changes. It then
|
||||
@@ -177,6 +261,208 @@ class GeneralTestsOptimizer(OptimizedBuildTarget):
|
||||
|
||||
return modules_to_build
|
||||
|
||||
def get_package_outputs_commands_impl(self):
|
||||
src_top = pathlib.Path(os.environ.get('TOP', os.getcwd()))
|
||||
dist_dir = pathlib.Path(os.environ.get('DIST_DIR'))
|
||||
|
||||
soong_vars = self._query_soong_vars(
|
||||
src_top,
|
||||
[
|
||||
'HOST_OUT_TESTCASES',
|
||||
'TARGET_OUT_TESTCASES',
|
||||
'PRODUCT_OUT',
|
||||
'SOONG_HOST_OUT',
|
||||
'HOST_OUT',
|
||||
],
|
||||
)
|
||||
host_out_testcases = pathlib.Path(soong_vars.get('HOST_OUT_TESTCASES'))
|
||||
target_out_testcases = pathlib.Path(soong_vars.get('TARGET_OUT_TESTCASES'))
|
||||
product_out = pathlib.Path(soong_vars.get('PRODUCT_OUT'))
|
||||
soong_host_out = pathlib.Path(soong_vars.get('SOONG_HOST_OUT'))
|
||||
host_out = pathlib.Path(soong_vars.get('HOST_OUT'))
|
||||
|
||||
host_paths = []
|
||||
target_paths = []
|
||||
host_config_files = []
|
||||
target_config_files = []
|
||||
for module in self.modules_to_build:
|
||||
host_path = host_out_testcases / module
|
||||
if os.path.exists(host_path):
|
||||
host_paths.append(host_path)
|
||||
self._collect_config_files(src_top, host_path, host_config_files)
|
||||
|
||||
target_path = target_out_testcases / module
|
||||
if os.path.exists(target_path):
|
||||
target_paths.append(target_path)
|
||||
self._collect_config_files(src_top, target_path, target_config_files)
|
||||
|
||||
if not os.path.exists(host_path) and not os.path.exists(target_path):
|
||||
logging.info(f'No host or target build outputs found for {module}.')
|
||||
|
||||
zip_commands = []
|
||||
|
||||
zip_commands.extend(
|
||||
self._get_zip_test_configs_zips_commands(
|
||||
dist_dir,
|
||||
host_out,
|
||||
product_out,
|
||||
host_config_files,
|
||||
target_config_files,
|
||||
)
|
||||
)
|
||||
|
||||
zip_command = self._base_zip_command(
|
||||
host_out, dist_dir, 'general-tests.zip'
|
||||
)
|
||||
|
||||
# Add host testcases.
|
||||
zip_command.extend(
|
||||
self._generate_zip_options_for_items(
|
||||
prefix='host',
|
||||
relative_root=f'{src_top / soong_host_out}',
|
||||
directories=host_paths,
|
||||
)
|
||||
)
|
||||
|
||||
# Add target testcases.
|
||||
zip_command.extend(
|
||||
self._generate_zip_options_for_items(
|
||||
prefix='target',
|
||||
relative_root=f'{src_top / product_out}',
|
||||
directories=target_paths,
|
||||
)
|
||||
)
|
||||
|
||||
# TODO(lucafarsi): Push this logic into a general-tests-minimal build command
|
||||
# Add necessary tools. These are also hardcoded in general-tests.mk.
|
||||
framework_path = soong_host_out / 'framework'
|
||||
|
||||
zip_command.extend(
|
||||
self._generate_zip_options_for_items(
|
||||
prefix='host/tools',
|
||||
relative_root=str(framework_path),
|
||||
files=[
|
||||
f"{framework_path / 'cts-tradefed.jar'}",
|
||||
f"{framework_path / 'compatibility-host-util.jar'}",
|
||||
f"{framework_path / 'vts-tradefed.jar'}",
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
zip_commands.append(zip_command)
|
||||
return zip_commands
|
||||
|
||||
def _collect_config_files(
|
||||
self,
|
||||
src_top: pathlib.Path,
|
||||
root_dir: pathlib.Path,
|
||||
config_files: list[str],
|
||||
):
|
||||
for root, dirs, files in os.walk(src_top / root_dir):
|
||||
for file in files:
|
||||
if file.endswith('.config'):
|
||||
config_files.append(root_dir / file)
|
||||
|
||||
def _get_zip_test_configs_zips_commands(
|
||||
self,
|
||||
dist_dir: pathlib.Path,
|
||||
host_out: pathlib.Path,
|
||||
product_out: pathlib.Path,
|
||||
host_config_files: list[str],
|
||||
target_config_files: list[str],
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Generate general-tests_configs.zip and general-tests_list.zip.
|
||||
|
||||
general-tests_configs.zip contains all of the .config files that were
|
||||
built and general-tests_list.zip contains a text file which lists
|
||||
all of the .config files that are in general-tests_configs.zip.
|
||||
|
||||
general-tests_configs.zip is organized as follows:
|
||||
/
|
||||
host/
|
||||
testcases/
|
||||
test_1.config
|
||||
test_2.config
|
||||
...
|
||||
target/
|
||||
testcases/
|
||||
test_1.config
|
||||
test_2.config
|
||||
...
|
||||
|
||||
So the process is we write out the paths to all the host config files into
|
||||
one
|
||||
file and all the paths to the target config files in another. We also write
|
||||
the paths to all the config files into a third file to use for
|
||||
general-tests_list.zip.
|
||||
|
||||
Args:
|
||||
dist_dir: dist directory.
|
||||
host_out: host out directory.
|
||||
product_out: product out directory.
|
||||
host_config_files: list of all host config files.
|
||||
target_config_files: list of all target config files.
|
||||
|
||||
Returns:
|
||||
The commands to generate general-tests_configs.zip and
|
||||
general-tests_list.zip
|
||||
"""
|
||||
with open(
|
||||
f"{host_out / 'host_general-tests_list'}", 'w'
|
||||
) as host_list_file, open(
|
||||
f"{product_out / 'target_general-tests_list'}", 'w'
|
||||
) as target_list_file, open(
|
||||
f"{host_out / 'general-tests_list'}", 'w'
|
||||
) as list_file:
|
||||
|
||||
for config_file in host_config_files:
|
||||
host_list_file.write(f'{config_file}' + '\n')
|
||||
list_file.write('host/' + os.path.relpath(config_file, host_out) + '\n')
|
||||
|
||||
for config_file in target_config_files:
|
||||
target_list_file.write(f'{config_file}' + '\n')
|
||||
list_file.write(
|
||||
'target/' + os.path.relpath(config_file, product_out) + '\n'
|
||||
)
|
||||
|
||||
zip_commands = []
|
||||
|
||||
tests_config_zip_command = self._base_zip_command(
|
||||
host_out, dist_dir, 'general-tests_configs.zip'
|
||||
)
|
||||
tests_config_zip_command.extend(
|
||||
self._generate_zip_options_for_items(
|
||||
prefix='host',
|
||||
relative_root=str(host_out),
|
||||
list_files=[f"{host_out / 'host_general-tests_list'}"],
|
||||
)
|
||||
)
|
||||
|
||||
tests_config_zip_command.extend(
|
||||
self._generate_zip_options_for_items(
|
||||
prefix='target',
|
||||
relative_root=str(product_out),
|
||||
list_files=[
|
||||
f"{product_out / 'target_general-tests_list'}"
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
zip_commands.append(tests_config_zip_command)
|
||||
|
||||
tests_list_zip_command = self._base_zip_command(
|
||||
host_out, dist_dir, 'general-tests_list.zip'
|
||||
)
|
||||
tests_list_zip_command.extend(
|
||||
self._generate_zip_options_for_items(
|
||||
relative_root=str(host_out),
|
||||
files=[f"{host_out / 'general-tests_list'}"],
|
||||
)
|
||||
)
|
||||
zip_commands.append(tests_list_zip_command)
|
||||
|
||||
return zip_commands
|
||||
|
||||
def get_enabled_flag(self):
|
||||
return 'general_tests_optimized'
|
||||
|
||||
|
||||
Reference in New Issue
Block a user