Stream build process output

Ensure that output and errors from the underlying build command are
streamed for real-time build progress updates and debugging.

This change strips out all code that is currently unecessary and adds
tests for the remaining functionality.

Test: atest --host build_test_suites_local_test build_test_suites_test
Bug: 330365727
Change-Id: I7ef98d6654fe1435cf67c15e2c516a0967e03a75

ci/Android.bp

@@ -0,0 +1,85 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+python_test_host {
+    name: "build_test_suites_test",
+    main: "build_test_suites_test.py",
+    pkg_path: "testdata",
+    srcs: [
+        "build_test_suites_test.py",
+    ],
+    libs: [
+        "build_test_suites",
+        "pyfakefs",
+        "ci_test_lib",
+    ],
+    test_options: {
+        unit_test: true,
+    },
+    data: [
+        ":py3-cmd",
+    ],
+    version: {
+        py3: {
+            embedded_launcher: true,
+        },
+    },
+}
+
+// This test is only intended to be run locally since it's slow, not hermetic,
+// and requires a lot of system state. It is therefore not marked as `unit_test`
+// and is not part of any test suite. Note that we also don't want to run this
+// test with Bazel since that would require disabling sandboxing and explicitly
+// passing in all the env vars we depend on via the command-line. The test
+// target could be configured to do so but it's not worth doing seeing that
+// we're moving away from Bazel.
+python_test_host {
+    name: "build_test_suites_local_test",
+    main: "build_test_suites_local_test.py",
+    srcs: [
+        "build_test_suites_local_test.py",
+    ],
+    libs: [
+        "build_test_suites",
+        "pyfakefs",
+        "ci_test_lib",
+    ],
+    test_config_template: "AndroidTest.xml.template",
+    test_options: {
+        unit_test: false,
+    },
+    version: {
+        py3: {
+            embedded_launcher: true,
+        },
+    },
+}
+
+python_library_host {
+    name: "build_test_suites",
+    srcs: [
+        "build_test_suites.py",
+    ],
+}
+
+python_library_host {
+    name: "ci_test_lib",
+    srcs: [
+        "ci_test_lib.py",
+    ],
+}

ci/AndroidTest.xml.template

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2020 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+<configuration>
+  <test class="com.android.tradefed.testtype.python.PythonBinaryHostTest">
+    <option name="par-file-name" value="{MODULE}"/>
+    <option name="use-test-output-file" value="false"/>
+    <option name="test-timeout" value="5m"/>
+  </test>
+</configuration>

ci/build_test_suites

@@ -1,4 +1,5 @@
 #!prebuilts/build-tools/linux-x86/bin/py3-cmd -B
+#
 # Copyright 2024, The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
 import build_test_suites
+import sys
 
-build_test_suites.main(sys.argv)
+build_test_suites.main(sys.argv[1:])

ci/build_test_suites.py

@@ -12,404 +12,115 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Script to build only the necessary modules for general-tests along
+"""Build script for the CI `test_suites` target."""
-
-with whatever other targets are passed in.
-"""
 
 import argparse
-from collections.abc import Sequence
+import logging
-import json
 import os
 import pathlib
-import re
 import subprocess
 import sys
-from typing import Any
-
-import test_mapping_module_retriever
 
 
-# List of modules that are always required to be in general-tests.zip
+class Error(Exception):
-REQUIRED_MODULES = frozenset(
+
-    ['cts-tradefed', 'vts-tradefed', 'compatibility-host-util', 'soong_zip']
+  def __init__(self, message):
-)
+    super().__init__(message)
 
 
-def build_test_suites(argv):
+class BuildFailureError(Error):
+
+  def __init__(self, return_code):
+    super().__init__(f'Build command failed with return code: f{return_code}')
+    self.return_code = return_code
+
+
+REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP'])
+SOONG_UI_EXE_REL_PATH = 'build/soong/soong_ui.bash'
+
+
+def get_top() -> pathlib.Path:
+  return pathlib.Path(os.environ['TOP'])
+
+
+def build_test_suites(argv: list[str]) -> int:
+  """Builds the general-tests and any other test suites passed in.
+
+  Args:
+    argv: The command line arguments passed in.
+
+  Returns:
+    The exit code of the build.
+  """
   args = parse_args(argv)
+  check_required_env()
 
-  if is_optimization_enabled():
+  try:
-    # Call the class to map changed files to modules to build.
-    # TODO(lucafarsi): Move this into a replaceable class.
-    build_affected_modules(args)
-  else:
     build_everything(args)
+  except BuildFailureError as e:
+    logging.error('Build command failed! Check build_log for details.')
+    return e.return_code
+
+  return 0
+
+
+def check_required_env():
+  """Check for required env vars.
+
+  Raises:
+    RuntimeError: If any required env vars are not found.
+  """
+  missing_env_vars = sorted(v for v in REQUIRED_ENV_VARS if v not in os.environ)
+
+  if not missing_env_vars:
+    return
+
+  t = ','.join(missing_env_vars)
+  raise Error(f'Missing required environment variables: {t}')
 
 
 def parse_args(argv):
   argparser = argparse.ArgumentParser()
+
   argparser.add_argument(
       'extra_targets', nargs='*', help='Extra test suites to build.'
   )
-  argparser.add_argument('--target_product')
-  argparser.add_argument('--target_release')
-  argparser.add_argument(
-      '--with_dexpreopt_boot_img_and_system_server_only', action='store_true'
-  )
-  argparser.add_argument('--change_info', nargs='?')
 
-  return argparser.parse_args()
+  return argparser.parse_args(argv)
-
-
-def is_optimization_enabled() -> bool:
-  # TODO(lucafarsi): switch back to building only affected general-tests modules
-  # in presubmit once ready.
-  # if os.environ.get('BUILD_NUMBER')[0] == 'P':
-  #   return True
-  return False
 
 
 def build_everything(args: argparse.Namespace):
+  """Builds all tests (regardless of whether they are needed).
+
+  Args:
+    args: The parsed arguments.
+
+  Raises:
+    BuildFailure: If the build command fails.
+  """
   build_command = base_build_command(args, args.extra_targets)
-  build_command.append('general-tests')
 
-  run_command(build_command, print_output=True)
+  try:
-
+    run_command(build_command)
-
+  except subprocess.CalledProcessError as e:
-def build_affected_modules(args: argparse.Namespace):
+    raise BuildFailureError(e.returncode) from e
-  modules_to_build = find_modules_to_build(
-      pathlib.Path(args.change_info), args.extra_required_modules
-  )
-
-  # Call the build command with everything.
-  build_command = base_build_command(args, args.extra_targets)
-  build_command.extend(modules_to_build)
-  # When not building general-tests we also have to build the general tests
-  # shared libs.
-  build_command.append('general-tests-shared-libs')
-
-  run_command(build_command, print_output=True)
-
-  zip_build_outputs(modules_to_build, args.target_release)
 
 
 def base_build_command(
     args: argparse.Namespace, extra_targets: set[str]
-) -> list:
+) -> list[str]:
+
   build_command = []
-  build_command.append('time')
+  build_command.append(get_top().joinpath(SOONG_UI_EXE_REL_PATH))
-  build_command.append('./build/soong/soong_ui.bash')
   build_command.append('--make-mode')
-  build_command.append('dist')
-  build_command.append('TARGET_PRODUCT=' + args.target_product)
-  build_command.append('TARGET_RELEASE=' + args.target_release)
-  if args.with_dexpreopt_boot_img_and_system_server_only:
-    build_command.append('WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY=true')
   build_command.extend(extra_targets)
 
   return build_command
 
 
-def run_command(
+def run_command(args: list[str], stdout=None):
-    args: list[str],
+  subprocess.run(args=args, check=True, stdout=stdout)
-    env: dict[str, str] = os.environ,
-    print_output: bool = False,
-) -> str:
-  result = subprocess.run(
-      args=args,
-      text=True,
-      capture_output=True,
-      check=False,
-      env=env,
-  )
-  # If the process failed, print its stdout and propagate the exception.
-  if not result.returncode == 0:
-    print('Build command failed! output:')
-    print('stdout: ' + result.stdout)
-    print('stderr: ' + result.stderr)
-
-  result.check_returncode()
-
-  if print_output:
-    print(result.stdout)
-
-  return result.stdout
-
-
-def find_modules_to_build(
-    change_info: pathlib.Path, extra_required_modules: list[str]
-) -> set[str]:
-  changed_files = find_changed_files(change_info)
-
-  test_mappings = test_mapping_module_retriever.GetTestMappings(
-      changed_files, set()
-  )
-
-  # Soong_zip is required to generate the output zip so always build it.
-  modules_to_build = set(REQUIRED_MODULES)
-  if extra_required_modules:
-    modules_to_build.update(extra_required_modules)
-
-  modules_to_build.update(find_affected_modules(test_mappings, changed_files))
-
-  return modules_to_build
-
-
-def find_changed_files(change_info: pathlib.Path) -> set[str]:
-  with open(change_info) as change_info_file:
-    change_info_contents = json.load(change_info_file)
-
-  changed_files = set()
-
-  for change in change_info_contents['changes']:
-    project_path = change.get('projectPath') + '/'
-
-    for revision in change.get('revisions'):
-      for file_info in revision.get('fileInfos'):
-        changed_files.add(project_path + file_info.get('path'))
-
-  return changed_files
-
-
-def find_affected_modules(
-    test_mappings: dict[str, Any], changed_files: set[str]
-) -> set[str]:
-  modules = set()
-
-  # The test_mappings object returned by GetTestMappings is organized as
-  # follows:
-  # {
-  #   'test_mapping_file_path': {
-  #     'group_name' : [
-  #       'name': 'module_name',
-  #     ],
-  #   }
-  # }
-  for test_mapping in test_mappings.values():
-    for group in test_mapping.values():
-      for entry in group:
-        module_name = entry.get('name', None)
-
-        if not module_name:
-          continue
-
-        file_patterns = entry.get('file_patterns')
-        if not file_patterns:
-          modules.add(module_name)
-          continue
-
-        if matches_file_patterns(file_patterns, changed_files):
-          modules.add(module_name)
-          continue
-
-  return modules
-
-
-# TODO(lucafarsi): Share this logic with the original logic in
-# test_mapping_test_retriever.py
-def matches_file_patterns(
-    file_patterns: list[set], changed_files: set[str]
-) -> bool:
-  for changed_file in changed_files:
-    for pattern in file_patterns:
-      if re.search(pattern, changed_file):
-        return True
-
-  return False
-
-
-def zip_build_outputs(
-    modules_to_build: set[str], target_release: str
-):
-  src_top = os.environ.get('TOP', os.getcwd())
-
-  # Call dumpvars to get the necessary things.
-  # TODO(lucafarsi): Don't call soong_ui 4 times for this, --dumpvars-mode can
-  # do it but it requires parsing.
-  host_out_testcases = pathlib.Path(
-      get_soong_var('HOST_OUT_TESTCASES', target_release)
-  )
-  target_out_testcases = pathlib.Path(
-      get_soong_var('TARGET_OUT_TESTCASES', target_release)
-  )
-  product_out = pathlib.Path(get_soong_var('PRODUCT_OUT', target_release))
-  soong_host_out = pathlib.Path(get_soong_var('SOONG_HOST_OUT', target_release))
-  host_out = pathlib.Path(get_soong_var('HOST_OUT', target_release))
-  dist_dir = pathlib.Path(get_soong_var('DIST_DIR', target_release))
-
-  # Call the class to package the outputs.
-  # TODO(lucafarsi): Move this code into a replaceable class.
-  host_paths = []
-  target_paths = []
-  host_config_files = []
-  target_config_files = []
-  for module in modules_to_build:
-    host_path = os.path.join(host_out_testcases, module)
-    if os.path.exists(host_path):
-      host_paths.append(host_path)
-      collect_config_files(src_top, host_path, host_config_files)
-
-    target_path = os.path.join(target_out_testcases, module)
-    if os.path.exists(target_path):
-      target_paths.append(target_path)
-      collect_config_files(src_top, target_path, target_config_files)
-
-  zip_test_configs_zips(
-      dist_dir, host_out, product_out, host_config_files, target_config_files
-  )
-
-  zip_command = base_zip_command(host_out, dist_dir, 'general-tests.zip')
-
-  # Add host testcases.
-  zip_command.append('-C')
-  zip_command.append(os.path.join(src_top, soong_host_out))
-  zip_command.append('-P')
-  zip_command.append('host/')
-  for path in host_paths:
-    zip_command.append('-D')
-    zip_command.append(path)
-
-  # Add target testcases.
-  zip_command.append('-C')
-  zip_command.append(os.path.join(src_top, product_out))
-  zip_command.append('-P')
-  zip_command.append('target')
-  for path in target_paths:
-    zip_command.append('-D')
-    zip_command.append(path)
-
-  # TODO(lucafarsi): Push this logic into a general-tests-minimal build command
-  # Add necessary tools. These are also hardcoded in general-tests.mk.
-  framework_path = os.path.join(soong_host_out, 'framework')
-
-  zip_command.append('-C')
-  zip_command.append(framework_path)
-  zip_command.append('-P')
-  zip_command.append('host/tools')
-  zip_command.append('-f')
-  zip_command.append(os.path.join(framework_path, 'cts-tradefed.jar'))
-  zip_command.append('-f')
-  zip_command.append(
-      os.path.join(framework_path, 'compatibility-host-util.jar')
-  )
-  zip_command.append('-f')
-  zip_command.append(os.path.join(framework_path, 'vts-tradefed.jar'))
-
-  run_command(zip_command, print_output=True)
-
-
-def collect_config_files(
-    src_top: pathlib.Path, root_dir: pathlib.Path, config_files: list[str]
-):
-  for root, dirs, files in os.walk(os.path.join(src_top, root_dir)):
-    for file in files:
-      if file.endswith('.config'):
-        config_files.append(os.path.join(root_dir, file))
-
-
-def base_zip_command(
-    host_out: pathlib.Path, dist_dir: pathlib.Path, name: str
-) -> list[str]:
-  return [
-      'time',
-      os.path.join(host_out, 'bin', 'soong_zip'),
-      '-d',
-      '-o',
-      os.path.join(dist_dir, name),
-  ]
-
-
-# generate general-tests_configs.zip which contains all of the .config files
-# that were built and general-tests_list.zip which contains a text file which
-# lists all of the .config files that are in general-tests_configs.zip.
-#
-# general-tests_comfigs.zip is organized as follows:
-# /
-#   host/
-#     testcases/
-#       test_1.config
-#       test_2.config
-#       ...
-#   target/
-#     testcases/
-#       test_1.config
-#       test_2.config
-#       ...
-#
-# So the process is we write out the paths to all the host config files into one
-# file and all the paths to the target config files in another. We also write
-# the paths to all the config files into a third file to use for
-# general-tests_list.zip.
-def zip_test_configs_zips(
-    dist_dir: pathlib.Path,
-    host_out: pathlib.Path,
-    product_out: pathlib.Path,
-    host_config_files: list[str],
-    target_config_files: list[str],
-):
-  with open(
-      os.path.join(host_out, 'host_general-tests_list'), 'w'
-  ) as host_list_file, open(
-      os.path.join(product_out, 'target_general-tests_list'), 'w'
-  ) as target_list_file, open(
-      os.path.join(host_out, 'general-tests_list'), 'w'
-  ) as list_file:
-
-    for config_file in host_config_files:
-      host_list_file.write(config_file + '\n')
-      list_file.write('host/' + os.path.relpath(config_file, host_out) + '\n')
-
-    for config_file in target_config_files:
-      target_list_file.write(config_file + '\n')
-      list_file.write(
-          'target/' + os.path.relpath(config_file, product_out) + '\n'
-      )
-
-  tests_config_zip_command = base_zip_command(
-      host_out, dist_dir, 'general-tests_configs.zip'
-  )
-  tests_config_zip_command.append('-P')
-  tests_config_zip_command.append('host')
-  tests_config_zip_command.append('-C')
-  tests_config_zip_command.append(host_out)
-  tests_config_zip_command.append('-l')
-  tests_config_zip_command.append(
-      os.path.join(host_out, 'host_general-tests_list')
-  )
-  tests_config_zip_command.append('-P')
-  tests_config_zip_command.append('target')
-  tests_config_zip_command.append('-C')
-  tests_config_zip_command.append(product_out)
-  tests_config_zip_command.append('-l')
-  tests_config_zip_command.append(
-      os.path.join(product_out, 'target_general-tests_list')
-  )
-  run_command(tests_config_zip_command, print_output=True)
-
-  tests_list_zip_command = base_zip_command(
-      host_out, dist_dir, 'general-tests_list.zip'
-  )
-  tests_list_zip_command.append('-C')
-  tests_list_zip_command.append(host_out)
-  tests_list_zip_command.append('-f')
-  tests_list_zip_command.append(os.path.join(host_out, 'general-tests_list'))
-  run_command(tests_list_zip_command, print_output=True)
-
-
-def get_soong_var(var: str, target_release: str) -> str:
-  new_env = os.environ.copy()
-  new_env['TARGET_RELEASE'] = target_release
-
-  value = run_command(
-      ['./build/soong/soong_ui.bash', '--dumpvar-mode', '--abs', var],
-      env=new_env,
-  ).strip()
-  if not value:
-    raise RuntimeError('Necessary soong variable ' + var + ' not found.')
-
-  return value
 
 
 def main(argv):
-  build_test_suites(argv)
+  sys.exit(build_test_suites(argv))

ci/build_test_suites_local_test.py

@@ -0,0 +1,123 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Integration tests for build_test_suites that require a local build env."""
+
+import os
+import pathlib
+import shutil
+import signal
+import subprocess
+import tempfile
+import time
+import ci_test_lib
+
+
+class BuildTestSuitesLocalTest(ci_test_lib.TestCase):
+
+  def setUp(self):
+    self.top_dir = pathlib.Path(os.environ['ANDROID_BUILD_TOP']).resolve()
+    self.executable = self.top_dir.joinpath('build/make/ci/build_test_suites')
+    self.process_session = ci_test_lib.TemporaryProcessSession(self)
+    self.temp_dir = ci_test_lib.TestTemporaryDirectory.create(self)
+
+  def build_subprocess_args(self, build_args: list[str]):
+    env = os.environ.copy()
+    env['TOP'] = str(self.top_dir)
+    env['OUT_DIR'] = self.temp_dir
+
+    args = ([self.executable] + build_args,)
+    kwargs = {
+        'cwd': self.top_dir,
+        'env': env,
+        'text': True,
+    }
+
+    return (args, kwargs)
+
+  def run_build(self, build_args: list[str]) -> subprocess.CompletedProcess:
+    args, kwargs = self.build_subprocess_args(build_args)
+
+    return subprocess.run(
+        *args,
+        **kwargs,
+        check=True,
+        capture_output=True,
+        timeout=5 * 60,
+    )
+
+  def assert_children_alive(self, children: list[int]):
+    for c in children:
+      self.assertTrue(ci_test_lib.process_alive(c))
+
+  def assert_children_dead(self, children: list[int]):
+    for c in children:
+      self.assertFalse(ci_test_lib.process_alive(c))
+
+  def test_fails_for_invalid_arg(self):
+    invalid_arg = '--invalid-arg'
+
+    with self.assertRaises(subprocess.CalledProcessError) as cm:
+      self.run_build([invalid_arg])
+
+    self.assertIn(invalid_arg, cm.exception.stderr)
+
+  def test_builds_successfully(self):
+    self.run_build(['nothing'])
+
+  def test_can_interrupt_build(self):
+    args, kwargs = self.build_subprocess_args(['general-tests'])
+    p = self.process_session.create(args, kwargs)
+
+    # TODO(lucafarsi): Replace this (and other instances) with a condition.
+    time.sleep(5)  # Wait for the build to get going.
+    self.assertIsNone(p.poll())  # Check that the process is still alive.
+    children = query_child_pids(p.pid)
+    self.assert_children_alive(children)
+
+    p.send_signal(signal.SIGINT)
+    p.wait()
+
+    time.sleep(5)  # Wait for things to die out.
+    self.assert_children_dead(children)
+
+  def test_can_kill_build_process_group(self):
+    args, kwargs = self.build_subprocess_args(['general-tests'])
+    p = self.process_session.create(args, kwargs)
+
+    time.sleep(5)  # Wait for the build to get going.
+    self.assertIsNone(p.poll())  # Check that the process is still alive.
+    children = query_child_pids(p.pid)
+    self.assert_children_alive(children)
+
+    os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+    p.wait()
+
+    time.sleep(5)  # Wait for things to die out.
+    self.assert_children_dead(children)
+
+
+# TODO(hzalek): Replace this with `psutils` once available in the tree.
+def query_child_pids(parent_pid: int) -> set[int]:
+  p = subprocess.run(
+      ['pgrep', '-P', str(parent_pid)],
+      check=True,
+      capture_output=True,
+      text=True,
+  )
+  return {int(pid) for pid in p.stdout.splitlines()}
+
+
+if __name__ == '__main__':
+  ci_test_lib.main()

ci/build_test_suites_test.py

@@ -0,0 +1,254 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for build_test_suites.py"""
+
+from importlib import resources
+import multiprocessing
+import os
+import pathlib
+import shutil
+import signal
+import stat
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+from typing import Callable
+from unittest import mock
+import build_test_suites
+import ci_test_lib
+from pyfakefs import fake_filesystem_unittest
+
+
+class BuildTestSuitesTest(fake_filesystem_unittest.TestCase):
+
+  def setUp(self):
+    self.setUpPyfakefs()
+
+    os_environ_patcher = mock.patch.dict('os.environ', {})
+    self.addCleanup(os_environ_patcher.stop)
+    self.mock_os_environ = os_environ_patcher.start()
+
+    subprocess_run_patcher = mock.patch('subprocess.run')
+    self.addCleanup(subprocess_run_patcher.stop)
+    self.mock_subprocess_run = subprocess_run_patcher.start()
+
+    self._setup_working_build_env()
+
+  def test_missing_target_release_env_var_raises(self):
+    del os.environ['TARGET_RELEASE']
+
+    with self.assert_raises_word(build_test_suites.Error, 'TARGET_RELEASE'):
+      build_test_suites.main([])
+
+  def test_missing_target_product_env_var_raises(self):
+    del os.environ['TARGET_PRODUCT']
+
+    with self.assert_raises_word(build_test_suites.Error, 'TARGET_PRODUCT'):
+      build_test_suites.main([])
+
+  def test_missing_top_env_var_raises(self):
+    del os.environ['TOP']
+
+    with self.assert_raises_word(build_test_suites.Error, 'TOP'):
+      build_test_suites.main([])
+
+  def test_invalid_arg_raises(self):
+    invalid_args = ['--invalid_arg']
+
+    with self.assertRaisesRegex(SystemExit, '2'):
+      build_test_suites.main(invalid_args)
+
+  def test_build_failure_returns(self):
+    self.mock_subprocess_run.side_effect = subprocess.CalledProcessError(
+        42, None
+    )
+
+    with self.assertRaisesRegex(SystemExit, '42'):
+      build_test_suites.main([])
+
+  def test_build_success_returns(self):
+    with self.assertRaisesRegex(SystemExit, '0'):
+      build_test_suites.main([])
+
+  def assert_raises_word(self, cls, word):
+    return self.assertRaisesRegex(build_test_suites.Error, rf'\b{word}\b')
+
+  def _setup_working_build_env(self):
+    self.fake_top = pathlib.Path('/fake/top')
+    self.fake_top.mkdir(parents=True)
+
+    self.soong_ui_dir = self.fake_top.joinpath('build/soong')
+    self.soong_ui_dir.mkdir(parents=True, exist_ok=True)
+
+    self.soong_ui = self.soong_ui_dir.joinpath('soong_ui.bash')
+    self.soong_ui.touch()
+
+    self.mock_os_environ.update({
+        'TARGET_RELEASE': 'release',
+        'TARGET_PRODUCT': 'product',
+        'TOP': str(self.fake_top),
+    })
+
+    self.mock_subprocess_run.return_value = 0
+
+
+class RunCommandIntegrationTest(ci_test_lib.TestCase):
+
+  def setUp(self):
+    self.temp_dir = ci_test_lib.TestTemporaryDirectory.create(self)
+
+    # Copy the Python executable from 'non-code' resources and make it
+    # executable for use by tests that launch a subprocess. Note that we don't
+    # use Python's native `sys.executable` property since that is not set when
+    # running via the embedded launcher.
+    base_name = 'py3-cmd'
+    dest_file = self.temp_dir.joinpath(base_name)
+    with resources.as_file(
+        resources.files('testdata').joinpath(base_name)
+    ) as p:
+      shutil.copy(p, dest_file)
+    dest_file.chmod(dest_file.stat().st_mode | stat.S_IEXEC)
+    self.python_executable = dest_file
+
+    self._managed_processes = []
+
+  def tearDown(self):
+    self._terminate_managed_processes()
+
+  def test_raises_on_nonzero_exit(self):
+    with self.assertRaises(Exception):
+      build_test_suites.run_command([
+          self.python_executable,
+          '-c',
+          textwrap.dedent(f"""\
+              import sys
+              sys.exit(1)
+              """),
+      ])
+
+  def test_streams_stdout(self):
+
+    def run_slow_command(stdout_file, marker):
+      with open(stdout_file, 'w') as f:
+        build_test_suites.run_command(
+            [
+                self.python_executable,
+                '-c',
+                textwrap.dedent(f"""\
+                  import time
+
+                  print('{marker}', end='', flush=True)
+
+                  # Keep process alive until we check stdout.
+                  time.sleep(10)
+                  """),
+            ],
+            stdout=f,
+        )
+
+    marker = 'Spinach'
+    stdout_file = self.temp_dir.joinpath('stdout.txt')
+
+    p = self.start_process(target=run_slow_command, args=[stdout_file, marker])
+
+    self.assert_file_eventually_contains(stdout_file, marker)
+
+  def test_propagates_interruptions(self):
+
+    def run(pid_file):
+      build_test_suites.run_command([
+          self.python_executable,
+          '-c',
+          textwrap.dedent(f"""\
+              import os
+              import pathlib
+              import time
+
+              pathlib.Path('{pid_file}').write_text(str(os.getpid()))
+
+              # Keep the process alive for us to explicitly interrupt it.
+              time.sleep(10)
+              """),
+      ])
+
+    pid_file = self.temp_dir.joinpath('pid.txt')
+    p = self.start_process(target=run, args=[pid_file])
+    subprocess_pid = int(read_eventual_file_contents(pid_file))
+
+    os.kill(p.pid, signal.SIGINT)
+    p.join()
+
+    self.assert_process_eventually_dies(p.pid)
+    self.assert_process_eventually_dies(subprocess_pid)
+
+  def start_process(self, *args, **kwargs) -> multiprocessing.Process:
+    p = multiprocessing.Process(*args, **kwargs)
+    self._managed_processes.append(p)
+    p.start()
+    return p
+
+  def assert_process_eventually_dies(self, pid: int):
+    try:
+      wait_until(lambda: not ci_test_lib.process_alive(pid))
+    except TimeoutError as e:
+      self.fail(f'Process {pid} did not die after a while: {e}')
+
+  def assert_file_eventually_contains(self, file: pathlib.Path, substring: str):
+    wait_until(lambda: file.is_file() and file.stat().st_size > 0)
+    self.assertIn(substring, read_file_contents(file))
+
+  def _terminate_managed_processes(self):
+    for p in self._managed_processes:
+      if not p.is_alive():
+        continue
+
+      # We terminate the process with `SIGINT` since using `terminate` or
+      # `SIGKILL` doesn't kill any grandchild processes and we don't have
+      # `psutil` available to easily query all children.
+      os.kill(p.pid, signal.SIGINT)
+
+
+def wait_until(
+    condition_function: Callable[[], bool],
+    timeout_secs: float = 3.0,
+    polling_interval_secs: float = 0.1,
+):
+  """Waits until a condition function returns True."""
+
+  start_time_secs = time.time()
+
+  while not condition_function():
+    if time.time() - start_time_secs > timeout_secs:
+      raise TimeoutError(
+          f'Condition not met within timeout: {timeout_secs} seconds'
+      )
+
+    time.sleep(polling_interval_secs)
+
+
+def read_file_contents(file: pathlib.Path) -> str:
+  with open(file, 'r') as f:
+    return f.read()
+
+
+def read_eventual_file_contents(file: pathlib.Path) -> str:
+  wait_until(lambda: file.is_file() and file.stat().st_size > 0)
+  return read_file_contents(file)
+
+
+if __name__ == '__main__':
+  ci_test_lib.main()

ci/ci_test_lib.py

@@ -0,0 +1,86 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Testing utilities for tests in the CI package."""
+
+import logging
+import os
+import unittest
+import subprocess
+import pathlib
+import shutil
+import tempfile
+
+
+# Export the TestCase class to reduce the number of imports tests have to list.
+TestCase = unittest.TestCase
+
+
+def process_alive(pid):
+  """Check For the existence of a pid."""
+
+  try:
+    os.kill(pid, 0)
+  except OSError:
+    return False
+
+  return True
+
+
+class TemporaryProcessSession:
+
+  def __init__(self, test_case: TestCase):
+    self._created_processes = []
+    test_case.addCleanup(self.cleanup)
+
+  def create(self, args, kwargs):
+    p = subprocess.Popen(*args, **kwargs, start_new_session=True)
+    self._created_processes.append(p)
+    return p
+
+  def cleanup(self):
+    for p in self._created_processes:
+      if not process_alive(p.pid):
+        return
+      os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+
+
+class TestTemporaryDirectory:
+
+  def __init__(self, delete: bool, ):
+    self._delete = delete
+
+  @classmethod
+  def create(cls, test_case: TestCase, delete: bool = True):
+    temp_dir = TestTemporaryDirectory(delete)
+    temp_dir._dir = pathlib.Path(tempfile.mkdtemp())
+    test_case.addCleanup(temp_dir.cleanup)
+    return temp_dir._dir
+
+  def get_dir(self):
+    return self._dir
+
+  def cleanup(self):
+    if not self._delete:
+      return
+    shutil.rmtree(self._dir, ignore_errors=True)
+
+
+def main():
+
+  # Disable logging since it breaks the TF Python test output parser.
+  # TODO(hzalek): Use TF's `test-output-file` option to re-enable logging.
+  logging.getLogger().disabled = True
+
+  unittest.main()