From fd2e967af05abd37983af4a2c62e7ba96f5577e4 Mon Sep 17 00:00:00 2001 From: Sangmin Lee Date: Wed, 21 Aug 2024 17:01:07 +0900 Subject: [PATCH 1/4] Optimize: Reduce vendor target build files Prevent system/% files from being built by BUILDING_SYSTEM_IMAGE and data/% files from being built by BUILDING_USERDATA_IMAGE 1) Address issue where system/% files were being built even when PRODUCT_BUILD_SYSTEM_IMAGE was false. This change reduces the ninja targets for aosp_cf_x86_64_phone_vendor-trunk_staging-userdebug from 140,427 to 57,303 Test: lunch aosp_cf_x86_64_phone_vendor-trunk_staging-userdebug ; m 2) Resolve issue where data/% files were being built during "m dist" even if PRODUCT_BUILD_USERDATA_IMAGE was set to false Test: Set any module to install at data partition using LOCAL_MODULE_PATH=$(TARGET_OUT_DATA)/app lunch aosp_cf_x86_64_phone_vendor-trunk_staging-userdebug ; m dist Check the contents at out/target/product/vsoc_x86_64 and out/dist Change-Id: I2a914339231d57598a6f30fc83fd8203d4107a11 --- core/Makefile | 4 +++- core/main.mk | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/Makefile b/core/Makefile index 18ee77d393..a25bef6a93 100644 --- a/core/Makefile +++ b/core/Makefile @@ -3416,8 +3416,10 @@ endif # PRODUCT_FSVERITY_GENERATE_METADATA # system image INSTALLED_FILES_OUTSIDE_IMAGES := $(filter-out $(TARGET_OUT)/%, $(INSTALLED_FILES_OUTSIDE_IMAGES)) +ifdef BUILDING_SYSTEM_IMAGE INTERNAL_SYSTEMIMAGE_FILES := $(sort $(filter $(TARGET_OUT)/%, \ $(ALL_DEFAULT_INSTALLED_MODULES))) +endif # Create symlink /system/vendor to /vendor if necessary. ifdef BOARD_USES_VENDORIMAGE @@ -3674,10 +3676,10 @@ platform-java: # ----------------------------------------------------------------- # data partition image INSTALLED_FILES_OUTSIDE_IMAGES := $(filter-out $(TARGET_OUT_DATA)/%, $(INSTALLED_FILES_OUTSIDE_IMAGES)) +ifdef BUILDING_USERDATA_IMAGE INTERNAL_USERDATAIMAGE_FILES := \ $(filter $(TARGET_OUT_DATA)/%,$(ALL_DEFAULT_INSTALLED_MODULES)) -ifdef BUILDING_USERDATA_IMAGE userdataimage_intermediates := \ $(call intermediates-dir-for,PACKAGING,userdata) BUILT_USERDATAIMAGE_TARGET := $(PRODUCT_OUT)/userdata.img diff --git a/core/main.mk b/core/main.mk index 85fd65c2f5..5c280da1a2 100644 --- a/core/main.mk +++ b/core/main.mk @@ -1860,6 +1860,11 @@ ifndef INSTALLED_RECOVERYIMAGE_TARGET filter_out_files += $(PRODUCT_OUT)/recovery/% endif +# userdata.img +ifndef BUILDING_USERDATA_IMAGE +filter_out_files += $(PRODUCT_OUT)/data/% +endif + installed_files := $(sort $(filter-out $(filter_out_files),$(filter $(PRODUCT_OUT)/%,$(modules_to_install)))) else installed_files := $(apps_only_installed_files) From b130e791b61eb3401d095ae1d610d5667961496f Mon Sep 17 00:00:00 2001 From: Luca Farsi Date: Thu, 22 Aug 2024 12:04:41 -0700 Subject: [PATCH 2/4] add BuildContext class and fix enabled features Add a BuildContext class to simplify parsing the build context dict, and fix the parsing of enabledBuildFeatures, which was being parsed as a list of strings when it's actually a list of dicts like: [{'name': ''}] Test: atest build_test_suites_test Bug: 361605425 Change-Id: I6424c444daf1582e92313c39f43207cb274aa78f --- ci/Android.bp | 1 + ci/build_context.py | 64 ++++++++++++++++++++++++++++++++++++ ci/build_test_suites.py | 50 +++++----------------------- ci/build_test_suites_test.py | 33 ++++++++++--------- ci/optimized_targets.py | 9 ++--- 5 files changed, 96 insertions(+), 61 deletions(-) create mode 100644 ci/build_context.py diff --git a/ci/Android.bp b/ci/Android.bp index 104f517ccd..22c4851bca 100644 --- a/ci/Android.bp +++ b/ci/Android.bp @@ -76,6 +76,7 @@ python_library_host { srcs: [ "build_test_suites.py", "optimized_targets.py", + "build_context.py", ], } diff --git a/ci/build_context.py b/ci/build_context.py new file mode 100644 index 0000000000..cc48d53992 --- /dev/null +++ b/ci/build_context.py @@ -0,0 +1,64 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Container class for build context with utility functions.""" + +import re + + +class BuildContext: + + def __init__(self, build_context_dict: dict[str, any]): + self.enabled_build_features = set() + for opt in build_context_dict.get('enabledBuildFeatures', []): + self.enabled_build_features.add(opt.get('name')) + self.test_infos = set() + for test_info_dict in build_context_dict.get('testContext', dict()).get( + 'testInfos', [] + ): + self.test_infos.add(self.TestInfo(test_info_dict)) + + def build_target_used(self, target: str) -> bool: + return any(test.build_target_used(target) for test in self.test_infos) + + class TestInfo: + + _DOWNLOAD_OPTS = { + 'test-config-only-zip', + 'test-zip-file-filter', + 'extra-host-shared-lib-zip', + 'sandbox-tests-zips', + 'additional-files-filter', + 'cts-package-name', + } + + def __init__(self, test_info_dict: dict[str, any]): + self.is_test_mapping = False + self.test_mapping_test_groups = set() + self.file_download_options = set() + for opt in test_info_dict.get('extraOptions', []): + key = opt.get('key') + if key == 'test-mapping-test-group': + self.is_test_mapping = True + self.test_mapping_test_groups.update(opt.get('values', set())) + + if key in self._DOWNLOAD_OPTS: + self.file_download_options.update(opt.get('values', set())) + + def build_target_used(self, target: str) -> bool: + # For all of a targets' outputs, check if any of the regexes used by tests + # to download artifacts would match it. If any of them do then this target + # is necessary. + regex = r'\b(%s)\b' % re.escape(target) + return any(re.search(regex, opt) for opt in self.file_download_options) diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py index deb1f1d0fb..402880c6ac 100644 --- a/ci/build_test_suites.py +++ b/ci/build_test_suites.py @@ -24,6 +24,7 @@ import re import subprocess import sys from typing import Callable +from build_context import BuildContext import optimized_targets @@ -53,18 +54,9 @@ class BuildPlanner: any output zip files needed by the build. """ - _DOWNLOAD_OPTS = { - 'test-config-only-zip', - 'test-zip-file-filter', - 'extra-host-shared-lib-zip', - 'sandbox-tests-zips', - 'additional-files-filter', - 'cts-package-name', - } - def __init__( self, - build_context: dict[str, any], + build_context: BuildContext, args: argparse.Namespace, target_optimizations: dict[str, optimized_targets.OptimizedBuildTarget], ): @@ -74,18 +66,15 @@ class BuildPlanner: def create_build_plan(self): - if 'optimized_build' not in self.build_context.get( - 'enabledBuildFeatures', [] - ): + if 'optimized_build' not in self.build_context.enabled_build_features: return BuildPlan(set(self.args.extra_targets), set()) build_targets = set() packaging_functions = set() - self.file_download_options = self._aggregate_file_download_options() for target in self.args.extra_targets: if self._unused_target_exclusion_enabled( target - ) and not self._build_target_used(target): + ) and not self.build_context.build_target_used(target): continue target_optimizer_getter = self.target_optimizations.get(target, None) @@ -102,34 +91,11 @@ class BuildPlanner: return BuildPlan(build_targets, packaging_functions) def _unused_target_exclusion_enabled(self, target: str) -> bool: - return f'{target}_unused_exclusion' in self.build_context.get( - 'enabledBuildFeatures', [] + return ( + f'{target}_unused_exclusion' + in self.build_context.enabled_build_features ) - def _build_target_used(self, target: str) -> bool: - """Determines whether this target's outputs are used by the test configurations listed in the build context.""" - # For all of a targets' outputs, check if any of the regexes used by tests - # to download artifacts would match it. If any of them do then this target - # is necessary. - regex = r'\b(%s)\b' % re.escape(target) - return any(re.search(regex, opt) for opt in self.file_download_options) - - def _aggregate_file_download_options(self) -> set[str]: - """Lists out all test config options to specify targets to download. - - These come in the form of regexes. - """ - all_options = set() - for test_info in self._get_test_infos(): - for opt in test_info.get('extraOptions', []): - # check the known list of options for downloading files. - if opt.get('key') in self._DOWNLOAD_OPTS: - all_options.update(opt.get('values', [])) - return all_options - - def _get_test_infos(self): - return self.build_context.get('testContext', dict()).get('testInfos', []) - @dataclass(frozen=True) class BuildPlan: @@ -148,7 +114,7 @@ def build_test_suites(argv: list[str]) -> int: """ args = parse_args(argv) check_required_env() - build_context = load_build_context() + build_context = BuildContext(load_build_context()) build_planner = BuildPlanner( build_context, args, optimized_targets.OPTIMIZED_BUILD_TARGETS ) diff --git a/ci/build_test_suites_test.py b/ci/build_test_suites_test.py index 463bdd0d73..f3ff6f4926 100644 --- a/ci/build_test_suites_test.py +++ b/ci/build_test_suites_test.py @@ -32,6 +32,7 @@ import time from typing import Callable import unittest from unittest import mock +from build_context import BuildContext import build_test_suites import ci_test_lib import optimized_targets @@ -282,7 +283,7 @@ class BuildPlannerTest(unittest.TestCase): build_planner = self.create_build_planner( build_targets=build_targets, build_context=self.create_build_context( - enabled_build_features={self.get_target_flag('target_1')} + enabled_build_features=[{'name': self.get_target_flag('target_1')}] ), ) @@ -297,7 +298,7 @@ class BuildPlannerTest(unittest.TestCase): build_planner = self.create_build_planner( build_targets=build_targets, build_context=self.create_build_context( - enabled_build_features={self.get_target_flag('target_1')}, + enabled_build_features=[{'name': self.get_target_flag('target_1')}] ), packaging_outputs=packaging_outputs, ) @@ -337,7 +338,7 @@ class BuildPlannerTest(unittest.TestCase): build_targets={build_target}, build_context=self.create_build_context( test_context=self.get_test_context(build_target), - enabled_build_features={'test_target_unused_exclusion'}, + enabled_build_features=[{'name': 'test_target_unused_exclusion'}], ), ) @@ -356,7 +357,7 @@ class BuildPlannerTest(unittest.TestCase): build_targets={build_target}, build_context=self.create_build_context( test_context=test_context, - enabled_build_features={'test_target_unused_exclusion'}, + enabled_build_features=[{'name': 'test_target_unused_exclusion'}], ), ) @@ -372,7 +373,7 @@ class BuildPlannerTest(unittest.TestCase): build_targets={build_target}, build_context=self.create_build_context( test_context=test_context, - enabled_build_features={'test_target_unused_exclusion'}, + enabled_build_features=[{'name': 'test_target_unused_exclusion'}], ), ) @@ -391,7 +392,7 @@ class BuildPlannerTest(unittest.TestCase): build_targets={build_target}, build_context=self.create_build_context( test_context=test_context, - enabled_build_features={'test_target_unused_exclusion'}, + enabled_build_features=[{'name': 'test_target_unused_exclusion'}], ), ) @@ -402,7 +403,7 @@ class BuildPlannerTest(unittest.TestCase): def create_build_planner( self, build_targets: set[str], - build_context: dict[str, any] = None, + build_context: BuildContext = None, args: argparse.Namespace = None, target_optimizations: dict[ str, optimized_targets.OptimizedBuildTarget @@ -426,15 +427,17 @@ class BuildPlannerTest(unittest.TestCase): def create_build_context( self, optimized_build_enabled: bool = True, - enabled_build_features: set[str] = set(), + enabled_build_features: list[dict[str, str]] = [], test_context: dict[str, any] = {}, - ) -> dict[str, any]: - build_context = {} - build_context['enabledBuildFeatures'] = enabled_build_features + ) -> BuildContext: + build_context_dict = {} + build_context_dict['enabledBuildFeatures'] = enabled_build_features if optimized_build_enabled: - build_context['enabledBuildFeatures'].add('optimized_build') - build_context['testContext'] = test_context - return build_context + build_context_dict['enabledBuildFeatures'].append( + {'name': 'optimized_build'} + ) + build_context_dict['testContext'] = test_context + return BuildContext(build_context_dict) def create_args( self, extra_build_targets: set[str] = set() @@ -445,7 +448,7 @@ class BuildPlannerTest(unittest.TestCase): def create_target_optimizations( self, - build_context: dict[str, any], + build_context: BuildContext, build_targets: set[str], packaging_outputs: set[str] = set(), ): diff --git a/ci/optimized_targets.py b/ci/optimized_targets.py index 8a529c7420..116d6f8882 100644 --- a/ci/optimized_targets.py +++ b/ci/optimized_targets.py @@ -14,9 +14,10 @@ # limitations under the License. from abc import ABC -from typing import Self import argparse import functools +from typing import Self +from build_context import BuildContext class OptimizedBuildTarget(ABC): @@ -30,7 +31,7 @@ class OptimizedBuildTarget(ABC): def __init__( self, target: str, - build_context: dict[str, any], + build_context: BuildContext, args: argparse.Namespace, ): self.target = target @@ -38,13 +39,13 @@ class OptimizedBuildTarget(ABC): self.args = args def get_build_targets(self) -> set[str]: - features = self.build_context.get('enabledBuildFeatures', []) + features = self.build_context.enabled_build_features if self.get_enabled_flag() in features: return self.get_build_targets_impl() return {self.target} def package_outputs(self): - features = self.build_context.get('enabledBuildFeatures', []) + features = self.build_context.enabled_build_features if self.get_enabled_flag() in features: return self.package_outputs_impl() From 0e695737e486f39aba1a4d27a803faa43cf33717 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Mon, 26 Aug 2024 21:28:00 +0000 Subject: [PATCH 3/4] Remove some files from packaging allowlist These files are symlinks. With https://r.android.com/3200722, these should be available in the soong built system image. Test: presubmits Change-Id: I4acb05ed8498dc8b7aac76e4c91b9173ed3d6574 --- tools/filelistdiff/allowlist | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tools/filelistdiff/allowlist b/tools/filelistdiff/allowlist index 943f9559f3..abe35bfce5 100644 --- a/tools/filelistdiff/allowlist +++ b/tools/filelistdiff/allowlist @@ -22,34 +22,6 @@ etc/vintf/compatibility_matrix.7.xml etc/vintf/compatibility_matrix.8.xml etc/vintf/compatibility_matrix.device.xml etc/vintf/manifest.xml -framework/boot-apache-xml.vdex -framework/boot-apache-xml.vdex.fsv_meta -framework/boot-bouncycastle.vdex -framework/boot-bouncycastle.vdex.fsv_meta -framework/boot-core-icu4j.vdex -framework/boot-core-icu4j.vdex.fsv_meta -framework/boot-core-libart.vdex -framework/boot-core-libart.vdex.fsv_meta -framework/boot-ext.vdex -framework/boot-ext.vdex.fsv_meta -framework/boot-framework-adservices.vdex -framework/boot-framework-adservices.vdex.fsv_meta -framework/boot-framework-graphics.vdex -framework/boot-framework-graphics.vdex.fsv_meta -framework/boot-framework-location.vdex -framework/boot-framework-location.vdex.fsv_meta -framework/boot-framework.vdex -framework/boot-framework.vdex.fsv_meta -framework/boot-ims-common.vdex -framework/boot-ims-common.vdex.fsv_meta -framework/boot-okhttp.vdex -framework/boot-okhttp.vdex.fsv_meta -framework/boot-telephony-common.vdex -framework/boot-telephony-common.vdex.fsv_meta -framework/boot-voip-common.vdex -framework/boot-voip-common.vdex.fsv_meta -framework/boot.vdex -framework/boot.vdex.fsv_meta framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex From 1c8d6ea6f8ca1cfbcfa8c80961b6bd67092062a8 Mon Sep 17 00:00:00 2001 From: Pindar Yang Date: Mon, 26 Aug 2024 06:52:12 +0000 Subject: [PATCH 4/4] Revert "Export variable to soong for converting vintf_compatibil..." Revert submission 3195743-compatibility_matrix Reason for revert: Build Break, Bug: 361227456 Reverted changes: /q/submissionid:3195743-compatibility_matrix Change-Id: I7dc43c32d3ada03fd319f47892049b5f33cfcfb4 --- core/soong_config.mk | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/soong_config.mk b/core/soong_config.mk index 874cbc3fc2..09ee938f73 100644 --- a/core/soong_config.mk +++ b/core/soong_config.mk @@ -353,11 +353,6 @@ _config_enable_uffd_gc := \ $(call add_json_str, EnableUffdGc, $(_config_enable_uffd_gc)) _config_enable_uffd_gc := -$(call add_json_list, DeviceFrameworkCompatibilityMatrixFile, $(DEVICE_FRAMEWORK_COMPATIBILITY_MATRIX_FILE)) -$(call add_json_list, DeviceProductCompatibilityMatrixFile, $(DEVICE_PRODUCT_COMPATIBILITY_MATRIX_FILE)) -$(call add_json_list, BoardAvbSystemAddHashtreeFooterArgs, $(BOARD_AVB_SYSTEM_ADD_HASHTREE_FOOTER_ARGS)) -$(call add_json_bool, BoardAvbEnable, $(filter true,$(BOARD_AVB_ENABLE))) - $(call json_end) $(file >$(SOONG_VARIABLES).tmp,$(json_contents))