From f5a52d9a4f012ec460d2d9f17f0efe99c1c5bb8b Mon Sep 17 00:00:00 2001 From: Kiyoung Kim Date: Fri, 28 Jun 2024 12:52:07 +0000 Subject: [PATCH] Generate system diff between KATI and SOONG There is an effort to generate system image with Soong, but there are some difference in installed files between Soong defined system image and KATI defined one. This change generates diffs between installed files from two sources. Bug: 346873717 Test: USE_SOONG_DEFINED_SYSTEM_IMAGE=true m -j on aosp_cf_x86_64 generated diff file. Ignore-AOSP-First: Resolve conflict Change-Id: I25c71f88d16a4efb873c21abe70ca9c41c6423ca --- core/Makefile | 18 ++++++ core/base_rules.mk | 5 ++ core/clear_vars.mk | 1 + core/product.mk | 3 + tools/filelistdiff/Android.bp | 27 +++++++++ tools/filelistdiff/allowlist | 83 ++++++++++++++++++++++++++++ tools/filelistdiff/file_list_diff.py | 66 ++++++++++++++++++++++ 7 files changed, 203 insertions(+) create mode 100644 tools/filelistdiff/Android.bp create mode 100644 tools/filelistdiff/allowlist create mode 100644 tools/filelistdiff/file_list_diff.py diff --git a/core/Makefile b/core/Makefile index b5a7e7428d..ccb4c3e3b3 100644 --- a/core/Makefile +++ b/core/Makefile @@ -3557,6 +3557,24 @@ define build-systemimage-target endef $(eval $(call write-partition-file-list,$(systemimage_intermediates)/file_list.txt,$(TARGET_OUT),$(FULL_SYSTEMIMAGE_DEPS))) + +ifneq ($(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE),) +file_list_diff := $(HOST_OUT_EXECUTABLES)/file_list_diff$(HOST_EXECUTABLE_SUFFIX) +system_file_diff_timestamp := $(systemimage_intermediates)/file_diff.timestamp + +$(system_file_diff_timestamp): \ + $(systemimage_intermediates)/file_list.txt \ + $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \ + $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \ + $(file_list_diff) + $(file_list_diff) $(systemimage_intermediates)/file_list.txt \ + $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \ + $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE) + touch $@ + +$(BUILT_SYSTEMIMAGE): $(system_file_diff_timestamp) +endif + # Used by soong sandwich to request the staging dir be built $(systemimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT)/%,$(FULL_SYSTEMIMAGE_DEPS)) touch $@ diff --git a/core/base_rules.mk b/core/base_rules.mk index a8cf67e4b0..a9d649446d 100644 --- a/core/base_rules.mk +++ b/core/base_rules.mk @@ -1059,6 +1059,11 @@ ifdef LOCAL_ACONFIG_FILES $(ALL_MODULES.$(my_register_name).ACONFIG_FILES) $(LOCAL_ACONFIG_FILES) endif +ifdef LOCAL_FILESYSTEM_FILELIST + ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST := \ + $(ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST) $(LOCAL_FILESYSTEM_FILELIST) +endif + ifndef LOCAL_SOONG_MODULE_INFO_JSON ALL_MAKE_MODULE_INFO_JSON_MODULES += $(my_register_name) ALL_MODULES.$(my_register_name).SHARED_LIBS := \ diff --git a/core/clear_vars.mk b/core/clear_vars.mk index 61926907c3..fed19e6d45 100644 --- a/core/clear_vars.mk +++ b/core/clear_vars.mk @@ -87,6 +87,7 @@ LOCAL_EXPORT_STATIC_LIBRARY_HEADERS:= LOCAL_EXTRA_FULL_TEST_CONFIGS:= LOCAL_EXTRACT_APK:= LOCAL_EXTRACT_DPI_APK:= +LOCAL_FILESYSTEM_FILELIST:= LOCAL_FINDBUGS_FLAGS:= LOCAL_FORCE_STATIC_EXECUTABLE:= LOCAL_FULL_CLASSES_JACOCO_JAR:= diff --git a/core/product.mk b/core/product.mk index acb559672a..832d0946e5 100644 --- a/core/product.mk +++ b/core/product.mk @@ -495,6 +495,9 @@ _product_single_value_vars += PRODUCT_NOT_DEBUGGABLE_IN_USERDEBUG # If set, the default value of the versionName of apps will include the build number. _product_single_value_vars += PRODUCT_BUILD_APPS_WITH_BUILD_NUMBER +# If set, build would generate system image from Soong-defined module. +_product_single_value_vars += PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE + .KATI_READONLY := _product_single_value_vars _product_list_vars _product_var_list :=$= $(_product_single_value_vars) $(_product_list_vars) diff --git a/tools/filelistdiff/Android.bp b/tools/filelistdiff/Android.bp new file mode 100644 index 0000000000..ab766d6d93 --- /dev/null +++ b/tools/filelistdiff/Android.bp @@ -0,0 +1,27 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +python_binary_host { + name: "file_list_diff", + srcs: ["file_list_diff.py"], +} + +prebuilt_etc_host { + name: "system_image_diff_allowlist", + src: "allowlist", +} \ No newline at end of file diff --git a/tools/filelistdiff/allowlist b/tools/filelistdiff/allowlist new file mode 100644 index 0000000000..0a51d0e255 --- /dev/null +++ b/tools/filelistdiff/allowlist @@ -0,0 +1,83 @@ +# Known diffs only in the KATI system image +etc/NOTICE.xml.gz +etc/compatconfig/TeleService-platform-compat-config.xml +etc/compatconfig/calendar-provider-compat-config.xml +etc/compatconfig/contacts-provider-platform-compat-config.xml +etc/compatconfig/documents-ui-compat-config.xml +etc/compatconfig/framework-location-compat-config.xml +etc/compatconfig/framework-platform-compat-config.xml +etc/compatconfig/icu4j-platform-compat-config.xml +etc/compatconfig/services-platform-compat-config.xml +etc/permissions/android.software.credentials.xml +etc/permissions/android.software.preview_sdk.xml +etc/permissions/android.software.webview.xml +etc/permissions/android.software.window_magnification.xml +etc/permissions/com.android.adservices.sdksandbox.xml +etc/security/otacerts.zip +etc/vintf/compatibility_matrix.202404.xml +etc/vintf/compatibility_matrix.202504.xml +etc/vintf/compatibility_matrix.5.xml +etc/vintf/compatibility_matrix.6.xml +etc/vintf/compatibility_matrix.7.xml +etc/vintf/compatibility_matrix.8.xml +etc/vintf/compatibility_matrix.device.xml +etc/vintf/manifest.xml +framework/boot-apache-xml.vdex +framework/boot-apache-xml.vdex.fsv_meta +framework/boot-bouncycastle.vdex +framework/boot-bouncycastle.vdex.fsv_meta +framework/boot-core-icu4j.vdex +framework/boot-core-icu4j.vdex.fsv_meta +framework/boot-core-libart.vdex +framework/boot-core-libart.vdex.fsv_meta +framework/boot-ext.vdex +framework/boot-ext.vdex.fsv_meta +framework/boot-framework-adservices.vdex +framework/boot-framework-adservices.vdex.fsv_meta +framework/boot-framework-graphics.vdex +framework/boot-framework-graphics.vdex.fsv_meta +framework/boot-framework-location.vdex +framework/boot-framework-location.vdex.fsv_meta +framework/boot-framework.vdex +framework/boot-framework.vdex.fsv_meta +framework/boot-ims-common.vdex +framework/boot-ims-common.vdex.fsv_meta +framework/boot-okhttp.vdex +framework/boot-okhttp.vdex.fsv_meta +framework/boot-telephony-common.vdex +framework/boot-telephony-common.vdex.fsv_meta +framework/boot-voip-common.vdex +framework/boot-voip-common.vdex.fsv_meta +framework/boot.vdex +framework/boot.vdex.fsv_meta +framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex +framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta +framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex +framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex.fsv_meta +lib/aaudio-aidl-cpp.so +lib/android.hardware.biometrics.fingerprint@2.1.so +lib/android.hardware.radio.config@1.0.so +lib/android.hardware.radio.deprecated@1.0.so +lib/android.hardware.radio@1.0.so +lib/android.hardware.radio@1.1.so +lib/android.hardware.radio@1.2.so +lib/android.hardware.radio@1.3.so +lib/android.hardware.radio@1.4.so +lib/android.hardware.secure_element@1.0.so +lib/com.android.media.aaudio-aconfig-cc.so +lib/heapprofd_client.so +lib/heapprofd_client_api.so +lib/libaaudio.so +lib/libaaudio_internal.so +lib/libalarm_jni.so +lib/libamidi.so +lib/libcups.so +lib/libjni_deviceAsWebcam.so +lib/libprintspooler_jni.so +lib/libvendorsupport.so +lib/libwfds.so +lib/libyuv.so + +# Known diffs only in the Soong system image +lib/libhidcommand_jni.so +lib/libuinputcommand_jni.so \ No newline at end of file diff --git a/tools/filelistdiff/file_list_diff.py b/tools/filelistdiff/file_list_diff.py new file mode 100644 index 0000000000..cdc5b2ee41 --- /dev/null +++ b/tools/filelistdiff/file_list_diff.py @@ -0,0 +1,66 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +COLOR_WARNING = '\033[93m' +COLOR_ERROR = '\033[91m' +COLOR_NORMAL = '\033[0m' + +def find_unique_items(kati_installed_files, soong_installed_files, allowlist, system_module_name): + with open(kati_installed_files, 'r') as kati_list_file, \ + open(soong_installed_files, 'r') as soong_list_file, \ + open(allowlist, 'r') as allowlist_file: + kati_files = set(kati_list_file.read().split()) + soong_files = set(soong_list_file.read().split()) + allowed_files = set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n')))) + + def is_unknown_diff(filepath): + return not filepath in allowed_files + + unique_in_kati = set(filter(is_unknown_diff, kati_files - soong_files)) + unique_in_soong = set(filter(is_unknown_diff, soong_files - kati_files)) + + if unique_in_kati: + print(f'{COLOR_ERROR}Please add following modules into system image module {system_module_name}.{COLOR_NORMAL}') + print(f'{COLOR_WARNING}KATI only module(s):{COLOR_NORMAL}') + for item in sorted(unique_in_kati): + print(item) + + if unique_in_soong: + if unique_in_kati: + print('') + + print(f'{COLOR_ERROR}Please add following modules into build/make/target/product/base_system.mk.{COLOR_NORMAL}') + print(f'{COLOR_WARNING}Soong only module(s):{COLOR_NORMAL}') + for item in sorted(unique_in_soong): + print(item) + + if unique_in_kati or unique_in_soong: + print('') + print(f'{COLOR_ERROR}FAILED: System image from KATI and SOONG differs from installed file list.{COLOR_NORMAL}') + sys.exit(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument('kati_installed_file_list') + parser.add_argument('soong_installed_file_list') + parser.add_argument('allowlist') + parser.add_argument('system_module_name') + args = parser.parse_args() + + find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.allowlist, args.system_module_name) \ No newline at end of file