From 8376b32e429d0da671c6813a149938dc92685ece Mon Sep 17 00:00:00 2001 From: Kiyoung Kim Date: Fri, 21 Jun 2024 14:15:49 +0900 Subject: [PATCH] Generate system diff between KATI and SOONG There is an effort to generate system image with Soong, but there are some difference in installed files between Soong defined system image and KATI defined one. This change generates diffs between installed files from two sources. Bug: 346873717 Test: USE_SOONG_DEFINED_SYSTEM_IMAGE=true m -j on aosp_cf_x86_64 generated diff file. Change-Id: I25c71f88d16a4efb873c21abe70ca9c41c6423ca Merged-In: I25c71f88d16a4efb873c21abe70ca9c41c6423ca --- core/Makefile | 18 ++++++ core/base_rules.mk | 5 ++ core/clear_vars.mk | 1 + core/product.mk | 3 + tools/filelistdiff/Android.bp | 27 +++++++++ tools/filelistdiff/allowlist | 83 ++++++++++++++++++++++++++++ tools/filelistdiff/file_list_diff.py | 66 ++++++++++++++++++++++ 7 files changed, 203 insertions(+) create mode 100644 tools/filelistdiff/Android.bp create mode 100644 tools/filelistdiff/allowlist create mode 100644 tools/filelistdiff/file_list_diff.py diff --git a/core/Makefile b/core/Makefile index dfce50ed8f..39bb5e6939 100644 --- a/core/Makefile +++ b/core/Makefile @@ -3558,6 +3558,24 @@ define build-systemimage-target endef $(eval $(call write-partition-file-list,$(systemimage_intermediates)/file_list.txt,$(TARGET_OUT),$(FULL_SYSTEMIMAGE_DEPS))) + +ifneq ($(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE),) +file_list_diff := $(HOST_OUT_EXECUTABLES)/file_list_diff$(HOST_EXECUTABLE_SUFFIX) +system_file_diff_timestamp := $(systemimage_intermediates)/file_diff.timestamp + +$(system_file_diff_timestamp): \ + $(systemimage_intermediates)/file_list.txt \ + $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \ + $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \ + $(file_list_diff) + $(file_list_diff) $(systemimage_intermediates)/file_list.txt \ + $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \ + $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE) + touch $@ + +$(BUILT_SYSTEMIMAGE): $(system_file_diff_timestamp) +endif + # Used by soong sandwich to request the staging dir be built $(systemimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT)/%,$(FULL_SYSTEMIMAGE_DEPS)) touch $@ diff --git a/core/base_rules.mk b/core/base_rules.mk index a8cf67e4b0..a9d649446d 100644 --- a/core/base_rules.mk +++ b/core/base_rules.mk @@ -1059,6 +1059,11 @@ ifdef LOCAL_ACONFIG_FILES $(ALL_MODULES.$(my_register_name).ACONFIG_FILES) $(LOCAL_ACONFIG_FILES) endif +ifdef LOCAL_FILESYSTEM_FILELIST + ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST := \ + $(ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST) $(LOCAL_FILESYSTEM_FILELIST) +endif + ifndef LOCAL_SOONG_MODULE_INFO_JSON ALL_MAKE_MODULE_INFO_JSON_MODULES += $(my_register_name) ALL_MODULES.$(my_register_name).SHARED_LIBS := \ diff --git a/core/clear_vars.mk b/core/clear_vars.mk index 61926907c3..fed19e6d45 100644 --- a/core/clear_vars.mk +++ b/core/clear_vars.mk @@ -87,6 +87,7 @@ LOCAL_EXPORT_STATIC_LIBRARY_HEADERS:= LOCAL_EXTRA_FULL_TEST_CONFIGS:= LOCAL_EXTRACT_APK:= LOCAL_EXTRACT_DPI_APK:= +LOCAL_FILESYSTEM_FILELIST:= LOCAL_FINDBUGS_FLAGS:= LOCAL_FORCE_STATIC_EXECUTABLE:= LOCAL_FULL_CLASSES_JACOCO_JAR:= diff --git a/core/product.mk b/core/product.mk index 5ceccbf087..ad80ee4599 100644 --- a/core/product.mk +++ b/core/product.mk @@ -492,6 +492,9 @@ _product_single_value_vars += PRODUCT_16K_DEVELOPER_OPTION # by this flag. _product_single_value_vars += PRODUCT_NOT_DEBUGGABLE_IN_USERDEBUG +# If set, build would generate system image from Soong-defined module. +_product_single_value_vars += PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE + .KATI_READONLY := _product_single_value_vars _product_list_vars _product_var_list :=$= $(_product_single_value_vars) $(_product_list_vars) diff --git a/tools/filelistdiff/Android.bp b/tools/filelistdiff/Android.bp new file mode 100644 index 0000000000..632ada348e --- /dev/null +++ b/tools/filelistdiff/Android.bp @@ -0,0 +1,27 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +python_binary_host { + name: "file_list_diff", + srcs: ["file_list_diff.py"], +} + +prebuilt_etc_host { + name: "system_image_diff_allowlist", + src: "allowlist", +} diff --git a/tools/filelistdiff/allowlist b/tools/filelistdiff/allowlist new file mode 100644 index 0000000000..0a51d0e255 --- /dev/null +++ b/tools/filelistdiff/allowlist @@ -0,0 +1,83 @@ +# Known diffs only in the KATI system image +etc/NOTICE.xml.gz +etc/compatconfig/TeleService-platform-compat-config.xml +etc/compatconfig/calendar-provider-compat-config.xml +etc/compatconfig/contacts-provider-platform-compat-config.xml +etc/compatconfig/documents-ui-compat-config.xml +etc/compatconfig/framework-location-compat-config.xml +etc/compatconfig/framework-platform-compat-config.xml +etc/compatconfig/icu4j-platform-compat-config.xml +etc/compatconfig/services-platform-compat-config.xml +etc/permissions/android.software.credentials.xml +etc/permissions/android.software.preview_sdk.xml +etc/permissions/android.software.webview.xml +etc/permissions/android.software.window_magnification.xml +etc/permissions/com.android.adservices.sdksandbox.xml +etc/security/otacerts.zip +etc/vintf/compatibility_matrix.202404.xml +etc/vintf/compatibility_matrix.202504.xml +etc/vintf/compatibility_matrix.5.xml +etc/vintf/compatibility_matrix.6.xml +etc/vintf/compatibility_matrix.7.xml +etc/vintf/compatibility_matrix.8.xml +etc/vintf/compatibility_matrix.device.xml +etc/vintf/manifest.xml +framework/boot-apache-xml.vdex +framework/boot-apache-xml.vdex.fsv_meta +framework/boot-bouncycastle.vdex +framework/boot-bouncycastle.vdex.fsv_meta +framework/boot-core-icu4j.vdex +framework/boot-core-icu4j.vdex.fsv_meta +framework/boot-core-libart.vdex +framework/boot-core-libart.vdex.fsv_meta +framework/boot-ext.vdex +framework/boot-ext.vdex.fsv_meta +framework/boot-framework-adservices.vdex +framework/boot-framework-adservices.vdex.fsv_meta +framework/boot-framework-graphics.vdex +framework/boot-framework-graphics.vdex.fsv_meta +framework/boot-framework-location.vdex +framework/boot-framework-location.vdex.fsv_meta +framework/boot-framework.vdex +framework/boot-framework.vdex.fsv_meta +framework/boot-ims-common.vdex +framework/boot-ims-common.vdex.fsv_meta +framework/boot-okhttp.vdex +framework/boot-okhttp.vdex.fsv_meta +framework/boot-telephony-common.vdex +framework/boot-telephony-common.vdex.fsv_meta +framework/boot-voip-common.vdex +framework/boot-voip-common.vdex.fsv_meta +framework/boot.vdex +framework/boot.vdex.fsv_meta +framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex +framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta +framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex +framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex.fsv_meta +lib/aaudio-aidl-cpp.so +lib/android.hardware.biometrics.fingerprint@2.1.so +lib/android.hardware.radio.config@1.0.so +lib/android.hardware.radio.deprecated@1.0.so +lib/android.hardware.radio@1.0.so +lib/android.hardware.radio@1.1.so +lib/android.hardware.radio@1.2.so +lib/android.hardware.radio@1.3.so +lib/android.hardware.radio@1.4.so +lib/android.hardware.secure_element@1.0.so +lib/com.android.media.aaudio-aconfig-cc.so +lib/heapprofd_client.so +lib/heapprofd_client_api.so +lib/libaaudio.so +lib/libaaudio_internal.so +lib/libalarm_jni.so +lib/libamidi.so +lib/libcups.so +lib/libjni_deviceAsWebcam.so +lib/libprintspooler_jni.so +lib/libvendorsupport.so +lib/libwfds.so +lib/libyuv.so + +# Known diffs only in the Soong system image +lib/libhidcommand_jni.so +lib/libuinputcommand_jni.so \ No newline at end of file diff --git a/tools/filelistdiff/file_list_diff.py b/tools/filelistdiff/file_list_diff.py new file mode 100644 index 0000000000..cdc5b2ee41 --- /dev/null +++ b/tools/filelistdiff/file_list_diff.py @@ -0,0 +1,66 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +COLOR_WARNING = '\033[93m' +COLOR_ERROR = '\033[91m' +COLOR_NORMAL = '\033[0m' + +def find_unique_items(kati_installed_files, soong_installed_files, allowlist, system_module_name): + with open(kati_installed_files, 'r') as kati_list_file, \ + open(soong_installed_files, 'r') as soong_list_file, \ + open(allowlist, 'r') as allowlist_file: + kati_files = set(kati_list_file.read().split()) + soong_files = set(soong_list_file.read().split()) + allowed_files = set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n')))) + + def is_unknown_diff(filepath): + return not filepath in allowed_files + + unique_in_kati = set(filter(is_unknown_diff, kati_files - soong_files)) + unique_in_soong = set(filter(is_unknown_diff, soong_files - kati_files)) + + if unique_in_kati: + print(f'{COLOR_ERROR}Please add following modules into system image module {system_module_name}.{COLOR_NORMAL}') + print(f'{COLOR_WARNING}KATI only module(s):{COLOR_NORMAL}') + for item in sorted(unique_in_kati): + print(item) + + if unique_in_soong: + if unique_in_kati: + print('') + + print(f'{COLOR_ERROR}Please add following modules into build/make/target/product/base_system.mk.{COLOR_NORMAL}') + print(f'{COLOR_WARNING}Soong only module(s):{COLOR_NORMAL}') + for item in sorted(unique_in_soong): + print(item) + + if unique_in_kati or unique_in_soong: + print('') + print(f'{COLOR_ERROR}FAILED: System image from KATI and SOONG differs from installed file list.{COLOR_NORMAL}') + sys.exit(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument('kati_installed_file_list') + parser.add_argument('soong_installed_file_list') + parser.add_argument('allowlist') + parser.add_argument('system_module_name') + args = parser.parse_args() + + find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.allowlist, args.system_module_name) \ No newline at end of file