Merge "Generate system diff between KATI and SOONG" into aosp-main-future

core/Makefile

@@ -3557,6 +3557,24 @@ define build-systemimage-target
 endef
 
 $(eval $(call write-partition-file-list,$(systemimage_intermediates)/file_list.txt,$(TARGET_OUT),$(FULL_SYSTEMIMAGE_DEPS)))
+
+ifneq ($(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE),)
+file_list_diff := $(HOST_OUT_EXECUTABLES)/file_list_diff$(HOST_EXECUTABLE_SUFFIX)
+system_file_diff_timestamp := $(systemimage_intermediates)/file_diff.timestamp
+
+$(system_file_diff_timestamp): \
+	    $(systemimage_intermediates)/file_list.txt \
+	    $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \
+	    $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \
+	    $(file_list_diff)
+	$(file_list_diff) $(systemimage_intermediates)/file_list.txt \
+	  $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \
+	  $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE)
+	touch $@
+
+$(BUILT_SYSTEMIMAGE): $(system_file_diff_timestamp)
+endif
+
 # Used by soong sandwich to request the staging dir be built
 $(systemimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT)/%,$(FULL_SYSTEMIMAGE_DEPS))
 	touch $@

core/base_rules.mk

@@ -1059,6 +1059,11 @@ ifdef LOCAL_ACONFIG_FILES
       $(ALL_MODULES.$(my_register_name).ACONFIG_FILES) $(LOCAL_ACONFIG_FILES)
 endif
 
+ifdef LOCAL_FILESYSTEM_FILELIST
+  ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST := \
+      $(ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST) $(LOCAL_FILESYSTEM_FILELIST)
+endif
+
 ifndef LOCAL_SOONG_MODULE_INFO_JSON
   ALL_MAKE_MODULE_INFO_JSON_MODULES += $(my_register_name)
   ALL_MODULES.$(my_register_name).SHARED_LIBS := \

core/clear_vars.mk

@@ -87,6 +87,7 @@ LOCAL_EXPORT_STATIC_LIBRARY_HEADERS:=
 LOCAL_EXTRA_FULL_TEST_CONFIGS:=
 LOCAL_EXTRACT_APK:=
 LOCAL_EXTRACT_DPI_APK:=
+LOCAL_FILESYSTEM_FILELIST:=
 LOCAL_FINDBUGS_FLAGS:=
 LOCAL_FORCE_STATIC_EXECUTABLE:=
 LOCAL_FULL_CLASSES_JACOCO_JAR:=

core/product.mk

@@ -498,6 +498,9 @@ _product_single_value_vars += PRODUCT_NOT_DEBUGGABLE_IN_USERDEBUG
 # If set, the default value of the versionName of apps will include the build number.
 _product_single_value_vars += PRODUCT_BUILD_APPS_WITH_BUILD_NUMBER
 
+# If set, build would generate system image from Soong-defined module.
+_product_single_value_vars += PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE
+
 .KATI_READONLY := _product_single_value_vars _product_list_vars
 _product_var_list :=$= $(_product_single_value_vars) $(_product_list_vars)
 

tools/filelistdiff/Android.bp

@@ -0,0 +1,27 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+python_binary_host {
+    name: "file_list_diff",
+    srcs: ["file_list_diff.py"],
+}
+
+prebuilt_etc_host {
+    name: "system_image_diff_allowlist",
+    src: "allowlist",
+}

tools/filelistdiff/allowlist

@@ -0,0 +1,83 @@
+# Known diffs only in the KATI system image
+etc/NOTICE.xml.gz
+etc/compatconfig/TeleService-platform-compat-config.xml
+etc/compatconfig/calendar-provider-compat-config.xml
+etc/compatconfig/contacts-provider-platform-compat-config.xml
+etc/compatconfig/documents-ui-compat-config.xml
+etc/compatconfig/framework-location-compat-config.xml
+etc/compatconfig/framework-platform-compat-config.xml
+etc/compatconfig/icu4j-platform-compat-config.xml
+etc/compatconfig/services-platform-compat-config.xml
+etc/permissions/android.software.credentials.xml
+etc/permissions/android.software.preview_sdk.xml
+etc/permissions/android.software.webview.xml
+etc/permissions/android.software.window_magnification.xml
+etc/permissions/com.android.adservices.sdksandbox.xml
+etc/security/otacerts.zip
+etc/vintf/compatibility_matrix.202404.xml
+etc/vintf/compatibility_matrix.202504.xml
+etc/vintf/compatibility_matrix.5.xml
+etc/vintf/compatibility_matrix.6.xml
+etc/vintf/compatibility_matrix.7.xml
+etc/vintf/compatibility_matrix.8.xml
+etc/vintf/compatibility_matrix.device.xml
+etc/vintf/manifest.xml
+framework/boot-apache-xml.vdex
+framework/boot-apache-xml.vdex.fsv_meta
+framework/boot-bouncycastle.vdex
+framework/boot-bouncycastle.vdex.fsv_meta
+framework/boot-core-icu4j.vdex
+framework/boot-core-icu4j.vdex.fsv_meta
+framework/boot-core-libart.vdex
+framework/boot-core-libart.vdex.fsv_meta
+framework/boot-ext.vdex
+framework/boot-ext.vdex.fsv_meta
+framework/boot-framework-adservices.vdex
+framework/boot-framework-adservices.vdex.fsv_meta
+framework/boot-framework-graphics.vdex
+framework/boot-framework-graphics.vdex.fsv_meta
+framework/boot-framework-location.vdex
+framework/boot-framework-location.vdex.fsv_meta
+framework/boot-framework.vdex
+framework/boot-framework.vdex.fsv_meta
+framework/boot-ims-common.vdex
+framework/boot-ims-common.vdex.fsv_meta
+framework/boot-okhttp.vdex
+framework/boot-okhttp.vdex.fsv_meta
+framework/boot-telephony-common.vdex
+framework/boot-telephony-common.vdex.fsv_meta
+framework/boot-voip-common.vdex
+framework/boot-voip-common.vdex.fsv_meta
+framework/boot.vdex
+framework/boot.vdex.fsv_meta
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex.fsv_meta
+lib/aaudio-aidl-cpp.so
+lib/android.hardware.biometrics.fingerprint@2.1.so
+lib/android.hardware.radio.config@1.0.so
+lib/android.hardware.radio.deprecated@1.0.so
+lib/android.hardware.radio@1.0.so
+lib/android.hardware.radio@1.1.so
+lib/android.hardware.radio@1.2.so
+lib/android.hardware.radio@1.3.so
+lib/android.hardware.radio@1.4.so
+lib/android.hardware.secure_element@1.0.so
+lib/com.android.media.aaudio-aconfig-cc.so
+lib/heapprofd_client.so
+lib/heapprofd_client_api.so
+lib/libaaudio.so
+lib/libaaudio_internal.so
+lib/libalarm_jni.so
+lib/libamidi.so
+lib/libcups.so
+lib/libjni_deviceAsWebcam.so
+lib/libprintspooler_jni.so
+lib/libvendorsupport.so
+lib/libwfds.so
+lib/libyuv.so
+
+# Known diffs only in the Soong system image
+lib/libhidcommand_jni.so
+lib/libuinputcommand_jni.so

tools/filelistdiff/file_list_diff.py

@@ -0,0 +1,66 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+
+COLOR_WARNING = '\033[93m'
+COLOR_ERROR = '\033[91m'
+COLOR_NORMAL = '\033[0m'
+
+def find_unique_items(kati_installed_files, soong_installed_files, allowlist, system_module_name):
+    with open(kati_installed_files, 'r') as kati_list_file, \
+            open(soong_installed_files, 'r') as soong_list_file, \
+            open(allowlist, 'r') as allowlist_file:
+        kati_files = set(kati_list_file.read().split())
+        soong_files = set(soong_list_file.read().split())
+        allowed_files = set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n'))))
+
+    def is_unknown_diff(filepath):
+        return not filepath in allowed_files
+
+    unique_in_kati = set(filter(is_unknown_diff, kati_files - soong_files))
+    unique_in_soong = set(filter(is_unknown_diff, soong_files - kati_files))
+
+    if unique_in_kati:
+        print(f'{COLOR_ERROR}Please add following modules into system image module {system_module_name}.{COLOR_NORMAL}')
+        print(f'{COLOR_WARNING}KATI only module(s):{COLOR_NORMAL}')
+        for item in sorted(unique_in_kati):
+            print(item)
+
+    if unique_in_soong:
+        if unique_in_kati:
+            print('')
+
+        print(f'{COLOR_ERROR}Please add following modules into build/make/target/product/base_system.mk.{COLOR_NORMAL}')
+        print(f'{COLOR_WARNING}Soong only module(s):{COLOR_NORMAL}')
+        for item in sorted(unique_in_soong):
+            print(item)
+
+    if unique_in_kati or unique_in_soong:
+        print('')
+        print(f'{COLOR_ERROR}FAILED: System image from KATI and SOONG differs from installed file list.{COLOR_NORMAL}')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('kati_installed_file_list')
+    parser.add_argument('soong_installed_file_list')
+    parser.add_argument('allowlist')
+    parser.add_argument('system_module_name')
+    args = parser.parse_args()
+
+    find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.allowlist, args.system_module_name)