Generate system diff between KATI and SOONG

There is an effort to generate system image with Soong, but there are
some difference in installed files between Soong defined system image
and KATI defined one. This change generates diffs between installed
files from two sources.

Bug: 346873717
Test: USE_SOONG_DEFINED_SYSTEM_IMAGE=true m -j on aosp_cf_x86_64
generated diff file.

Change-Id: I25c71f88d16a4efb873c21abe70ca9c41c6423ca
Merged-In: I25c71f88d16a4efb873c21abe70ca9c41c6423ca

core/Makefile

@@ -3558,6 +3558,24 @@ define build-systemimage-target
 endef
 
 $(eval $(call write-partition-file-list,$(systemimage_intermediates)/file_list.txt,$(TARGET_OUT),$(FULL_SYSTEMIMAGE_DEPS)))
+
+ifneq ($(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE),)
+file_list_diff := $(HOST_OUT_EXECUTABLES)/file_list_diff$(HOST_EXECUTABLE_SUFFIX)
+system_file_diff_timestamp := $(systemimage_intermediates)/file_diff.timestamp
+
+$(system_file_diff_timestamp): \
+	    $(systemimage_intermediates)/file_list.txt \
+	    $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \
+	    $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \
+	    $(file_list_diff)
+	$(file_list_diff) $(systemimage_intermediates)/file_list.txt \
+	  $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \
+	  $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE)
+	touch $@
+
+$(BUILT_SYSTEMIMAGE): $(system_file_diff_timestamp)
+endif
+
 # Used by soong sandwich to request the staging dir be built
 $(systemimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT)/%,$(FULL_SYSTEMIMAGE_DEPS))
 	touch $@

core/base_rules.mk

@@ -1059,6 +1059,11 @@ ifdef LOCAL_ACONFIG_FILES
       $(ALL_MODULES.$(my_register_name).ACONFIG_FILES) $(LOCAL_ACONFIG_FILES)
 endif
 
+ifdef LOCAL_FILESYSTEM_FILELIST
+  ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST := \
+      $(ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST) $(LOCAL_FILESYSTEM_FILELIST)
+endif
+
 ifndef LOCAL_SOONG_MODULE_INFO_JSON
   ALL_MAKE_MODULE_INFO_JSON_MODULES += $(my_register_name)
   ALL_MODULES.$(my_register_name).SHARED_LIBS := \

core/clear_vars.mk

@@ -87,6 +87,7 @@ LOCAL_EXPORT_STATIC_LIBRARY_HEADERS:=
 LOCAL_EXTRA_FULL_TEST_CONFIGS:=
 LOCAL_EXTRACT_APK:=
 LOCAL_EXTRACT_DPI_APK:=
+LOCAL_FILESYSTEM_FILELIST:=
 LOCAL_FINDBUGS_FLAGS:=
 LOCAL_FORCE_STATIC_EXECUTABLE:=
 LOCAL_FULL_CLASSES_JACOCO_JAR:=

core/product.mk

@@ -492,6 +492,9 @@ _product_single_value_vars += PRODUCT_16K_DEVELOPER_OPTION
 # by this flag.
 _product_single_value_vars += PRODUCT_NOT_DEBUGGABLE_IN_USERDEBUG
 
+# If set, build would generate system image from Soong-defined module.
+_product_single_value_vars += PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE
+
 .KATI_READONLY := _product_single_value_vars _product_list_vars
 _product_var_list :=$= $(_product_single_value_vars) $(_product_list_vars)
 

tools/filelistdiff/Android.bp

@@ -0,0 +1,27 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+python_binary_host {
+    name: "file_list_diff",
+    srcs: ["file_list_diff.py"],
+}
+
+prebuilt_etc_host {
+    name: "system_image_diff_allowlist",
+    src: "allowlist",
+}

tools/filelistdiff/allowlist

@@ -0,0 +1,83 @@
+# Known diffs only in the KATI system image
+etc/NOTICE.xml.gz
+etc/compatconfig/TeleService-platform-compat-config.xml
+etc/compatconfig/calendar-provider-compat-config.xml
+etc/compatconfig/contacts-provider-platform-compat-config.xml
+etc/compatconfig/documents-ui-compat-config.xml
+etc/compatconfig/framework-location-compat-config.xml
+etc/compatconfig/framework-platform-compat-config.xml
+etc/compatconfig/icu4j-platform-compat-config.xml
+etc/compatconfig/services-platform-compat-config.xml
+etc/permissions/android.software.credentials.xml
+etc/permissions/android.software.preview_sdk.xml
+etc/permissions/android.software.webview.xml
+etc/permissions/android.software.window_magnification.xml
+etc/permissions/com.android.adservices.sdksandbox.xml
+etc/security/otacerts.zip
+etc/vintf/compatibility_matrix.202404.xml
+etc/vintf/compatibility_matrix.202504.xml
+etc/vintf/compatibility_matrix.5.xml
+etc/vintf/compatibility_matrix.6.xml
+etc/vintf/compatibility_matrix.7.xml
+etc/vintf/compatibility_matrix.8.xml
+etc/vintf/compatibility_matrix.device.xml
+etc/vintf/manifest.xml
+framework/boot-apache-xml.vdex
+framework/boot-apache-xml.vdex.fsv_meta
+framework/boot-bouncycastle.vdex
+framework/boot-bouncycastle.vdex.fsv_meta
+framework/boot-core-icu4j.vdex
+framework/boot-core-icu4j.vdex.fsv_meta
+framework/boot-core-libart.vdex
+framework/boot-core-libart.vdex.fsv_meta
+framework/boot-ext.vdex
+framework/boot-ext.vdex.fsv_meta
+framework/boot-framework-adservices.vdex
+framework/boot-framework-adservices.vdex.fsv_meta
+framework/boot-framework-graphics.vdex
+framework/boot-framework-graphics.vdex.fsv_meta
+framework/boot-framework-location.vdex
+framework/boot-framework-location.vdex.fsv_meta
+framework/boot-framework.vdex
+framework/boot-framework.vdex.fsv_meta
+framework/boot-ims-common.vdex
+framework/boot-ims-common.vdex.fsv_meta
+framework/boot-okhttp.vdex
+framework/boot-okhttp.vdex.fsv_meta
+framework/boot-telephony-common.vdex
+framework/boot-telephony-common.vdex.fsv_meta
+framework/boot-voip-common.vdex
+framework/boot-voip-common.vdex.fsv_meta
+framework/boot.vdex
+framework/boot.vdex.fsv_meta
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex.fsv_meta
+lib/aaudio-aidl-cpp.so
+lib/android.hardware.biometrics.fingerprint@2.1.so
+lib/android.hardware.radio.config@1.0.so
+lib/android.hardware.radio.deprecated@1.0.so
+lib/android.hardware.radio@1.0.so
+lib/android.hardware.radio@1.1.so
+lib/android.hardware.radio@1.2.so
+lib/android.hardware.radio@1.3.so
+lib/android.hardware.radio@1.4.so
+lib/android.hardware.secure_element@1.0.so
+lib/com.android.media.aaudio-aconfig-cc.so
+lib/heapprofd_client.so
+lib/heapprofd_client_api.so
+lib/libaaudio.so
+lib/libaaudio_internal.so
+lib/libalarm_jni.so
+lib/libamidi.so
+lib/libcups.so
+lib/libjni_deviceAsWebcam.so
+lib/libprintspooler_jni.so
+lib/libvendorsupport.so
+lib/libwfds.so
+lib/libyuv.so
+
+# Known diffs only in the Soong system image
+lib/libhidcommand_jni.so
+lib/libuinputcommand_jni.so

tools/filelistdiff/file_list_diff.py

@@ -0,0 +1,66 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+
+COLOR_WARNING = '\033[93m'
+COLOR_ERROR = '\033[91m'
+COLOR_NORMAL = '\033[0m'
+
+def find_unique_items(kati_installed_files, soong_installed_files, allowlist, system_module_name):
+    with open(kati_installed_files, 'r') as kati_list_file, \
+            open(soong_installed_files, 'r') as soong_list_file, \
+            open(allowlist, 'r') as allowlist_file:
+        kati_files = set(kati_list_file.read().split())
+        soong_files = set(soong_list_file.read().split())
+        allowed_files = set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n'))))
+
+    def is_unknown_diff(filepath):
+        return not filepath in allowed_files
+
+    unique_in_kati = set(filter(is_unknown_diff, kati_files - soong_files))
+    unique_in_soong = set(filter(is_unknown_diff, soong_files - kati_files))
+
+    if unique_in_kati:
+        print(f'{COLOR_ERROR}Please add following modules into system image module {system_module_name}.{COLOR_NORMAL}')
+        print(f'{COLOR_WARNING}KATI only module(s):{COLOR_NORMAL}')
+        for item in sorted(unique_in_kati):
+            print(item)
+
+    if unique_in_soong:
+        if unique_in_kati:
+            print('')
+
+        print(f'{COLOR_ERROR}Please add following modules into build/make/target/product/base_system.mk.{COLOR_NORMAL}')
+        print(f'{COLOR_WARNING}Soong only module(s):{COLOR_NORMAL}')
+        for item in sorted(unique_in_soong):
+            print(item)
+
+    if unique_in_kati or unique_in_soong:
+        print('')
+        print(f'{COLOR_ERROR}FAILED: System image from KATI and SOONG differs from installed file list.{COLOR_NORMAL}')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('kati_installed_file_list')
+    parser.add_argument('soong_installed_file_list')
+    parser.add_argument('allowlist')
+    parser.add_argument('system_module_name')
+    args = parser.parse_args()
+
+    find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.allowlist, args.system_module_name)