diff --git a/core/Makefile b/core/Makefile index 821c27a2f3..3e4bf964c9 100644 --- a/core/Makefile +++ b/core/Makefile @@ -2981,7 +2981,7 @@ endif # BUILDING_DEBUG_VENDOR_BOOT_IMAGE endif # BUILDING_DEBUG_BOOT_IMAGE || BUILDING_DEBUG_VENDOR_BOOT_IMAGE - +PARTITION_COMPAT_SYMLINKS := # Creates a compatibility symlink between two partitions, e.g. /system/vendor to /vendor # $1: from location (e.g $(TARGET_OUT)/vendor) # $2: destination location (e.g. /vendor) @@ -2999,6 +2999,7 @@ $1: ln -sfn $2 $1 $1: .KATI_SYMLINK_OUTPUTS := $1 ) +$(eval PARTITION_COMPAT_SYMLINKS += $1) $1 endef @@ -3109,17 +3110,23 @@ INTERNAL_SYSTEMIMAGE_FILES := $(sort $(filter $(TARGET_OUT)/%, \ # Create symlink /system/vendor to /vendor if necessary. ifdef BOARD_USES_VENDORIMAGE - INTERNAL_SYSTEMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT)/vendor,/vendor,vendor.img) + _vendor_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT)/vendor,/vendor,vendor.img) + INTERNAL_SYSTEMIMAGE_FILES += $(_vendor_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_vendor_symlink) endif # Create symlink /system/product to /product if necessary. ifdef BOARD_USES_PRODUCTIMAGE - INTERNAL_SYSTEMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT)/product,/product,product.img) + _product_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT)/product,/product,product.img) + INTERNAL_SYSTEMIMAGE_FILES += $(_product_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_product_symlink) endif # Create symlink /system/system_ext to /system_ext if necessary. ifdef BOARD_USES_SYSTEM_EXTIMAGE - INTERNAL_SYSTEMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT)/system_ext,/system_ext,system_ext.img) + _systemext_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT)/system_ext,/system_ext,system_ext.img) + INTERNAL_SYSTEMIMAGE_FILES += $(_systemext_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_systemext_symlink) endif # ----------------------------------------------------------------- @@ -3132,7 +3139,9 @@ endif # - /system/lib/modules is a symlink to a directory that stores system DLKMs. # - The system_dlkm partition is mounted at /system_dlkm at runtime. ifdef BOARD_USES_SYSTEM_DLKMIMAGE - INTERNAL_SYSTEMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT)/lib/modules,/system_dlkm/lib/modules,system_dlkm.img) + _system_dlkm_lib_modules_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT)/lib/modules,/system_dlkm/lib/modules,system_dlkm.img) + INTERNAL_SYSTEMIMAGE_FILES += $(_system_dlkm_lib_modules_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_system_dlkm_lib_modules_symlink) endif FULL_SYSTEMIMAGE_DEPS := $(INTERNAL_SYSTEMIMAGE_FILES) $(INTERNAL_USERIMAGES_DEPS) @@ -3521,7 +3530,9 @@ INTERNAL_VENDORIMAGE_FILES := \ # Create symlink /vendor/odm to /odm if necessary. ifdef BOARD_USES_ODMIMAGE - INTERNAL_VENDORIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT_VENDOR)/odm,/odm,odm.img) + _odm_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT_VENDOR)/odm,/odm,odm.img) + INTERNAL_VENDORIMAGE_FILES += $(_odm_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_odm_symlink) endif # Create symlinks for vendor_dlkm on devices with a vendor_dlkm partition: @@ -3539,7 +3550,9 @@ endif # The vendor DLKMs and other vendor_dlkm files must not be accessed using other paths because they # are not guaranteed to exist on all devices. ifdef BOARD_USES_VENDOR_DLKMIMAGE - INTERNAL_VENDORIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT_VENDOR)/lib/modules,/vendor_dlkm/lib/modules,vendor_dlkm.img) + _vendor_dlkm_lib_modules_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT_VENDOR)/lib/modules,/vendor_dlkm/lib/modules,vendor_dlkm.img) + INTERNAL_VENDORIMAGE_FILES += $(_vendor_dlkm_lib_modules_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_vendor_dlkm_lib_modules_symlink) endif # Install vendor/etc/linker.config.pb with PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS and STUB_LIBRARIES @@ -3760,7 +3773,9 @@ INTERNAL_ODMIMAGE_FILES := \ # The odm DLKMs and other odm_dlkm files must not be accessed using other paths because they # are not guaranteed to exist on all devices. ifdef BOARD_USES_ODM_DLKMIMAGE - INTERNAL_ODMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT_ODM)/lib/modules,/odm_dlkm/lib/modules,odm_dlkm.img) + _odm_dlkm_lib_modules_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT_ODM)/lib/modules,/odm_dlkm/lib/modules,odm_dlkm.img) + INTERNAL_ODMIMAGE_FILES += $(_odm_dlkm_lib_modules_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_odm_dlkm_lib_modules_symlink) endif INSTALLED_FILES_FILE_ODM := $(PRODUCT_OUT)/installed-files-odm.txt diff --git a/core/build_id.mk b/core/build_id.mk index 29a07b99b3..81e8713e38 100644 --- a/core/build_id.mk +++ b/core/build_id.mk @@ -18,4 +18,4 @@ # (like "CRB01"). It must be a single word, and is # capitalized by convention. -BUILD_ID=UP1A.230407.001 +BUILD_ID=UP1A.230408.001 diff --git a/core/config.mk b/core/config.mk index 91c8bd81aa..26e90efe9e 100644 --- a/core/config.mk +++ b/core/config.mk @@ -863,7 +863,6 @@ endif # A list of SEPolicy versions, besides PLATFORM_SEPOLICY_VERSION, that the framework supports. PLATFORM_SEPOLICY_COMPAT_VERSIONS := \ - 28.0 \ 29.0 \ 30.0 \ 31.0 \ diff --git a/core/main.mk b/core/main.mk index d0f30a254d..9ddd990b34 100644 --- a/core/main.mk +++ b/core/main.mk @@ -2178,7 +2178,8 @@ $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) $(eval _is_kernel_modules_blocklist := $(if $(findstring $f,$(ALL_KERNEL_MODULES_BLOCKLIST)),Y)) \ $(eval _is_fsverity_build_manifest_apk := $(if $(findstring $f,$(ALL_FSVERITY_BUILD_MANIFEST_APK)),Y)) \ $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \ - $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)) \ + $(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \ + $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \ @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ $(newline) \ $(if $(_post_installed_dexpreopt_zip), \ for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ ; done $(newline) \ diff --git a/tools/finalization/README.md b/tools/finalization/README.md new file mode 100644 index 0000000000..501f2605a8 --- /dev/null +++ b/tools/finalization/README.md @@ -0,0 +1,22 @@ +# Finalization tools +This folder contains automation and CI scripts for [finalizing](https://go/android-finalization) Android before release. + +## Automation: +1. [Environment setup](./environment.sh). Set values for varios finalization constants. +2. [Finalize SDK](./finalize-aidl-vndk-sdk-resources.sh). Prepare the branch for SDK release. SDK contains Android Java APIs and other stable APIs. Commonly referred as a 1st step. +3. [Finalize Android](./finalize-sdk-rel.sh). Mark branch as "REL", i.e. prepares for Android release. Any signed build containing these changes will be considered an official Android Release. Referred as a 2nd finalization step. +4. [Finalize SDK and submit](./step-1.sh). Do [Finalize SDK](./finalize-aidl-vndk-sdk-resources.sh) step, create CLs, organize them into topic and send to Gerrit. + a. [Update SDK and submit](./update-step-1.sh). Same as above, but updates the existings CLs. +5. [Finalize Android and submit](./step-2.sh). Do [Finalize Android](./finalize-sdk-rel.sh) step, create CLs, organize them into topic and send to Gerrit. + a. [Update Android and submit](./update-step-2.sh). Same as above, but updates the existings CLs. + +## CI: +Performed in build targets in Finalization branches. +1. [Finalization Step 1 for Main, git_main-fina-1-release](https://android-build.googleplex.com/builds/branches/git_main-fina-1-release/grid). Test [1st step/Finalize SDK](./finalize-aidl-vndk-sdk-resources.sh). +2. [Finalization Step 1 for UDC, git_udc-fina-1-release](https://android-build.googleplex.com/builds/branches/git_udc-fina-1-release/grid). Same but for udc-dev. +3. [Finalization Step 2 for Main, git_main-fina-2-release](https://android-build.googleplex.com/builds/branches/git_main-fina-2-release/grid). Test [1st step/Finalize SDK](./finalize-aidl-vndk-sdk-resources.sh) and [2nd step/Finalize Android](./finalize-sdk-rel.sh). Use [local finalization](./localonly-steps.sh) to build and copy presubmits. +4. [Finalization Step 2 for UDC, git_udc-fina-2-release](https://android-build.googleplex.com/builds/branches/git_udc-fina-2-release/grid). Same but for udc-dev. +5. [Local finalization steps](./localonly-steps.sh) are done only during local testing or in the CI lab. Normally these steps use artifacts from other builds. + +## Utility: +[Full cleanup](./cleanup.sh). Remove all local changes and switch each project into head-less state. This is the best state to sync/rebase/finalize the branch. diff --git a/tools/finalization/build-step-1-and-2.sh b/tools/finalization/build-step-1-and-2.sh index 1b749b106f..a5aba0806b 100755 --- a/tools/finalization/build-step-1-and-2.sh +++ b/tools/finalization/build-step-1-and-2.sh @@ -9,8 +9,8 @@ function finalize_main_step12() { # SDK codename -> int source $top/build/make/tools/finalization/finalize-aidl-vndk-sdk-resources.sh - # Platform/Mainline SDKs build and move to prebuilts - source $top/build/make/tools/finalization/localonly-finalize-mainline-sdk.sh + # ADB, Platform/Mainline SDKs build and move to prebuilts + source $top/build/make/tools/finalization/localonly-steps.sh # REL source $top/build/make/tools/finalization/finalize-sdk-rel.sh diff --git a/tools/finalization/localonly-finalize-mainline-sdk.sh b/tools/finalization/localonly-steps.sh similarity index 93% rename from tools/finalization/localonly-finalize-mainline-sdk.sh rename to tools/finalization/localonly-steps.sh index 2b77c5dbbd..6107b3e090 100755 --- a/tools/finalization/localonly-finalize-mainline-sdk.sh +++ b/tools/finalization/localonly-steps.sh @@ -2,7 +2,7 @@ set -ex -function finalize_locally_mainline_sdk() { +function finalize_locally() { local top="$(dirname "$0")"/../../../.. source $top/build/make/tools/finalization/environment.sh @@ -23,5 +23,4 @@ function finalize_locally_mainline_sdk() { "$top/prebuilts/build-tools/path/linux-x86/python3" -W ignore::DeprecationWarning "$top/prebuilts/sdk/update_prebuilts.py" --local_mode -f ${FINAL_PLATFORM_SDK_VERSION} -e ${FINAL_MAINLINE_EXTENSION} --bug 1 1 } -finalize_locally_mainline_sdk - +finalize_locally diff --git a/tools/finalization/update-step-1.sh b/tools/finalization/update-step-1.sh new file mode 100644 index 0000000000..fd07b7b22a --- /dev/null +++ b/tools/finalization/update-step-1.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Script to perform a 1st step of Android Finalization: API/SDK finalization, update CLs and upload to Gerrit. + +set -ex + +function update_step_1_changes() { + set +e + repo forall -c '\ + if [[ $(git status --short) ]]; then + git stash -u ; + repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization" ; + git stash pop ; + git add -A . ; + git commit --amend --no-edit ; + repo upload --cbr --no-verify -o nokeycheck -t -y . ; + fi' +} + +function update_step_1_main() { + local top="$(dirname "$0")"/../../../.. + source $top/build/make/tools/finalization/environment.sh + + + local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug" + + # vndk etc finalization + source $top/build/make/tools/finalization/finalize-aidl-vndk-sdk-resources.sh + + # update existing CLs and upload to gerrit + update_step_1_changes + + # build to confirm everything is OK + AIDL_FROZEN_REL=true $m +} + +update_step_1_main diff --git a/tools/finalization/update-step-2.sh b/tools/finalization/update-step-2.sh new file mode 100755 index 0000000000..e65d35a3ee --- /dev/null +++ b/tools/finalization/update-step-2.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Script to perform a 2nd step of Android Finalization: REL finalization, create CLs and upload to Gerrit. + +function update_step_2_changes() { + set +e + repo forall -c '\ + if [[ $(git status --short) ]]; then + git stash -u ; + repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization-Rel" ; + git stash pop ; + git add -A . ; + git commit --amend --no-edit ; + repo upload --cbr --no-verify -o nokeycheck -t -y . ; + fi' +} + +function update_step_2_main() { + local top="$(dirname "$0")"/../../../.. + source $top/build/make/tools/finalization/environment.sh + + local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug" + + # prebuilts etc + source $top/build/make/tools/finalization/finalize-sdk-rel.sh + + # move all changes to finalization branch/topic and upload to gerrit + update_step_2_changes + + # build to confirm everything is OK + AIDL_FROZEN_REL=true $m +} + +update_step_2_main diff --git a/tools/generate-sbom.py b/tools/generate-sbom.py index eae7945588..c22175c9af 100755 --- a/tools/generate-sbom.py +++ b/tools/generate-sbom.py @@ -87,6 +87,7 @@ ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:' ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:' ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:' ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:' +ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-exist installed files:' INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:' @@ -597,11 +598,12 @@ def main(): # Report on some issues and information report = { - ISSUE_NO_METADATA: [], - ISSUE_NO_METADATA_FILE: [], - ISSUE_METADATA_FILE_INCOMPLETE: [], - ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [], - INFO_METADATA_FOUND_FOR_PACKAGE: [] + ISSUE_NO_METADATA: [], + ISSUE_NO_METADATA_FILE: [], + ISSUE_METADATA_FILE_INCOMPLETE: [], + ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [], + ISSUE_INSTALLED_FILE_NOT_EXIST: [], + INFO_METADATA_FOUND_FOR_PACKAGE: [], } # Scan the metadata in CSV file and create the corresponding package and file records in SPDX @@ -619,6 +621,9 @@ def main(): if not installed_file_has_metadata(installed_file_metadata, report): continue + if not os.path.isfile(installed_file): + report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file) + continue file_id = new_file_id(installed_file) product_files.append(new_file_record(file_id, installed_file, checksum(installed_file)))