From 038091ddea34971a3aacbe16e426c4f460671e22 Mon Sep 17 00:00:00 2001 From: Jiyong Park Date: Mon, 26 Sep 2022 14:48:56 +0900 Subject: [PATCH 01/20] Use PackedAdditionalOutputs when reading apk_set for canned_fs_config When an app_set module is included in an APEX, we don't use its `builtFile` which is the main APK, , but `PackedAdditionalOutputs()` which is the zip file where all APK files (main + additional) files are packed. We then unzip them into the corresponding directory inside the APEX. However, `buildFile` still has been used when constructing canned_fs_config whose content should match with what are actually included in the APEX. Fixing the bug by using `PackedAdditionalOutputs()` instead. Bug: 236299724 Bug: 254192829 Bug: 240449675 Test: m nothing (apex_test.go amended) Test: follow https://buganizer.corp.google.com/issues/236299724#comment34 * mkdir for-236299724 * cd for-236299724 * repo init -u https://android.googlesource.com/platform/manifest -b * master --partial-clone --clone-filter=blob:none * repo sync -c -j8 * unzip -l ~/Downloads/mainline_t_2022_jun_t1004275.zip | cut -c31- | grep | xargs rm -f * unzip ~/Downloads/mainline_t_2022_jun_t1004275.zip * source build/envsetup.sh * choosecombo 1 aosp_arm64 userdebug * m com.android.extservices.gms Merged-In: If9752c07748300dbb963568de4e879b041b0a206 Change-Id: If9752c07748300dbb963568de4e879b041b0a206 (cherry picked from commit e1b6914f58d61fb7279d704dbadc1193744315c3) --- apex/apex_test.go | 5 +++++ apex/builder.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 83adeca6d..579025635 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -7291,6 +7291,11 @@ func TestAppSetBundle(t *testing.T) { ensureMatches(t, copyCmds[0], "^rm -rf .*/app/AppSet@TEST.BUILD_ID$") ensureMatches(t, copyCmds[1], "^mkdir -p .*/app/AppSet@TEST.BUILD_ID$") ensureMatches(t, copyCmds[2], "^unzip .*-d .*/app/AppSet@TEST.BUILD_ID .*/AppSet.zip$") + + // Ensure that canned_fs_config has an entry for the app set zip file + generateFsRule := mod.Rule("generateFsConfig") + cmd := generateFsRule.RuleParams.Command + ensureContains(t, cmd, "AppSet.zip") } func TestAppSetBundlePrebuilt(t *testing.T) { diff --git a/apex/builder.go b/apex/builder.go index 0c1f3ba89..ef8d4663b 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -1091,7 +1091,7 @@ func (a *apexBundle) buildCannedFsConfig(ctx android.ModuleContext) android.Outp } } else if f.class == appSet { appSetDirs = append(appSetDirs, f.installDir) - appSetFiles[f.installDir] = f.builtFile + appSetFiles[f.installDir] = f.module.(*java.AndroidAppSet).PackedAdditionalOutputs() } else { readOnlyPaths = append(readOnlyPaths, pathInApex) } From e7cc3513f8c8d2c347cb39c030c445a516755b90 Mon Sep 17 00:00:00 2001 From: Jiyong Park Date: Thu, 29 Sep 2022 21:30:25 +0900 Subject: [PATCH 02/20] Put base APK when AppSet is included in APEX This change fixes a bug that base APK of an AppSet is not put into the APEX. Bug: 236299724 Bug: 254192829 Bug: 240449675 Test: m nothing Merged-In: I1010ec66d8551fcc1e2d9119be7ecdba22186607 Change-Id: I1010ec66d8551fcc1e2d9119be7ecdba22186607 (cherry picked from commit 4169a25f547c3d8d36289a87ae76d61104589c54) --- apex/apex_test.go | 7 ++++--- apex/builder.go | 9 ++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 579025635..d0cb6dfed 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -7285,12 +7285,13 @@ func TestAppSetBundle(t *testing.T) { ensureContains(t, content, `"compression":{"uncompressed_glob":["apex_payload.img","apex_manifest.*"]}`) s := mod.Rule("apexRule").Args["copy_commands"] copyCmds := regexp.MustCompile(" *&& *").Split(s, -1) - if len(copyCmds) != 3 { - t.Fatalf("Expected 3 commands, got %d in:\n%s", len(copyCmds), s) + if len(copyCmds) != 4 { + t.Fatalf("Expected 4 commands, got %d in:\n%s", len(copyCmds), s) } ensureMatches(t, copyCmds[0], "^rm -rf .*/app/AppSet@TEST.BUILD_ID$") ensureMatches(t, copyCmds[1], "^mkdir -p .*/app/AppSet@TEST.BUILD_ID$") - ensureMatches(t, copyCmds[2], "^unzip .*-d .*/app/AppSet@TEST.BUILD_ID .*/AppSet.zip$") + ensureMatches(t, copyCmds[2], "^cp -f .*/app/AppSet@TEST.BUILD_ID/AppSet.apk$") + ensureMatches(t, copyCmds[3], "^unzip .*-d .*/app/AppSet@TEST.BUILD_ID .*/AppSet.zip$") // Ensure that canned_fs_config has an entry for the app set zip file generateFsRule := mod.Rule("generateFsConfig") diff --git a/apex/builder.go b/apex/builder.go index ef8d4663b..3040ac7da 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -463,8 +463,13 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { pathOnDevice := filepath.Join("/system", fi.path()) copyCommands = append(copyCommands, "ln -sfn "+pathOnDevice+" "+destPath) } else { + // Copy the file into APEX + copyCommands = append(copyCommands, "cp -f "+fi.builtFile.String()+" "+destPath) + var installedPath android.InstallPath if fi.class == appSet { + // In case of AppSet, we need to copy additional APKs as well. They + // are zipped. So we need to unzip them. copyCommands = append(copyCommands, fmt.Sprintf("unzip -qDD -d %s %s", destPathDir, fi.module.(*java.AndroidAppSet).PackedAdditionalOutputs().String())) @@ -473,7 +478,6 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { fi.stem(), fi.builtFile, fi.module.(*java.AndroidAppSet).PackedAdditionalOutputs()) } } else { - copyCommands = append(copyCommands, "cp -f "+fi.builtFile.String()+" "+destPath) if installSymbolFiles { installedPath = ctx.InstallFile(pathWhenActivated.Join(ctx, fi.installDir), fi.stem(), fi.builtFile) } @@ -1090,6 +1094,9 @@ func (a *apexBundle) buildCannedFsConfig(ctx android.ModuleContext) android.Outp executablePaths = append(executablePaths, filepath.Join(f.installDir, s)) } } else if f.class == appSet { + // base APK + readOnlyPaths = append(readOnlyPaths, pathInApex) + // Additional APKs appSetDirs = append(appSetDirs, f.installDir) appSetFiles[f.installDir] = f.module.(*java.AndroidAppSet).PackedAdditionalOutputs() } else { From 79435deb8df0c3f303844c7c1ea5b85365d1cc05 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Tue, 23 May 2023 17:32:30 +0000 Subject: [PATCH 03/20] Increase R8 max heap size to 4GB The size was increased for D8 in an earlier change, but is more critical for R8. Match the D8 configuration. Bug: 241220562,283888124 Test: m (cherry picked from https://android-review.googlesource.com/q/commit:3e65f2a53c6e5fd82ddad403360bb87b986ad0d8) Merged-In: I3b98bf6098f2196c87a97f6c8f7308d9299170d8 Change-Id: I3b98bf6098f2196c87a97f6c8f7308d9299170d8 --- java/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/config/config.go b/java/config/config.go index b82a137e7..195dae16b 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -98,7 +98,7 @@ func init() { "-JDcom.android.tools.r8.emitRecordAnnotationsExInDex", }, dexerJavaVmFlagsList...)) exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{ - "-JXmx2048M", + "-JXmx4096M", "-JDcom.android.tools.r8.emitRecordAnnotationsInDex", "-JDcom.android.tools.r8.emitPermittedSubclassesAnnotationsInDex", "-JDcom.android.tools.r8.emitRecordAnnotationsExInDex", From 373e894e3ccbb6806a4310581e5135c1d7ba25ec Mon Sep 17 00:00:00 2001 From: Alexei Nicoara Date: Tue, 27 Jun 2023 16:17:44 +0000 Subject: [PATCH 04/20] DO NOT MERGE Increasing the dev branch default module version After the prebuilt drop the modules under test in udc-dev can not be installed anymore because their version is lower than the prebuilt version. Increasing the version so the modules built from source would have a higher version. Bug: 288871663 Change-Id: If5cfd3ec62860a6f26bdd87ee6c77248c2bea0fb Ignore-AOSP-First: change needed on udc-dev only Merged-In: Iad9079f026f46da35f36723937db9bf5771580d1 --- android/updatable_modules.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/updatable_modules.go b/android/updatable_modules.go index 6d0eeb716..e6ad3fea5 100644 --- a/android/updatable_modules.go +++ b/android/updatable_modules.go @@ -29,8 +29,8 @@ package android // based on the branch such that the builds from testing and development // branches will have a version higher than the prebuilts. // Versions per branch: -// * x-dev - xx0090000 (where xx is the branch SDK level) +// * x-dev - xx2090000 (where xx is the branch SDK level) // * AOSP - xx9990000 // * x-mainline-prod - xx9990000 // * master - 990090000 -const DefaultUpdatableModuleVersion = "340090000" +const DefaultUpdatableModuleVersion = "342090000" From 754b054d634e1712d38a586692b0bb7d0bc5c31b Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Fri, 30 Jun 2023 14:40:10 -0400 Subject: [PATCH 05/20] don't export systemserverclasspath_fragment if contents are empty If a systemserverclasspath_fragment only contains libraries that have a higher min_sdk_version than the target build release version, then we should not export the systemserverclasspath_fragment. Before this change, the fragment was exported with an empty `contents` property which caused errors after being dropped as a prebuilt. Bug: 289183551 Test: go test ./sdk (cherry picked from https://android-review.googlesource.com/q/commit:35881365b49235b2311f48bfcbaff8f873f19b88) Merged-In: Ifefc6880228e4dd37f5e42b2bda31a83df785375 Change-Id: Ifefc6880228e4dd37f5e42b2bda31a83df785375 --- sdk/bootclasspath_fragment_sdk_test.go | 55 +++++++++++++++++++ sdk/bp.go | 10 ++-- ...systemserverclasspath_fragment_sdk_test.go | 45 +++++++++++++++ sdk/update.go | 36 ++++++++++-- 4 files changed, 136 insertions(+), 10 deletions(-) diff --git a/sdk/bootclasspath_fragment_sdk_test.go b/sdk/bootclasspath_fragment_sdk_test.go index bef82d601..b6611e0fa 100644 --- a/sdk/bootclasspath_fragment_sdk_test.go +++ b/sdk/bootclasspath_fragment_sdk_test.go @@ -1194,3 +1194,58 @@ java_sdk_library_import { expectedSnapshot, expectedCopyRules, expectedStubFlagsInputs, "") }) } + +func TestSnapshotWithEmptyBootClasspathFragment(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForSdkTestWithJava, + java.PrepareForTestWithJavaDefaultModules, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.FixtureWithLastReleaseApis("mysdklibrary", "mynewsdklibrary"), + java.FixtureConfigureApexBootJars("myapex:mysdklibrary", "myapex:mynewsdklibrary"), + prepareForSdkTestWithApex, + // Add a platform_bootclasspath that depends on the fragment. + fixtureAddPlatformBootclasspathForBootclasspathFragment("myapex", "mybootclasspathfragment"), + android.FixtureMergeEnv(map[string]string{ + "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": "S", + }), + android.FixtureWithRootAndroidBp(` + sdk { + name: "mysdk", + apexes: ["myapex"], + } + apex { + name: "myapex", + key: "myapex.key", + min_sdk_version: "S", + bootclasspath_fragments: ["mybootclasspathfragment"], + } + bootclasspath_fragment { + name: "mybootclasspathfragment", + apex_available: ["myapex"], + contents: ["mysdklibrary", "mynewsdklibrary"], + hidden_api: { + split_packages: [], + }, + } + java_sdk_library { + name: "mysdklibrary", + apex_available: ["myapex"], + srcs: ["Test.java"], + shared_library: false, + public: {enabled: true}, + min_sdk_version: "Tiramisu", + } + java_sdk_library { + name: "mynewsdklibrary", + apex_available: ["myapex"], + srcs: ["Test.java"], + compile_dex: true, + public: {enabled: true}, + min_sdk_version: "Tiramisu", + permitted_packages: ["mynewsdklibrary"], + } + `), + ).RunTest(t) + + CheckSnapshot(t, result, "mysdk", "", checkAndroidBpContents(`// This is auto-generated. DO NOT EDIT.`)) +} diff --git a/sdk/bp.go b/sdk/bp.go index 7ff85a121..57eb2ca38 100644 --- a/sdk/bp.go +++ b/sdk/bp.go @@ -311,13 +311,15 @@ func (t identityTransformation) transformProperty(_ string, value interface{}, t } func (m *bpModule) deepCopy() *bpModule { - return m.transform(deepCopyTransformer) + return transformModule(m, deepCopyTransformer) } -func (m *bpModule) transform(transformer bpTransformer) *bpModule { +func transformModule(m *bpModule, transformer bpTransformer) *bpModule { transformedModule := transformer.transformModule(m) - // Copy the contents of the returned property set into the module and then transform that. - transformedModule.bpPropertySet, _ = transformPropertySet(transformer, "", transformedModule.bpPropertySet, nil) + if transformedModule != nil { + // Copy the contents of the returned property set into the module and then transform that. + transformedModule.bpPropertySet, _ = transformPropertySet(transformer, "", transformedModule.bpPropertySet, nil) + } return transformedModule } diff --git a/sdk/systemserverclasspath_fragment_sdk_test.go b/sdk/systemserverclasspath_fragment_sdk_test.go index 66c44c843..7ccc11413 100644 --- a/sdk/systemserverclasspath_fragment_sdk_test.go +++ b/sdk/systemserverclasspath_fragment_sdk_test.go @@ -86,6 +86,51 @@ func testSnapshotWithSystemServerClasspathFragment(t *testing.T, sdk string, tar ) } +func TestSnapshotWithEmptySystemServerClasspathFragment(t *testing.T) { + commonSdk := ` + apex { + name: "myapex", + key: "myapex.key", + min_sdk_version: "Tiramisu", + systemserverclasspath_fragments: ["mysystemserverclasspathfragment"], + } + systemserverclasspath_fragment { + name: "mysystemserverclasspathfragment", + apex_available: ["myapex"], + contents: ["mysdklibrary"], + } + java_sdk_library { + name: "mysdklibrary", + apex_available: ["myapex"], + srcs: ["Test.java"], + min_sdk_version: "34", // UpsideDownCake + } + sdk { + name: "mysdk", + apexes: ["myapex"], + } + ` + + result := android.GroupFixturePreparers( + prepareForSdkTestWithJava, + java.PrepareForTestWithJavaDefaultModules, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.FixtureWithLastReleaseApis("mysdklibrary"), + dexpreopt.FixtureSetApexSystemServerJars("myapex:mysdklibrary"), + android.FixtureModifyEnv(func(env map[string]string) { + // targeting Tiramisu here means that we won't export mysdklibrary + env["SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE"] = "Tiramisu" + }), + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.Platform_version_active_codenames = []string{"UpsideDownCake"} + }), + prepareForSdkTestWithApex, + android.FixtureWithRootAndroidBp(commonSdk), + ).RunTest(t) + + CheckSnapshot(t, result, "mysdk", "", checkAndroidBpContents(`// This is auto-generated. DO NOT EDIT.`)) +} + func TestSnapshotWithSystemServerClasspathFragment(t *testing.T) { commonSdk := ` diff --git a/sdk/update.go b/sdk/update.go index d98ab683d..4aacd0637 100644 --- a/sdk/update.go +++ b/sdk/update.go @@ -455,11 +455,14 @@ be unnecessary as every module in the sdk already has its own licenses property. for _, module := range builder.prebuiltOrder { // Prune any empty property sets. - module = module.transform(pruneEmptySetTransformer{}) + module = transformModule(module, pruneEmptySetTransformer{}) // Transform the module module to make it suitable for use in the snapshot. - module.transform(snapshotTransformer) - bpFile.AddModule(module) + module = transformModule(module, snapshotTransformer) + module = transformModule(module, emptyClasspathContentsTransformation{}) + if module != nil { + bpFile.AddModule(module) + } } // generate Android.bp @@ -835,9 +838,11 @@ type snapshotTransformation struct { } func (t snapshotTransformation) transformModule(module *bpModule) *bpModule { - // If the module is an internal member then use a unique name for it. - name := module.Name() - module.setProperty("name", t.builder.snapshotSdkMemberName(name, true)) + if module != nil { + // If the module is an internal member then use a unique name for it. + name := module.Name() + module.setProperty("name", t.builder.snapshotSdkMemberName(name, true)) + } return module } @@ -850,6 +855,25 @@ func (t snapshotTransformation) transformProperty(_ string, value interface{}, t } } +type emptyClasspathContentsTransformation struct { + identityTransformation +} + +func (t emptyClasspathContentsTransformation) transformModule(module *bpModule) *bpModule { + classpathModuleTypes := []string{ + "prebuilt_bootclasspath_fragment", + "prebuilt_systemserverclasspath_fragment", + } + if module != nil && android.InList(module.moduleType, classpathModuleTypes) { + if contents, ok := module.bpPropertySet.properties["contents"].([]string); ok { + if len(contents) == 0 { + return nil + } + } + } + return module +} + type pruneEmptySetTransformer struct { identityTransformation } From 92275ff913c6d41d6e85f5f622735df69bb9ad5f Mon Sep 17 00:00:00 2001 From: Wei Li Date: Thu, 25 May 2023 16:29:20 -0700 Subject: [PATCH 06/20] Exclude diffs caught by sbom_test. Bug: 284279034 Test: build/soong/tests/sbom_test.sh (cherry picked from https://android-review.googlesource.com/q/commit:704146094a262f1f7e99ee8606469834ec142a34) Merged-In: I152c2e2c55b6547b2dddf7b1b2524681c9af952e Change-Id: I152c2e2c55b6547b2dddf7b1b2524681c9af952e --- tests/sbom_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/sbom_test.sh b/tests/sbom_test.sh index 2f154cde5..94fe51d2d 100755 --- a/tests/sbom_test.sh +++ b/tests/sbom_test.sh @@ -90,10 +90,12 @@ diff_excludes[system]=\ -I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \ -I /system/lib64/android.security.compat-ndk.so \ -I /system/lib64/libkeymaster4_1support.so \ + -I /system/lib64/libkeymaster4support.so \ -I /system/lib64/libkeymint.so \ -I /system/lib64/libkeystore2_aaid.so \ -I /system/lib64/libkeystore2_apc_compat.so \ -I /system/lib64/libkeystore2_crypto.so \ + -I /system/lib64/libkeystore-attestation-application-id.so \ -I /system/lib64/libkm_compat_service.so \ -I /system/lib64/libkm_compat.so \ -I /system/lib64/vndk-29 \ From 1b2badc435c135e3e2c5996f4a5a352da1bb179a Mon Sep 17 00:00:00 2001 From: Wei Li Date: Thu, 22 Jun 2023 17:52:31 -0700 Subject: [PATCH 07/20] Cleanup SBOM test script for later adding tests of generating SBOM of unbundled APKs/APEXs. Bug: 266726655 Test: build/soong/tests/sbom_test.sh (cherry picked from https://android-review.googlesource.com/q/commit:cd9948f66e9686e83cbe86d2b8ee998564caff0f) Merged-In: I2bb3e8405dad716837d3004d4c6fd54ae80527c1 Change-Id: I2bb3e8405dad716837d3004d4c6fd54ae80527c1 --- tests/sbom_test.sh | 366 +++++++++++++++++++++++---------------------- 1 file changed, 188 insertions(+), 178 deletions(-) diff --git a/tests/sbom_test.sh b/tests/sbom_test.sh index 94fe51d2d..19987f204 100755 --- a/tests/sbom_test.sh +++ b/tests/sbom_test.sh @@ -23,194 +23,204 @@ if [ ! -e "build/make/core/Makefile" ]; then exit 1 fi -tmp_dir="$(mktemp -d tmp.XXXXXX)" +function setup { + tmp_dir="$(mktemp -d tmp.XXXXXX)" + trap 'cleanup "${tmp_dir}"' EXIT + echo "${tmp_dir}" +} + function cleanup { + tmp_dir="$1"; shift rm -rf "${tmp_dir}" } -trap cleanup EXIT - -out_dir=$tmp_dir -droid_target=droid - -debug=false -if [ $debug = "true" ]; then - out_dir=out - droid_target= -fi function run_soong { - TARGET_PRODUCT="aosp_cf_x86_64_phone" TARGET_BUILD_VARIANT=userdebug OUT_DIR=$out_dir \ - build/soong/soong_ui.bash --make-mode "$@" + target_product="$1";shift + out_dir="$1"; shift + targets="$1"; shift + if [ "$#" -ge 1 ]; then + apps=$1; shift + TARGET_PRODUCT="${target_product}" TARGET_BUILD_VARIANT=userdebug OUT_DIR="${out_dir}" TARGET_BUILD_UNBUNDLED=true TARGET_BUILD_APPS=$apps build/soong/soong_ui.bash --make-mode ${targets} + else + TARGET_PRODUCT="${target_product}" TARGET_BUILD_VARIANT=userdebug OUT_DIR="${out_dir}" build/soong/soong_ui.bash --make-mode ${targets} + fi } -# m droid, build sbom later in case additional dependencies might be built and included in partition images. -run_soong $droid_target dump.erofs lz4 - -product_out=$out_dir/target/product/vsoc_x86_64 -sbom_test=$product_out/sbom_test -mkdir $sbom_test -cp $product_out/*.img $sbom_test - -# m sbom -run_soong sbom - -# Generate installed file list from .img files in PRODUCT_OUT -dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs -lz4=$out_dir/host/linux-x86/bin/lz4 - -declare -A diff_excludes -diff_excludes[odm]="-I /odm/lib/modules" -diff_excludes[vendor]=\ -"-I /vendor/lib64/libkeystore2_crypto.so \ - -I /vendor/lib/modules \ - -I /vendor/odm" -diff_excludes[system]=\ -"-I /bin \ - -I /bugreports \ - -I /cache \ - -I /d \ - -I /etc \ - -I /init \ - -I /odm/app \ - -I /odm/bin \ - -I /odm_dlkm/etc \ - -I /odm/etc \ - -I /odm/firmware \ - -I /odm/framework \ - -I /odm/lib \ - -I /odm/lib64 \ - -I /odm/overlay \ - -I /odm/priv-app \ - -I /odm/usr \ - -I /sdcard \ - -I /system/lib64/android.hardware.confirmationui@1.0.so \ - -I /system/lib64/android.hardware.confirmationui-V1-ndk.so \ - -I /system/lib64/android.hardware.keymaster@4.1.so \ - -I /system/lib64/android.hardware.security.rkp-V3-ndk.so \ - -I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \ - -I /system/lib64/android.security.compat-ndk.so \ - -I /system/lib64/libkeymaster4_1support.so \ - -I /system/lib64/libkeymaster4support.so \ - -I /system/lib64/libkeymint.so \ - -I /system/lib64/libkeystore2_aaid.so \ - -I /system/lib64/libkeystore2_apc_compat.so \ - -I /system/lib64/libkeystore2_crypto.so \ - -I /system/lib64/libkeystore-attestation-application-id.so \ - -I /system/lib64/libkm_compat_service.so \ - -I /system/lib64/libkm_compat.so \ - -I /system/lib64/vndk-29 \ - -I /system/lib64/vndk-sp-29 \ - -I /system/lib/vndk-29 \ - -I /system/lib/vndk-sp-29 \ - -I /system/usr/icu \ - -I /vendor_dlkm/etc" - function diff_files { - file_list_file="$1"; shift - files_in_spdx_file="$1"; shift - partition_name="$1"; shift - exclude= - if [ -v 'diff_excludes[$partition_name]' ]; then - exclude=${diff_excludes[$partition_name]} - fi - - diff "$file_list_file" "$files_in_spdx_file" $exclude - if [ $? != "0" ]; then - echo Found diffs in $f and SBOM. - exit 1 - else - echo No diffs. - fi - } - -# Example output of dump.erofs is as below, and the data used in the test start -# at line 11. Column 1 is inode id, column 2 is inode type and column 3 is name. -# Each line is captured in variable "entry", awk is used to get type and name. -# Output of dump.erofs: -# File : / -# Size: 160 On-disk size: 160 directory -# NID: 39 Links: 10 Layout: 2 Compression ratio: 100.00% -# Inode size: 64 Extent size: 0 Xattr size: 16 -# Uid: 0 Gid: 0 Access: 0755/rwxr-xr-x -# Timestamp: 2023-02-14 01:15:54.000000000 -# -# NID TYPE FILENAME -# 39 2 . -# 39 2 .. -# 47 2 app -# 1286748 2 bin -# 1286754 2 etc -# 5304814 2 lib -# 5309056 2 lib64 -# 5309130 2 media -# 5388910 2 overlay -# 5479537 2 priv-app -EROFS_IMAGES="\ - $sbom_test/product.img \ - $sbom_test/system.img \ - $sbom_test/system_ext.img \ - $sbom_test/system_dlkm.img \ - $sbom_test/system_other.img \ - $sbom_test/odm.img \ - $sbom_test/odm_dlkm.img \ - $sbom_test/vendor.img \ - $sbom_test/vendor_dlkm.img" -for f in $EROFS_IMAGES; do - partition_name=$(basename $f | cut -d. -f1) - file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" - files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt" - rm "$file_list_file" > /dev/null 2>&1 - all_dirs="/" - while [ ! -z "$all_dirs" ]; do - dir=$(echo "$all_dirs" | cut -d ' ' -f1) - all_dirs=$(echo "$all_dirs" | cut -d ' ' -f1 --complement -s) - entries=$($dump_erofs --ls --path "$dir" $f | tail -n +11) - while read -r entry; do - inode_type=$(echo $entry | awk -F ' ' '{print $2}') - name=$(echo $entry | awk -F ' ' '{print $3}') - case $inode_type in - "2") # directory - all_dirs=$(echo "$all_dirs $dir/$name" | sed 's/^\s*//') - ;; - "1"|"7") # 1: file, 7: symlink - ( - if [ "$partition_name" != "system" ]; then - # system partition is mounted to /, not to prepend partition name. - printf %s "/$partition_name" - fi - echo "$dir/$name" | sed 's#^//#/#' - ) >> "$file_list_file" - ;; - esac - done <<< "$entries" - done - sort -n -o "$file_list_file" "$file_list_file" - - grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file" - if [ "$partition_name" = "system" ]; then - # system partition is mounted to /, so include FileName starts with /root/ too. - grep "FileName: /root/" $product_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_spdx_file" + file_list_file="$1"; shift + files_in_spdx_file="$1"; shift + partition_name="$1"; shift + exclude= + if [ -v 'diff_excludes[$partition_name]' ]; then + exclude=${diff_excludes[$partition_name]} fi - sort -n -o "$files_in_spdx_file" "$files_in_spdx_file" - echo ============ Diffing files in $f and SBOM - diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" -done + diff "$file_list_file" "$files_in_spdx_file" $exclude + if [ $? != "0" ]; then + echo Found diffs in $f and SBOM. + exit 1 + else + echo No diffs. + fi +} -RAMDISK_IMAGES="$product_out/ramdisk.img" -for f in $RAMDISK_IMAGES; do - partition_name=$(basename $f | cut -d. -f1) - file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" - files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt" - # lz4 decompress $f to stdout - # cpio list all entries like ls -l - # grep filter normal files and symlinks - # awk get entry names - # sed remove partition name from entry names - $lz4 -c -d $f | cpio -tv 2>/dev/null | grep '^[-l]' | awk -F ' ' '{print $9}' | sed "s:^:/$partition_name/:" | sort -n > "$file_list_file" +function test_sbom_aosp_cf_x86_64_phone { + # Setup + out_dir="$(setup)" - grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file" + # Test + # m droid, build sbom later in case additional dependencies might be built and included in partition images. + run_soong "aosp_cf_x86_64_phone" "${out_dir}" "droid dump.erofs lz4" - echo ============ Diffing files in $f and SBOM - diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" -done \ No newline at end of file + product_out=$out_dir/target/product/vsoc_x86_64 + sbom_test=$product_out/sbom_test + mkdir -p $sbom_test + cp $product_out/*.img $sbom_test + + # m sbom + run_soong "aosp_cf_x86_64_phone" "${out_dir}" sbom + + # Generate installed file list from .img files in PRODUCT_OUT + dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs + lz4=$out_dir/host/linux-x86/bin/lz4 + + declare -A diff_excludes + diff_excludes[vendor]="-I /vendor/lib64/libkeystore2_crypto.so" + diff_excludes[system]="\ + -I /bin \ + -I /bugreports \ + -I /cache \ + -I /d \ + -I /etc \ + -I /init \ + -I /odm/app \ + -I /odm/bin \ + -I /odm_dlkm/etc \ + -I /odm/etc \ + -I /odm/firmware \ + -I /odm/framework \ + -I /odm/lib \ + -I /odm/lib64 \ + -I /odm/overlay \ + -I /odm/priv-app \ + -I /odm/usr \ + -I /sdcard \ + -I /system/lib64/android.hardware.confirmationui@1.0.so \ + -I /system/lib64/android.hardware.confirmationui-V1-ndk.so \ + -I /system/lib64/android.hardware.keymaster@4.1.so \ + -I /system/lib64/android.hardware.security.rkp-V3-ndk.so \ + -I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \ + -I /system/lib64/android.security.compat-ndk.so \ + -I /system/lib64/libkeymaster4_1support.so \ + -I /system/lib64/libkeymaster4support.so \ + -I /system/lib64/libkeymint.so \ + -I /system/lib64/libkeystore2_aaid.so \ + -I /system/lib64/libkeystore2_apc_compat.so \ + -I /system/lib64/libkeystore2_crypto.so \ + -I /system/lib64/libkeystore-attestation-application-id.so \ + -I /system/lib64/libkm_compat_service.so \ + -I /system/lib64/libkm_compat.so \ + -I /system/lib64/vndk-29 \ + -I /system/lib64/vndk-sp-29 \ + -I /system/lib/vndk-29 \ + -I /system/lib/vndk-sp-29 \ + -I /system/usr/icu \ + -I /vendor_dlkm/etc" + + # Example output of dump.erofs is as below, and the data used in the test start + # at line 11. Column 1 is inode id, column 2 is inode type and column 3 is name. + # Each line is captured in variable "entry", awk is used to get type and name. + # Output of dump.erofs: + # File : / + # Size: 160 On-disk size: 160 directory + # NID: 39 Links: 10 Layout: 2 Compression ratio: 100.00% + # Inode size: 64 Extent size: 0 Xattr size: 16 + # Uid: 0 Gid: 0 Access: 0755/rwxr-xr-x + # Timestamp: 2023-02-14 01:15:54.000000000 + # + # NID TYPE FILENAME + # 39 2 . + # 39 2 .. + # 47 2 app + # 1286748 2 bin + # 1286754 2 etc + # 5304814 2 lib + # 5309056 2 lib64 + # 5309130 2 media + # 5388910 2 overlay + # 5479537 2 priv-app + EROFS_IMAGES="\ + $sbom_test/product.img \ + $sbom_test/system.img \ + $sbom_test/system_ext.img \ + $sbom_test/system_dlkm.img \ + $sbom_test/system_other.img \ + $sbom_test/odm.img \ + $sbom_test/odm_dlkm.img \ + $sbom_test/vendor.img \ + $sbom_test/vendor_dlkm.img" + for f in $EROFS_IMAGES; do + partition_name=$(basename $f | cut -d. -f1) + file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" + files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt" + rm "$file_list_file" > /dev/null 2>&1 || true + all_dirs="/" + while [ ! -z "$all_dirs" ]; do + dir=$(echo "$all_dirs" | cut -d ' ' -f1) + all_dirs=$(echo "$all_dirs" | cut -d ' ' -f1 --complement -s) + entries=$($dump_erofs --ls --path "$dir" $f | tail -n +11) + while read -r entry; do + inode_type=$(echo $entry | awk -F ' ' '{print $2}') + name=$(echo $entry | awk -F ' ' '{print $3}') + case $inode_type in + "2") # directory + all_dirs=$(echo "$all_dirs $dir/$name" | sed 's/^\s*//') + ;; + "1"|"7") # 1: file, 7: symlink + ( + if [ "$partition_name" != "system" ]; then + # system partition is mounted to /, not to prepend partition name. + printf %s "/$partition_name" + fi + echo "$dir/$name" | sed 's#^//#/#' + ) >> "$file_list_file" + ;; + esac + done <<< "$entries" + done + sort -n -o "$file_list_file" "$file_list_file" + + grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file" + if [ "$partition_name" = "system" ]; then + # system partition is mounted to /, so include FileName starts with /root/ too. + grep "FileName: /root/" $product_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_spdx_file" + fi + sort -n -o "$files_in_spdx_file" "$files_in_spdx_file" + + echo ============ Diffing files in $f and SBOM + diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" + done + + RAMDISK_IMAGES="$product_out/ramdisk.img" + for f in $RAMDISK_IMAGES; do + partition_name=$(basename $f | cut -d. -f1) + file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" + files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt" + # lz4 decompress $f to stdout + # cpio list all entries like ls -l + # grep filter normal files and symlinks + # awk get entry names + # sed remove partition name from entry names + $lz4 -c -d $f | cpio -tv 2>/dev/null | grep '^[-l]' | awk -F ' ' '{print $9}' | sed "s:^:/$partition_name/:" | sort -n > "$file_list_file" + + grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file" + + echo ============ Diffing files in $f and SBOM + diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" + done + + # Teardown + cleanup "${out_dir}" +} + +test_sbom_aosp_cf_x86_64_phone \ No newline at end of file From b68bfc65d82c360656adeec56efaf24775658d30 Mon Sep 17 00:00:00 2001 From: Wei Li Date: Mon, 10 Jul 2023 15:50:38 -0700 Subject: [PATCH 08/20] Add integration test for m build SBOM of APEXs/APKs. Bug: 266726655 Test: build/soong/tests/sbom_test.sh (cherry picked from https://android-review.googlesource.com/q/commit:59df0ee44206557cbe18ce42117b201f4d858bee) Merged-In: Id8002042f607f8e158f561ca47e0a84a8b8927e2 Change-Id: Id8002042f607f8e158f561ca47e0a84a8b8927e2 --- tests/sbom_test.sh | 63 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/sbom_test.sh b/tests/sbom_test.sh index 19987f204..30a1d377f 100755 --- a/tests/sbom_test.sh +++ b/tests/sbom_test.sh @@ -223,4 +223,65 @@ function test_sbom_aosp_cf_x86_64_phone { cleanup "${out_dir}" } -test_sbom_aosp_cf_x86_64_phone \ No newline at end of file +function test_sbom_unbundled_apex { + # Setup + out_dir="$(setup)" + + # run_soong to build com.android.adbd.apex + run_soong "module_arm64" "${out_dir}" "sbom deapexer" "com.android.adbd" + + deapexer=${out_dir}/host/linux-x86/bin/deapexer + debugfs=${out_dir}/host/linux-x86/bin/debugfs_static + apex_file=${out_dir}/target/product/module_arm64/system/apex/com.android.adbd.apex + echo "============ Diffing files in $apex_file and SBOM" + set +e + # deapexer prints the list of all files and directories + # sed extracts the file/directory names + # grep removes directories + # sed removes leading ./ in file names + diff -I /system/apex/com.android.adbd.apex -I apex_manifest.pb \ + <($deapexer --debugfs_path=$debugfs list --extents ${apex_file} | sed -E 's#(.*) \[.*\]$#\1#' | grep -v "/$" | sed -E 's#^\./(.*)#\1#' | sort -n) \ + <(grep '"fileName": ' ${apex_file}.spdx.json | sed -E 's/.*"fileName": "(.*)",/\1/' | sort -n ) + + if [ $? != "0" ]; then + echo "Diffs found in $apex_file and SBOM" + exit 1 + else + echo "No diffs." + fi + set -e + + # Teardown + cleanup "${out_dir}" +} + +function test_sbom_unbundled_apk { + # Setup + out_dir="$(setup)" + + # run_soong to build Browser2.apk + run_soong "module_arm64" "${out_dir}" "sbom" "Browser2" + + sbom_file=${out_dir}/target/product/module_arm64/system/product/app/Browser2/Browser2.apk.spdx.json + echo "============ Diffing files in Browser2.apk and SBOM" + set +e + # There is only one file in SBOM of APKs + diff \ + <(echo "/system/product/app/Browser2/Browser2.apk" ) \ + <(grep '"fileName": ' ${sbom_file} | sed -E 's/.*"fileName": "(.*)",/\1/' ) + + if [ $? != "0" ]; then + echo "Diffs found in $sbom_file" + exit 1 + else + echo "No diffs." + fi + set -e + + # Teardown + cleanup "${out_dir}" +} + +test_sbom_aosp_cf_x86_64_phone +test_sbom_unbundled_apex +test_sbom_unbundled_apk \ No newline at end of file From 3d6f3a02f87ed4179925962695b8e5d299e639bb Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 25 Apr 2023 11:30:51 -0700 Subject: [PATCH 09/20] Install sdk variants in unbundled builds and package uninstallable variants This effectively undoes both If6c3ee82d588e2742c85cef7244c090c93f38b8e and I682e4f1f477f3024f7719dfaa67006ef335e0640. SDK variants are now installed again, which will fix unbundled builds of cc_test modules. The platform variants used by com.android.virt are now packagable even though they are not installable. Fix the original problem in b/194403710 by adding a flag to platform variants of modules in apexes that are not platform available, and using that to prevent install and packaging dependencies. That allows the HideFromMake flag to go back to being used for preventing install dependencies but not packaging dependencies. Test: TestPackagingWithSkipInstallDeps Test: TestFileSystemShouldInstallCoreVariantIfTargetBuildAppsIsSet Test: TestFileSystemShouldSkipApexLibraries Bug: 194403710 Bug: 268582372 Fixes: 274443025 Bug: 274028926 Change-Id: If5418df3ddbb940bd631caebdf38daa81e71f40e Merged-In: If5418df3ddbb940bd631caebdf38daa81e71f40e (cherry picked from commit bd3a16b5e75e15d91d6a8cb16ccc53d9d8e202a6) --- android/deptag.go | 4 +- android/license_metadata.go | 2 +- android/module.go | 30 ++++++++++-- android/packaging_test.go | 4 +- apex/apex_test.go | 52 +++++++++++++++++++++ cc/androidmk.go | 13 ++---- cc/sdk.go | 6 +-- cc/sdk_test.go | 92 ------------------------------------- 8 files changed, 92 insertions(+), 111 deletions(-) diff --git a/android/deptag.go b/android/deptag.go index be5c35c8d..a15443b4a 100644 --- a/android/deptag.go +++ b/android/deptag.go @@ -34,10 +34,10 @@ func (i InstallAlwaysNeededDependencyTag) InstallDepNeeded() bool { var _ InstallNeededDependencyTag = InstallAlwaysNeededDependencyTag{} -// IsInstallDepNeeded returns true if the dependency tag implements the InstallNeededDependencyTag +// IsInstallDepNeededTag returns true if the dependency tag implements the InstallNeededDependencyTag // interface and the InstallDepNeeded returns true, meaning that the installed files of the parent // should depend on the installed files of the child. -func IsInstallDepNeeded(tag blueprint.DependencyTag) bool { +func IsInstallDepNeededTag(tag blueprint.DependencyTag) bool { if i, ok := tag.(InstallNeededDependencyTag); ok { return i.InstallDepNeeded() } diff --git a/android/license_metadata.go b/android/license_metadata.go index 18b63d310..73000a9fa 100644 --- a/android/license_metadata.go +++ b/android/license_metadata.go @@ -74,7 +74,7 @@ func buildLicenseMetadata(ctx ModuleContext, licenseMetadataFile WritablePath) { if ctx.OtherModuleHasProvider(dep, LicenseMetadataProvider) { info := ctx.OtherModuleProvider(dep, LicenseMetadataProvider).(*LicenseMetadataInfo) allDepMetadataFiles = append(allDepMetadataFiles, info.LicenseMetadataPath) - if isContainer || IsInstallDepNeeded(ctx.OtherModuleDependencyTag(dep)) { + if isContainer || isInstallDepNeeded(dep, ctx.OtherModuleDependencyTag(dep)) { allDepMetadataDepSets = append(allDepMetadataDepSets, info.LicenseMetadataDepSet) } diff --git a/android/module.go b/android/module.go index ba474530d..76b4e3d11 100644 --- a/android/module.go +++ b/android/module.go @@ -925,6 +925,12 @@ type commonProperties struct { // and don't create a rule to install the file. SkipInstall bool `blueprint:"mutated"` + // UninstallableApexPlatformVariant is set by MakeUninstallable called by the apex + // mutator. MakeUninstallable also sets HideFromMake. UninstallableApexPlatformVariant + // is used to avoid adding install or packaging dependencies into libraries provided + // by apexes. + UninstallableApexPlatformVariant bool `blueprint:"mutated"` + // Whether the module has been replaced by a prebuilt ReplacedByPrebuilt bool `blueprint:"mutated"` @@ -2009,6 +2015,7 @@ func (m *ModuleBase) IsSkipInstall() bool { // have other side effects, in particular when it adds a NOTICE file target, // which other install targets might depend on. func (m *ModuleBase) MakeUninstallable() { + m.commonProperties.UninstallableApexPlatformVariant = true m.HideFromMake() } @@ -2038,13 +2045,19 @@ func (m *ModuleBase) EffectiveLicenseFiles() Paths { } // computeInstallDeps finds the installed paths of all dependencies that have a dependency -// tag that is annotated as needing installation via the IsInstallDepNeeded method. +// tag that is annotated as needing installation via the isInstallDepNeeded method. func (m *ModuleBase) computeInstallDeps(ctx ModuleContext) ([]*installPathsDepSet, []*packagingSpecsDepSet) { var installDeps []*installPathsDepSet var packagingSpecs []*packagingSpecsDepSet ctx.VisitDirectDeps(func(dep Module) { - if IsInstallDepNeeded(ctx.OtherModuleDependencyTag(dep)) && !dep.IsHideFromMake() && !dep.IsSkipInstall() { - installDeps = append(installDeps, dep.base().installFilesDepSet) + if isInstallDepNeeded(dep, ctx.OtherModuleDependencyTag(dep)) { + // Installation is still handled by Make, so anything hidden from Make is not + // installable. + if !dep.IsHideFromMake() && !dep.IsSkipInstall() { + installDeps = append(installDeps, dep.base().installFilesDepSet) + } + // Add packaging deps even when the dependency is not installed so that uninstallable + // modules can still be packaged. Often the package will be installed instead. packagingSpecs = append(packagingSpecs, dep.base().packagingSpecsDepSet) } }) @@ -2052,6 +2065,17 @@ func (m *ModuleBase) computeInstallDeps(ctx ModuleContext) ([]*installPathsDepSe return installDeps, packagingSpecs } +// isInstallDepNeeded returns true if installing the output files of the current module +// should also install the output files of the given dependency and dependency tag. +func isInstallDepNeeded(dep Module, tag blueprint.DependencyTag) bool { + // Don't add a dependency from the platform to a library provided by an apex. + if dep.base().commonProperties.UninstallableApexPlatformVariant { + return false + } + // Only install modules if the dependency tag is an InstallDepNeeded tag. + return IsInstallDepNeededTag(tag) +} + func (m *ModuleBase) FilesToInstall() InstallPaths { return m.installFiles } diff --git a/android/packaging_test.go b/android/packaging_test.go index 91ac1f386..383343723 100644 --- a/android/packaging_test.go +++ b/android/packaging_test.go @@ -373,7 +373,7 @@ func TestPackagingBaseSingleTarget(t *testing.T) { func TestPackagingWithSkipInstallDeps(t *testing.T) { // package -[dep]-> foo -[dep]-> bar -[dep]-> baz - // OK SKIPPED + // Packaging should continue transitively through modules that are not installed. multiTarget := false runPackagingTest(t, multiTarget, ` @@ -396,5 +396,5 @@ func TestPackagingWithSkipInstallDeps(t *testing.T) { name: "package", deps: ["foo"], } - `, []string{"lib64/foo"}) + `, []string{"lib64/foo", "lib64/bar", "lib64/baz"}) } diff --git a/apex/apex_test.go b/apex/apex_test.go index 139b77ef6..e12f758b1 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -33,6 +33,7 @@ import ( "android/soong/cc" "android/soong/dexpreopt" prebuilt_etc "android/soong/etc" + "android/soong/filesystem" "android/soong/java" "android/soong/rust" "android/soong/sh" @@ -10421,3 +10422,54 @@ func TestCannedFsConfig_HasCustomConfig(t *testing.T) { // Ensure that canned_fs_config has "cat my_config" at the end ensureContains(t, cmd, `( echo '/ 1000 1000 0755'; echo '/apex_manifest.json 1000 1000 0644'; echo '/apex_manifest.pb 1000 1000 0644'; cat my_config ) >`) } + +func TestFileSystemShouldSkipApexLibraries(t *testing.T) { + context := android.GroupFixturePreparers( + android.PrepareForIntegrationTestWithAndroid, + cc.PrepareForIntegrationTestWithCc, + PrepareForTestWithApexBuildComponents, + prepareForTestWithMyapex, + filesystem.PrepareForTestWithFilesystemBuildComponents, + ) + result := context.RunTestWithBp(t, ` + android_system_image { + name: "myfilesystem", + deps: [ + "libfoo", + ], + linker_config_src: "linker.config.json", + } + + cc_library { + name: "libfoo", + shared_libs: [ + "libbar", + ], + stl: "none", + } + + cc_library { + name: "libbar", + stl: "none", + apex_available: ["myapex"], + } + + apex { + name: "myapex", + native_shared_libs: ["libbar"], + key: "myapex.key", + updatable: false, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + + inputs := result.ModuleForTests("myfilesystem", "android_common").Output("deps.zip").Implicits + android.AssertStringListDoesNotContain(t, "filesystem should not have libbar", + inputs.Strings(), + "out/soong/.intermediates/libbar/android_arm64_armv8-a_shared/libbar.so") +} diff --git a/cc/androidmk.go b/cc/androidmk.go index 980dd0762..ce35b5c44 100644 --- a/cc/androidmk.go +++ b/cc/androidmk.go @@ -124,17 +124,14 @@ func (c *Module) AndroidMkEntries() []android.AndroidMkEntries { } } } - if c.Properties.IsSdkVariant { + if c.Properties.IsSdkVariant && c.Properties.SdkAndPlatformVariantVisibleToMake { // Make the SDK variant uninstallable so that there are not two rules to install // to the same location. entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) - - if c.Properties.SdkAndPlatformVariantVisibleToMake { - // Add the unsuffixed name to SOONG_SDK_VARIANT_MODULES so that Make can rewrite - // dependencies to the .sdk suffix when building a module that uses the SDK. - entries.SetString("SOONG_SDK_VARIANT_MODULES", - "$(SOONG_SDK_VARIANT_MODULES) $(patsubst %.sdk,%,$(LOCAL_MODULE))") - } + // Add the unsuffixed name to SOONG_SDK_VARIANT_MODULES so that Make can rewrite + // dependencies to the .sdk suffix when building a module that uses the SDK. + entries.SetString("SOONG_SDK_VARIANT_MODULES", + "$(SOONG_SDK_VARIANT_MODULES) $(patsubst %.sdk,%,$(LOCAL_MODULE))") } }, }, diff --git a/cc/sdk.go b/cc/sdk.go index 4f361eb16..6341926ff 100644 --- a/cc/sdk.go +++ b/cc/sdk.go @@ -47,16 +47,16 @@ func sdkMutator(ctx android.BottomUpMutatorContext) { // Mark the SDK variant. modules[1].(*Module).Properties.IsSdkVariant = true - // SDK variant is not supposed to be installed - modules[1].(*Module).Properties.PreventInstall = true if ctx.Config().UnbundledBuildApps() { // For an unbundled apps build, hide the platform variant from Make. modules[0].(*Module).Properties.HideFromMake = true + modules[0].(*Module).Properties.PreventInstall = true } else { // For a platform build, mark the SDK variant so that it gets a ".sdk" suffix when // exposed to Make. modules[1].(*Module).Properties.SdkAndPlatformVariantVisibleToMake = true + modules[1].(*Module).Properties.PreventInstall = true } ctx.AliasVariation("") } else if isCcModule && ccModule.isImportedApiLibrary() { @@ -74,8 +74,8 @@ func sdkMutator(ctx android.BottomUpMutatorContext) { if apiLibrary.hasApexStubs() { // For an unbundled apps build, hide the platform variant from Make. modules[1].(*Module).Properties.HideFromMake = true - modules[1].(*Module).Properties.PreventInstall = true } + modules[1].(*Module).Properties.PreventInstall = true } else { // For a platform build, mark the SDK variant so that it gets a ".sdk" suffix when // exposed to Make. diff --git a/cc/sdk_test.go b/cc/sdk_test.go index 790440cb9..61925e30c 100644 --- a/cc/sdk_test.go +++ b/cc/sdk_test.go @@ -101,95 +101,3 @@ func TestSdkMutator(t *testing.T) { assertDep(t, libsdkNDK, libcxxNDK) assertDep(t, libsdkPlatform, libcxxPlatform) } - -func TestMakeModuleNameForSdkVariant(t *testing.T) { - bp := ` - cc_library { - name: "libfoo", - srcs: ["main_test.cpp"], - sdk_version: "current", - stl: "none", - } - ` - platformVariant := "android_arm64_armv8-a_shared" - sdkVariant := "android_arm64_armv8-a_sdk_shared" - testCases := []struct { - name string - unbundledApps []string - variant string - skipInstall bool // soong skips install - hideFromMake bool // no make entry - makeUninstallable bool // make skips install - makeModuleName string - }{ - { - name: "platform variant in normal builds", - unbundledApps: nil, - variant: platformVariant, - // installable in soong - skipInstall: false, - // visiable in Make as "libfoo" - hideFromMake: false, - makeModuleName: "libfoo", - // installable in Make - makeUninstallable: false, - }, - { - name: "sdk variant in normal builds", - unbundledApps: nil, - variant: sdkVariant, - // soong doesn't install - skipInstall: true, - // visible in Make as "libfoo.sdk" - hideFromMake: false, - makeModuleName: "libfoo.sdk", - // but not installed - makeUninstallable: true, - }, - { - name: "platform variant in unbunded builds", - unbundledApps: []string{"bar"}, - variant: platformVariant, - // installable in soong - skipInstall: false, - // hidden from make - hideFromMake: true, - }, - { - name: "sdk variant in unbunded builds", - unbundledApps: []string{"bar"}, - variant: sdkVariant, - // soong doesn't install - skipInstall: true, - // visible in Make as "libfoo" - hideFromMake: false, - makeModuleName: "libfoo", - // but not installed - makeUninstallable: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - fixture := android.GroupFixturePreparers(prepareForCcTest, - android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { - variables.Unbundled_build_apps = tc.unbundledApps - }), - ) - ctx := fixture.RunTestWithBp(t, bp).TestContext - module := ctx.ModuleForTests("libfoo", tc.variant).Module().(*Module) - android.AssertBoolEquals(t, "IsSkipInstall", tc.skipInstall, module.IsSkipInstall()) - android.AssertBoolEquals(t, "HideFromMake", tc.hideFromMake, module.HiddenFromMake()) - if !tc.hideFromMake { - entries := android.AndroidMkEntriesForTest(t, ctx, module)[0] - android.AssertStringEquals(t, "LOCAL_MODULE", - tc.makeModuleName, entries.EntryMap["LOCAL_MODULE"][0]) - actualUninstallable := false - if actual, ok := entries.EntryMap["LOCAL_UNINSTALLABLE_MODULE"]; ok { - actualUninstallable = "true" == actual[0] - } - android.AssertBoolEquals(t, "LOCAL_UNINSTALLABLE_MODULE", - tc.makeUninstallable, actualUninstallable) - } - }) - } -} From d3b1b77184b1c002136039e70074899a35b1ed77 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 18 Jul 2023 22:11:07 -0700 Subject: [PATCH 10/20] Strip META-INF/services from implementation jars when using as header jars If a header jar couldn't be built (for example when an API generating annoation processor is in use) the implementation jar is reused as the header jar. If the implementation jar contains an annotation processor listed in META-INF/services/javax.annotation.processing.Processor then later javac executions with the implementation jar in the classpath could attempt to run the annotation processors unexpectedly. Remove the META-INF/services directory when using an implementation jar as a header jar. Bug: 290933559 Test: builds Change-Id: I40d48644bc5a09a9564dc2c4b38f627edd00fcf8 Merged-In: I40d48644bc5a09a9564dc2c4b38f627edd00fcf8 (cherry picked from commit f06d8dc8e3816cef2d3c0ba072ad1fc24bc44de0) --- java/androidmk.go | 3 +++ java/base.go | 8 +++++++- java/builder.go | 15 +++++++++++++++ java/device_host_converter_test.go | 3 ++- java/fuzz_test.go | 4 ++-- sdk/java_sdk_test.go | 8 ++++---- 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/java/androidmk.go b/java/androidmk.go index 9c21633fb..291ed90ee 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -340,6 +340,9 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries { // App module names can be overridden. entries.SetString("LOCAL_MODULE", app.installApkName) entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", app.appProperties.PreventInstall) + if app.headerJarFile != nil { + entries.SetPath("LOCAL_SOONG_HEADER_JAR", app.headerJarFile) + } entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", app.exportPackage) if app.dexJarFile.IsSet() { entries.SetPath("LOCAL_SOONG_DEX_JAR", app.dexJarFile.Path()) diff --git a/java/base.go b/java/base.go index 991132321..e29e94ae3 100644 --- a/java/base.go +++ b/java/base.go @@ -1446,7 +1446,13 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { j.implementationJarFile = outputFile if j.headerJarFile == nil { - j.headerJarFile = j.implementationJarFile + // If this module couldn't generate a header jar (for example due to api generating annotation processors) + // then use the implementation jar. Run it through zip2zip first to remove any files in META-INF/services + // so that javac on modules that depend on this module don't pick up annotation processors (which may be + // missing their implementations) from META-INF/services/javax.annotation.processing.Processor. + headerJarFile := android.PathForModuleOut(ctx, "javac-header", jarName) + convertImplementationJarToHeaderJar(ctx, j.implementationJarFile, headerJarFile) + j.headerJarFile = headerJarFile } // enforce syntax check to jacoco filters for any build (http://b/183622051) diff --git a/java/builder.go b/java/builder.go index 462626712..8cc0c2587 100644 --- a/java/builder.go +++ b/java/builder.go @@ -246,6 +246,12 @@ var ( CommandDeps: []string{"${config.ZipAlign}"}, }, ) + + convertImplementationJarToHeaderJarRule = pctx.AndroidStaticRule("convertImplementationJarToHeaderJar", + blueprint.RuleParams{ + Command: `${config.Zip2ZipCmd} -i ${in} -o ${out} -x 'META-INF/services/**/*'`, + CommandDeps: []string{"${config.Zip2ZipCmd}"}, + }) ) func init() { @@ -600,6 +606,15 @@ func TransformJarsToJar(ctx android.ModuleContext, outputFile android.WritablePa }) } +func convertImplementationJarToHeaderJar(ctx android.ModuleContext, implementationJarFile android.Path, + headerJarFile android.WritablePath) { + ctx.Build(pctx, android.BuildParams{ + Rule: convertImplementationJarToHeaderJarRule, + Input: implementationJarFile, + Output: headerJarFile, + }) +} + func TransformJarJar(ctx android.ModuleContext, outputFile android.WritablePath, classesJar android.Path, rulesFile android.Path) { ctx.Build(pctx, android.BuildParams{ diff --git a/java/device_host_converter_test.go b/java/device_host_converter_test.go index 3c9a0f3f1..3413da03d 100644 --- a/java/device_host_converter_test.go +++ b/java/device_host_converter_test.go @@ -135,6 +135,7 @@ func TestHostForDevice(t *testing.T) { hostModule := ctx.ModuleForTests("host_module", config.BuildOSCommonTarget.String()) hostJavac := hostModule.Output("javac/host_module.jar") + hostJavacHeader := hostModule.Output("javac-header/host_module.jar") hostRes := hostModule.Output("res/host_module.jar") hostImportModule := ctx.ModuleForTests("host_import_module", config.BuildOSCommonTarget.String()) @@ -148,7 +149,7 @@ func TestHostForDevice(t *testing.T) { // check classpath of device module with dependency on host_for_device_module expectedClasspath := "-classpath " + strings.Join(android.Paths{ - hostJavac.Output, + hostJavacHeader.Output, hostImportCombined.Output, }.Strings(), ":") diff --git a/java/fuzz_test.go b/java/fuzz_test.go index dd1e96b3e..f29c91327 100644 --- a/java/fuzz_test.go +++ b/java/fuzz_test.go @@ -71,8 +71,8 @@ func TestJavaFuzz(t *testing.T) { } baz := result.ModuleForTests("baz", osCommonTarget).Rule("javac").Output.String() - barOut := filepath.Join("out", "soong", ".intermediates", "bar", osCommonTarget, "javac", "bar.jar") - bazOut := filepath.Join("out", "soong", ".intermediates", "baz", osCommonTarget, "javac", "baz.jar") + barOut := filepath.Join("out", "soong", ".intermediates", "bar", osCommonTarget, "javac-header", "bar.jar") + bazOut := filepath.Join("out", "soong", ".intermediates", "baz", osCommonTarget, "javac-header", "baz.jar") android.AssertStringDoesContain(t, "foo classpath", javac.Args["classpath"], barOut) android.AssertStringDoesContain(t, "foo classpath", javac.Args["classpath"], bazOut) diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 6159ea9c2..c8cc83476 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -160,7 +160,7 @@ java_import { } `), checkAllCopyRules(` -.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/myjavalib.jar +.intermediates/myjavalib/linux_glibc_common/javac-header/myjavalib.jar -> java/myjavalib.jar aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl `), ) @@ -206,7 +206,7 @@ java_import { `), checkAllCopyRules(` .intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/android/myjavalib.jar -.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar +.intermediates/myjavalib/linux_glibc_common/javac-header/myjavalib.jar -> java/linux_glibc/myjavalib.jar `), ) } @@ -799,7 +799,7 @@ java_system_modules_import { libs: ["mysdk_system-module"], } `), - checkAllCopyRules(".intermediates/system-module/linux_glibc_common/javac/system-module.jar -> java/system-module.jar"), + checkAllCopyRules(".intermediates/system-module/linux_glibc_common/javac-header/system-module.jar -> java/system-module.jar"), ) } @@ -879,7 +879,7 @@ java_import { } `), checkAllCopyRules(` -.intermediates/hostjavalib/linux_glibc_common/javac/hostjavalib.jar -> java/hostjavalib.jar +.intermediates/hostjavalib/linux_glibc_common/javac-header/hostjavalib.jar -> java/hostjavalib.jar .intermediates/androidjavalib/android_common/turbine-combined/androidjavalib.jar -> java/androidjavalib.jar .intermediates/myjavalib/android_common/javac/myjavalib.jar -> java/android/myjavalib.jar .intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar From 47efcdcf6bc845608f137df06d3b1d4426088839 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 18 Jul 2023 15:57:09 -0700 Subject: [PATCH 11/20] Merge META-INF/services/* files in merge_zips -jar kotlinx_coroutines_test and kotlinx_coroutine_android each provide a META-INF/services/kotlinx.coroutines.CoroutineExceptionHandler with different contents, and the final contents needs to be the combination of the two files. Implement service merging in merge_zips when the -jar argument is provided. Bug: 290933559 Test: TestMergeZips Change-Id: I69f80d1265c64c671d308ef4cdccfa1564abe056 Merged-In: I69f80d1265c64c671d308ef4cdccfa1564abe056 (cherry picked from commit 7592d5a0bdec6848b1679eb29a28eb8dddfe4c87) --- cmd/merge_zips/merge_zips.go | 22 ++++- cmd/merge_zips/merge_zips_test.go | 66 ++++++++++----- jar/Android.bp | 1 + jar/services.go | 128 ++++++++++++++++++++++++++++++ third_party/zip/android.go | 2 +- 5 files changed, 197 insertions(+), 22 deletions(-) create mode 100644 jar/services.go diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go index e3d1179b4..a70a9d158 100644 --- a/cmd/merge_zips/merge_zips.go +++ b/cmd/merge_zips/merge_zips.go @@ -122,7 +122,7 @@ func (be ZipEntryFromBuffer) Size() uint64 { } func (be ZipEntryFromBuffer) WriteToZip(dest string, zw *zip.Writer) error { - w, err := zw.CreateHeader(be.fh) + w, err := zw.CreateHeaderAndroid(be.fh) if err != nil { return err } @@ -562,6 +562,8 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } } + var jarServices jar.Services + // Finally, add entries from all the input zips. for _, inputZip := range inputZips { _, copyFully := zipsToNotStrip[inputZip.Name()] @@ -570,6 +572,14 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } for i, entry := range inputZip.Entries() { + if emulateJar && jarServices.IsServiceFile(entry) { + // If this is a jar, collect service files to combine instead of adding them to the zip. + err := jarServices.AddServiceFile(entry) + if err != nil { + return err + } + continue + } if copyFully || !out.isEntryExcluded(entry.Name) { if err := out.copyEntry(inputZip, i); err != nil { return err @@ -585,6 +595,16 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } if emulateJar { + // Combine all the service files into a single list of combined service files and add them to the zip. + for _, serviceFile := range jarServices.ServiceFiles() { + _, err := out.addZipEntry(serviceFile.Name, ZipEntryFromBuffer{ + fh: serviceFile.FileHeader, + content: serviceFile.Contents, + }) + if err != nil { + return err + } + } return out.writeEntries(out.jarSorted()) } else if sortEntries { return out.writeEntries(out.alphanumericSorted()) diff --git a/cmd/merge_zips/merge_zips_test.go b/cmd/merge_zips/merge_zips_test.go index cb5843607..767d4e61f 100644 --- a/cmd/merge_zips/merge_zips_test.go +++ b/cmd/merge_zips/merge_zips_test.go @@ -17,6 +17,7 @@ package main import ( "bytes" "fmt" + "hash/crc32" "os" "strconv" "strings" @@ -27,28 +28,34 @@ import ( ) type testZipEntry struct { - name string - mode os.FileMode - data []byte + name string + mode os.FileMode + data []byte + method uint16 } var ( - A = testZipEntry{"A", 0755, []byte("foo")} - a = testZipEntry{"a", 0755, []byte("foo")} - a2 = testZipEntry{"a", 0755, []byte("FOO2")} - a3 = testZipEntry{"a", 0755, []byte("Foo3")} - bDir = testZipEntry{"b/", os.ModeDir | 0755, nil} - bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil} - bbb = testZipEntry{"b/b/b", 0755, nil} - ba = testZipEntry{"b/a", 0755, []byte("foob")} - bc = testZipEntry{"b/c", 0755, []byte("bar")} - bd = testZipEntry{"b/d", 0700, []byte("baz")} - be = testZipEntry{"b/e", 0700, []byte("")} + A = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate} + a = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate} + a2 = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate} + a3 = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate} + bDir = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate} + bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate} + bbb = testZipEntry{"b/b/b", 0755, nil, zip.Deflate} + ba = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate} + bc = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate} + bd = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate} + be = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate} - metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil} - manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest")} - manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2")} - moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")} + service1a = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store} + service1b = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate} + service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store} + service2 = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate} + + metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate} + manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate} + manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate} + moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate} ) type testInputZip struct { @@ -236,6 +243,15 @@ func TestMergeZips(t *testing.T) { "in1": true, }, }, + { + name: "services", + in: [][]testZipEntry{ + {service1a, service2}, + {service1b}, + }, + jar: true, + out: []testZipEntry{service1combined, service2}, + }, } for _, test := range testCases { @@ -256,7 +272,7 @@ func TestMergeZips(t *testing.T) { closeErr := writer.Close() if closeErr != nil { - t.Fatal(err) + t.Fatal(closeErr) } if test.err != "" { @@ -266,12 +282,16 @@ func TestMergeZips(t *testing.T) { t.Fatal("incorrect err, want:", test.err, "got:", err) } return + } else if err != nil { + t.Fatal("unexpected err: ", err) } if !bytes.Equal(want, out.Bytes()) { t.Error("incorrect zip output") t.Errorf("want:\n%s", dumpZip(want)) t.Errorf("got:\n%s", dumpZip(out.Bytes())) + os.WriteFile("/tmp/got.zip", out.Bytes(), 0755) + os.WriteFile("/tmp/want.zip", want, 0755) } }) } @@ -286,8 +306,14 @@ func testZipEntriesToBuf(entries []testZipEntry) []byte { Name: e.name, } fh.SetMode(e.mode) + fh.Method = e.method + fh.UncompressedSize64 = uint64(len(e.data)) + fh.CRC32 = crc32.ChecksumIEEE(e.data) + if fh.Method == zip.Store { + fh.CompressedSize64 = fh.UncompressedSize64 + } - w, err := zw.CreateHeader(&fh) + w, err := zw.CreateHeaderAndroid(&fh) if err != nil { panic(err) } diff --git a/jar/Android.bp b/jar/Android.bp index 46113d877..c03e49174 100644 --- a/jar/Android.bp +++ b/jar/Android.bp @@ -21,6 +21,7 @@ bootstrap_go_package { pkgPath: "android/soong/jar", srcs: [ "jar.go", + "services.go", ], testSrcs: [ "jar_test.go", diff --git a/jar/services.go b/jar/services.go new file mode 100644 index 000000000..d06a6dc99 --- /dev/null +++ b/jar/services.go @@ -0,0 +1,128 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jar + +import ( + "android/soong/third_party/zip" + "bufio" + "hash/crc32" + "sort" + "strings" +) + +const servicesPrefix = "META-INF/services/" + +// Services is used to collect service files from multiple zip files and produce a list of ServiceFiles containing +// the unique lines from all the input zip entries with the same name. +type Services struct { + services map[string]*ServiceFile +} + +// ServiceFile contains the combined contents of all input zip entries with a single name. +type ServiceFile struct { + Name string + FileHeader *zip.FileHeader + Contents []byte + Lines []string +} + +// IsServiceFile returns true if the zip entry is in the META-INF/services/ directory. +func (Services) IsServiceFile(entry *zip.File) bool { + return strings.HasPrefix(entry.Name, servicesPrefix) +} + +// AddServiceFile adds a zip entry in the META-INF/services/ directory to the list of service files that need +// to be combined. +func (j *Services) AddServiceFile(entry *zip.File) error { + if j.services == nil { + j.services = map[string]*ServiceFile{} + } + + service := entry.Name + serviceFile := j.services[service] + fh := entry.FileHeader + if serviceFile == nil { + serviceFile = &ServiceFile{ + Name: service, + FileHeader: &fh, + } + j.services[service] = serviceFile + } + + f, err := entry.Open() + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + serviceFile.Lines = append(serviceFile.Lines, line) + } + } + + if err := scanner.Err(); err != nil { + return err + } + + return nil +} + +// ServiceFiles returns the list of combined service files, each containing all the unique lines from the +// corresponding service files in the input zip entries. +func (j *Services) ServiceFiles() []ServiceFile { + services := make([]ServiceFile, 0, len(j.services)) + + for _, serviceFile := range j.services { + serviceFile.Lines = dedupServicesLines(serviceFile.Lines) + serviceFile.Lines = append(serviceFile.Lines, "") + serviceFile.Contents = []byte(strings.Join(serviceFile.Lines, "\n")) + + serviceFile.FileHeader.UncompressedSize64 = uint64(len(serviceFile.Contents)) + serviceFile.FileHeader.CRC32 = crc32.ChecksumIEEE(serviceFile.Contents) + if serviceFile.FileHeader.Method == zip.Store { + serviceFile.FileHeader.CompressedSize64 = serviceFile.FileHeader.UncompressedSize64 + } + + services = append(services, *serviceFile) + } + + sort.Slice(services, func(i, j int) bool { + return services[i].Name < services[j].Name + }) + + return services +} + +func dedupServicesLines(in []string) []string { + writeIndex := 0 +outer: + for readIndex := 0; readIndex < len(in); readIndex++ { + for compareIndex := 0; compareIndex < writeIndex; compareIndex++ { + if interface{}(in[readIndex]) == interface{}(in[compareIndex]) { + // The value at readIndex already exists somewhere in the output region + // of the slice before writeIndex, skip it. + continue outer + } + } + if readIndex != writeIndex { + in[writeIndex] = in[readIndex] + } + writeIndex++ + } + return in[0:writeIndex] +} diff --git a/third_party/zip/android.go b/third_party/zip/android.go index f8e45c56d..0f41f6200 100644 --- a/third_party/zip/android.go +++ b/third_party/zip/android.go @@ -170,7 +170,7 @@ func (w *Writer) CreateCompressedHeader(fh *FileHeader) (io.WriteCloser, error) func (w *Writer) CreateHeaderAndroid(fh *FileHeader) (io.Writer, error) { writeDataDescriptor := fh.Method != Store if writeDataDescriptor { - fh.Flags &= DataDescriptorFlag + fh.Flags |= DataDescriptorFlag } else { fh.Flags &= ^uint16(DataDescriptorFlag) } From 6e0dbf887e330d0ff99bab8120f5df0a6da2802f Mon Sep 17 00:00:00 2001 From: Kevin Jeon Date: Mon, 31 Jul 2023 12:23:03 -0400 Subject: [PATCH 12/20] Add libdumpstateutil to vendor variant list This change adds libdumpstateutil to VndkMustUseVendorVariantList. This is done to avoid a VNDK re-spin on aosp/2679475 and aosp/2681060, which add the dumpstate util function IsStrictRun and make it framework-only. Test: Presubmit; verify that builds still work on internal (refer to aosp/2681060 test). Ignore-AOSP-First: This is already in AOSP, but needs to be cherry-picked here first before being removed from AOSP. Bug: 283326935 Change-Id: I3c60dd919a65182937d6e0b233635ff6ca9414d3 --- cc/config/vndk.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cc/config/vndk.go b/cc/config/vndk.go index dd612ce63..f9b3eac82 100644 --- a/cc/config/vndk.go +++ b/cc/config/vndk.go @@ -21,6 +21,7 @@ package config var VndkMustUseVendorVariantList = []string{ "android.hardware.nfc@1.2", "libbinder", + "libdumpstateutil", "libcrypto", "libexpat", "libgatekeeper", From bdc0578df1b8df4d61d412df00be896ba3434068 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 9 Aug 2023 14:51:13 -0700 Subject: [PATCH 13/20] Re-enable checks that pass in udc-dev-plus-aosp and downstream Re-enable ForegroundServiceType and MutableImplicitPendingIntent, which have been fixed in udc-dev-plus-aosp and downstream. Bug: 294098365 Test: treehugger Ignore-AOSP-First: re-enabling checks that fail in AOSP Change-Id: Ifa581e8d346cf7e70387d6aa3135203d6a7d621b --- java/lint_defaults.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/lint_defaults.txt b/java/lint_defaults.txt index 8494d029e..a4dbb17e4 100644 --- a/java/lint_defaults.txt +++ b/java/lint_defaults.txt @@ -123,10 +123,6 @@ --warning_check SupportAnnotationUsage --warning_check UniqueConstants -# TODO(b/294098365): these checks fail in AOSP, but pass downstream ---warning_check ForegroundServiceType ---warning_check MutableImplicitPendingIntent - --warning_check ExactAlarm --warning_check ExpiredTargetSdkVersion --warning_check ForegroundServicePermission From 58c2388f92ba65240048450372eb4754c5ea5920 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Wed, 16 Aug 2023 08:04:26 +0000 Subject: [PATCH 14/20] Revert "Re-enable checks that pass in udc-dev-plus-aosp and downstream" This reverts commit bdc0578df1b8df4d61d412df00be896ba3434068. Reason for revert: b/296088866 Change-Id: I053314736758f48ea99920a141443c1489d2477b --- java/lint_defaults.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/lint_defaults.txt b/java/lint_defaults.txt index a4dbb17e4..8494d029e 100644 --- a/java/lint_defaults.txt +++ b/java/lint_defaults.txt @@ -123,6 +123,10 @@ --warning_check SupportAnnotationUsage --warning_check UniqueConstants +# TODO(b/294098365): these checks fail in AOSP, but pass downstream +--warning_check ForegroundServiceType +--warning_check MutableImplicitPendingIntent + --warning_check ExactAlarm --warning_check ExpiredTargetSdkVersion --warning_check ForegroundServicePermission From 7b9c1e906542ba1abf9d55cae4e8dc503d55bd16 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 22 Aug 2023 18:33:58 +0000 Subject: [PATCH 15/20] Reland: Re-enable checks that pass in udc-dev-plus-aosp and downstream This relands Ifa581e8d346cf7e70387d6aa3135203d6a7d621b after fixes to the manifests of modules that didn't pass the new lint checks. Bug: 294098365 Test: treehugger Test: https://android-build.googleplex.com/builds/abtd/run/L56200000962810233 Test: https://android-build.googleplex.com/builds/abtd/run/L49700000962810033 Change-Id: I82129ef29025f35c0f4fc20f2d892c0af39d76ae --- java/lint_defaults.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/lint_defaults.txt b/java/lint_defaults.txt index 8494d029e..a4dbb17e4 100644 --- a/java/lint_defaults.txt +++ b/java/lint_defaults.txt @@ -123,10 +123,6 @@ --warning_check SupportAnnotationUsage --warning_check UniqueConstants -# TODO(b/294098365): these checks fail in AOSP, but pass downstream ---warning_check ForegroundServiceType ---warning_check MutableImplicitPendingIntent - --warning_check ExactAlarm --warning_check ExpiredTargetSdkVersion --warning_check ForegroundServicePermission From 818f3d948595bc6e43ef4aa033fafb6584315cd7 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Fri, 22 Sep 2023 00:28:20 +0000 Subject: [PATCH 16/20] RESTRICT AUTOMERGE Move java_api_library modules to f/b/api/StubLibraries.bp This change was already submitted with https://android-review.git.corp.google.com/q/topic:%22revert-2713677-revert-2655262-move_java_api_libraries-JTESUMBERD-FPSEKJYXCE%22 but was never propagated to downstream. Ignore-AOSP-First: change already merged in aosp Test: m nothing --build-from-text-stub Change-Id: I1f2f33e1a5df6f7b8f8215ab41b41228fa873ea1 --- java/core-libraries/Android.bp | 54 +++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/java/core-libraries/Android.bp b/java/core-libraries/Android.bp index de9a82d09..4380f4f5c 100644 --- a/java/core-libraries/Android.bp +++ b/java/core-libraries/Android.bp @@ -55,6 +55,19 @@ java_library { ], } +java_api_library { + name: "core.current.stubs.from-text", + api_surface: "core", + api_contributions: [ + "art.module.public.api.stubs.source.api.contribution", + "conscrypt.module.public.api.stubs.source.api.contribution", + "i18n.module.public.api.stubs.source.api.contribution", + ], + libs: [ + "stub-annotations", + ], +} + java_library { name: "core.current.stubs", defaults: [ @@ -153,7 +166,6 @@ java_defaults { system_modules: "none", } - // A stubs target containing the parts of the public SDK & @SystemApi(MODULE_LIBRARIES) API // provided by the core libraries. // @@ -265,6 +277,32 @@ java_library { ], } +java_defaults { + name: "android_core_platform_stubs_current_contributions", + api_surface: "core_platform", + api_contributions: [ + "art.module.public.api.stubs.source.api.contribution", + "art.module.public.api.stubs.source.system.api.contribution", + "art.module.public.api.stubs.source.module_lib.api.contribution", + "conscrypt.module.platform.api.stubs.source.api.contribution", + "i18n.module.public.api.stubs.source.api.contribution", + ], +} + +java_api_library { + name: "legacy.core.platform.api.stubs.from-text", + api_surface: "core_platform", + defaults: [ + "android_core_platform_stubs_current_contributions", + ], + api_contributions: [ + "legacy.i18n.module.platform.api.stubs.source.api.contribution", + ], + libs: [ + "stub-annotations", + ], +} + java_library { name: "legacy.core.platform.api.stubs", visibility: core_platform_visibility, @@ -328,6 +366,20 @@ java_library { ], } +java_api_library { + name: "stable.core.platform.api.stubs.from-text", + api_surface: "core_platform", + defaults: [ + "android_core_platform_stubs_current_contributions", + ], + api_contributions: [ + "stable.i18n.module.platform.api.stubs.source.api.contribution", + ], + libs: [ + "stub-annotations", + ], +} + java_library { name: "stable.core.platform.api.stubs", visibility: core_platform_visibility, From f8c3957415a987c58fbb4a04b2f0803365054735 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 18 Oct 2023 19:50:47 +0000 Subject: [PATCH 17/20] Move non-vendor modules to the build/soong allowlist Otherwise, builds fail on git_main-without-vendor. Bug: 290816499 Change-Id: I78c8c359779d99d26a7b621f89d3feb4c2cf62d1 Merged-In: I78c8c359779d99d26a7b621f89d3feb4c2cf62d1 Test: Presubmits Ignore-AOSP-First: These modules are not in aosp yet (cherry picked from commit eb51e88205f1e8d187d95bb64128b3aae504d3ce) --- genrule/allowlists.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 8552173a3..06869a281 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -109,6 +109,8 @@ var ( "camera-its", "checkIn-service-stub-lite", "chre_atoms_log.h", + "cobalt-registry-binarypb-gen", + "cobalt-registry-validated-java", "com.android.apex.apkrollback.test.pem", "com.android.apex.apkrollback.test.pubkey", "com.android.apex.cts.shim.debug.pem", @@ -150,6 +152,7 @@ var ( "egl_functions_hdr", "emp_ematch.yacc.c", "emp_ematch.yacc.h", + "exercise_type_data_type_config-pb", "fdt_test_tree_empty_memory_range_dtb", "fdt_test_tree_multiple_memory_ranges_dtb", "fdt_test_tree_one_memory_range_dtb", @@ -236,6 +239,10 @@ var ( "pixelstatsatoms.cpp", "pixelstatsatoms.h", "pvmfw_fdt_template_rs", + "pw_log_log_proto_pwpb_h", + "pw_log_log_raw_rpc_h", + "pw_log_log_rpc_pwpb_h", + "pw_tokenizer_proto_options_pwpb_h", "r8retrace-dexdump-sample-app", "r8retrace-run-retrace", "rootcanal_bredr_bb_packets_cxx_gen", From 0ec29e44ed80f001e68706836c53333e4d25a5c8 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 26 Oct 2023 11:00:34 -0700 Subject: [PATCH 18/20] Remove cobalt genrules from denylist These genrules only existed on udc-mainline-prod and main. Ignore-AOSP-First: This change is not on aosp Bug: 307824623 Test: Presubmits Change-Id: I15ce8e04546cd3bfb7c777adc27abef3c4903a08 --- genrule/allowlists.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 715ff1e14..048b971db 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -109,8 +109,6 @@ var ( "camera-its", "checkIn-service-stub-lite", "chre_atoms_log.h", - "cobalt-registry-binarypb-gen", - "cobalt-registry-validated-java", "com.android.apex.apkrollback.test.pem", "com.android.apex.apkrollback.test.pubkey", "com.android.apex.cts.shim.debug.pem", From bfff3ee343dade8887491b22a01153935ac5b4e6 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 26 Oct 2023 12:47:16 -0700 Subject: [PATCH 19/20] Enable sandboxing for exercise_type_data_type_config-pb go/roboleaf-busy-beavers-sandboxing Ignore-AOSP-First: exercise_type_data_type_config-pb only exists on internal Bug: 307824623 Test: m exercise_type_data_type_config-pb Change-Id: I2fb6e8254c124c4c0382b294a2e5a86a50be02b9 (cherry picked from commit 90498937ed9ae9735a21087daad9f24fb67de50e) --- genrule/allowlists.go | 1 - 1 file changed, 1 deletion(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index adf27862f..cf7a8e8e3 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -57,7 +57,6 @@ var ( "deqp_spvtools_update_build_version", "emp_ematch.yacc.c", "emp_ematch.yacc.h", - "exercise_type_data_type_config-pb", "fdt_test_tree_empty_memory_range_dtb", "fdt_test_tree_multiple_memory_ranges_dtb", "fdt_test_tree_one_memory_range_dtb", From eda7420e0edf72fb8b94e403a889929852c193f3 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 26 Oct 2023 18:09:06 -0700 Subject: [PATCH 20/20] Enable sandboxing for pigweed gnerules go/roboleaf-busy-beavers-sandboxing Ignore-AOSP-First: these modules only exist on internal Bug: 307824623 Test: m pw_log_log_proto_pwpb_h pw_log_log_raw_rpc_h pw_log_log_rpc_pwpb_h pw_tokenizer_proto_options_pwpb_h Change-Id: I5f4c68657a9440a3db339e7a51a4714c3bc5df55 (cherry picked from commit 50005b08c26992eca5af99dbfa79dca0197e5105) --- genrule/allowlists.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 284a3415c..297976879 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -53,10 +53,6 @@ var ( "libc_musl_sysroot_bits", "libcore-non-cts-tests-txt", "pvmfw_fdt_template_rs", - "pw_log_log_proto_pwpb_h", - "pw_log_log_raw_rpc_h", - "pw_log_log_rpc_pwpb_h", - "pw_tokenizer_proto_options_pwpb_h", "r8retrace-dexdump-sample-app", "r8retrace-run-retrace", "seller-frontend-service-stub-lite",