From 598f045c4af3fefd08c9253206af2b3a50cdfb87 Mon Sep 17 00:00:00 2001 From: Alexei Nicoara Date: Thu, 9 Mar 2023 16:12:25 +0000 Subject: [PATCH 01/80] DO NOT MERGE Update the module versions in udc-mainline-prod Bug: 258850762 Change-Id: I8844c712bc9f94cecfe15ce343fe45f008244145 Ignore-AOSP-First: Version bump only needed on udc-mainline-prod branch not on git_master --- android/updatable_modules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/updatable_modules.go b/android/updatable_modules.go index 6d0eeb716..db4563750 100644 --- a/android/updatable_modules.go +++ b/android/updatable_modules.go @@ -33,4 +33,4 @@ package android // * AOSP - xx9990000 // * x-mainline-prod - xx9990000 // * master - 990090000 -const DefaultUpdatableModuleVersion = "340090000" +const DefaultUpdatableModuleVersion = "349990000" From ab959882b39f43a65b9bbc63e6688bb43825263b Mon Sep 17 00:00:00 2001 From: Eric Miao Date: Thu, 1 Dec 2022 09:35:41 -0800 Subject: [PATCH 02/80] Enable compact resource entries for aapt linking Bug: 237583012 This enables compact resource entries by default when linking with aapt2, which helps generating resource file with smaller size. Change-Id: Ibf22f301491dcc56346a5947ee115e3059eed6aa Ignore-AOSP-First: this CL has dependency on non-AOSP-First repos --- java/aar.go | 1 + 1 file changed, 1 insertion(+) diff --git a/java/aar.go b/java/aar.go index f162a1722..324e0af2e 100644 --- a/java/aar.go +++ b/java/aar.go @@ -176,6 +176,7 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext android.SdkConte linkFlags = append(linkFlags, a.aaptProperties.Aaptflags...) linkFlags = append(linkFlags, "--no-static-lib-packages") + linkFlags = append(linkFlags, "--enable-compact-entries") // Find implicit or explicit asset and resource dirs assetDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Asset_dirs, "assets") From 825cff4169ef07880ae4ba51b978873ed276eb73 Mon Sep 17 00:00:00 2001 From: Beatrice Marchegiani Date: Tue, 28 Mar 2023 11:30:12 +0000 Subject: [PATCH 03/80] Revert "Enable compact resource entries for aapt linking" This reverts commit ab959882b39f43a65b9bbc63e6688bb43825263b. Reason for revert: DroidMonitor-triggered revert due to breakage BackupFrameworksServicesRoboTests, bug b/275422786 Identified in culprit search session (https://android-build.googleplex.com/builds/culprit-assistant/run/531ea159-f8c1-4ddf-a29f-307c3a9c057e) Bug:275422786 Change-Id: I79b550ba109ce0da23074c143c65511b6f193df4 --- java/aar.go | 1 - 1 file changed, 1 deletion(-) diff --git a/java/aar.go b/java/aar.go index 324e0af2e..f162a1722 100644 --- a/java/aar.go +++ b/java/aar.go @@ -176,7 +176,6 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext android.SdkConte linkFlags = append(linkFlags, a.aaptProperties.Aaptflags...) linkFlags = append(linkFlags, "--no-static-lib-packages") - linkFlags = append(linkFlags, "--enable-compact-entries") // Find implicit or explicit asset and resource dirs assetDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Asset_dirs, "assets") From 40eab20c34ca2084d030f0af3ef3b5a237263df3 Mon Sep 17 00:00:00 2001 From: Eric Miao Date: Thu, 30 Mar 2023 16:57:17 +0000 Subject: [PATCH 04/80] Revert "Revert "Enable compact resource entries for aapt linking"" This reverts commit 825cff4169ef07880ae4ba51b978873ed276eb73. Reason for revert: Failed test has been fixed - b/275572186 Change-Id: I661206f4e0e783a5c403ac15ed33a93ba3acaf40 --- java/aar.go | 1 + 1 file changed, 1 insertion(+) diff --git a/java/aar.go b/java/aar.go index 47e6efae3..4fdcb80b8 100644 --- a/java/aar.go +++ b/java/aar.go @@ -176,6 +176,7 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext android.SdkConte linkFlags = append(linkFlags, a.aaptProperties.Aaptflags...) linkFlags = append(linkFlags, "--no-static-lib-packages") + linkFlags = append(linkFlags, "--enable-compact-entries") // Find implicit or explicit asset and resource dirs assetDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Asset_dirs, "assets") From 1fb4ece718bd34f9c47754c14281c7ad644bb174 Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Wed, 26 Apr 2023 21:06:24 +0900 Subject: [PATCH 05/80] BUILD_NUMBER and BUILD_HOSTNAME doesn't affect kati regeneration * Extract BUILD_NUMBER, BUILD_HOSTNAME to file to avoid kati change * Handle FILE_NAME_TAG_PLACEHOLDER string in dist in build/make/packaging/distdir.mk Ignore-AOSP-First: there are internal use cases of FILE_NAME_TAG Test: check if kati isn't invoked even though BUILD_NUMBER, BUILD_HOSTNAME is changed Test: m && m, and check if the second m is no-op Bug: 278060169 Change-Id: I65eefc6bb86d4076098a1bf8b317b4cb88201499 --- android_sdk/sdk_repo_host.go | 2 +- ui/build/kati.go | 60 +++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/android_sdk/sdk_repo_host.go b/android_sdk/sdk_repo_host.go index 61058df09..9623a8bc4 100644 --- a/android_sdk/sdk_repo_host.go +++ b/android_sdk/sdk_repo_host.go @@ -242,7 +242,7 @@ func (s *sdkRepoHost) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, ".PHONY:", name, "sdk_repo", "sdk-repo-"+name) fmt.Fprintln(w, "sdk_repo", "sdk-repo-"+name+":", strings.Join(s.FilesToInstall().Strings(), " ")) - fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-$(FILE_NAME_TAG).zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) + fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-FILE_NAME_TAG_PLACEHOLDER.zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) }, } } diff --git a/ui/build/kati.go b/ui/build/kati.go index dad68fac2..4297378b3 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -22,6 +22,7 @@ import ( "os/user" "path/filepath" "strings" + "time" "android/soong/ui/metrics" "android/soong/ui/status" @@ -66,6 +67,21 @@ func genKatiSuffix(ctx Context, config Config) { } } +func writeValueIfChanged(ctx Context, config Config, dir string, filename string, value string) { + filePath := filepath.Join(dir, filename) + previousValue := "" + rawPreviousValue, err := ioutil.ReadFile(filePath) + if err == nil { + previousValue = string(rawPreviousValue) + } + + if previousValue != value { + if err = ioutil.WriteFile(filePath, []byte(value), 0666); err != nil { + ctx.Fatalf("Failed to write: %v", err) + } + } +} + // Base function to construct and run the Kati command line with additional // arguments, and a custom function closure to mutate the environment Kati runs // in. @@ -157,28 +173,57 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF } cmd.Stderr = cmd.Stdout - // Apply the caller's function closure to mutate the environment variables. - envFunc(cmd.Environment) - + var username string // Pass on various build environment metadata to Kati. - if _, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { - username := "unknown" + if usernameFromEnv, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { + username = "unknown" if u, err := user.Current(); err == nil { username = u.Username } else { ctx.Println("Failed to get current user:", err) } cmd.Environment.Set("BUILD_USERNAME", username) + } else { + username = usernameFromEnv } - if _, ok := cmd.Environment.Get("BUILD_HOSTNAME"); !ok { - hostname, err := os.Hostname() + hostname, ok := cmd.Environment.Get("BUILD_HOSTNAME") + if !ok { + hostname, err = os.Hostname() if err != nil { ctx.Println("Failed to read hostname:", err) hostname = "unknown" } cmd.Environment.Set("BUILD_HOSTNAME", hostname) } + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_hostname.txt", hostname) + + // BUILD_NUMBER should be set to the source control value that + // represents the current state of the source code. E.g., a + // perforce changelist number or a git hash. Can be an arbitrary string + // (to allow for source control that uses something other than numbers), + // but must be a single word and a valid file name. + // + // If no BUILD_NUMBER is set, create a useful "I am an engineering build + // from this date/time" value. Make it start with a non-digit so that + // anyone trying to parse it as an integer will probably get "0". + cmd.Environment.Unset("HAS_BUILD_NUMBER") + buildNumber, ok := cmd.Environment.Get("BUILD_NUMBER") + if ok { + cmd.Environment.Set("HAS_BUILD_NUMBER", "true") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", buildNumber) + } else { + buildNumber = fmt.Sprintf("eng.%.6s.%s", username, time.Now().Format("20060102.150405" /* YYYYMMDD.HHMMSS */)) + cmd.Environment.Set("HAS_BUILD_NUMBER", "false") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", username) + } + // Write the build number to a file so it can be read back in + // without changing the command line every time. Avoids rebuilds + // when using ninja. + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_number.txt", buildNumber) + + // Apply the caller's function closure to mutate the environment variables. + envFunc(cmd.Environment) cmd.StartOrFatal() // Set up the ToolStatus command line reader for Kati for a consistent UI @@ -336,6 +381,7 @@ func runKatiPackage(ctx Context, config Config) { "ANDROID_BUILD_SHELL", "DIST_DIR", "OUT_DIR", + "FILE_NAME_TAG", }...) if config.Dist() { From ed885bb7dd15d8c3b099ba504ce050f6508f62c1 Mon Sep 17 00:00:00 2001 From: Gurpreet Singh Date: Fri, 21 Apr 2023 16:30:03 +0000 Subject: [PATCH 06/80] Add genrule to build *.latest.version build target. Build a new target *.latest.version which will contain a text file containing the last finalized version. Bug: 242316893, 282140551 Test: atest prebuilt_apis_test (cherry picked from https://android-review.googlesource.com/q/commit:daa314ac97d5ab8076113ee7f269b4e4cc3cf620) Merged-In: I41fa91c9ec273f342b7807c66c4d65ba13260124 Change-Id: I41fa91c9ec273f342b7807c66c4d65ba13260124 --- java/prebuilt_apis.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go index 206d99527..0740467eb 100644 --- a/java/prebuilt_apis.go +++ b/java/prebuilt_apis.go @@ -135,6 +135,19 @@ func createApiModule(mctx android.LoadHookContext, name string, path string) { mctx.CreateModule(genrule.GenRuleFactory, &genruleProps) } +func createLatestApiModuleExtensionVersionFile(mctx android.LoadHookContext, name string, version string) { + genruleProps := struct { + Name *string + Srcs []string + Out []string + Cmd *string + }{} + genruleProps.Name = proptools.StringPtr(name) + genruleProps.Out = []string{name} + genruleProps.Cmd = proptools.StringPtr("echo " + version + " > $(out)") + mctx.CreateModule(genrule.GenRuleFactory, &genruleProps) +} + func createEmptyFile(mctx android.LoadHookContext, name string) { props := struct { Name *string @@ -233,9 +246,10 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { type latestApiInfo struct { module, scope, path string version int + isExtensionApiFile bool } - getLatest := func(files []string) map[string]latestApiInfo { + getLatest := func(files []string, isExtensionApiFile bool) map[string]latestApiInfo { m := make(map[string]latestApiInfo) for _, f := range files { module, version, scope := parseFinalizedPrebuiltPath(mctx, f) @@ -245,16 +259,16 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { key := module + "." + scope info, exists := m[key] if !exists || version > info.version { - m[key] = latestApiInfo{module, scope, f, version} + m[key] = latestApiInfo{module, scope, f, version, isExtensionApiFile} } } return m } - latest := getLatest(apiLevelFiles) + latest := getLatest(apiLevelFiles, false) if p.properties.Extensions_dir != nil { extensionApiFiles := globExtensionDirs(mctx, p, "api/*.txt") - for k, v := range getLatest(extensionApiFiles) { + for k, v := range getLatest(extensionApiFiles, true) { if _, exists := latest[k]; !exists { mctx.ModuleErrorf("Module %v finalized for extension %d but never during an API level; likely error", v.module, v.version) } @@ -267,6 +281,12 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { for _, k := range android.SortedKeys(latest) { info := latest[k] name := PrebuiltApiModuleName(info.module, info.scope, "latest") + latestExtensionVersionModuleName := PrebuiltApiModuleName(info.module, info.scope, "latest.extension_version") + if info.isExtensionApiFile { + createLatestApiModuleExtensionVersionFile(mctx, latestExtensionVersionModuleName, strconv.Itoa(info.version)) + } else { + createLatestApiModuleExtensionVersionFile(mctx, latestExtensionVersionModuleName, "-1") + } createApiModule(mctx, name, info.path) } From 069010cc7442dbc8393f833a098a81be545b7c89 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 15 May 2023 17:21:47 -0400 Subject: [PATCH 07/80] don't require package_name for non-override android_apps For a non-override android_app, we can assume that the privapp_allowlist already contains the correct package_name, and so we don't need to overwrite it in this case. Bug: 242509786 Test: go test Ignore-AOSP-First: need to submit here to update PermissionController in sync with internal version Change-Id: I0f137e34cae3478dc8b9178d138121ff1d936f07 --- apex/apex_test.go | 3 +-- java/app.go | 10 +++++++++- java/app_test.go | 18 ++++++------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 6ca5afb84..cd0c8a3ba 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6222,8 +6222,7 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, - privapp_allowlist: "perms.xml", - package_name: "com.android.AppFooPriv", + privapp_allowlist: "privapp_allowlist_com.android.AppFooPriv.xml", stl: "none", apex_available: [ "myapex" ], } diff --git a/java/app.go b/java/app.go index 334464772..f9454edb6 100755 --- a/java/app.go +++ b/java/app.go @@ -620,13 +620,21 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } -func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) android.Path { if a.appProperties.Privapp_allowlist == nil { return nil } + + isOverrideApp := a.GetOverriddenBy() != "" + if !isOverrideApp { + // if this is not an override, we don't need to rewrite the existing privapp allowlist + return android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist) + } + if a.overridableAppProperties.Package_name == nil { ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") } + packageName := *a.overridableAppProperties.Package_name fileName := "privapp_allowlist_" + packageName + ".xml" outPath := android.PathForModuleOut(ctx, fileName).OutputPath diff --git a/java/app_test.go b/java/app_test.go index daff94ca9..b250bc1b0 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3555,9 +3555,8 @@ func TestPrivappAllowlist(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - privapp_allowlist: "perms.xml", + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", privileged: true, - package_name: "com.android.foo", sdk_version: "current", } override_android_app { @@ -3570,17 +3569,12 @@ func TestPrivappAllowlist(t *testing.T) { app := result.ModuleForTests("foo", "android_common") overrideApp := result.ModuleForTests("foo", "android_common_bar") - // verify that privapp allowlist is created - app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + // verify that privapp allowlist is created for override apps overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") - expectedAllowlist := "perms.xml" - actualAllowlist := app.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != actualAllowlist { - t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) - } - overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != overrideActualAllowlist { - t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + expectedAllowlistInput := "privapp_allowlist_com.android.foo.xml" + overrideActualAllowlistInput := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlistInput != overrideActualAllowlistInput { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlistInput, overrideActualAllowlistInput) } // verify that permissions are copied to device From 480943ff647fdab582ddadc7a435bbc54ef53301 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Fri, 19 May 2023 06:02:08 +0000 Subject: [PATCH 08/80] Generate java_api_library from java_sdk_library This change enables java_sdk_library to generate java_api_library modules per api surface, so that from-text stubs can be generated per api domain scope. This module is only created when `--build-from-text-stub` flag is passed during build. Ignore-AOSP-First: config.apiLibraries list differ between AOSP and internal master Test: enable disabled modules in java/core-libraries/TxtStubLibraries.bp then m art.module.public.api.stubs.from-text --build-from-text-stub Bug: 276957733 Change-Id: Ic1ead15b3d0bcb921ca8d31bcaeeb4cd9ee8715c --- android/config.go | 45 +++++++++++++++++++++++ java/sdk_library.go | 78 +++++++++++++++++++++++++++++++++++++++- java/sdk_library_test.go | 63 ++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) diff --git a/android/config.go b/android/config.go index 01b8cfaa3..a60ceedf2 100644 --- a/android/config.go +++ b/android/config.go @@ -302,6 +302,9 @@ type config struct { // modules that aren't mixed-built for at least one variant will cause a build // failure ensureAllowlistIntegrity bool + + // List of Api libraries that contribute to Api surfaces. + apiLibraries map[string]struct{} } type deviceConfig struct { @@ -622,6 +625,36 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) config.BazelContext, err = NewBazelContext(config) config.Bp2buildPackageConfig = GetBp2BuildAllowList() + // TODO(b/276958307): Replace the hardcoded list to a sdk_library local prop. + config.apiLibraries = map[string]struct{}{ + "android.net.ipsec.ike": {}, + "art.module.public.api": {}, + "conscrypt.module.public.api": {}, + "framework-adservices": {}, + "framework-appsearch": {}, + "framework-bluetooth": {}, + "framework-configinfrastructure": {}, + "framework-connectivity": {}, + "framework-connectivity-t": {}, + "framework-devicelock": {}, + "framework-graphics": {}, + "framework-healthfitness": {}, + "framework-media": {}, + "framework-mediaprovider": {}, + "framework-ondevicepersonalization": {}, + "framework-permission": {}, + "framework-permission-s": {}, + "framework-scheduling": {}, + "framework-sdkextensions": {}, + "framework-statsd": {}, + "framework-sdksandbox": {}, + "framework-tethering": {}, + "framework-uwb": {}, + "framework-virtualization": {}, + "framework-wifi": {}, + "i18n.module.public.api": {}, + } + return Config{config}, err } @@ -1983,8 +2016,20 @@ func (c *config) BuildFromTextStub() bool { func (c *config) SetBuildFromTextStub(b bool) { c.buildFromTextStub = b } + func (c *config) AddForceEnabledModules(forceEnabled []string) { for _, forceEnabledModule := range forceEnabled { c.bazelForceEnabledModules[forceEnabledModule] = struct{}{} } } + +func (c *config) SetApiLibraries(libs []string) { + c.apiLibraries = make(map[string]struct{}) + for _, lib := range libs { + c.apiLibraries[lib] = struct{}{} + } +} + +func (c *config) GetApiLibraries() map[string]struct{} { + return c.apiLibraries +} diff --git a/java/sdk_library.go b/java/sdk_library.go index 103f1ace7..89da19a19 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -156,6 +156,9 @@ type apiScope struct { // Whether the api scope can be treated as unstable, and should skip compat checks. unstable bool + + // Represents the SDK kind of this scope. + kind android.SdkKind } // Initialize a scope, creating and adding appropriate dependency tags @@ -229,6 +232,10 @@ func (scope *apiScope) stubsLibraryModuleNameSuffix() string { return ".stubs" + scope.moduleSuffix } +func (scope *apiScope) apiLibraryModuleName(baseName string) string { + return scope.stubsLibraryModuleName(baseName) + ".from-text" +} + func (scope *apiScope) stubsLibraryModuleName(baseName string) string { return baseName + scope.stubsLibraryModuleNameSuffix() } @@ -289,6 +296,7 @@ var ( return &module.sdkLibraryProperties.Public }, sdkVersion: "current", + kind: android.SdkPublic, }) apiScopeSystem = initApiScope(&apiScope{ name: "system", @@ -301,6 +309,7 @@ var ( moduleSuffix: ".system", sdkVersion: "system_current", annotation: "android.annotation.SystemApi(client=android.annotation.SystemApi.Client.PRIVILEGED_APPS)", + kind: android.SdkSystem, }) apiScopeTest = initApiScope(&apiScope{ name: "test", @@ -314,6 +323,7 @@ var ( sdkVersion: "test_current", annotation: "android.annotation.TestApi", unstable: true, + kind: android.SdkTest, }) apiScopeModuleLib = initApiScope(&apiScope{ name: "module-lib", @@ -331,6 +341,7 @@ var ( moduleSuffix: ".module_lib", sdkVersion: "module_current", annotation: "android.annotation.SystemApi(client=android.annotation.SystemApi.Client.MODULE_LIBRARIES)", + kind: android.SdkModule, }) apiScopeSystemServer = initApiScope(&apiScope{ name: "system-server", @@ -361,6 +372,7 @@ var ( // com.android.* classes are okay in this interface" "--hide", "InternalClasses", }, + kind: android.SdkSystemServer, }) allApiScopes = apiScopes{ apiScopePublic, @@ -842,6 +854,13 @@ func (c *commonToSdkLibraryAndImport) stubsSourceModuleName(apiScope *apiScope) return c.namingScheme.stubsSourceModuleName(apiScope, baseName) } +// Name of the java_api_library module that generates the from-text stubs source +// and compiles to a jar file. +func (c *commonToSdkLibraryAndImport) apiLibraryModuleName(apiScope *apiScope) string { + baseName := c.module.BaseModuleName() + return c.namingScheme.apiLibraryModuleName(apiScope, baseName) +} + // The component names for different outputs of the java_sdk_library. // // They are similar to the names used for the child modules it creates @@ -1269,7 +1288,9 @@ func (module *SdkLibrary) ComponentDepsMutator(ctx android.BottomUpMutatorContex // Add dependencies to the stubs library stubModuleName := module.stubsLibraryModuleName(apiScope) // Use JavaApiLibraryName function to be redirected to stubs generated from .txt if applicable - stubModuleName = android.JavaApiLibraryName(ctx.Config(), stubModuleName) + if module.contributesToApiSurface(ctx.Config()) { + stubModuleName = android.JavaApiLibraryName(ctx.Config(), stubModuleName) + } ctx.AddVariationDependencies(nil, apiScope.stubsTag, stubModuleName) // Add a dependency on the stubs source in order to access both stubs source and api information. @@ -1477,6 +1498,11 @@ func (module *SdkLibrary) latestIncompatibilitiesModuleName(apiScope *apiScope) return latestPrebuiltApiModuleName(module.distStem()+"-incompatibilities", apiScope) } +func (module *SdkLibrary) contributesToApiSurface(c android.Config) bool { + _, exists := c.GetApiLibraries()[module.Name()] + return exists +} + func childModuleVisibility(childVisibility []string) []string { if childVisibility == nil { // No child visibility set. The child will use the visibility of the sdk_library. @@ -1758,6 +1784,46 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC mctx.CreateModule(DroidstubsFactory, &props).(*Droidstubs).CallHookIfAvailable(mctx) } +func (module *SdkLibrary) createApiLibrary(mctx android.DefaultableHookContext, apiScope *apiScope) { + props := struct { + Name *string + Visibility []string + Api_contributions []string + Libs []string + Static_libs []string + Dep_api_srcs *string + }{} + + props.Name = proptools.StringPtr(module.apiLibraryModuleName(apiScope)) + props.Visibility = childModuleVisibility(module.sdkLibraryProperties.Stubs_library_visibility) + + apiContributions := []string{} + + // Api surfaces are not independent of each other, but have subset relationships, + // and so does the api files. To generate from-text stubs for api surfaces other than public, + // all subset api domains' api_contriubtions must be added as well. + scope := apiScope + for scope != nil { + apiContributions = append(apiContributions, module.stubsSourceModuleName(scope)+".api.contribution") + scope = scope.extends + } + + props.Api_contributions = apiContributions + props.Libs = module.properties.Libs + props.Libs = append(props.Libs, module.sdkLibraryProperties.Stub_only_libs...) + props.Libs = append(props.Libs, "stub-annotations") + props.Static_libs = module.sdkLibraryProperties.Stub_only_static_libs + props.Dep_api_srcs = proptools.StringPtr(apiScope.kind.DefaultJavaLibraryName() + ".from-text") + + // android_module_lib_stubs_current.from-text only comprises api contributions from art, conscrypt and i18n. + // Thus, replace with android_module_lib_stubs_current_full.from-text, which comprises every api domains. + if apiScope.kind == android.SdkModule { + props.Dep_api_srcs = proptools.StringPtr(apiScope.kind.DefaultJavaLibraryName() + "_full.from-text") + } + + mctx.CreateModule(ApiLibraryFactory, &props) +} + func (module *SdkLibrary) compareAgainstLatestApi(apiScope *apiScope) bool { return !(apiScope.unstable || module.sdkLibraryProperties.Unsafe_ignore_missing_latest_api) } @@ -1954,6 +2020,10 @@ func (module *SdkLibrary) CreateInternalModules(mctx android.DefaultableHookCont module.createStubsSourcesAndApi(mctx, scope, module.stubsSourceModuleName(scope), scope.droidstubsArgs) module.createStubsLibrary(mctx, scope) + + if module.contributesToApiSurface(mctx.Config()) { + module.createApiLibrary(mctx, scope) + } } if module.requiresRuntimeImplementationLibrary() { @@ -2006,6 +2076,8 @@ type sdkLibraryComponentNamingScheme interface { stubsLibraryModuleName(scope *apiScope, baseName string) string stubsSourceModuleName(scope *apiScope, baseName string) string + + apiLibraryModuleName(scope *apiScope, baseName string) string } type defaultNamingScheme struct { @@ -2019,6 +2091,10 @@ func (s *defaultNamingScheme) stubsSourceModuleName(scope *apiScope, baseName st return scope.stubsSourceModuleName(baseName) } +func (s *defaultNamingScheme) apiLibraryModuleName(scope *apiScope, baseName string) string { + return scope.apiLibraryModuleName(baseName) +} + var _ sdkLibraryComponentNamingScheme = (*defaultNamingScheme)(nil) func moduleStubLinkType(name string) (stub bool, ret sdkLinkType) { diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 1d0c13d4b..141e16bf7 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -35,6 +35,9 @@ func TestJavaSdkLibrary(t *testing.T) { "29": {"foo"}, "30": {"bar", "barney", "baz", "betty", "foo", "fred", "quuz", "wilma"}, }), + android.FixtureModifyConfig(func(config android.Config) { + config.SetApiLibraries([]string{"foo"}) + }), ).RunTestWithBp(t, ` droiddoc_exported_dir { name: "droiddoc-templates-sdk", @@ -121,6 +124,7 @@ func TestJavaSdkLibrary(t *testing.T) { result.ModuleForTests(apiScopeSystem.stubsSourceModuleName("foo"), "android_common") result.ModuleForTests(apiScopeTest.stubsSourceModuleName("foo"), "android_common") result.ModuleForTests(apiScopePublic.stubsSourceModuleName("foo")+".api.contribution", "") + result.ModuleForTests(apiScopePublic.apiLibraryModuleName("foo"), "android_common") result.ModuleForTests("foo"+sdkXmlFileSuffix, "android_common") result.ModuleForTests("foo.api.public.28", "") result.ModuleForTests("foo.api.system.28", "") @@ -1412,3 +1416,62 @@ func TestJavaSdkLibrary_StubOnlyLibs_PassedToDroidstubs(t *testing.T) { fooStubsSources := result.ModuleForTests("foo.stubs.source", "android_common").Module().(*Droidstubs) android.AssertStringListContains(t, "foo stubs should depend on bar-lib", fooStubsSources.Javadoc.properties.Libs, "bar-lib") } + +func TestJavaSdkLibrary_ApiLibrary(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForJavaTest, + PrepareForTestWithJavaSdkLibraryFiles, + FixtureWithLastReleaseApis("foo"), + android.FixtureModifyConfig(func(config android.Config) { + config.SetApiLibraries([]string{"foo"}) + }), + ).RunTestWithBp(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java", "b.java"], + api_packages: ["foo"], + system: { + enabled: true, + }, + module_lib: { + enabled: true, + }, + test: { + enabled: true, + }, + } + `) + + testCases := []struct { + scope *apiScope + apiContributions []string + depApiSrcs string + }{ + { + scope: apiScopePublic, + apiContributions: []string{"foo.stubs.source.api.contribution"}, + depApiSrcs: "android_stubs_current.from-text", + }, + { + scope: apiScopeSystem, + apiContributions: []string{"foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_system_stubs_current.from-text", + }, + { + scope: apiScopeTest, + apiContributions: []string{"foo.stubs.source.test.api.contribution", "foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_test_stubs_current.from-text", + }, + { + scope: apiScopeModuleLib, + apiContributions: []string{"foo.stubs.source.module_lib.api.contribution", "foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_module_lib_stubs_current_full.from-text", + }, + } + + for _, c := range testCases { + m := result.ModuleForTests(c.scope.apiLibraryModuleName("foo"), "android_common").Module().(*ApiLibrary) + android.AssertArrayString(t, "Module expected to contain api contributions", c.apiContributions, m.properties.Api_contributions) + android.AssertStringEquals(t, "Module expected to contain full api surface api library", c.depApiSrcs, *m.properties.Dep_api_srcs) + } +} From 30bf7fe32d0ba644cd181645fab3c714a9a11b2f Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Mon, 22 May 2023 19:06:56 +0000 Subject: [PATCH 09/80] Run "plugins" singleton in parallel with the rest. This completes the changes in aosp/q/topic:"parallel-singletons". Ignore-AOSP-First: File only exists internally Bug: 281536768 Test: manual, presubmits Change-Id: I6529e957fbcca6c18441f823a00061db7556e303 --- android/plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/plugin.go b/android/plugin.go index c9d1338f8..4672453ee 100644 --- a/android/plugin.go +++ b/android/plugin.go @@ -29,7 +29,7 @@ func init() { } func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { - ctx.RegisterSingletonType("plugins", pluginSingletonFactory) + ctx.RegisterParallelSingletonType("plugins", pluginSingletonFactory) } // pluginSingleton is a singleton to handle allowlisting of the final Android-.mk file From 4c5da94fa4320aa868203ad2128fb49492274c73 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Thu, 30 Mar 2023 16:40:24 -0700 Subject: [PATCH 10/80] Enable referencing of aapt-generated proguard flags Some targets may implement classes declared elsewhere, e.g., system server implements services defined in framework-res.apk's manifest. Allow depending on the aapt-generated proguard flags for a given target to support this. Bug: 272495195 Test: m + reference ":framework-res{.aapt.proguardOptionsFile}" (cherry picked from https://android-review.googlesource.com/q/commit:34b080ddcf7757bba3dcb938df46aa660b2564a0) Merged-In: I2a16632ed6e5a9bddbe326bdb7ab0dd79b45e587 Change-Id: I2a16632ed6e5a9bddbe326bdb7ab0dd79b45e587 --- java/app.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/app.go b/java/app.go index 706f99a83..d333a8977 100755 --- a/java/app.go +++ b/java/app.go @@ -925,6 +925,10 @@ func (a *AndroidApp) DepIsInSameApex(ctx android.BaseModuleContext, dep android. // For OutputFileProducer interface func (a *AndroidApp) OutputFiles(tag string) (android.Paths, error) { switch tag { + // In some instances, it can be useful to reference the aapt-generated flags from another + // target, e.g., system server implements services declared in the framework-res manifest. + case ".aapt.proguardOptionsFile": + return []android.Path{a.proguardOptionsFile}, nil case ".aapt.srcjar": return []android.Path{a.aaptSrcJar}, nil case ".export-package.apk": From d87d22372b34bcdeab6184fcb56391aeed06a84a Mon Sep 17 00:00:00 2001 From: Andrei Onea Date: Wed, 17 Aug 2022 16:50:48 +0000 Subject: [PATCH 11/80] Add script for modifying privapp permission allowlists This script will be used to change the package that is being allowed to use privileged permissions. Test: python scripts/modify_permissions_allowlist.py Bug: 242509786 (cherry picked from https://android-review.googlesource.com/q/commit:580f7b0094bd98d3769b6cdeaaab6a9748621253) Merged-In: I1d7cb3c906ad39aef42ad4394953251db40d1bf1 Change-Id: I1d7cb3c906ad39aef42ad4394953251db40d1bf1 --- scripts/Android.bp | 17 +++++ scripts/modify_permissions_allowlist.py | 70 ++++++++++++++++++ scripts/modify_permissions_allowlist_test.py | 76 ++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100755 scripts/modify_permissions_allowlist.py create mode 100755 scripts/modify_permissions_allowlist_test.py diff --git a/scripts/Android.bp b/scripts/Android.bp index 9367ff06a..97f6ab424 100644 --- a/scripts/Android.bp +++ b/scripts/Android.bp @@ -237,3 +237,20 @@ sh_binary_host { name: "jars-to-module-info-java", src: "jars-to-module-info-java.sh", } + +python_binary_host { + name: "modify_permissions_allowlist", + main: "modify_permissions_allowlist.py", + srcs: [ + "modify_permissions_allowlist.py", + ], +} + +python_test_host { + name: "modify_permissions_allowlist_test", + main: "modify_permissions_allowlist_test.py", + srcs: [ + "modify_permissions_allowlist_test.py", + "modify_permissions_allowlist.py", + ], +} diff --git a/scripts/modify_permissions_allowlist.py b/scripts/modify_permissions_allowlist.py new file mode 100755 index 000000000..38ec7ec86 --- /dev/null +++ b/scripts/modify_permissions_allowlist.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for modifying privileged permission allowlists.""" + +from __future__ import print_function + +import argparse +import sys +from xml.dom import minidom + + +class InvalidRootNodeException(Exception): + pass + + +class InvalidNumberOfPrivappPermissionChildren(Exception): + pass + + +def modify_allowlist(allowlist_dom, package_name): + if allowlist_dom.documentElement.tagName != 'permissions': + raise InvalidRootNodeException + nodes = allowlist_dom.getElementsByTagName('privapp-permissions') + if nodes.length != 1: + raise InvalidNumberOfPrivappPermissionChildren + privapp_permissions = nodes[0] + privapp_permissions.setAttribute('package', package_name) + + +def parse_args(): + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument('input', help='input allowlist template file') + parser.add_argument( + 'package_name', help='package name to use in the allowlist' + ) + parser.add_argument('output', help='output allowlist file') + + return parser.parse_args() + + +def main(): + try: + args = parse_args() + doc = minidom.parse(args.input) + modify_allowlist(doc, args.package_name) + with open(args.output, 'w') as output_file: + doc.writexml(output_file, encoding='utf-8') + except Exception as err: + print('error: ' + str(err), file=sys.stderr) + sys.exit(-1) + + +if __name__ == '__main__': + main() diff --git a/scripts/modify_permissions_allowlist_test.py b/scripts/modify_permissions_allowlist_test.py new file mode 100755 index 000000000..ee8b12cc8 --- /dev/null +++ b/scripts/modify_permissions_allowlist_test.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unit tests for modify_permissions_allowlist.py.""" + +from __future__ import print_function + +import unittest + +from xml.dom import minidom + +from modify_permissions_allowlist import InvalidRootNodeException, InvalidNumberOfPrivappPermissionChildren, modify_allowlist + + +class ModifyPermissionsAllowlistTest(unittest.TestCase): + + def test_invalid_root(self): + xml_data = '' + xml_dom = minidom.parseString(xml_data) + self.assertRaises(InvalidRootNodeException, modify_allowlist, xml_dom, 'x') + + def test_no_packages(self): + xml_data = '' + xml_dom = minidom.parseString(xml_data) + self.assertRaises( + InvalidNumberOfPrivappPermissionChildren, modify_allowlist, xml_dom, 'x' + ) + + def test_multiple_packages(self): + xml_data = ( + '' + ' ' + ' ' + '' + ) + xml_dom = minidom.parseString(xml_data) + self.assertRaises( + InvalidNumberOfPrivappPermissionChildren, modify_allowlist, xml_dom, 'x' + ) + + def test_modify_package_name(self): + xml_data = ( + '' + ' ' + ' ' + ' ' + '' + ) + xml_dom = minidom.parseString(xml_data) + modify_allowlist(xml_dom, 'bar.baz') + expected_data = ( + '' + '' + ' ' + ' ' + ' ' + '' + ) + self.assertEqual(expected_data, xml_dom.toxml()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From c53cfd54d937c34920d84c3ef2d6685a71dcd624 Mon Sep 17 00:00:00 2001 From: Andrei Onea Date: Wed, 17 Aug 2022 16:53:46 +0000 Subject: [PATCH 12/80] add privapp_allowlist property to android_app This change allows override_android_app to use the same privapp_allowlist as the non-override module so that they will always remain in sync. Test: go test ./java -v -run TestPrivappAllowlist Test: go test ./apex -v -run TestApexWithApps Test: m com.android.permission com.google.android.permission and verify manually that apex_payload.img contains correct privapp_allowlist Test: m com.android.permission before and after change && `diffoscope apex_payload_reference.img apex_payload_with_change.img` && verify that there are no semantic changes Bug: 242509786 (cherry picked from https://android-review.googlesource.com/q/commit:580636bdd23171f31bfedd773c065e0861dd5c4a) Merged-In: Ifdcb28af40763aed7a4aac9a7f681153554bc256 Change-Id: Ifdcb28af40763aed7a4aac9a7f681153554bc256 --- apex/apex.go | 19 +++++++++++---- apex/apex_test.go | 3 +++ java/app.go | 58 +++++++++++++++++++++++++++++++++++++++++++--- java/app_import.go | 4 ++++ java/app_test.go | 48 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 8 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index c1c9e5c88..1773fcf23 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1823,6 +1823,7 @@ type androidApp interface { Certificate() java.Certificate BaseModuleName() string LintDepSets() java.LintDepSets + PrivAppAllowlist() android.OptionalPath } var _ androidApp = (*java.AndroidApp)(nil) @@ -1843,7 +1844,7 @@ func sanitizedBuildIdForPath(ctx android.BaseModuleContext) string { return buildId } -func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexFile { +func apexFilesForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) []apexFile { appDir := "app" if aapp.Privileged() { appDir = "priv-app" @@ -1865,7 +1866,15 @@ func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexF }); ok { af.overriddenPackageName = app.OverriddenManifestPackageName() } - return af + apexFiles := []apexFile{af} + + if allowlist := aapp.PrivAppAllowlist(); allowlist.Valid() { + dirInApex := filepath.Join("etc", "permissions") + privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"privapp", dirInApex, etc, aapp) + apexFiles = append(apexFiles, privAppAllowlist) + } + + return apexFiles } func apexFileForRuntimeResourceOverlay(ctx android.BaseModuleContext, rro java.RuntimeResourceOverlayModule) apexFile { @@ -2310,12 +2319,12 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, case androidAppTag: switch ap := child.(type) { case *java.AndroidApp: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) return true // track transitive dependencies case *java.AndroidAppImport: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) case *java.AndroidTestHelperApp: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) case *java.AndroidAppSet: appDir := "app" if ap.Privileged() { diff --git a/apex/apex_test.go b/apex/apex_test.go index 139b77ef6..139d23d49 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6165,6 +6165,8 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, + privapp_allowlist: "perms.xml", + package_name: "com.android.AppFooPriv", stl: "none", apex_available: [ "myapex" ], } @@ -6194,6 +6196,7 @@ func TestApexWithApps(t *testing.T) { ensureContains(t, copyCmds, "image.apex/app/AppFoo@TEST.BUILD_ID/AppFoo.apk") ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPriv@TEST.BUILD_ID/AppFooPriv.apk") + ensureContains(t, copyCmds, "image.apex/etc/permissions/privapp_allowlist_com.android.AppFooPriv.xml") appZipRule := ctx.ModuleForTests("AppFoo", "android_common_apex10000").Description("zip jni libs") // JNI libraries are uncompressed diff --git a/java/app.go b/java/app.go index 706f99a83..e4c5b706a 100755 --- a/java/app.go +++ b/java/app.go @@ -33,8 +33,17 @@ import ( func init() { RegisterAppBuildComponents(android.InitRegistrationContext) + pctx.HostBinToolVariable("ModifyAllowlistCmd", "modify_permissions_allowlist") } +var ( + modifyAllowlist = pctx.AndroidStaticRule("modifyAllowlist", + blueprint.RuleParams{ + Command: "${ModifyAllowlistCmd} $in $packageName $out", + CommandDeps: []string{"${ModifyAllowlistCmd}"}, + }, "packageName") +) + func RegisterAppBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("android_app", AndroidAppFactory) ctx.RegisterModuleType("android_test", AndroidTestFactory) @@ -115,6 +124,9 @@ type appProperties struct { // Prefer using other specific properties if build behaviour must be changed; avoid using this // flag for anything but neverallow rules (unless the behaviour change is invisible to owners). Updatable *bool + + // Specifies the file that contains the allowlist for this app. + Privapp_allowlist *string `android:"path"` } // android_app properties that can be overridden by override_android_app @@ -179,6 +191,8 @@ type AndroidApp struct { android.ApexBundleDepsInfo javaApiUsedByOutputFile android.ModuleOutPath + + privAppAllowlist android.OptionalPath } func (a *AndroidApp) IsInstallable() bool { @@ -205,6 +219,10 @@ func (a *AndroidApp) JniCoverageOutputs() android.Paths { return a.jniCoverageOutputs } +func (a *AndroidApp) PrivAppAllowlist() android.OptionalPath { + return a.privAppAllowlist +} + var _ AndroidLibraryDependency = (*AndroidApp)(nil) type Certificate struct { @@ -269,6 +287,10 @@ func (a *AndroidApp) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato ctx.AddDependency(ctx.Module(), certificateTag, cert) } + if a.appProperties.Privapp_allowlist != nil && !Bool(a.appProperties.Privileged) { + ctx.PropertyErrorf("privapp_allowlist", "privileged must be set in order to use privapp_allowlist") + } + for _, cert := range a.appProperties.Additional_certificates { cert = android.SrcIsModule(cert) if cert != "" { @@ -599,6 +621,27 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { + if a.appProperties.Privapp_allowlist == nil { + return nil + } + if a.overridableAppProperties.Package_name == nil { + ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") + } + packageName := *a.overridableAppProperties.Package_name + fileName := "privapp_allowlist_" + packageName + ".xml" + outPath := android.PathForModuleOut(ctx, fileName).OutputPath + ctx.Build(pctx, android.BuildParams{ + Rule: modifyAllowlist, + Input: android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist), + Output: outPath, + Args: map[string]string{ + "packageName": packageName, + }, + }) + return &outPath +} + func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { var apkDeps android.Paths @@ -734,18 +777,27 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { BuildBundleModule(ctx, bundleFile, a.exportPackage, jniJarFile, dexJarFile) a.bundleFile = bundleFile + allowlist := a.createPrivappAllowlist(ctx) + if allowlist != nil { + a.privAppAllowlist = android.OptionalPathForPath(allowlist) + } + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) // Install the app package. - if (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && - !a.appProperties.PreventInstall { - + shouldInstallAppPackage := (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && !a.appProperties.PreventInstall + if shouldInstallAppPackage { var extraInstalledPaths android.Paths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) } ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile, extraInstalledPaths...) + + if a.privAppAllowlist.Valid() { + installPath := android.PathForModuleInstall(ctx, "etc", "permissions") + ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) + } } a.buildAppDependencyInfo(ctx) diff --git a/java/app_import.go b/java/app_import.go index 85b35ebaa..3097d7fd5 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -376,6 +376,10 @@ func (a *AndroidAppImport) ProvenanceMetaDataFile() android.OutputPath { return a.provenanceMetaDataFile } +func (a *AndroidAppImport) PrivAppAllowlist() android.OptionalPath { + return android.OptionalPath{} +} + var dpiVariantGroupType reflect.Type var archVariantGroupType reflect.Type var supportedDpis = []string{"ldpi", "mdpi", "hdpi", "xhdpi", "xxhdpi", "xxxhdpi"} diff --git a/java/app_test.go b/java/app_test.go index b154bc990..f17284dd4 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3547,3 +3547,51 @@ func TestTargetSdkVersionMtsTests(t *testing.T) { android.AssertStringDoesContain(t, testCase.desc, manifestFixerArgs, "--targetSdkVersion "+testCase.targetSdkVersionExpected) } } + +func TestPrivappAllowlist(t *testing.T) { + testJavaError(t, "privileged must be set in order to use privapp_allowlist", ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "perms.xml", + } + `) + + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp( + t, + ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "perms.xml", + privileged: true, + package_name: "com.android.foo", + sdk_version: "current", + } + override_android_app { + name: "bar", + base: "foo", + package_name: "com.google.android.foo", + } + `, + ) + app := result.ModuleForTests("foo", "android_common") + overrideApp := result.ModuleForTests("foo", "android_common_bar") + + // verify that privapp allowlist is created + app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") + expectedAllowlist := "perms.xml" + actualAllowlist := app.Rule("modifyAllowlist").Input.String() + if expectedAllowlist != actualAllowlist { + t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) + } + overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlist != overrideActualAllowlist { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + } + + // verify that permissions are copied to device + app.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml") + overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml") +} From 502d807ae9ab60786108c38320c7618a5e1c65ec Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 15 May 2023 17:21:47 -0400 Subject: [PATCH 13/80] don't require package_name for non-override android_apps For a non-override android_app, we can assume that the privapp_allowlist already contains the correct package_name, and so we don't need to overwrite it in this case. Bug: 242509786 Test: go test Ignore-AOSP-First: need to submit here to update PermissionController in sync with internal version Merged-In: I0f137e34cae3478dc8b9178d138121ff1d936f07 Change-Id: I0f137e34cae3478dc8b9178d138121ff1d936f07 --- apex/apex_test.go | 3 +-- java/app.go | 10 +++++++++- java/app_test.go | 18 ++++++------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 139d23d49..6e440ffe2 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6165,8 +6165,7 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, - privapp_allowlist: "perms.xml", - package_name: "com.android.AppFooPriv", + privapp_allowlist: "privapp_allowlist_com.android.AppFooPriv.xml", stl: "none", apex_available: [ "myapex" ], } diff --git a/java/app.go b/java/app.go index e4c5b706a..d2e9a4121 100755 --- a/java/app.go +++ b/java/app.go @@ -621,13 +621,21 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } -func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) android.Path { if a.appProperties.Privapp_allowlist == nil { return nil } + + isOverrideApp := a.GetOverriddenBy() != "" + if !isOverrideApp { + // if this is not an override, we don't need to rewrite the existing privapp allowlist + return android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist) + } + if a.overridableAppProperties.Package_name == nil { ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") } + packageName := *a.overridableAppProperties.Package_name fileName := "privapp_allowlist_" + packageName + ".xml" outPath := android.PathForModuleOut(ctx, fileName).OutputPath diff --git a/java/app_test.go b/java/app_test.go index f17284dd4..6e5870f9e 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3563,9 +3563,8 @@ func TestPrivappAllowlist(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - privapp_allowlist: "perms.xml", + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", privileged: true, - package_name: "com.android.foo", sdk_version: "current", } override_android_app { @@ -3578,17 +3577,12 @@ func TestPrivappAllowlist(t *testing.T) { app := result.ModuleForTests("foo", "android_common") overrideApp := result.ModuleForTests("foo", "android_common_bar") - // verify that privapp allowlist is created - app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + // verify that privapp allowlist is created for override apps overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") - expectedAllowlist := "perms.xml" - actualAllowlist := app.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != actualAllowlist { - t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) - } - overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != overrideActualAllowlist { - t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + expectedAllowlistInput := "privapp_allowlist_com.android.foo.xml" + overrideActualAllowlistInput := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlistInput != overrideActualAllowlistInput { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlistInput, overrideActualAllowlistInput) } // verify that permissions are copied to device From 022adb0b8cbe8ce17c1e612115068e327b0b4786 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Thu, 25 May 2023 14:45:30 -0400 Subject: [PATCH 14/80] ensure that privapp_allowlist is installed before android_app AndroidMk assumes that the app is the last file installed, and it uses this assumption to populate the LOCAL_SOONG_INSTALLED_MODULE entry. This CL moves the privapp_allowlist installation to before the app installation to respect this assumption. Bug: 242509786 Test: go test Test: OUT_DIR=out.ref m nothing && cp aosp/2562351 && OUT_DIR=out.change m nothing && GOWORK=$PWD/build/bazel/mkcompare/go.work \ go run android/bazel/mkcompare/cmd -json \ <(sed -e "s/out\.ref/out/g" out.ref/soong/Android-aosp_cheetah.mk) \ <(sed -e "s/out\.change/out/g" out.change/soong/Android-aosp_cheetah.mk) && verify manually that the only diffs are related to the removal of the prebuilt_etc module. (cherry picked from https://android-review.googlesource.com/q/commit:b1daccdc97ae08603b0e2f5eafe97bd11238ee58) Merged-In: I95ec27070f575e79fb976de68493a219717ed89a Change-Id: I95ec27070f575e79fb976de68493a219717ed89a --- android/test_asserts.go | 15 ++++++++ apex/apex.go | 7 ++-- apex/apex_test.go | 12 +++++++ java/app.go | 11 +++--- java/app_test.go | 79 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 8 deletions(-) diff --git a/android/test_asserts.go b/android/test_asserts.go index 4143f150d..5cc7e4ac0 100644 --- a/android/test_asserts.go +++ b/android/test_asserts.go @@ -17,6 +17,7 @@ package android import ( "fmt" "reflect" + "regexp" "strings" "testing" ) @@ -137,6 +138,20 @@ func AssertStringContainsEquals(t *testing.T, message string, s string, substrin } } +// AssertStringMatches checks if the string matches the given regular expression. If it does not match, +// then an error is reported with the supplied message including a reason for why it failed. +func AssertStringMatches(t *testing.T, message, s, expectedRex string) { + t.Helper() + ok, err := regexp.MatchString(expectedRex, s) + if err != nil { + t.Fatalf("regexp failure trying to match %s against `%s` expression: %s", s, expectedRex, err) + return + } + if !ok { + t.Errorf("%s does not match regular expression %s", s, expectedRex) + } +} + // AssertStringListContains checks if the list of strings contains the expected string. If it does // not then it reports an error prefixed with the supplied message and including a reason for why it // failed. diff --git a/apex/apex.go b/apex/apex.go index 1773fcf23..85be920ee 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1866,14 +1866,17 @@ func apexFilesForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) []ap }); ok { af.overriddenPackageName = app.OverriddenManifestPackageName() } - apexFiles := []apexFile{af} + + apexFiles := []apexFile{} if allowlist := aapp.PrivAppAllowlist(); allowlist.Valid() { dirInApex := filepath.Join("etc", "permissions") - privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"privapp", dirInApex, etc, aapp) + privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"_privapp", dirInApex, etc, aapp) apexFiles = append(apexFiles, privAppAllowlist) } + apexFiles = append(apexFiles, af) + return apexFiles } diff --git a/apex/apex_test.go b/apex/apex_test.go index 6e440ffe2..66ba0a8cf 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6210,6 +6210,18 @@ func TestApexWithApps(t *testing.T) { // ... and not directly inside the APEX ensureNotContains(t, copyCmds, "image.apex/lib64/"+jni+".so") } + + apexBundle := module.Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, ctx, apexBundle) + var builder strings.Builder + data.Custom(&builder, apexBundle.Name(), "TARGET_", "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := AppFooPriv.myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := AppFoo.myapex") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALLED_MODULE := \\S+AppFooPriv.apk") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALLED_MODULE := \\S+AppFoo.apk") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALL_PAIRS := \\S+AppFooPriv.apk") + ensureContains(t, androidMk, "LOCAL_SOONG_INSTALL_PAIRS := privapp_allowlist_com.android.AppFooPriv.xml:$(PRODUCT_OUT)/apex/myapex/etc/permissions/privapp_allowlist_com.android.AppFooPriv.xml") } func TestApexWithAppImportBuildId(t *testing.T) { diff --git a/java/app.go b/java/app.go index e08e01048..cbdb50221 100755 --- a/java/app.go +++ b/java/app.go @@ -613,7 +613,6 @@ func processMainCert(m android.ModuleBase, certPropValue string, certificates [] } } - return mainCertificate, certificates } @@ -795,17 +794,17 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // Install the app package. shouldInstallAppPackage := (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && !a.appProperties.PreventInstall if shouldInstallAppPackage { + if a.privAppAllowlist.Valid() { + installPath := android.PathForModuleInstall(ctx, "etc", "permissions") + ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) + } + var extraInstalledPaths android.Paths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) } ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile, extraInstalledPaths...) - - if a.privAppAllowlist.Valid() { - installPath := android.PathForModuleInstall(ctx, "etc", "permissions") - ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) - } } a.buildAppDependencyInfo(ctx) diff --git a/java/app_test.go b/java/app_test.go index 6e5870f9e..71cd38c22 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3589,3 +3589,82 @@ func TestPrivappAllowlist(t *testing.T) { app.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml") overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml") } + +func TestPrivappAllowlistAndroidMk(t *testing.T) { + result := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + android.PrepareForTestWithAndroidMk, + ).RunTestWithBp( + t, + ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", + privileged: true, + sdk_version: "current", + } + override_android_app { + name: "bar", + base: "foo", + package_name: "com.google.android.foo", + } + `, + ) + baseApp := result.ModuleForTests("foo", "android_common") + overrideApp := result.ModuleForTests("foo", "android_common_bar") + + baseAndroidApp := baseApp.Module().(*AndroidApp) + baseEntries := android.AndroidMkEntriesForTest(t, result.TestContext, baseAndroidApp)[0] + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALLED_MODULE; expected to find foo.apk", + baseEntries.EntryMap["LOCAL_SOONG_INSTALLED_MODULE"][0], + "\\S+foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include foo.apk", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include app", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+foo.apk:\\S+/target/product/test_device/system/priv-app/foo/foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "privapp_allowlist_com.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml", + ) + + overrideAndroidApp := overrideApp.Module().(*AndroidApp) + overrideEntries := android.AndroidMkEntriesForTest(t, result.TestContext, overrideAndroidApp)[0] + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALLED_MODULE; expected to find bar.apk", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALLED_MODULE"][0], + "\\S+bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include bar.apk", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include app", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+bar.apk:\\S+/target/product/test_device/system/priv-app/bar/bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml", + ) +} From 0a3bb7dc5e242fda5ebfe9e8bc67464f0c1a7f55 Mon Sep 17 00:00:00 2001 From: Anton Hansson Date: Thu, 1 Jun 2023 17:07:58 +0000 Subject: [PATCH 15/80] Change install name of privapp allowlists Use the APK's filename as the on-device filename for the allowlist. Previously the filename of the source xml file was used, which leads to a problem when the same source file is used for different apps (even if only one of those modules ends up being installed). Bug: 284500052 Test: app_test.go & presubmit boot tests (cherry picked from https://android-review.googlesource.com/q/commit:92d1f52790e83952bf305464f0083d8c05f4c557) Merged-In: I0b43a35e3a310464d79d7f37e0900d639f4a479c Change-Id: I0b43a35e3a310464d79d7f37e0900d639f4a479c --- java/app.go | 5 +++-- java/app_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/java/app.go b/java/app.go index cbdb50221..0eb37c889 100755 --- a/java/app.go +++ b/java/app.go @@ -795,8 +795,9 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { shouldInstallAppPackage := (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && !a.appProperties.PreventInstall if shouldInstallAppPackage { if a.privAppAllowlist.Valid() { - installPath := android.PathForModuleInstall(ctx, "etc", "permissions") - ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) + allowlistInstallPath := android.PathForModuleInstall(ctx, "etc", "permissions") + allowlistInstallFilename := a.installApkName + ".xml" + ctx.InstallFile(allowlistInstallPath, allowlistInstallFilename, a.privAppAllowlist.Path()) } var extraInstalledPaths android.Paths diff --git a/java/app_test.go b/java/app_test.go index 71cd38c22..746ab75e7 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3586,8 +3586,8 @@ func TestPrivappAllowlist(t *testing.T) { } // verify that permissions are copied to device - app.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml") - overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml") + app.Output("out/soong/target/product/test_device/system/etc/permissions/foo.xml") + overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/bar.xml") } func TestPrivappAllowlistAndroidMk(t *testing.T) { @@ -3638,7 +3638,7 @@ func TestPrivappAllowlistAndroidMk(t *testing.T) { t, "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], - "privapp_allowlist_com.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml", + "privapp_allowlist_com.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/foo.xml", ) overrideAndroidApp := overrideApp.Module().(*AndroidApp) @@ -3665,6 +3665,6 @@ func TestPrivappAllowlistAndroidMk(t *testing.T) { t, "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], - "\\S+soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml", + "\\S+soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/bar.xml", ) } From a91268a690e2aa98a0fa7b376a948a48f41b1265 Mon Sep 17 00:00:00 2001 From: Anton Hansson Date: Fri, 2 Jun 2023 09:30:08 +0000 Subject: [PATCH 16/80] Allow Bluetooth to use privapp_allowlist Apparently there's some special case-code in package manager that scans apps as privileged regardless of their location. BT is one such app -- so allow it to specify a privapp allowlist even though it isn't in priv-app. Bug: 284500052 Test: use privapp_allowlist in Bluetooth and its overriddes Change-Id: Ib9ac86ebc45087a176100fe8df07db7bf81a85a7 --- java/app.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/java/app.go b/java/app.go index cbdb50221..7c4a0bfac 100755 --- a/java/app.go +++ b/java/app.go @@ -288,7 +288,13 @@ func (a *AndroidApp) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato } if a.appProperties.Privapp_allowlist != nil && !Bool(a.appProperties.Privileged) { - ctx.PropertyErrorf("privapp_allowlist", "privileged must be set in order to use privapp_allowlist") + // There are a few uids that are explicitly considered privileged regardless of their + // app's location. Bluetooth is one such app. It should arguably be moved to priv-app, + // but for now, allow it not to be in priv-app. + privilegedBecauseOfUid := ctx.ModuleName() == "Bluetooth" + if !privilegedBecauseOfUid { + ctx.PropertyErrorf("privapp_allowlist", "privileged must be set in order to use privapp_allowlist (with a few exceptions)") + } } for _, cert := range a.appProperties.Additional_certificates { From e5e07831eb4e521d4dd461dd5d756492e56a4c6a Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 12 Apr 2023 19:05:49 +0000 Subject: [PATCH 17/80] Ignore test apexes from bp2build generated tags The core problem I am trying to solve is making sure that stub libraries in Bazel have a single apex available. In Soong, this will be enforced using some graph walk which is not easy to port to Bazel. However, we might need to revisit this when we build the enforcement mechanism in Bazel. We likely need a `test_for` on the top level apex_test Bazel rule so that the test apex is _allowed_ to link against impl of the library it is trying to test. (This CL retricts this to cc_library, I can expand this to other modules if this is the right approach.) Bug: 277651159 (cherry picked from https://android-review.googlesource.com/q/commit:f57a966b6694da970e99f519919f9b845b713163) Merged-In: Iaeec22c5626df79a33785c766ed29102b1da403e Change-Id: Iaeec22c5626df79a33785c766ed29102b1da403e --- android/apex.go | 14 ++++++++++++++ android/mutator.go | 29 +++++++++++++++++++++++++++++ apex/apex.go | 4 ++++ cc/library.go | 6 +++--- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/android/apex.go b/android/apex.go index c9b4a0b7c..35e1d025a 100644 --- a/android/apex.go +++ b/android/apex.go @@ -451,6 +451,14 @@ const ( AvailableToGkiApex = "com.android.gki.*" ) +var ( + AvailableToRecognziedWildcards = []string{ + AvailableToPlatform, + AvailableToAnyApex, + AvailableToGkiApex, + } +) + // CheckAvailableForApex provides the default algorithm for checking the apex availability. When the // availability is empty, it defaults to ["//apex_available:platform"] which means "available to the // platform but not available to any APEX". When the list is not empty, `what` is matched against @@ -909,3 +917,9 @@ func CheckMinSdkVersion(ctx ModuleContext, minSdkVersion ApiLevel, walk WalkPayl return true }) } + +// Implemented by apexBundle. +type ApexTestInterface interface { + // Return true if the apex bundle is an apex_test + IsTestApex() bool +} diff --git a/android/mutator.go b/android/mutator.go index 4ec960472..c0040b7b9 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -758,6 +758,35 @@ func ApexAvailableTags(mod Module) bazel.StringListAttribute { return attr } +func ApexAvailableTagsWithoutTestApexes(ctx BaseModuleContext, mod Module) bazel.StringListAttribute { + attr := bazel.StringListAttribute{} + if am, ok := mod.(ApexModule); ok { + apexAvailableWithoutTestApexes := removeTestApexes(ctx, am.apexModuleBase().ApexAvailable()) + // If a user does not specify apex_available in Android.bp, then soong provides a default. + // To avoid verbosity of BUILD files, remove this default from user-facing BUILD files. + if len(am.apexModuleBase().ApexProperties.Apex_available) == 0 { + apexAvailableWithoutTestApexes = []string{} + } + attr.Value = ConvertApexAvailableToTags(apexAvailableWithoutTestApexes) + } + return attr +} + +func removeTestApexes(ctx BaseModuleContext, apex_available []string) []string { + testApexes := []string{} + for _, aa := range apex_available { + // ignore the wildcards + if InList(aa, AvailableToRecognziedWildcards) { + continue + } + mod, _ := ctx.ModuleFromName(aa) + if apex, ok := mod.(ApexTestInterface); ok && apex.IsTestApex() { + testApexes = append(testApexes, aa) + } + } + return RemoveListFromList(CopyOf(apex_available), testApexes) +} + func ConvertApexAvailableToTags(apexAvailable []string) []string { if len(apexAvailable) == 0 { // We need nil specifically to make bp2build not add the tags property at all, diff --git a/apex/apex.go b/apex/apex.go index 85be920ee..fa9233e79 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -3802,3 +3802,7 @@ func makeSharedLibsAttributes(config string, libsLabelList bazel.LabelList, func invalidCompileMultilib(ctx android.TopDownMutatorContext, value string) { ctx.PropertyErrorf("compile_multilib", "Invalid value: %s", value) } + +func (a *apexBundle) IsTestApex() bool { + return a.testApex +} diff --git a/cc/library.go b/cc/library.go index 7051f723c..d11887b10 100644 --- a/cc/library.go +++ b/cc/library.go @@ -428,11 +428,11 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { var tagsForStaticVariant bazel.StringListAttribute if compilerAttrs.stubsSymbolFile == nil && len(compilerAttrs.stubsVersions.Value) == 0 { - tagsForStaticVariant = android.ApexAvailableTags(m) + tagsForStaticVariant = android.ApexAvailableTagsWithoutTestApexes(ctx, m) } tagsForStaticVariant.Append(bazel.StringListAttribute{Value: staticAttrs.Apex_available}) - tagsForSharedVariant := android.ApexAvailableTags(m) + tagsForSharedVariant := android.ApexAvailableTagsWithoutTestApexes(ctx, m) tagsForSharedVariant.Append(bazel.StringListAttribute{Value: sharedAttrs.Apex_available}) ctx.CreateBazelTargetModuleWithRestrictions(staticProps, @@ -2994,7 +2994,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Bzl_load_location: fmt.Sprintf("//build/bazel/rules/cc:%s.bzl", modType), } - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name(), Tags: tags}, attrs) } From deaba3de51b8d7d04c772023bab900528d628460 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 12 Apr 2023 19:05:49 +0000 Subject: [PATCH 18/80] Ignore test apexes from bp2build generated tags Soong does not enforce apex_available on the contents of test apex. To prevent special-casing test apexes in the apex validation aspect in Bazel, drop the test apexes from the tags altogether. ( The core problem I am trying to solve is making sure that stub libraries in Bazel have a single apex available. apex validation happens to be a nice side benefit) Bug: 277651159 Test: go test ./bp2build (cherry picked from https://android-review.googlesource.com/q/commit:39b6cc53369a6ec132b58d99514cd7a2ea499efd) Merged-In: Ibb3cfedb5c0f2cda0464bf3758c70b67cb5885d1 Change-Id: Ibb3cfedb5c0f2cda0464bf3758c70b67cb5885d1 --- android/mutator.go | 7 +++++++ android/proto.go | 2 +- cc/binary.go | 2 +- cc/bp2build.go | 4 ++-- cc/library_headers.go | 2 +- cc/object.go | 2 +- cc/prebuilt.go | 8 ++++---- cc/proto.go | 2 +- cc/sysprop.go | 2 +- genrule/genrule.go | 2 +- java/java.go | 2 +- 11 files changed, 21 insertions(+), 14 deletions(-) diff --git a/android/mutator.go b/android/mutator.go index c0040b7b9..013fa77cf 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -800,6 +800,13 @@ func ConvertApexAvailableToTags(apexAvailable []string) []string { return result } +// ConvertApexAvailableToTagsWithoutTestApexes converts a list of apex names to a list of bazel tags +// This function drops any test apexes from the input. +func ConvertApexAvailableToTagsWithoutTestApexes(ctx BaseModuleContext, apexAvailable []string) []string { + noTestApexes := removeTestApexes(ctx, apexAvailable) + return ConvertApexAvailableToTags(noTestApexes) +} + func (t *topDownMutatorContext) createBazelTargetModule( bazelProps bazel.BazelTargetModuleProperties, commonAttrs CommonAttributes, diff --git a/android/proto.go b/android/proto.go index 09e50c8c3..cebbd59cd 100644 --- a/android/proto.go +++ b/android/proto.go @@ -234,7 +234,7 @@ func Bp2buildProtoProperties(ctx Bp2buildMutatorContext, m *ModuleBase, srcs baz } } - tags := ApexAvailableTags(ctx.Module()) + tags := ApexAvailableTagsWithoutTestApexes(ctx.(TopDownMutatorContext), ctx.Module()) ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{Rule_class: "proto_library"}, diff --git a/cc/binary.go b/cc/binary.go index 097f82252..98b9923ee 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -661,7 +661,7 @@ func binaryBp2build(ctx android.TopDownMutatorContext, m *Module) { // shared with cc_test binaryAttrs := binaryBp2buildAttrs(ctx, m) - tags := android.ApexAvailableTags(m) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m) ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ Rule_class: "cc_binary", Bzl_load_location: "//build/bazel/rules/cc:cc_binary.bzl", diff --git a/cc/bp2build.go b/cc/bp2build.go index adf5a08ec..15feb9e1a 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -268,7 +268,7 @@ func bp2buildParseStaticOrSharedProps(ctx android.BazelConversionPathContext, mo attrs.Srcs_c = partitionedSrcs[cSrcPartition] attrs.Srcs_as = partitionedSrcs[asSrcPartition] - attrs.Apex_available = android.ConvertApexAvailableToTags(apexAvailable) + attrs.Apex_available = android.ConvertApexAvailableToTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), apexAvailable) if !partitionedSrcs[protoSrcPartition].IsEmpty() { // TODO(b/208815215): determine whether this is used and add support if necessary @@ -912,7 +912,7 @@ func bp2buildCcAidlLibrary( return false }) - apexAvailableTags := android.ApexAvailableTags(ctx.Module()) + apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) sdkAttrs := bp2BuildParseSdkAttributes(m) if !aidlSrcs.IsEmpty() { diff --git a/cc/library_headers.go b/cc/library_headers.go index 1dee72679..ce9c4aacf 100644 --- a/cc/library_headers.go +++ b/cc/library_headers.go @@ -151,7 +151,7 @@ func libraryHeadersBp2Build(ctx android.TopDownMutatorContext, module *Module) { Bzl_load_location: "//build/bazel/rules/cc:cc_library_headers.bzl", } - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModule(props, android.CommonAttributes{ Name: module.Name(), diff --git a/cc/object.go b/cc/object.go index d65cdea74..5d6187233 100644 --- a/cc/object.go +++ b/cc/object.go @@ -226,7 +226,7 @@ func objectBp2Build(ctx android.TopDownMutatorContext, m *Module) { Bzl_load_location: "//build/bazel/rules/cc:cc_object.bzl", } - tags := android.ApexAvailableTags(m) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m) ctx.CreateBazelTargetModule(props, android.CommonAttributes{ Name: m.Name(), diff --git a/cc/prebuilt.go b/cc/prebuilt.go index 0b5841ef0..44cd0d73f 100644 --- a/cc/prebuilt.go +++ b/cc/prebuilt.go @@ -389,7 +389,7 @@ func prebuiltLibraryStaticBp2Build(ctx android.TopDownMutatorContext, module *Mo name += "_bp2build_cc_library_static" } - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModuleWithRestrictions(props, android.CommonAttributes{Name: name, Tags: tags}, attrs, prebuiltAttrs.Enabled) _true := true @@ -420,7 +420,7 @@ func prebuiltLibrarySharedBp2Build(ctx android.TopDownMutatorContext, module *Mo } name := android.RemoveOptionalPrebuiltPrefix(module.Name()) - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModuleWithRestrictions(props, android.CommonAttributes{Name: name, Tags: tags}, attrs, prebuiltAttrs.Enabled) } @@ -650,7 +650,7 @@ func prebuiltObjectBp2Build(ctx android.TopDownMutatorContext, module *Module) { } name := android.RemoveOptionalPrebuiltPrefix(module.Name()) - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: name, Tags: tags}, attrs) } @@ -813,7 +813,7 @@ func prebuiltBinaryBp2Build(ctx android.TopDownMutatorContext, module *Module) { } name := android.RemoveOptionalPrebuiltPrefix(module.Name()) - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: name, Tags: tags}, attrs) } diff --git a/cc/proto.go b/cc/proto.go index 97470e5ea..5d9aef60f 100644 --- a/cc/proto.go +++ b/cc/proto.go @@ -207,7 +207,7 @@ func bp2buildProto(ctx android.Bp2buildMutatorContext, m *Module, protoSrcs baze protoAttrs.Min_sdk_version = m.Properties.Min_sdk_version name := m.Name() + suffix - tags := android.ApexAvailableTags(m) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), m) ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: rule_class, diff --git a/cc/sysprop.go b/cc/sysprop.go index 0df290afa..7ddd4760e 100644 --- a/cc/sysprop.go +++ b/cc/sysprop.go @@ -38,7 +38,7 @@ type SyspropLibraryLabels struct { } func Bp2buildSysprop(ctx android.Bp2buildMutatorContext, labels SyspropLibraryLabels, srcs bazel.LabelListAttribute, minSdkVersion *string) { - apexAvailableTags := android.ApexAvailableTags(ctx.Module()) + apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: "sysprop_library", diff --git a/genrule/genrule.go b/genrule/genrule.go index f5da50ef3..00adb7025 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -940,7 +940,7 @@ func (m *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { } } - tags := android.ApexAvailableTags(m) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m) if ctx.ModuleType() == "gensrcs" { // The Output_extension prop is not in an immediately accessible field diff --git a/java/java.go b/java/java.go index 683e54682..ccb150747 100644 --- a/java/java.go +++ b/java/java.go @@ -2836,7 +2836,7 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) return android.IsConvertedToAidlLibrary(ctx, src.OriginalModuleName) }) - apexAvailableTags := android.ApexAvailableTags(ctx.Module()) + apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx, ctx.Module()) if !aidlSrcs.IsEmpty() { aidlLibName := m.Name() + "_aidl_library" From 46004f15112dbcce4d6508d5372b7f3e3977f237 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 19/80] Implement aidl_library module type We currently specifies aidl files directly to the srcs prop on a filegroup or other module types such as cc_library or java_library. We use aidl.include_dirs prop to specify paths to aidl headers. This include_dirs pattern isn't migratable to Bazel because Bazel requires explicit dependencies. This CL introduces aidl_library to better map with Bazel's aidl_library rule and to enable aidl headers to be specified in a separate aidl_library or the hdrs prop. A follow-up CL will turn on inputs sandbox to enforce all aidl headers be explicitly specified in Android.bp Test: go test Bug: 278704136 (cherry picked from https://android-review.googlesource.com/q/commit:0e7fd8a14b5e1ebc2f2516639e9cea5dfdefce63) Merged-In: I2c99af080525bf8a6c5724ed5ee2001842969098 Change-Id: I2c99af080525bf8a6c5724ed5ee2001842969098 --- aidl_library/Android.bp | 32 ++++++++ aidl_library/aidl_library.go | 120 ++++++++++++++++++++++++++++ aidl_library/aidl_library_test.go | 126 ++++++++++++++++++++++++++++++ 3 files changed, 278 insertions(+) create mode 100644 aidl_library/Android.bp create mode 100644 aidl_library/aidl_library.go create mode 100644 aidl_library/aidl_library_test.go diff --git a/aidl_library/Android.bp b/aidl_library/Android.bp new file mode 100644 index 000000000..ec2150427 --- /dev/null +++ b/aidl_library/Android.bp @@ -0,0 +1,32 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-aidl-library", + pkgPath: "android/soong/aidl_library", + deps: [ + "soong-android", + ], + srcs: [ + "aidl_library.go", + ], + testSrcs: [ + "aidl_library_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go new file mode 100644 index 000000000..2a42170f2 --- /dev/null +++ b/aidl_library/aidl_library.go @@ -0,0 +1,120 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aidl_library + +import ( + "android/soong/android" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +func init() { + registerAidlLibraryBuildComponents(android.InitRegistrationContext) +} + +func registerAidlLibraryBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("aidl_library", AidlLibraryFactory) +} + +type aidlLibraryProperties struct { + // srcs lists files that are included in this module for aidl compilation + Srcs []string `android:"path"` + + // hdrs lists the headers that are imported by srcs but are not compiled by aidl to language binding code + // hdrs is provided to support Bazel migration. It is a no-op until + // we enable input sandbox in aidl compilation action + Hdrs []string `android:"path"` + + // The prefix to strip from the paths of the .aidl files + // The remaining path is the package path of the aidl interface + Strip_import_prefix *string + + // List of aidl files or aidl_library depended on by the module + Deps []string `android:"arch_variant"` +} + +type AidlLibrary struct { + android.ModuleBase + properties aidlLibraryProperties +} + +type AidlLibraryInfo struct { + // The direct aidl files of the module + Srcs android.Paths + // The include dirs to the direct aidl files and those provided from aidl_library deps + IncludeDirs android.DepSet +} + +// AidlLibraryProvider provides the srcs and the transitive include dirs +var AidlLibraryProvider = blueprint.NewProvider(AidlLibraryInfo{}) + +func (lib *AidlLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + includeDirsDepSetBuilder := android.NewDepSetBuilder(android.PREORDER) + + if len(lib.properties.Srcs) == 0 && len(lib.properties.Hdrs) == 0 { + ctx.ModuleErrorf("at least srcs or hdrs prop must be non-empty") + } + + srcs := android.PathsForModuleSrc(ctx, lib.properties.Srcs) + if lib.properties.Strip_import_prefix != nil { + srcs = android.PathsWithModuleSrcSubDir( + ctx, + srcs, + android.String(lib.properties.Strip_import_prefix)) + } + + includeDir := android.PathForModuleSrc( + ctx, + proptools.StringDefault(lib.properties.Strip_import_prefix, ""), + ) + + includeDirsDepSetBuilder.Direct(includeDir) + + for _, dep := range ctx.GetDirectDepsWithTag(aidlLibraryTag) { + if ctx.OtherModuleHasProvider(dep, AidlLibraryProvider) { + info := ctx.OtherModuleProvider(dep, AidlLibraryProvider).(AidlLibraryInfo) + includeDirsDepSetBuilder.Transitive(&info.IncludeDirs) + } + } + + // TODO(b/279960133) Propagate direct and transitive headers/srcs when aidl action sandboxes inputs + ctx.SetProvider(AidlLibraryProvider, AidlLibraryInfo{ + Srcs: srcs, + IncludeDirs: *includeDirsDepSetBuilder.Build(), + }) +} + +// aidl_library contains a list of .aidl files and the strip_import_prefix to +// to strip from the paths of the .aidl files. The sub-path left-over after stripping +// corresponds to the aidl package path the aidl interfaces are scoped in +func AidlLibraryFactory() android.Module { + module := &AidlLibrary{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + return module +} + +type aidlDependencyTag struct { + blueprint.BaseDependencyTag +} + +var aidlLibraryTag = aidlDependencyTag{} + +func (lib *AidlLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { + for _, dep := range lib.properties.Deps { + ctx.AddDependency(lib, aidlLibraryTag, dep) + } +} diff --git a/aidl_library/aidl_library_test.go b/aidl_library/aidl_library_test.go new file mode 100644 index 000000000..42fa5367e --- /dev/null +++ b/aidl_library/aidl_library_test.go @@ -0,0 +1,126 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aidl_library + +import ( + "android/soong/android" + "testing" +) + +var PrepareForTestWithAidlLibrary = android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + registerAidlLibraryBuildComponents(ctx) +}) + +func TestAidlLibrary(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + PrepareForTestWithAidlLibrary, + android.MockFS{ + "package_bar/Android.bp": []byte(` + aidl_library { + name: "bar", + srcs: ["x/y/Bar.aidl"], + strip_import_prefix: "x", + } + `), + }.AddToFixture(), + android.MockFS{ + "package_foo/Android.bp": []byte(` + aidl_library { + name: "foo", + srcs: ["a/b/Foo.aidl"], + hdrs: ["Header.aidl"], + strip_import_prefix: "a", + deps: ["bar"], + } + `), + }.AddToFixture(), + ).RunTest(t).TestContext + + foo := ctx.ModuleForTests("foo", "").Module().(*AidlLibrary) + actualInfo := ctx.ModuleProvider(foo, AidlLibraryProvider).(AidlLibraryInfo) + + android.AssertArrayString( + t, + "aidl include dirs", + []string{"package_foo/a", "package_bar/x"}, + actualInfo.IncludeDirs.ToList().Strings(), + ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl srcs paths", + []string{"package_foo/a/b/Foo.aidl"}, + actualInfo.Srcs, + ) +} + +func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + PrepareForTestWithAidlLibrary, + android.MockFS{ + "package_bar/Android.bp": []byte(` + aidl_library { + name: "bar", + srcs: ["x/y/Bar.aidl"], + } + `), + }.AddToFixture(), + android.MockFS{ + "package_foo/Android.bp": []byte(` + aidl_library { + name: "foo", + srcs: ["a/b/Foo.aidl"], + hdrs: ["Header.aidl"], + deps: ["bar"], + } + `), + }.AddToFixture(), + ).RunTest(t).TestContext + + foo := ctx.ModuleForTests("foo", "").Module().(*AidlLibrary) + actualInfo := ctx.ModuleProvider(foo, AidlLibraryProvider).(AidlLibraryInfo) + + android.AssertArrayString( + t, + "aidl include dirs", + []string{"package_foo", "package_bar"}, + actualInfo.IncludeDirs.ToList().Strings(), + ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl srcs paths", + []string{"package_foo/a/b/Foo.aidl"}, + actualInfo.Srcs, + ) +} + +func TestAidlLibraryWithNoSrcsHdrsDeps(t *testing.T) { + t.Parallel() + android.GroupFixturePreparers( + PrepareForTestWithAidlLibrary, + android.MockFS{ + "package_bar/Android.bp": []byte(` + aidl_library { + name: "bar", + } + `), + }.AddToFixture(), + ). + ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern("at least srcs or hdrs prop must be non-empty")). + RunTest(t) +} From 4c43b2febcc489afe6c3e24b4f1475cdd8bbb7c8 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 20/80] Implement bp2build converter for aidl_library Test: go test Bug: 278704136 (cherry picked from https://android-review.googlesource.com/q/commit:3d16990b29fe2a4a8deed9769908f8496755adde) Merged-In: Ia9c3772257af58e1de9041ba465130740b555fe4 Change-Id: Ia9c3772257af58e1de9041ba465130740b555fe4 --- aidl_library/aidl_library.go | 50 ++++++++++ bp2build/Android.bp | 2 + bp2build/aidl_library_conversion_test.go | 119 +++++++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 bp2build/aidl_library_conversion_test.go diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go index 2a42170f2..9b5f0a814 100644 --- a/aidl_library/aidl_library.go +++ b/aidl_library/aidl_library.go @@ -16,6 +16,7 @@ package aidl_library import ( "android/soong/android" + "android/soong/bazel" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -48,9 +49,57 @@ type aidlLibraryProperties struct { type AidlLibrary struct { android.ModuleBase + android.BazelModuleBase properties aidlLibraryProperties } +type bazelAidlLibraryAttributes struct { + Srcs bazel.LabelListAttribute + Hdrs bazel.LabelListAttribute + Strip_import_prefix *string + Deps bazel.LabelListAttribute +} + +func (lib *AidlLibrary) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + srcs := bazel.MakeLabelListAttribute( + android.BazelLabelForModuleSrc( + ctx, + lib.properties.Srcs, + ), + ) + + hdrs := bazel.MakeLabelListAttribute( + android.BazelLabelForModuleSrc( + ctx, + lib.properties.Hdrs, + ), + ) + + tags := []string{"apex_available=//apex_available:anyapex"} + deps := bazel.MakeLabelListAttribute(android.BazelLabelForModuleDeps(ctx, lib.properties.Deps)) + + attrs := &bazelAidlLibraryAttributes{ + Srcs: srcs, + Hdrs: hdrs, + Strip_import_prefix: lib.properties.Strip_import_prefix, + Deps: deps, + } + + props := bazel.BazelTargetModuleProperties{ + Rule_class: "aidl_library", + Bzl_load_location: "//build/bazel/rules/aidl:aidl_library.bzl", + } + + ctx.CreateBazelTargetModule( + props, + android.CommonAttributes{ + Name: lib.Name(), + Tags: bazel.MakeStringListAttribute(tags), + }, + attrs, + ) +} + type AidlLibraryInfo struct { // The direct aidl files of the module Srcs android.Paths @@ -104,6 +153,7 @@ func AidlLibraryFactory() android.Module { module := &AidlLibrary{} module.AddProperties(&module.properties) android.InitAndroidModule(module) + android.InitBazelModule(module) return module } diff --git a/bp2build/Android.bp b/bp2build/Android.bp index b6635c430..9ec3a40b9 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -19,6 +19,7 @@ bootstrap_go_package { "testing.go", ], deps: [ + "soong-aidl-library", "soong-android", "soong-android-allowlists", "soong-android-soongconfig", @@ -37,6 +38,7 @@ bootstrap_go_package { ], testSrcs: [ "aar_conversion_test.go", + "aidl_library_conversion_test.go", "android_app_certificate_conversion_test.go", "android_app_conversion_test.go", "apex_conversion_test.go", diff --git a/bp2build/aidl_library_conversion_test.go b/bp2build/aidl_library_conversion_test.go new file mode 100644 index 000000000..0522da45e --- /dev/null +++ b/bp2build/aidl_library_conversion_test.go @@ -0,0 +1,119 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "testing" + + "android/soong/aidl_library" + "android/soong/android" +) + +func runAidlLibraryTestCase(t *testing.T, tc Bp2buildTestCase) { + t.Helper() + (&tc).ModuleTypeUnderTest = "aidl_library" + (&tc).ModuleTypeUnderTestFactory = aidl_library.AidlLibraryFactory + RunBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, tc) +} + +func TestAidlLibrary(t *testing.T) { + testcases := []struct { + name string + bp string + expectedBazelAttrs AttrNameToString + }{ + { + name: "aidl_library with strip_import_prefix", + bp: ` + aidl_library { + name: "foo", + srcs: ["aidl/foo.aidl"], + hdrs: ["aidl/header.aidl"], + strip_import_prefix: "aidl", + }`, + expectedBazelAttrs: AttrNameToString{ + "srcs": `["aidl/foo.aidl"]`, + "hdrs": `["aidl/header.aidl"]`, + "strip_import_prefix": `"aidl"`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }, + }, + { + name: "aidl_library without strip_import_prefix", + bp: ` + aidl_library { + name: "foo", + srcs: ["aidl/foo.aidl"], + hdrs: ["aidl/header.aidl"], + }`, + expectedBazelAttrs: AttrNameToString{ + "srcs": `["aidl/foo.aidl"]`, + "hdrs": `["aidl/header.aidl"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }, + }, + } + + for _, test := range testcases { + t.Run(test.name, func(t *testing.T) { + expectedBazelTargets := []string{ + MakeBazelTargetNoRestrictions("aidl_library", "foo", test.expectedBazelAttrs), + } + runAidlLibraryTestCase(t, Bp2buildTestCase{ + Description: test.name, + Blueprint: test.bp, + ExpectedBazelTargets: expectedBazelTargets, + }) + }) + } +} + +func TestAidlLibraryWithDeps(t *testing.T) { + bp := ` + aidl_library { + name: "bar", + srcs: ["Bar.aidl"], + hdrs: ["aidl/BarHeader.aidl"], + } + aidl_library { + name: "foo", + srcs: ["aidl/Foo.aidl"], + hdrs: ["aidl/FooHeader.aidl"], + strip_import_prefix: "aidl", + deps: ["bar"], + }` + + t.Run("aidl_library with deps", func(t *testing.T) { + expectedBazelTargets := []string{ + MakeBazelTargetNoRestrictions("aidl_library", "bar", AttrNameToString{ + "srcs": `["Bar.aidl"]`, + "hdrs": `["aidl/BarHeader.aidl"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + MakeBazelTargetNoRestrictions("aidl_library", "foo", AttrNameToString{ + "srcs": `["aidl/Foo.aidl"]`, + "hdrs": `["aidl/FooHeader.aidl"]`, + "strip_import_prefix": `"aidl"`, + "deps": `[":bar"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + } + runAidlLibraryTestCase(t, Bp2buildTestCase{ + Description: "aidl_library with deps", + Blueprint: bp, + ExpectedBazelTargets: expectedBazelTargets, + }) + }) +} From 0059b69829eb633724cf908919f8a87b9150b423 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 21/80] Use aidl_library in cc libraries Introduce aidl.libs prop on cc libraries to pass in aidl_library. The goal is to eventually disallow aidl.include_dirs (a pattern for passing aidl headers dir for aidl compilation) and enforce aidl headers to be explicitly specified in Android.bp. Bug: 278704136 Test: go test (cherry picked from https://android-review.googlesource.com/q/commit:367d89da7887825f74b0c9554b2e1b9619d94f08) Merged-In: Ia78bc11dfa12f47d2d1bb90dc65372ddb17f7e14 Change-Id: Ia78bc11dfa12f47d2d1bb90dc65372ddb17f7e14 --- aidl_library/aidl_library.go | 4 + aidl_library/aidl_library_test.go | 4 - bp2build/cc_library_conversion_test.go | 142 +++++++++++++++++++------ cc/Android.bp | 2 +- cc/bp2build.go | 102 ++++++++++-------- cc/cc.go | 27 +++++ cc/cc_test.go | 78 +++++++++++++- cc/compiler.go | 20 +++- cc/gen.go | 33 ++++-- cc/library.go | 2 +- 10 files changed, 324 insertions(+), 90 deletions(-) diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go index 9b5f0a814..8a84e6bfa 100644 --- a/aidl_library/aidl_library.go +++ b/aidl_library/aidl_library.go @@ -22,6 +22,10 @@ import ( "github.com/google/blueprint/proptools" ) +var PrepareForTestWithAidlLibrary = android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + registerAidlLibraryBuildComponents(ctx) +}) + func init() { registerAidlLibraryBuildComponents(android.InitRegistrationContext) } diff --git a/aidl_library/aidl_library_test.go b/aidl_library/aidl_library_test.go index 42fa5367e..d9b410acc 100644 --- a/aidl_library/aidl_library_test.go +++ b/aidl_library/aidl_library_test.go @@ -19,10 +19,6 @@ import ( "testing" ) -var PrepareForTestWithAidlLibrary = android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { - registerAidlLibraryBuildComponents(ctx) -}) - func TestAidlLibrary(t *testing.T) { t.Parallel() ctx := android.GroupFixturePreparers( diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 7165ac45b..bd8450fba 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -18,6 +18,7 @@ import ( "fmt" "testing" + "android/soong/aidl_library" "android/soong/android" "android/soong/cc" ) @@ -63,6 +64,7 @@ func registerCcLibraryModuleTypes(ctx android.RegistrationContext) { ctx.RegisterModuleType("cc_library_static", cc.LibraryStaticFactory) ctx.RegisterModuleType("cc_prebuilt_library_static", cc.PrebuiltStaticLibraryFactory) ctx.RegisterModuleType("cc_library_headers", cc.LibraryHeaderFactory) + ctx.RegisterModuleType("aidl_library", aidl_library.AidlLibraryFactory) } func TestCcLibrarySimple(t *testing.T) { @@ -3310,6 +3312,46 @@ func TestCcLibraryArchVariantSuffix(t *testing.T) { }) } +func TestCcLibraryWithAidlLibrary(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library with aidl_library", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: ` +aidl_library { + name: "A_aidl", + srcs: ["aidl/A.aidl"], + hdrs: ["aidl/Header.aidl"], + strip_import_prefix: "aidl", +} +cc_library { + name: "foo", + aidl: { + libs: ["A_aidl"], + } +}`, + ExpectedBazelTargets: []string{ + MakeBazelTargetNoRestrictions("aidl_library", "A_aidl", AttrNameToString{ + "srcs": `["aidl/A.aidl"]`, + "hdrs": `["aidl/Header.aidl"]`, + "strip_import_prefix": `"aidl"`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ + "deps": `[":A_aidl"]`, + }), + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, + "local_includes": `["."]`, + }), + }, + }) +} + func TestCcLibraryWithAidlSrcs(t *testing.T) { runCcLibraryTestCase(t, Bp2buildTestCase{ Description: "cc_library with aidl srcs", @@ -3392,37 +3434,77 @@ cc_library { } func TestCcLibraryWithExportAidlHeaders(t *testing.T) { - runCcLibraryTestCase(t, Bp2buildTestCase{ - Description: "cc_library with export aidl headers", - ModuleTypeUnderTest: "cc_library", - ModuleTypeUnderTestFactory: cc.LibraryFactory, - Blueprint: ` -cc_library { - name: "foo", - srcs: [ - "Foo.aidl", - ], - aidl: { - export_aidl_headers: true, - } -}`, - ExpectedBazelTargets: []string{ - MakeBazelTarget("aidl_library", "foo_aidl_library", AttrNameToString{ - "srcs": `["Foo.aidl"]`, - }), - MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `[":foo_aidl_library"]`, - }), - MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ - "whole_archive_deps": `[":foo_cc_aidl_library"]`, - "local_includes": `["."]`, - }), - MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ - "whole_archive_deps": `[":foo_cc_aidl_library"]`, - "local_includes": `["."]`, - }), + t.Parallel() + + expectedBazelTargets := []string{ + MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ + "deps": `[":foo_aidl_library"]`, + }), + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "whole_archive_deps": `[":foo_cc_aidl_library"]`, + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "whole_archive_deps": `[":foo_cc_aidl_library"]`, + "local_includes": `["."]`, + }), + } + testCases := []struct { + description string + bp string + expectedBazelTargets []string + }{ + { + description: "cc_library with aidl srcs and aidl.export_aidl_headers set", + bp: ` + cc_library { + name: "foo", + srcs: [ + "Foo.aidl", + ], + aidl: { + export_aidl_headers: true, + } + }`, + expectedBazelTargets: append( + expectedBazelTargets, + MakeBazelTarget("aidl_library", "foo_aidl_library", AttrNameToString{ + "srcs": `["Foo.aidl"]`, + })), }, - }) + { + description: "cc_library with aidl.libs and aidl.export_aidl_headers set", + bp: ` + aidl_library { + name: "foo_aidl_library", + srcs: ["Foo.aidl"], + } + cc_library { + name: "foo", + aidl: { + libs: ["foo_aidl_library"], + export_aidl_headers: true, + } + }`, + expectedBazelTargets: append( + expectedBazelTargets, + MakeBazelTargetNoRestrictions("aidl_library", "foo_aidl_library", AttrNameToString{ + "srcs": `["Foo.aidl"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + ), + }, + } + + for _, testCase := range testCases { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library with export aidl headers", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: testCase.bp, + ExpectedBazelTargets: testCase.expectedBazelTargets, + }) + } } func TestCcLibraryWithTargetApex(t *testing.T) { diff --git a/cc/Android.bp b/cc/Android.bp index be2cc5a34..f49dc1a9e 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -9,6 +9,7 @@ bootstrap_go_package { "blueprint", "blueprint-pathtools", "soong", + "soong-aidl-library", "soong-android", "soong-bazel", "soong-cc-config", @@ -22,7 +23,6 @@ bootstrap_go_package { srcs: [ "afdo.go", "fdo_profile.go", - "androidmk.go", "api_level.go", "bp2build.go", diff --git a/cc/bp2build.go b/cc/bp2build.go index 15feb9e1a..e82527706 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -717,6 +717,8 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) compilerAttrs := compilerAttributes{} linkerAttrs := linkerAttributes{} + var aidlLibs bazel.LabelList + // Iterate through these axes in a deterministic order. This is required // because processing certain dependencies may result in concatenating // elements along other axes. (For example, processing NoConfig may result @@ -732,6 +734,7 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) compilerAttrs.lexopts.SetSelectValue(axis, cfg, baseCompilerProps.Lex.Flags) } (&compilerAttrs).bp2buildForAxisAndConfig(ctx, axis, cfg, baseCompilerProps) + aidlLibs.Append(android.BazelLabelForModuleDeps(ctx, baseCompilerProps.Aidl.Libs)) } var exportHdrs []string @@ -804,7 +807,14 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) (&linkerAttrs).wholeArchiveDeps.Add(protoDep.wholeStaticLib) (&linkerAttrs).implementationWholeArchiveDeps.Add(protoDep.implementationWholeStaticLib) - aidlDep := bp2buildCcAidlLibrary(ctx, module, compilerAttrs.aidlSrcs, linkerAttrs) + aidlDep := bp2buildCcAidlLibrary( + ctx, module, + compilerAttrs.aidlSrcs, + bazel.LabelListAttribute{ + Value: aidlLibs, + }, + linkerAttrs, + ) if aidlDep != nil { if lib, ok := module.linker.(*libraryDecorator); ok { if proptools.Bool(lib.Properties.Aidl.Export_aidl_headers) { @@ -900,11 +910,15 @@ func bp2buildFdoProfile( func bp2buildCcAidlLibrary( ctx android.Bp2buildMutatorContext, m *Module, - aidlLabelList bazel.LabelListAttribute, + aidlSrcs bazel.LabelListAttribute, + aidlLibs bazel.LabelListAttribute, linkerAttrs linkerAttributes, ) *bazel.LabelAttribute { - if !aidlLabelList.IsEmpty() { - aidlLibs, aidlSrcs := aidlLabelList.Partition(func(src bazel.Label) bool { + var aidlLibsFromSrcs, aidlFiles bazel.LabelListAttribute + apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) + + if !aidlSrcs.IsEmpty() { + aidlLibsFromSrcs, aidlFiles = aidlSrcs.Partition(func(src bazel.Label) bool { if fg, ok := android.ToFileGroupAsLibrary(ctx, src.OriginalModuleName); ok && fg.ShouldConvertToAidlLibrary(ctx) { return true @@ -912,57 +926,61 @@ func bp2buildCcAidlLibrary( return false }) - apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) - sdkAttrs := bp2BuildParseSdkAttributes(m) - - if !aidlSrcs.IsEmpty() { + if !aidlFiles.IsEmpty() { aidlLibName := m.Name() + "_aidl_library" ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: "aidl_library", Bzl_load_location: "//build/bazel/rules/aidl:aidl_library.bzl", }, - android.CommonAttributes{Name: aidlLibName}, - &aidlLibraryAttributes{ - Srcs: aidlSrcs, + android.CommonAttributes{ + Name: aidlLibName, Tags: apexAvailableTags, }, - ) - aidlLibs.Add(&bazel.LabelAttribute{Value: &bazel.Label{Label: ":" + aidlLibName}}) - } - - if !aidlLibs.IsEmpty() { - ccAidlLibrarylabel := m.Name() + "_cc_aidl_library" - // Since parent cc_library already has these dependencies, we can add them as implementation - // deps so that they don't re-export - implementationDeps := linkerAttrs.deps.Clone() - implementationDeps.Append(linkerAttrs.implementationDeps) - implementationDynamicDeps := linkerAttrs.dynamicDeps.Clone() - implementationDynamicDeps.Append(linkerAttrs.implementationDynamicDeps) - - ctx.CreateBazelTargetModule( - bazel.BazelTargetModuleProperties{ - Rule_class: "cc_aidl_library", - Bzl_load_location: "//build/bazel/rules/cc:cc_aidl_library.bzl", - }, - android.CommonAttributes{Name: ccAidlLibrarylabel}, - &ccAidlLibraryAttributes{ - Deps: aidlLibs, - Implementation_deps: *implementationDeps, - Implementation_dynamic_deps: *implementationDynamicDeps, - Tags: apexAvailableTags, - sdkAttributes: sdkAttrs, + &aidlLibraryAttributes{ + Srcs: aidlFiles, }, ) - label := &bazel.LabelAttribute{ - Value: &bazel.Label{ - Label: ":" + ccAidlLibrarylabel, - }, - } - return label + aidlLibsFromSrcs.Add(&bazel.LabelAttribute{Value: &bazel.Label{Label: ":" + aidlLibName}}) } } + allAidlLibs := aidlLibs.Clone() + allAidlLibs.Append(aidlLibsFromSrcs) + + if !allAidlLibs.IsEmpty() { + ccAidlLibrarylabel := m.Name() + "_cc_aidl_library" + // Since parent cc_library already has these dependencies, we can add them as implementation + // deps so that they don't re-export + implementationDeps := linkerAttrs.deps.Clone() + implementationDeps.Append(linkerAttrs.implementationDeps) + implementationDynamicDeps := linkerAttrs.dynamicDeps.Clone() + implementationDynamicDeps.Append(linkerAttrs.implementationDynamicDeps) + + sdkAttrs := bp2BuildParseSdkAttributes(m) + + ctx.CreateBazelTargetModule( + bazel.BazelTargetModuleProperties{ + Rule_class: "cc_aidl_library", + Bzl_load_location: "//build/bazel/rules/cc:cc_aidl_library.bzl", + }, + android.CommonAttributes{Name: ccAidlLibrarylabel}, + &ccAidlLibraryAttributes{ + Deps: *allAidlLibs, + Implementation_deps: *implementationDeps, + Implementation_dynamic_deps: *implementationDynamicDeps, + Tags: apexAvailableTags, + sdkAttributes: sdkAttrs, + }, + ) + label := &bazel.LabelAttribute{ + Value: &bazel.Label{ + Label: ":" + ccAidlLibrarylabel, + }, + } + return label + } + return nil } diff --git a/cc/cc.go b/cc/cc.go index 0addb60b4..307441d32 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -27,6 +27,7 @@ import ( "github.com/google/blueprint" "github.com/google/blueprint/proptools" + "android/soong/aidl_library" "android/soong/android" "android/soong/bazel/cquery" "android/soong/cc/config" @@ -110,6 +111,9 @@ type Deps struct { // Used by DepsMutator to pass system_shared_libs information to check_elf_file.py. SystemSharedLibs []string + // Used by DepMutator to pass aidl_library modules to aidl compiler + AidlLibs []string + // If true, statically link the unwinder into native libraries/binaries. StaticUnwinderIfLegacy bool @@ -182,6 +186,9 @@ type PathDeps struct { // For Darwin builds, the path to the second architecture's output that should // be combined with this architectures's output into a FAT MachO file. DarwinSecondArchOutput android.OptionalPath + + // Paths to direct srcs and transitive include dirs from direct aidl_library deps + AidlLibraryInfos []aidl_library.AidlLibraryInfo } // LocalOrGlobalFlags contains flags that need to have values set globally by the build system or locally by the module @@ -765,6 +772,7 @@ var ( stubImplDepTag = dependencyTag{name: "stub_impl"} JniFuzzLibTag = dependencyTag{name: "jni_fuzz_lib_tag"} FdoProfileTag = dependencyTag{name: "fdo_profile"} + aidlLibraryTag = dependencyTag{name: "aidl_library"} ) func IsSharedDepTag(depTag blueprint.DependencyTag) bool { @@ -2685,6 +2693,14 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { } } + if len(deps.AidlLibs) > 0 { + actx.AddDependency( + c, + aidlLibraryTag, + deps.AidlLibs..., + ) + } + updateImportedLibraryDependency(ctx) } @@ -2989,6 +3005,17 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { return } + if depTag == aidlLibraryTag { + if ctx.OtherModuleHasProvider(dep, aidl_library.AidlLibraryProvider) { + depPaths.AidlLibraryInfos = append( + depPaths.AidlLibraryInfos, + ctx.OtherModuleProvider( + dep, + aidl_library.AidlLibraryProvider).(aidl_library.AidlLibraryInfo), + ) + } + } + ccDep, ok := dep.(LinkableInterface) if !ok { diff --git a/cc/cc_test.go b/cc/cc_test.go index 3ae4b1561..2fd012170 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -24,6 +24,7 @@ import ( "strings" "testing" + "android/soong/aidl_library" "android/soong/android" "android/soong/bazel/cquery" ) @@ -4419,9 +4420,65 @@ func TestStubsLibReexportsHeaders(t *testing.T) { } } +func TestAidlLibraryWithHeader(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary, + android.MockFS{ + "package_bar/Android.bp": []byte(` + aidl_library { + name: "bar", + srcs: ["x/y/Bar.aidl"], + strip_import_prefix: "x", + } + `)}.AddToFixture(), + android.MockFS{ + "package_foo/Android.bp": []byte(` + aidl_library { + name: "foo", + srcs: ["a/b/Foo.aidl"], + strip_import_prefix: "a", + deps: ["bar"], + } + cc_library { + name: "libfoo", + aidl: { + libs: ["foo"], + } + } + `), + }.AddToFixture(), + ).RunTest(t).TestContext + + libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static") + manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl.sbox.textproto")) + aidlCommand := manifest.Commands[0].GetCommand() + + expectedAidlFlags := "-Ipackage_foo/a -Ipackage_bar/x" + if !strings.Contains(aidlCommand, expectedAidlFlags) { + t.Errorf("aidl command %q does not contain %q", aidlCommand, expectedAidlFlags) + } + + outputs := strings.Join(libfoo.AllOutputs(), " ") + + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BpFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BnFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/Foo.h") + android.AssertStringDoesContain(t, "aidl-generated cpp", outputs, "b/Foo.cpp") + // Confirm that the aidl header doesn't get compiled to cpp and h files + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BpBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BnBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/Bar.h") + android.AssertStringDoesNotContain(t, "aidl-generated cpp", outputs, "y/Bar.cpp") +} + func TestAidlFlagsPassedToTheAidlCompiler(t *testing.T) { t.Parallel() - ctx := testCc(t, ` + ctx := android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary, + ).RunTestWithBp(t, ` cc_library { name: "libfoo", srcs: ["a/Foo.aidl"], @@ -4706,7 +4763,15 @@ func TestIncludeDirsExporting(t *testing.T) { }) t.Run("ensure only aidl headers are exported", func(t *testing.T) { - ctx := testCc(t, genRuleModules+` + ctx := android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary, + ).RunTestWithBp(t, ` + aidl_library { + name: "libfoo_aidl", + srcs: ["x/y/Bar.aidl"], + strip_import_prefix: "x", + } cc_library_shared { name: "libfoo", srcs: [ @@ -4715,10 +4780,11 @@ func TestIncludeDirsExporting(t *testing.T) { "a.proto", ], aidl: { + libs: ["libfoo_aidl"], export_aidl_headers: true, } } - `) + `).TestContext foo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_shared").Module() checkIncludeDirs(t, ctx, foo, expectedIncludeDirs(` @@ -4729,11 +4795,17 @@ func TestIncludeDirsExporting(t *testing.T) { .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h `), expectedOrderOnlyDeps(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h `), ) }) diff --git a/cc/compiler.go b/cc/compiler.go index 88985b6f9..5da745e02 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -120,6 +120,9 @@ type BaseCompilerProperties struct { Lex *LexProperties Aidl struct { + // List of aidl_library modules + Libs []string + // list of directories that will be added to the aidl include paths. Include_dirs []string @@ -272,6 +275,7 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { deps.GeneratedSources = append(deps.GeneratedSources, compiler.Properties.Generated_sources...) deps.GeneratedSources = removeListFromList(deps.GeneratedSources, compiler.Properties.Exclude_generated_sources) deps.GeneratedHeaders = append(deps.GeneratedHeaders, compiler.Properties.Generated_headers...) + deps.AidlLibs = append(deps.AidlLibs, compiler.Properties.Aidl.Libs...) android.ProtoDeps(ctx, &compiler.Proto) if compiler.hasSrcExt(".proto") { @@ -561,7 +565,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps "-I"+android.PathForModuleGen(ctx, "yacc", ctx.ModuleDir()).String()) } - if compiler.hasSrcExt(".aidl") { + if compiler.hasAidl(deps) { flags.aidlFlags = append(flags.aidlFlags, compiler.Properties.Aidl.Flags...) if len(compiler.Properties.Aidl.Local_include_dirs) > 0 { localAidlIncludeDirs := android.PathsForModuleSrc(ctx, compiler.Properties.Aidl.Local_include_dirs) @@ -572,6 +576,14 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps flags.aidlFlags = append(flags.aidlFlags, includeDirsToFlags(rootAidlIncludeDirs)) } + var rootAidlIncludeDirs android.Paths + for _, aidlLibraryInfo := range deps.AidlLibraryInfos { + rootAidlIncludeDirs = append(rootAidlIncludeDirs, aidlLibraryInfo.IncludeDirs.ToList()...) + } + if len(rootAidlIncludeDirs) > 0 { + flags.aidlFlags = append(flags.aidlFlags, includeDirsToFlags(rootAidlIncludeDirs)) + } + if proptools.BoolDefault(compiler.Properties.Aidl.Generate_traces, true) { flags.aidlFlags = append(flags.aidlFlags, "-t") } @@ -660,6 +672,10 @@ func ndkPathDeps(ctx ModuleContext) android.Paths { return nil } +func (compiler *baseCompiler) hasAidl(deps PathDeps) bool { + return len(deps.AidlLibraryInfos) > 0 || compiler.hasSrcExt(".aidl") +} + func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { pathDeps := deps.GeneratedDeps pathDeps = append(pathDeps, ndkPathDeps(ctx)...) @@ -668,7 +684,7 @@ func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathD srcs := append(android.Paths(nil), compiler.srcsBeforeGen...) - srcs, genDeps, info := genSources(ctx, srcs, buildFlags) + srcs, genDeps, info := genSources(ctx, deps.AidlLibraryInfos, srcs, buildFlags) pathDeps = append(pathDeps, genDeps...) compiler.pathDeps = pathDeps diff --git a/cc/gen.go b/cc/gen.go index dfbb177e6..dbb9560ba 100644 --- a/cc/gen.go +++ b/cc/gen.go @@ -18,7 +18,9 @@ import ( "path/filepath" "strings" + "android/soong/aidl_library" "android/soong/bazel" + "github.com/google/blueprint" "android/soong/android" @@ -124,11 +126,6 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr headerBn := outDir.Join(ctx, aidlPackage, "Bn"+shortName+".h") headerBp := outDir.Join(ctx, aidlPackage, "Bp"+shortName+".h") - baseDir := strings.TrimSuffix(aidlFile.String(), aidlFile.Rel()) - if baseDir != "" { - aidlFlags += " -I" + baseDir - } - cmd := rule.Command() cmd.BuiltTool("aidl-cpp"). FlagWithDepFile("-d", depFile). @@ -282,7 +279,10 @@ type generatedSourceInfo struct { syspropOrderOnlyDeps android.Paths } -func genSources(ctx android.ModuleContext, srcFiles android.Paths, +func genSources( + ctx android.ModuleContext, + aidlLibraryInfos []aidl_library.AidlLibraryInfo, + srcFiles android.Paths, buildFlags builderFlags) (android.Paths, android.Paths, generatedSourceInfo) { var info generatedSourceInfo @@ -330,7 +330,8 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths, aidlRule = android.NewRuleBuilder(pctx, ctx).Sbox(android.PathForModuleGen(ctx, "aidl"), android.PathForModuleGen(ctx, "aidl.sbox.textproto")) } - cppFile, aidlHeaders := genAidl(ctx, aidlRule, srcFile, buildFlags.aidlFlags) + baseDir := strings.TrimSuffix(srcFile.String(), srcFile.Rel()) + cppFile, aidlHeaders := genAidl(ctx, aidlRule, srcFile, buildFlags.aidlFlags+" -I"+baseDir) srcFiles[i] = cppFile info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) @@ -352,6 +353,24 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths, } } + for _, aidlLibraryInfo := range aidlLibraryInfos { + for _, aidlSrc := range aidlLibraryInfo.Srcs { + if aidlRule == nil { + // TODO(b/279960133): Sandbox inputs to ensure aidl headers are explicitly specified + aidlRule = android.NewRuleBuilder(pctx, ctx).Sbox(android.PathForModuleGen(ctx, "aidl"), + android.PathForModuleGen(ctx, "aidl.sbox.textproto")) + } + cppFile, aidlHeaders := genAidl(ctx, aidlRule, aidlSrc, buildFlags.aidlFlags) + + srcFiles = append(srcFiles, cppFile) + info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) + // Use the generated headers as order only deps to ensure that they are up to date when + // needed. + // TODO: Reduce the size of the ninja file by using one order only dep for the whole rule + info.aidlOrderOnlyDeps = append(info.aidlOrderOnlyDeps, aidlHeaders...) + } + } + if aidlRule != nil { aidlRule.Build("aidl", "gen aidl") } diff --git a/cc/library.go b/cc/library.go index d11887b10..542e67515 100644 --- a/cc/library.go +++ b/cc/library.go @@ -2109,7 +2109,7 @@ func (library *libraryDecorator) link(ctx ModuleContext, // Optionally export aidl headers. if Bool(library.Properties.Aidl.Export_aidl_headers) { - if library.baseCompiler.hasSrcExt(".aidl") { + if library.baseCompiler.hasAidl(deps) { dir := android.PathForModuleGen(ctx, "aidl") library.reexportDirs(dir) From 732cf871fd5f7ce2fabce779cddca6df4cd775ae Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 22/80] Pass includes attrs to cc_aidl_library https://source.android.com/docs/core/architecture/aidl/aidl-backends#custom-parcelables Some aidl files have custom parcelables with references to cpp headers. Hence, we need to pass the include dirs from cc libs to cc_aidl_library so that custom cpp headers are findable by the generated cpp/h files. Test: go test Bug: 278704136 (cherry picked from https://android-review.googlesource.com/q/commit:e684294fb2f0407cd2e0cbfd8663542d3185b570) Merged-In: I4ede1d23fbc04bdbfb69823d955c6df6b0a7b4ee Change-Id: I4ede1d23fbc04bdbfb69823d955c6df6b0a7b4ee --- bp2build/cc_library_conversion_test.go | 19 ++++-- cc/bp2build.go | 12 ++++ cc/cc_test.go | 1 + cc/library.go | 93 +++++++++++++------------- 4 files changed, 75 insertions(+), 50 deletions(-) diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index bd8450fba..86aa4aa66 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -3328,7 +3328,8 @@ cc_library { name: "foo", aidl: { libs: ["A_aidl"], - } + }, + export_include_dirs: ["include"], }`, ExpectedBazelTargets: []string{ MakeBazelTargetNoRestrictions("aidl_library", "A_aidl", AttrNameToString{ @@ -3338,15 +3339,19 @@ cc_library { "tags": `["apex_available=//apex_available:anyapex"]`, }), MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `[":A_aidl"]`, + "deps": `[":A_aidl"]`, + "local_includes": `["."]`, + "export_includes": `["include"]`, }), MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, "local_includes": `["."]`, + "export_includes": `["include"]`, }), MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, "local_includes": `["."]`, + "export_includes": `["include"]`, }), }, }) @@ -3380,6 +3385,7 @@ cc_library { "srcs": `["B.aidl"]`, }), MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ + "local_includes": `["."]`, "deps": `[ ":A_aidl", ":foo_aidl_library", @@ -3419,7 +3425,8 @@ cc_library { }`, ExpectedBazelTargets: []string{ MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `["//path/to/A:A_aidl"]`, + "local_includes": `["."]`, + "deps": `["//path/to/A:A_aidl"]`, }), MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, @@ -3438,7 +3445,8 @@ func TestCcLibraryWithExportAidlHeaders(t *testing.T) { expectedBazelTargets := []string{ MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `[":foo_aidl_library"]`, + "local_includes": `["."]`, + "deps": `[":foo_aidl_library"]`, }), MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ "whole_archive_deps": `[":foo_cc_aidl_library"]`, @@ -3719,7 +3727,8 @@ cc_library_static { "srcs": `["Foo.aidl"]`, }), MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `[":foo_aidl_library"]`, + "local_includes": `["."]`, + "deps": `[":foo_aidl_library"]`, "implementation_deps": `[ ":baz-static", ":bar-static", diff --git a/cc/bp2build.go b/cc/bp2build.go index e82527706..5e1c2d3cf 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -814,6 +814,7 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) Value: aidlLibs, }, linkerAttrs, + compilerAttrs, ) if aidlDep != nil { if lib, ok := module.linker.(*libraryDecorator); ok { @@ -913,6 +914,7 @@ func bp2buildCcAidlLibrary( aidlSrcs bazel.LabelListAttribute, aidlLibs bazel.LabelListAttribute, linkerAttrs linkerAttributes, + compilerAttrs compilerAttributes, ) *bazel.LabelAttribute { var aidlLibsFromSrcs, aidlFiles bazel.LabelListAttribute apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) @@ -959,6 +961,15 @@ func bp2buildCcAidlLibrary( sdkAttrs := bp2BuildParseSdkAttributes(m) + exportedIncludes := bp2BuildParseExportedIncludes(ctx, m, &compilerAttrs.includes) + includeAttrs := includesAttributes{ + Export_includes: exportedIncludes.Includes, + Export_absolute_includes: exportedIncludes.AbsoluteIncludes, + Export_system_includes: exportedIncludes.SystemIncludes, + Local_includes: compilerAttrs.localIncludes, + Absolute_includes: compilerAttrs.absoluteIncludes, + } + ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: "cc_aidl_library", @@ -971,6 +982,7 @@ func bp2buildCcAidlLibrary( Implementation_dynamic_deps: *implementationDynamicDeps, Tags: apexAvailableTags, sdkAttributes: sdkAttrs, + includesAttributes: includeAttrs, }, ) label := &bazel.LabelAttribute{ diff --git a/cc/cc_test.go b/cc/cc_test.go index 2fd012170..81212c84b 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -39,6 +39,7 @@ func TestMain(m *testing.M) { var prepareForCcTest = android.GroupFixturePreparers( PrepareForTestWithCcIncludeVndk, + aidl_library.PrepareForTestWithAidlLibrary, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { variables.DeviceVndkVersion = StringPtr("current") variables.ProductVndkVersion = StringPtr("current") diff --git a/cc/library.go b/cc/library.go index 542e67515..313a1ee93 100644 --- a/cc/library.go +++ b/cc/library.go @@ -271,7 +271,9 @@ type ccAidlLibraryAttributes struct { Implementation_deps bazel.LabelListAttribute Implementation_dynamic_deps bazel.LabelListAttribute Tags bazel.StringListAttribute + sdkAttributes + includesAttributes } type stripAttributes struct { @@ -330,6 +332,14 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { Native_coverage: baseAttributes.Native_coverage, } + includeAttrs := includesAttributes{ + Export_includes: exportedIncludes.Includes, + Export_absolute_includes: exportedIncludes.AbsoluteIncludes, + Export_system_includes: exportedIncludes.SystemIncludes, + Local_includes: compilerAttrs.localIncludes, + Absolute_includes: compilerAttrs.absoluteIncludes, + } + sharedCommonAttrs := staticOrSharedAttributes{ Srcs: *srcs.Clone().Append(sharedAttrs.Srcs), Srcs_c: *compilerAttrs.cSrcs.Clone().Append(sharedAttrs.Srcs_c), @@ -351,41 +361,34 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { staticTargetAttrs := &bazelCcLibraryStaticAttributes{ staticOrSharedAttributes: staticCommonAttrs, + includesAttributes: includeAttrs, Cppflags: compilerAttrs.cppFlags, Conlyflags: compilerAttrs.conlyFlags, Asflags: asFlags, - Export_includes: exportedIncludes.Includes, - Export_absolute_includes: exportedIncludes.AbsoluteIncludes, - Export_system_includes: exportedIncludes.SystemIncludes, - Local_includes: compilerAttrs.localIncludes, - Absolute_includes: compilerAttrs.absoluteIncludes, - Rtti: compilerAttrs.rtti, - Stl: compilerAttrs.stl, - Cpp_std: compilerAttrs.cppStd, - C_std: compilerAttrs.cStd, + Rtti: compilerAttrs.rtti, + Stl: compilerAttrs.stl, + Cpp_std: compilerAttrs.cppStd, + C_std: compilerAttrs.cStd, Features: baseAttributes.features, } sharedTargetAttrs := &bazelCcLibrarySharedAttributes{ staticOrSharedAttributes: sharedCommonAttrs, - Cppflags: compilerAttrs.cppFlags, - Conlyflags: compilerAttrs.conlyFlags, - Asflags: asFlags, + includesAttributes: includeAttrs, - Export_includes: exportedIncludes.Includes, - Export_absolute_includes: exportedIncludes.AbsoluteIncludes, - Export_system_includes: exportedIncludes.SystemIncludes, - Local_includes: compilerAttrs.localIncludes, - Absolute_includes: compilerAttrs.absoluteIncludes, - Linkopts: linkerAttrs.linkopts, - Rtti: compilerAttrs.rtti, - Stl: compilerAttrs.stl, - Cpp_std: compilerAttrs.cppStd, - C_std: compilerAttrs.cStd, - Use_version_lib: linkerAttrs.useVersionLib, + Cppflags: compilerAttrs.cppFlags, + Conlyflags: compilerAttrs.conlyFlags, + Asflags: asFlags, + + Linkopts: linkerAttrs.linkopts, + Rtti: compilerAttrs.rtti, + Stl: compilerAttrs.stl, + Cpp_std: compilerAttrs.cppStd, + C_std: compilerAttrs.cStd, + Use_version_lib: linkerAttrs.useVersionLib, Additional_linker_inputs: linkerAttrs.additionalLinkerInputs, @@ -2875,6 +2878,13 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo linkerAttrs := baseAttributes.linkerAttributes exportedIncludes := bp2BuildParseExportedIncludes(ctx, module, &compilerAttrs.includes) + includeAttrs := includesAttributes{ + Export_includes: exportedIncludes.Includes, + Export_absolute_includes: exportedIncludes.AbsoluteIncludes, + Export_system_includes: exportedIncludes.SystemIncludes, + Local_includes: compilerAttrs.localIncludes, + Absolute_includes: compilerAttrs.absoluteIncludes, + } // Append shared/static{} stanza properties. These won't be specified on // cc_library_* itself, but may be specified in cc_defaults that this module @@ -2929,11 +2939,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Cpp_std: compilerAttrs.cppStd, C_std: compilerAttrs.cStd, - Export_includes: exportedIncludes.Includes, - Export_absolute_includes: exportedIncludes.AbsoluteIncludes, - Export_system_includes: exportedIncludes.SystemIncludes, - Local_includes: compilerAttrs.localIncludes, - Absolute_includes: compilerAttrs.absoluteIncludes, + includesAttributes: includeAttrs, Cppflags: compilerAttrs.cppFlags, Conlyflags: compilerAttrs.conlyFlags, @@ -2959,11 +2965,8 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Cpp_std: compilerAttrs.cppStd, C_std: compilerAttrs.cStd, - Export_includes: exportedIncludes.Includes, - Export_absolute_includes: exportedIncludes.AbsoluteIncludes, - Export_system_includes: exportedIncludes.SystemIncludes, - Local_includes: compilerAttrs.localIncludes, - Absolute_includes: compilerAttrs.absoluteIncludes, + includesAttributes: includeAttrs, + Additional_linker_inputs: linkerAttrs.additionalLinkerInputs, Strip: stripAttrsFromLinkerAttrs(&linkerAttrs), @@ -2999,9 +3002,18 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name(), Tags: tags}, attrs) } +type includesAttributes struct { + Export_includes bazel.StringListAttribute + Export_absolute_includes bazel.StringListAttribute + Export_system_includes bazel.StringListAttribute + Local_includes bazel.StringListAttribute + Absolute_includes bazel.StringListAttribute +} + // TODO(b/199902614): Can this be factored to share with the other Attributes? type bazelCcLibraryStaticAttributes struct { staticOrSharedAttributes + includesAttributes Use_version_lib bazel.BoolAttribute Rtti bazel.BoolAttribute @@ -3009,12 +3021,7 @@ type bazelCcLibraryStaticAttributes struct { Cpp_std *string C_std *string - Export_includes bazel.StringListAttribute - Export_absolute_includes bazel.StringListAttribute - Export_system_includes bazel.StringListAttribute - Local_includes bazel.StringListAttribute - Absolute_includes bazel.StringListAttribute - Hdrs bazel.LabelListAttribute + Hdrs bazel.LabelListAttribute Cppflags bazel.StringListAttribute Conlyflags bazel.StringListAttribute @@ -3026,6 +3033,7 @@ type bazelCcLibraryStaticAttributes struct { // TODO(b/199902614): Can this be factored to share with the other Attributes? type bazelCcLibrarySharedAttributes struct { staticOrSharedAttributes + includesAttributes Linkopts bazel.StringListAttribute Use_version_lib bazel.BoolAttribute @@ -3035,12 +3043,7 @@ type bazelCcLibrarySharedAttributes struct { Cpp_std *string C_std *string - Export_includes bazel.StringListAttribute - Export_absolute_includes bazel.StringListAttribute - Export_system_includes bazel.StringListAttribute - Local_includes bazel.StringListAttribute - Absolute_includes bazel.StringListAttribute - Hdrs bazel.LabelListAttribute + Hdrs bazel.LabelListAttribute Strip stripAttributes Additional_linker_inputs bazel.LabelListAttribute From 6d16b8894ce2acd4b4c3e9dc328b4d3cbc41d71f Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 23/80] Allowlist libservices Test: CI (cherry picked from https://android-review.googlesource.com/q/commit:320aecfbc81a545d3a7c5f84b74118cef29b5cbc) Merged-In: I6e7a969228ca491344520bba0d66159d5060e996 Change-Id: I6e7a969228ca491344520bba0d66159d5060e996 --- android/allowlists/allowlists.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 63d4e11e0..b9bbcdcd0 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -201,6 +201,7 @@ var ( "frameworks/av/media/module/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/av/services/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/base/libs/androidfw": Bp2BuildDefaultTrue, + "frameworks/base/libs/services": Bp2BuildDefaultTrue, "frameworks/base/media/tests/MediaDump": Bp2BuildDefaultTrue, "frameworks/base/services/tests/servicestests/aidl": Bp2BuildDefaultTrue, "frameworks/base/proto": Bp2BuildDefaultTrue, @@ -559,6 +560,9 @@ var ( //external/fec "libfec_rs", + //frameworks/base/core/java + "IDropBoxManagerService_aidl", + //system/core/libsparse "libsparse", From 0358fb73e8970dba08bef74d7f55f736436b86de Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Tue, 16 May 2023 16:03:20 -0400 Subject: [PATCH 24/80] Sandbox inputs to aidl rule in cc Bug: 279960133 Test: go test Test: Remove hdrs prop from IDropBoxManagerService_aidl && run BUILD_BROKEN_DISABLE_BAZEL=true m libservices && Expect an error from aidl (cherry picked from https://android-review.googlesource.com/q/commit:095819530af067884e1b131d9332c8c4ace31295) Merged-In: Ifdb260d8e2da9a5767f1e212393de4134b210616 Change-Id: Ifdb260d8e2da9a5767f1e212393de4134b210616 --- aidl_library/aidl_library.go | 21 +++++-- aidl_library/aidl_library_test.go | 18 +++++- android/config.go | 8 +++ cc/cc_test.go | 94 ++++++++++++++++++++++++++----- cc/compiler.go | 15 ++++- cc/gen.go | 57 +++++++++++++++---- cc/library.go | 10 +++- 7 files changed, 188 insertions(+), 35 deletions(-) diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go index 8a84e6bfa..598510386 100644 --- a/aidl_library/aidl_library.go +++ b/aidl_library/aidl_library.go @@ -107,8 +107,10 @@ func (lib *AidlLibrary) ConvertWithBp2build(ctx android.TopDownMutatorContext) { type AidlLibraryInfo struct { // The direct aidl files of the module Srcs android.Paths - // The include dirs to the direct aidl files and those provided from aidl_library deps + // The include dirs to the direct aidl files and those provided from transitive aidl_library deps IncludeDirs android.DepSet + // The direct hdrs and hdrs from transitive deps + Hdrs android.DepSet } // AidlLibraryProvider provides the srcs and the transitive include dirs @@ -116,37 +118,48 @@ var AidlLibraryProvider = blueprint.NewProvider(AidlLibraryInfo{}) func (lib *AidlLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { includeDirsDepSetBuilder := android.NewDepSetBuilder(android.PREORDER) + hdrsDepSetBuilder := android.NewDepSetBuilder(android.PREORDER) if len(lib.properties.Srcs) == 0 && len(lib.properties.Hdrs) == 0 { ctx.ModuleErrorf("at least srcs or hdrs prop must be non-empty") } srcs := android.PathsForModuleSrc(ctx, lib.properties.Srcs) + hdrs := android.PathsForModuleSrc(ctx, lib.properties.Hdrs) + if lib.properties.Strip_import_prefix != nil { srcs = android.PathsWithModuleSrcSubDir( ctx, srcs, - android.String(lib.properties.Strip_import_prefix)) + android.String(lib.properties.Strip_import_prefix), + ) + + hdrs = android.PathsWithModuleSrcSubDir( + ctx, + hdrs, + android.String(lib.properties.Strip_import_prefix), + ) } + hdrsDepSetBuilder.Direct(hdrs...) includeDir := android.PathForModuleSrc( ctx, proptools.StringDefault(lib.properties.Strip_import_prefix, ""), ) - includeDirsDepSetBuilder.Direct(includeDir) for _, dep := range ctx.GetDirectDepsWithTag(aidlLibraryTag) { if ctx.OtherModuleHasProvider(dep, AidlLibraryProvider) { info := ctx.OtherModuleProvider(dep, AidlLibraryProvider).(AidlLibraryInfo) includeDirsDepSetBuilder.Transitive(&info.IncludeDirs) + hdrsDepSetBuilder.Transitive(&info.Hdrs) } } - // TODO(b/279960133) Propagate direct and transitive headers/srcs when aidl action sandboxes inputs ctx.SetProvider(AidlLibraryProvider, AidlLibraryInfo{ Srcs: srcs, IncludeDirs: *includeDirsDepSetBuilder.Build(), + Hdrs: *hdrsDepSetBuilder.Build(), }) } diff --git a/aidl_library/aidl_library_test.go b/aidl_library/aidl_library_test.go index d9b410acc..d9dd24515 100644 --- a/aidl_library/aidl_library_test.go +++ b/aidl_library/aidl_library_test.go @@ -37,7 +37,7 @@ func TestAidlLibrary(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], - hdrs: ["Header.aidl"], + hdrs: ["a/Header.aidl"], strip_import_prefix: "a", deps: ["bar"], } @@ -61,6 +61,13 @@ func TestAidlLibrary(t *testing.T) { []string{"package_foo/a/b/Foo.aidl"}, actualInfo.Srcs, ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl hdrs paths", + []string{"package_foo/a/Header.aidl"}, + actualInfo.Hdrs.ToList(), + ) } func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { @@ -72,6 +79,7 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { aidl_library { name: "bar", srcs: ["x/y/Bar.aidl"], + hdrs: ["BarHeader.aidl"], } `), }.AddToFixture(), @@ -80,7 +88,6 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], - hdrs: ["Header.aidl"], deps: ["bar"], } `), @@ -103,6 +110,13 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { []string{"package_foo/a/b/Foo.aidl"}, actualInfo.Srcs, ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl hdrs paths", + []string{"package_bar/BarHeader.aidl"}, + actualInfo.Hdrs.ToList(), + ) } func TestAidlLibraryWithNoSrcsHdrsDeps(t *testing.T) { diff --git a/android/config.go b/android/config.go index 980460a7b..dfbb46fb7 100644 --- a/android/config.go +++ b/android/config.go @@ -702,6 +702,14 @@ func (c *config) HostJavaToolPath(ctx PathContext, tool string) Path { return path } +func (c *config) HostCcSharedLibPath(ctx PathContext, lib string) Path { + libDir := "lib" + if ctx.Config().BuildArch.Multilib == "lib64" { + libDir = "lib64" + } + return pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, libDir, false, lib+".so") +} + // PrebuiltOS returns the name of the host OS used in prebuilts directories. func (c *config) PrebuiltOS() string { switch runtime.GOOS { diff --git a/cc/cc_test.go b/cc/cc_test.go index 81212c84b..d6b969a3b 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -39,7 +39,6 @@ func TestMain(m *testing.M) { var prepareForCcTest = android.GroupFixturePreparers( PrepareForTestWithCcIncludeVndk, - aidl_library.PrepareForTestWithAidlLibrary, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { variables.DeviceVndkVersion = StringPtr("current") variables.ProductVndkVersion = StringPtr("current") @@ -4421,7 +4420,7 @@ func TestStubsLibReexportsHeaders(t *testing.T) { } } -func TestAidlLibraryWithHeader(t *testing.T) { +func TestAidlLibraryWithHeaders(t *testing.T) { t.Parallel() ctx := android.GroupFixturePreparers( prepareForCcTest, @@ -4431,6 +4430,7 @@ func TestAidlLibraryWithHeader(t *testing.T) { aidl_library { name: "bar", srcs: ["x/y/Bar.aidl"], + hdrs: ["x/HeaderBar.aidl"], strip_import_prefix: "x", } `)}.AddToFixture(), @@ -4439,6 +4439,7 @@ func TestAidlLibraryWithHeader(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], + hdrs: ["a/HeaderFoo.aidl"], strip_import_prefix: "a", deps: ["bar"], } @@ -4453,7 +4454,20 @@ func TestAidlLibraryWithHeader(t *testing.T) { ).RunTest(t).TestContext libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static") - manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl.sbox.textproto")) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl headers", + []string{ + "package_bar/x/HeaderBar.aidl", + "package_foo/a/HeaderFoo.aidl", + "package_foo/a/b/Foo.aidl", + "out/soong/.intermediates/package_foo/libfoo/android_arm64_armv8-a_static/gen/aidl_library.sbox.textproto", + }, + libfoo.Rule("aidl_library").Implicits, + ) + + manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl_library.sbox.textproto")) aidlCommand := manifest.Commands[0].GetCommand() expectedAidlFlags := "-Ipackage_foo/a -Ipackage_bar/x" @@ -4463,14 +4477,14 @@ func TestAidlLibraryWithHeader(t *testing.T) { outputs := strings.Join(libfoo.AllOutputs(), " ") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BpFoo.h") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BnFoo.h") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/Foo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/BpFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/BnFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/Foo.h") android.AssertStringDoesContain(t, "aidl-generated cpp", outputs, "b/Foo.cpp") // Confirm that the aidl header doesn't get compiled to cpp and h files - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BpBar.h") - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BnBar.h") - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/Bar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/BpBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/BnBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/Bar.h") android.AssertStringDoesNotContain(t, "aidl-generated cpp", outputs, "y/Bar.cpp") } @@ -4549,6 +4563,55 @@ func TestAidlFlagsWithMinSdkVersion(t *testing.T) { } } +func TestInvalidAidlProp(t *testing.T) { + t.Parallel() + + testCases := []struct { + description string + bp string + }{ + { + description: "Invalid use of aidl.libs and aidl.include_dirs", + bp: ` + cc_library { + name: "foo", + aidl: { + libs: ["foo_aidl"], + include_dirs: ["bar/include"], + } + } + `, + }, + { + description: "Invalid use of aidl.libs and aidl.local_include_dirs", + bp: ` + cc_library { + name: "foo", + aidl: { + libs: ["foo_aidl"], + local_include_dirs: ["include"], + } + } + `, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + bp := ` + aidl_library { + name: "foo_aidl", + srcs: ["Foo.aidl"], + } ` + testCase.bp + android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary. + ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern("For aidl headers, please only use aidl.libs prop")), + ).RunTestWithBp(t, bp) + }) + } +} + func TestMinSdkVersionInClangTriple(t *testing.T) { t.Parallel() ctx := testCc(t, ` @@ -4790,23 +4853,24 @@ func TestIncludeDirsExporting(t *testing.T) { checkIncludeDirs(t, ctx, foo, expectedIncludeDirs(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library `), expectedSystemIncludeDirs(``), expectedGeneratedHeaders(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BpBar.h `), expectedOrderOnlyDeps(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BpBar.h `), ) }) diff --git a/cc/compiler.go b/cc/compiler.go index 5da745e02..16f4a6ec0 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -565,6 +565,11 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps "-I"+android.PathForModuleGen(ctx, "yacc", ctx.ModuleDir()).String()) } + if len(compiler.Properties.Aidl.Libs) > 0 && + (len(compiler.Properties.Aidl.Include_dirs) > 0 || len(compiler.Properties.Aidl.Local_include_dirs) > 0) { + ctx.ModuleErrorf("aidl.libs and (aidl.include_dirs or aidl.local_include_dirs) can't be set at the same time. For aidl headers, please only use aidl.libs prop") + } + if compiler.hasAidl(deps) { flags.aidlFlags = append(flags.aidlFlags, compiler.Properties.Aidl.Flags...) if len(compiler.Properties.Aidl.Local_include_dirs) > 0 { @@ -594,8 +599,14 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps } flags.aidlFlags = append(flags.aidlFlags, "--min_sdk_version="+aidlMinSdkVersion) - flags.Local.CommonFlags = append(flags.Local.CommonFlags, - "-I"+android.PathForModuleGen(ctx, "aidl").String()) + if compiler.hasSrcExt(".aidl") { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, + "-I"+android.PathForModuleGen(ctx, "aidl").String()) + } + if len(deps.AidlLibraryInfos) > 0 { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, + "-I"+android.PathForModuleGen(ctx, "aidl_library").String()) + } } if compiler.hasSrcExt(".rscript") || compiler.hasSrcExt(".fs") { diff --git a/cc/gen.go b/cc/gen.go index dbb9560ba..b15f16496 100644 --- a/cc/gen.go +++ b/cc/gen.go @@ -107,7 +107,14 @@ func genYacc(ctx android.ModuleContext, rule *android.RuleBuilder, yaccFile andr return ret } -func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile android.Path, aidlFlags string) (cppFile android.OutputPath, headerFiles android.Paths) { +func genAidl( + ctx android.ModuleContext, + rule *android.RuleBuilder, + outDirBase string, + aidlFile android.Path, + aidlHdrs android.Paths, + aidlFlags string, +) (cppFile android.OutputPath, headerFiles android.Paths) { aidlPackage := strings.TrimSuffix(aidlFile.Rel(), aidlFile.Base()) baseName := strings.TrimSuffix(aidlFile.Base(), aidlFile.Ext()) shortName := baseName @@ -119,7 +126,7 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr shortName = strings.TrimPrefix(baseName, "I") } - outDir := android.PathForModuleGen(ctx, "aidl") + outDir := android.PathForModuleGen(ctx, outDirBase) cppFile = outDir.Join(ctx, aidlPackage, baseName+".cpp") depFile := outDir.Join(ctx, aidlPackage, baseName+".cpp.d") headerI := outDir.Join(ctx, aidlPackage, baseName+".h") @@ -128,6 +135,8 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr cmd := rule.Command() cmd.BuiltTool("aidl-cpp"). + // libc++ is default stl for aidl-cpp (a cc_binary_host module) + ImplicitTool(ctx.Config().HostCcSharedLibPath(ctx, "libc++")). FlagWithDepFile("-d", depFile). Flag("--ninja"). Flag(aidlFlags). @@ -140,6 +149,10 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr headerBp, }) + if aidlHdrs != nil { + cmd.Implicits(aidlHdrs) + } + return cppFile, android.Paths{ headerI, headerBn, @@ -283,14 +296,19 @@ func genSources( ctx android.ModuleContext, aidlLibraryInfos []aidl_library.AidlLibraryInfo, srcFiles android.Paths, - buildFlags builderFlags) (android.Paths, android.Paths, generatedSourceInfo) { + buildFlags builderFlags, +) (android.Paths, android.Paths, generatedSourceInfo) { var info generatedSourceInfo var deps android.Paths var rsFiles android.Paths + // aidlRule supports compiling aidl files from srcs prop while aidlLibraryRule supports + // compiling aidl files from aidl_library modules specified in aidl.libs prop. + // The rules are separated so that they don't wipe out the other's outputDir var aidlRule *android.RuleBuilder + var aidlLibraryRule *android.RuleBuilder var yaccRule_ *android.RuleBuilder yaccRule := func() *android.RuleBuilder { @@ -331,7 +349,14 @@ func genSources( android.PathForModuleGen(ctx, "aidl.sbox.textproto")) } baseDir := strings.TrimSuffix(srcFile.String(), srcFile.Rel()) - cppFile, aidlHeaders := genAidl(ctx, aidlRule, srcFile, buildFlags.aidlFlags+" -I"+baseDir) + cppFile, aidlHeaders := genAidl( + ctx, + aidlRule, + "aidl", + srcFile, + nil, + buildFlags.aidlFlags+" -I"+baseDir, + ) srcFiles[i] = cppFile info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) @@ -354,13 +379,21 @@ func genSources( } for _, aidlLibraryInfo := range aidlLibraryInfos { + if aidlLibraryRule == nil { + aidlLibraryRule = android.NewRuleBuilder(pctx, ctx).Sbox( + android.PathForModuleGen(ctx, "aidl_library"), + android.PathForModuleGen(ctx, "aidl_library.sbox.textproto"), + ).SandboxInputs() + } for _, aidlSrc := range aidlLibraryInfo.Srcs { - if aidlRule == nil { - // TODO(b/279960133): Sandbox inputs to ensure aidl headers are explicitly specified - aidlRule = android.NewRuleBuilder(pctx, ctx).Sbox(android.PathForModuleGen(ctx, "aidl"), - android.PathForModuleGen(ctx, "aidl.sbox.textproto")) - } - cppFile, aidlHeaders := genAidl(ctx, aidlRule, aidlSrc, buildFlags.aidlFlags) + cppFile, aidlHeaders := genAidl( + ctx, + aidlLibraryRule, + "aidl_library", + aidlSrc, + aidlLibraryInfo.Hdrs.ToList(), + buildFlags.aidlFlags, + ) srcFiles = append(srcFiles, cppFile) info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) @@ -375,6 +408,10 @@ func genSources( aidlRule.Build("aidl", "gen aidl") } + if aidlLibraryRule != nil { + aidlLibraryRule.Build("aidl_library", "gen aidl_library") + } + if yaccRule_ != nil { yaccRule_.Build("yacc", "gen yacc") } diff --git a/cc/library.go b/cc/library.go index 313a1ee93..d5f5b404c 100644 --- a/cc/library.go +++ b/cc/library.go @@ -2113,8 +2113,14 @@ func (library *libraryDecorator) link(ctx ModuleContext, // Optionally export aidl headers. if Bool(library.Properties.Aidl.Export_aidl_headers) { if library.baseCompiler.hasAidl(deps) { - dir := android.PathForModuleGen(ctx, "aidl") - library.reexportDirs(dir) + if library.baseCompiler.hasSrcExt(".aidl") { + dir := android.PathForModuleGen(ctx, "aidl") + library.reexportDirs(dir) + } + if len(deps.AidlLibraryInfos) > 0 { + dir := android.PathForModuleGen(ctx, "aidl_library") + library.reexportDirs(dir) + } library.reexportDeps(library.baseCompiler.aidlOrderOnlyDeps...) library.addExportedGeneratedHeaders(library.baseCompiler.aidlHeaders...) From 1dc63117f20e281afdad2100042756ff03a75f01 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Fri, 2 Jun 2023 14:09:50 -0400 Subject: [PATCH 25/80] add apex_available_name property to apexBundle This property allows developers to create multiple modules that provide the same APEX implemenation with some differences. For example, an APEX that targets the current SDK can be developed alongside an APEX that targets a pre-S SDK. Common dependencies of these APEX modules can both use the value of the apex_available_name property in their apex_available lists and be packaged in either APEX module. Bug: 285138555 (cherry picked from https://android-review.googlesource.com/q/commit:ca8165373070b45919e6a1879ebc11705b6c425c) Merged-In: Ia2cb027e2f19bd6d3d1c02e3d3742a0122a43c97 Change-Id: Ia2cb027e2f19bd6d3d1c02e3d3742a0122a43c97 --- apex/apex.go | 11 +++ apex/apex_test.go | 177 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) diff --git a/apex/apex.go b/apex/apex.go index fa9233e79..a314cc5ea 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -220,6 +220,10 @@ type apexBundleProperties struct { // imageApex or flattenedApex depending on Config.FlattenApex(). When payload_type is zip, // this becomes zipApex. ApexType apexPackaging `blueprint:"mutated"` + + // Name that dependencies can specify in their apex_available properties to refer to this module. + // If not specified, this defaults to Soong module name. + Apex_available_name *string } type ApexNativeDependencies struct { @@ -3095,6 +3099,13 @@ func (a *apexBundle) checkApexAvailability(ctx android.ModuleContext) { } apexName := ctx.ModuleName() + for _, props := range ctx.Module().GetProperties() { + if apexProps, ok := props.(*apexBundleProperties); ok { + if apexProps.Apex_available_name != nil { + apexName = *apexProps.Apex_available_name + } + } + } fromName := ctx.OtherModuleName(from) toName := ctx.OtherModuleName(to) diff --git a/apex/apex_test.go b/apex/apex_test.go index 66ba0a8cf..d5af826e4 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6556,6 +6556,183 @@ func TestApexAvailable_InvalidApexName(t *testing.T) { }`) } +func TestApexAvailable_ApexAvailableName(t *testing.T) { + t.Run("using name of apex that sets apex_available_name is not allowed", func(t *testing.T) { + testApexError(t, "Consider adding \"myapex\" to 'apex_available' property of \"AppFoo\"", ` + apex { + name: "myapex_sminus", + key: "myapex.key", + apps: ["AppFoo"], + apex_available_name: "myapex", + updatable: false, + } + apex { + name: "myapex", + key: "myapex.key", + apps: ["AppFoo"], + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + android_app { + name: "AppFoo", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex_sminus" ], + }`, + android.FixtureMergeMockFs(android.MockFS{ + "system/sepolicy/apex/myapex_sminus-file_contexts": nil, + }), + ) + }) + + t.Run("apex_available_name allows module to be used in two different apexes", func(t *testing.T) { + testApex(t, ` + apex { + name: "myapex_sminus", + key: "myapex.key", + apps: ["AppFoo"], + apex_available_name: "myapex", + updatable: false, + } + apex { + name: "myapex", + key: "myapex.key", + apps: ["AppFoo"], + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + android_app { + name: "AppFoo", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + }`, + android.FixtureMergeMockFs(android.MockFS{ + "system/sepolicy/apex/myapex_sminus-file_contexts": nil, + }), + ) + }) + + t.Run("override_apexes work with apex_available_name", func(t *testing.T) { + testApex(t, ` + override_apex { + name: "myoverrideapex_sminus", + base: "myapex_sminus", + key: "myapex.key", + apps: ["AppFooOverride"], + } + override_apex { + name: "myoverrideapex", + base: "myapex", + key: "myapex.key", + apps: ["AppFooOverride"], + } + apex { + name: "myapex_sminus", + key: "myapex.key", + apps: ["AppFoo"], + apex_available_name: "myapex", + updatable: false, + } + apex { + name: "myapex", + key: "myapex.key", + apps: ["AppFoo"], + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + android_app { + name: "AppFooOverride", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + android_app { + name: "AppFoo", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + }`, + android.FixtureMergeMockFs(android.MockFS{ + "system/sepolicy/apex/myapex_sminus-file_contexts": nil, + }), + ) + }) +} + +func TestApexAvailable_ApexAvailableNameWithOverrides(t *testing.T) { + context := android.GroupFixturePreparers( + android.PrepareForIntegrationTestWithAndroid, + PrepareForTestWithApexBuildComponents, + java.PrepareForTestWithDexpreopt, + android.FixtureMergeMockFs(android.MockFS{ + "system/sepolicy/apex/myapex-file_contexts": nil, + "system/sepolicy/apex/myapex_sminus-file_contexts": nil, + }), + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.BuildId = proptools.StringPtr("buildid") + }), + ) + context.RunTestWithBp(t, ` + override_apex { + name: "myoverrideapex_sminus", + base: "myapex_sminus", + } + override_apex { + name: "myoverrideapex", + base: "myapex", + } + apex { + name: "myapex", + key: "myapex.key", + apps: ["AppFoo"], + updatable: false, + } + apex { + name: "myapex_sminus", + apex_available_name: "myapex", + key: "myapex.key", + apps: ["AppFoo_sminus"], + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + android_app { + name: "AppFoo", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + android_app { + name: "AppFoo_sminus", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + min_sdk_version: "29", + system_modules: "none", + apex_available: [ "myapex" ], + }`) +} + func TestApexAvailable_CheckForPlatform(t *testing.T) { ctx := testApex(t, ` apex { From aae62179cee6171b9aafa8580d957c23e05b6301 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 5 Jun 2023 15:55:57 -0400 Subject: [PATCH 26/80] add variant_version property to apexBundle This property allows modifying the variant version that is put into the APEX manifest. Bug: 285138555 (cherry picked from https://android-review.googlesource.com/q/commit:6d65a0fc3dde0e488b2af91d166c8d37630a5b2e) Merged-In: I42b8c0ddad8822a0a329e99ec4ec62f8c3546ee9 Change-Id: I42b8c0ddad8822a0a329e99ec4ec62f8c3546ee9 --- apex/apex.go | 3 ++ apex/apex_test.go | 83 +++++++++++++++++++++++++++++++++++++++++++++++ apex/builder.go | 16 +++++++++ 3 files changed, 102 insertions(+) diff --git a/apex/apex.go b/apex/apex.go index a314cc5ea..f7c74440c 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -224,6 +224,9 @@ type apexBundleProperties struct { // Name that dependencies can specify in their apex_available properties to refer to this module. // If not specified, this defaults to Soong module name. Apex_available_name *string + + // Variant version of the mainline module. Must be an integer between 0-9 + Variant_version *string } type ApexNativeDependencies struct { diff --git a/apex/apex_test.go b/apex/apex_test.go index d5af826e4..d5493dd32 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6556,6 +6556,89 @@ func TestApexAvailable_InvalidApexName(t *testing.T) { }`) } +func TestApexAvailable_ApexAvailableNameWithVersionCodeError(t *testing.T) { + t.Run("negative variant_version produces error", func(t *testing.T) { + testApexError(t, "expected an integer between 0-9; got -1", ` + apex { + name: "myapex", + key: "myapex.key", + apex_available_name: "com.android.foo", + variant_version: "-1", + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + }) + + t.Run("variant_version greater than 9 produces error", func(t *testing.T) { + testApexError(t, "expected an integer between 0-9; got 10", ` + apex { + name: "myapex", + key: "myapex.key", + apex_available_name: "com.android.foo", + variant_version: "10", + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + }) +} + +func TestApexAvailable_ApexAvailableNameWithVersionCode(t *testing.T) { + context := android.GroupFixturePreparers( + android.PrepareForIntegrationTestWithAndroid, + PrepareForTestWithApexBuildComponents, + android.FixtureMergeMockFs(android.MockFS{ + "system/sepolicy/apex/foo-file_contexts": nil, + "system/sepolicy/apex/bar-file_contexts": nil, + }), + ) + result := context.RunTestWithBp(t, ` + apex { + name: "foo", + key: "myapex.key", + apex_available_name: "com.android.foo", + variant_version: "0", + updatable: false, + } + apex { + name: "bar", + key: "myapex.key", + apex_available_name: "com.android.foo", + variant_version: "3", + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + + fooManifestRule := result.ModuleForTests("foo", "android_common_foo_image").Rule("apexManifestRule") + fooExpectedDefaultVersion := android.DefaultUpdatableModuleVersion + fooActualDefaultVersion := fooManifestRule.Args["default_version"] + if fooActualDefaultVersion != fooExpectedDefaultVersion { + t.Errorf("expected to find defaultVersion %q; got %q", fooExpectedDefaultVersion, fooActualDefaultVersion) + } + + barManifestRule := result.ModuleForTests("bar", "android_common_bar_image").Rule("apexManifestRule") + defaultVersionInt, _ := strconv.Atoi(android.DefaultUpdatableModuleVersion) + barExpectedDefaultVersion := fmt.Sprint(defaultVersionInt + 3) + barActualDefaultVersion := barManifestRule.Args["default_version"] + if barActualDefaultVersion != barExpectedDefaultVersion { + t.Errorf("expected to find defaultVersion %q; got %q", barExpectedDefaultVersion, barActualDefaultVersion) + } +} + func TestApexAvailable_ApexAvailableName(t *testing.T) { t.Run("using name of apex that sets apex_available_name is not allowed", func(t *testing.T) { testApexError(t, "Consider adding \"myapex\" to 'apex_available' property of \"AppFoo\"", ` diff --git a/apex/builder.go b/apex/builder.go index 3c7671b00..f49bed93c 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -275,6 +275,22 @@ func (a *apexBundle) buildManifest(ctx android.ModuleContext, provideNativeLibs, manifestJsonFullOut := android.PathForModuleOut(ctx, "apex_manifest_full.json") defaultVersion := android.DefaultUpdatableModuleVersion + if a.properties.Variant_version != nil { + defaultVersionInt, err := strconv.Atoi(defaultVersion) + if err != nil { + ctx.ModuleErrorf("expected DefaultUpdatableModuleVersion to be an int, but got %s", defaultVersion) + } + if defaultVersionInt%10 != 0 { + ctx.ModuleErrorf("expected DefaultUpdatableModuleVersion to end in a zero, but got %s", defaultVersion) + } + variantVersion := []rune(*a.properties.Variant_version) + if len(variantVersion) != 1 || variantVersion[0] < '0' || variantVersion[0] > '9' { + ctx.PropertyErrorf("variant_version", "expected an integer between 0-9; got %s", *a.properties.Variant_version) + } + defaultVersionRunes := []rune(defaultVersion) + defaultVersionRunes[len(defaultVersion)-1] = []rune(variantVersion)[0] + defaultVersion = string(defaultVersionRunes) + } if override := ctx.Config().Getenv("OVERRIDE_APEX_MANIFEST_DEFAULT_VERSION"); override != "" { defaultVersion = override } From 8816bbd48933a8350f8c112979f44535b24bd199 Mon Sep 17 00:00:00 2001 From: Joe Onorato Date: Mon, 5 Jun 2023 16:41:45 -0700 Subject: [PATCH 27/80] Rename DEFAULT_PLATFORM_VERSION to RELEASE_PLATFORM_VERSION It is now a release config build flag. Bug: 282838389 Test: Treehugger Change-Id: I8bfeba5f6cf3a3f9e0c2f8bb3e182c25e3602d26 --- mk2rbc/test/version_defaults.mk.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mk2rbc/test/version_defaults.mk.test b/mk2rbc/test/version_defaults.mk.test index 166639278..3ce60bcdb 100644 --- a/mk2rbc/test/version_defaults.mk.test +++ b/mk2rbc/test/version_defaults.mk.test @@ -3,8 +3,8 @@ ifdef INTERNAL_BUILD_ID_MAKEFILE include $(INTERNAL_BUILD_ID_MAKEFILE) endif -DEFAULT_PLATFORM_VERSION := TP1A -.KATI_READONLY := DEFAULT_PLATFORM_VERSION +RELEASE_PLATFORM_VERSION := TP1A +.KATI_READONLY := RELEASE_PLATFORM_VERSION MIN_PLATFORM_VERSION := TP1A MAX_PLATFORM_VERSION := TP1A PLATFORM_VERSION_LAST_STABLE := 12 From d2f675d86c3cfcada8e0e09319904800b32d38f6 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Thu, 8 Jun 2023 23:29:18 +0000 Subject: [PATCH 28/80] Revert "Add aidl.deps prop to include aidl implicit deps explicitly in Android.bp" This reverts commit 256258587865f6e414e57faf3af256981e1df46a. Reason for revert: We now have aidl_library to remove this workaround to support specifying aidl headers (cherry picked from https://android-review.googlesource.com/q/commit:8106bd0b00039e207893749517d52d682d9f47df) Merged-In: Idf03ebbea764f8c5b3c4fc1623f8f02aa321c8c1 Change-Id: Idf03ebbea764f8c5b3c4fc1623f8f02aa321c8c1 --- android/filegroup.go | 12 --------- bp2build/filegroup_conversion_test.go | 36 --------------------------- 2 files changed, 48 deletions(-) diff --git a/android/filegroup.go b/android/filegroup.go index f30ee5143..121368d7b 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -82,7 +82,6 @@ type bazelFilegroupAttributes struct { type bazelAidlLibraryAttributes struct { Srcs bazel.LabelListAttribute Strip_import_prefix *string - Deps bazel.LabelListAttribute } // api srcs can be contained in filegroups. @@ -121,12 +120,9 @@ func (fg *fileGroup) ConvertWithBp2build(ctx TopDownMutatorContext) { // and then convert if fg.ShouldConvertToAidlLibrary(ctx) { tags := []string{"apex_available=//apex_available:anyapex"} - deps := bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, fg.properties.Aidl.Deps)) - attrs := &bazelAidlLibraryAttributes{ Srcs: srcs, Strip_import_prefix: fg.properties.Path, - Deps: deps, } props := bazel.BazelTargetModuleProperties{ @@ -192,14 +188,6 @@ type fileGroupProperties struct { // Create a make variable with the specified name that contains the list of files in the // filegroup, relative to the root of the source tree. Export_to_make_var *string - - // aidl is explicitly provided for implicit aidl dependencies - // TODO(b/278298615): aidl prop is a no-op in Soong and is an escape hatch - // to include implicit aidl dependencies for bazel migration compatibility - Aidl struct { - // List of aidl files or filegroup depended on by srcs - Deps []string `android:"path"` - } } type fileGroup struct { diff --git a/bp2build/filegroup_conversion_test.go b/bp2build/filegroup_conversion_test.go index 273d55636..7ce559d9b 100644 --- a/bp2build/filegroup_conversion_test.go +++ b/bp2build/filegroup_conversion_test.go @@ -105,42 +105,6 @@ func TestFilegroupWithAidlSrcs(t *testing.T) { } } -func TestFilegroupWithAidlDeps(t *testing.T) { - bp := ` - filegroup { - name: "bar", - srcs: ["bar.aidl"], - } - filegroup { - name: "foo", - srcs: ["aidl/foo.aidl"], - path: "aidl", - aidl: { - deps: [":bar"], - } - }` - - t.Run("filegroup with aidl deps", func(t *testing.T) { - expectedBazelTargets := []string{ - MakeBazelTargetNoRestrictions("aidl_library", "bar", AttrNameToString{ - "srcs": `["bar.aidl"]`, - "tags": `["apex_available=//apex_available:anyapex"]`, - }), - MakeBazelTargetNoRestrictions("aidl_library", "foo", AttrNameToString{ - "srcs": `["aidl/foo.aidl"]`, - "strip_import_prefix": `"aidl"`, - "deps": `[":bar"]`, - "tags": `["apex_available=//apex_available:anyapex"]`, - }), - } - runFilegroupTestCase(t, Bp2buildTestCase{ - Description: "filegroup with aidl deps", - Blueprint: bp, - ExpectedBazelTargets: expectedBazelTargets, - }) - }) -} - func TestFilegroupWithAidlAndNonAidlSrcs(t *testing.T) { runFilegroupTestCase(t, Bp2buildTestCase{ Description: "filegroup with aidl and non-aidl srcs", From 992d5142d24590d2321e4fe12becab1c80a7e930 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Mon, 12 Jun 2023 21:07:40 +0000 Subject: [PATCH 29/80] Revert "Revert "Add aidl.deps prop to include aidl implicit deps explicitly in Android.bp"" This reverts commit 8106bd0b00039e207893749517d52d682d9f47df. Reason for revert: Fix CI breakage (cherry picked from https://android-review.googlesource.com/q/commit:e3e7711ac60adea25eb64c95a6fad0b1f7005d7f) Merged-In: I5f0e72988cbb6845628d0ec7e82292c0aa1914a7 Change-Id: I5f0e72988cbb6845628d0ec7e82292c0aa1914a7 --- android/filegroup.go | 12 +++++++++ bp2build/filegroup_conversion_test.go | 36 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/android/filegroup.go b/android/filegroup.go index 121368d7b..f30ee5143 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -82,6 +82,7 @@ type bazelFilegroupAttributes struct { type bazelAidlLibraryAttributes struct { Srcs bazel.LabelListAttribute Strip_import_prefix *string + Deps bazel.LabelListAttribute } // api srcs can be contained in filegroups. @@ -120,9 +121,12 @@ func (fg *fileGroup) ConvertWithBp2build(ctx TopDownMutatorContext) { // and then convert if fg.ShouldConvertToAidlLibrary(ctx) { tags := []string{"apex_available=//apex_available:anyapex"} + deps := bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, fg.properties.Aidl.Deps)) + attrs := &bazelAidlLibraryAttributes{ Srcs: srcs, Strip_import_prefix: fg.properties.Path, + Deps: deps, } props := bazel.BazelTargetModuleProperties{ @@ -188,6 +192,14 @@ type fileGroupProperties struct { // Create a make variable with the specified name that contains the list of files in the // filegroup, relative to the root of the source tree. Export_to_make_var *string + + // aidl is explicitly provided for implicit aidl dependencies + // TODO(b/278298615): aidl prop is a no-op in Soong and is an escape hatch + // to include implicit aidl dependencies for bazel migration compatibility + Aidl struct { + // List of aidl files or filegroup depended on by srcs + Deps []string `android:"path"` + } } type fileGroup struct { diff --git a/bp2build/filegroup_conversion_test.go b/bp2build/filegroup_conversion_test.go index 7ce559d9b..273d55636 100644 --- a/bp2build/filegroup_conversion_test.go +++ b/bp2build/filegroup_conversion_test.go @@ -105,6 +105,42 @@ func TestFilegroupWithAidlSrcs(t *testing.T) { } } +func TestFilegroupWithAidlDeps(t *testing.T) { + bp := ` + filegroup { + name: "bar", + srcs: ["bar.aidl"], + } + filegroup { + name: "foo", + srcs: ["aidl/foo.aidl"], + path: "aidl", + aidl: { + deps: [":bar"], + } + }` + + t.Run("filegroup with aidl deps", func(t *testing.T) { + expectedBazelTargets := []string{ + MakeBazelTargetNoRestrictions("aidl_library", "bar", AttrNameToString{ + "srcs": `["bar.aidl"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + MakeBazelTargetNoRestrictions("aidl_library", "foo", AttrNameToString{ + "srcs": `["aidl/foo.aidl"]`, + "strip_import_prefix": `"aidl"`, + "deps": `[":bar"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + } + runFilegroupTestCase(t, Bp2buildTestCase{ + Description: "filegroup with aidl deps", + Blueprint: bp, + ExpectedBazelTargets: expectedBazelTargets, + }) + }) +} + func TestFilegroupWithAidlAndNonAidlSrcs(t *testing.T) { runFilegroupTestCase(t, Bp2buildTestCase{ Description: "filegroup with aidl and non-aidl srcs", From 10cd5f9e4d5a038cd903317799045a9333dc0289 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Mon, 12 Jun 2023 21:43:34 +0000 Subject: [PATCH 30/80] Revert "Revert "Revert "Add aidl.deps prop to include aidl implicit deps explicitly in Android.bp""" This reverts commit e3e7711ac60adea25eb64c95a6fad0b1f7005d7f. Reason for revert: CI fixed (cherry picked from https://android-review.googlesource.com/q/commit:21bfa5816dab69110c345b9d020dcab3df1402e8) Merged-In: Ib038241eae5ec6af78ac24105c891db55a4d0544 Change-Id: Ib038241eae5ec6af78ac24105c891db55a4d0544 --- android/filegroup.go | 12 --------- bp2build/filegroup_conversion_test.go | 36 --------------------------- 2 files changed, 48 deletions(-) diff --git a/android/filegroup.go b/android/filegroup.go index f30ee5143..121368d7b 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -82,7 +82,6 @@ type bazelFilegroupAttributes struct { type bazelAidlLibraryAttributes struct { Srcs bazel.LabelListAttribute Strip_import_prefix *string - Deps bazel.LabelListAttribute } // api srcs can be contained in filegroups. @@ -121,12 +120,9 @@ func (fg *fileGroup) ConvertWithBp2build(ctx TopDownMutatorContext) { // and then convert if fg.ShouldConvertToAidlLibrary(ctx) { tags := []string{"apex_available=//apex_available:anyapex"} - deps := bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, fg.properties.Aidl.Deps)) - attrs := &bazelAidlLibraryAttributes{ Srcs: srcs, Strip_import_prefix: fg.properties.Path, - Deps: deps, } props := bazel.BazelTargetModuleProperties{ @@ -192,14 +188,6 @@ type fileGroupProperties struct { // Create a make variable with the specified name that contains the list of files in the // filegroup, relative to the root of the source tree. Export_to_make_var *string - - // aidl is explicitly provided for implicit aidl dependencies - // TODO(b/278298615): aidl prop is a no-op in Soong and is an escape hatch - // to include implicit aidl dependencies for bazel migration compatibility - Aidl struct { - // List of aidl files or filegroup depended on by srcs - Deps []string `android:"path"` - } } type fileGroup struct { diff --git a/bp2build/filegroup_conversion_test.go b/bp2build/filegroup_conversion_test.go index 273d55636..7ce559d9b 100644 --- a/bp2build/filegroup_conversion_test.go +++ b/bp2build/filegroup_conversion_test.go @@ -105,42 +105,6 @@ func TestFilegroupWithAidlSrcs(t *testing.T) { } } -func TestFilegroupWithAidlDeps(t *testing.T) { - bp := ` - filegroup { - name: "bar", - srcs: ["bar.aidl"], - } - filegroup { - name: "foo", - srcs: ["aidl/foo.aidl"], - path: "aidl", - aidl: { - deps: [":bar"], - } - }` - - t.Run("filegroup with aidl deps", func(t *testing.T) { - expectedBazelTargets := []string{ - MakeBazelTargetNoRestrictions("aidl_library", "bar", AttrNameToString{ - "srcs": `["bar.aidl"]`, - "tags": `["apex_available=//apex_available:anyapex"]`, - }), - MakeBazelTargetNoRestrictions("aidl_library", "foo", AttrNameToString{ - "srcs": `["aidl/foo.aidl"]`, - "strip_import_prefix": `"aidl"`, - "deps": `[":bar"]`, - "tags": `["apex_available=//apex_available:anyapex"]`, - }), - } - runFilegroupTestCase(t, Bp2buildTestCase{ - Description: "filegroup with aidl deps", - Blueprint: bp, - ExpectedBazelTargets: expectedBazelTargets, - }) - }) -} - func TestFilegroupWithAidlAndNonAidlSrcs(t *testing.T) { runFilegroupTestCase(t, Bp2buildTestCase{ Description: "filegroup with aidl and non-aidl srcs", From 9036f731d0206fc81b1f99ba298b8f2115f14083 Mon Sep 17 00:00:00 2001 From: Mitch Phillips Date: Mon, 26 Jun 2023 08:08:56 +0000 Subject: [PATCH 31/80] Merge "[MTE] Add memtag_globals as a sanitizer target." am: 03d510404c am: f0a50d44f2 am: e0dc3cad79 Original change: https://android-review.googlesource.com/c/platform/build/soong/+/2625692 Bug: b/288828741 1. There was a topic that contained this build system glue, and a dependent change in external/cronet. 2. external/cronet is setup to automerge into udc-mainline-prod from aosp. 3. The build system isn't. 4. Boom because of the missing dependency. The build system glue should be low-risk, and I'd like to unbrick the build, and so merging this here on the desires of the cronet team. Follow ups to come in that bug. Change-Id: Ia92251cfaa3af5e3136eb28570c82c04101fb9d9 Merged-In: Ie7882d474d4d776232de3a3d571a82274df14bf0 Signed-off-by: Automerger Merge Worker (cherry picked from commit 69501ae617a86faa66358e32b72e83dba07be453) --- cc/sanitize.go | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/cc/sanitize.go b/cc/sanitize.go index 7fddc1b83..e28f53718 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -99,6 +99,7 @@ const ( Fuzzer Memtag_heap Memtag_stack + Memtag_globals cfi // cfi is last to prevent it running before incompatible mutators ) @@ -111,6 +112,7 @@ var Sanitizers = []SanitizerType{ Fuzzer, Memtag_heap, Memtag_stack, + Memtag_globals, cfi, // cfi is last to prevent it running before incompatible mutators } @@ -133,6 +135,8 @@ func (t SanitizerType) variationName() string { return "memtag_heap" case Memtag_stack: return "memtag_stack" + case Memtag_globals: + return "memtag_globals" case Fuzzer: return "fuzzer" default: @@ -151,6 +155,8 @@ func (t SanitizerType) name() string { return "memtag_heap" case Memtag_stack: return "memtag_stack" + case Memtag_globals: + return "memtag_globals" case tsan: return "thread" case intOverflow: @@ -172,7 +178,7 @@ func (t SanitizerType) registerMutators(ctx android.RegisterMutatorsContext) { sanitizer := &sanitizerSplitMutator{t} ctx.TopDown(t.variationName()+"_markapexes", sanitizer.markSanitizableApexesMutator) ctx.Transition(t.variationName(), sanitizer) - case Memtag_heap, Memtag_stack, intOverflow: + case Memtag_heap, Memtag_stack, Memtag_globals, intOverflow: // do nothing default: panic(fmt.Errorf("unknown SanitizerType %d", t)) @@ -213,6 +219,8 @@ func (*Module) SanitizerSupported(t SanitizerType) bool { return true case Memtag_stack: return true + case Memtag_globals: + return true default: return false } @@ -267,6 +275,9 @@ type SanitizeUserProps struct { // Memory-tagging stack instrumentation, only available on arm64 // Adds instrumentation to detect stack buffer overflows and use-after-scope using MTE. Memtag_stack *bool `android:"arch_variant"` + // Memory-tagging globals instrumentation, only available on arm64 + // Adds instrumentation to detect global buffer overflows using MTE. + Memtag_globals *bool `android:"arch_variant"` // A modifier for ASAN and HWASAN for write only instrumentation Writeonly *bool `android:"arch_variant"` @@ -342,6 +353,8 @@ type sanitizeMutatedProperties struct { Memtag_heap *bool `blueprint:"mutated"` // Whether Memory-tagging stack instrumentation is enabled for this module Memtag_stack *bool `blueprint:"mutated"` + // Whether Memory-tagging globals instrumentation is enabled for this module + Memtag_globals *bool `android:"arch_variant"` // Whether a modifier for ASAN and HWASAN for write only instrumentation is enabled for this // module @@ -425,6 +438,7 @@ func (p *sanitizeMutatedProperties) copyUserPropertiesToMutated(userProps *Sanit p.Integer_overflow = userProps.Integer_overflow p.Memtag_heap = userProps.Memtag_heap p.Memtag_stack = userProps.Memtag_stack + p.Memtag_globals = userProps.Memtag_globals p.Safestack = userProps.Safestack p.Scs = userProps.Scs p.Scudo = userProps.Scudo @@ -552,6 +566,10 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Memtag_stack = proptools.BoolPtr(true) } + if found, globalSanitizers = removeFromList("memtag_globals", globalSanitizers); found && s.Memtag_globals == nil { + s.Memtag_globals = proptools.BoolPtr(true) + } + if len(globalSanitizers) > 0 { ctx.ModuleErrorf("unknown global sanitizer option %s", globalSanitizers[0]) } @@ -632,6 +650,7 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { if ctx.Arch().ArchType != android.Arm64 || !ctx.toolchain().Bionic() || ctx.Host() { s.Memtag_heap = nil s.Memtag_stack = nil + s.Memtag_globals = nil } // Also disable CFI if ASAN is enabled. @@ -641,6 +660,7 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { // HWASAN and ASAN win against MTE. s.Memtag_heap = nil s.Memtag_stack = nil + s.Memtag_globals = nil } // Disable sanitizers that depend on the UBSan runtime for windows/darwin builds. @@ -696,7 +716,8 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { if ctx.Os() != android.Windows && (Bool(s.All_undefined) || Bool(s.Undefined) || Bool(s.Address) || Bool(s.Thread) || Bool(s.Fuzzer) || Bool(s.Safestack) || Bool(s.Cfi) || Bool(s.Integer_overflow) || len(s.Misc_undefined) > 0 || - Bool(s.Scudo) || Bool(s.Hwaddress) || Bool(s.Scs) || Bool(s.Memtag_heap) || Bool(s.Memtag_stack)) { + Bool(s.Scudo) || Bool(s.Hwaddress) || Bool(s.Scs) || Bool(s.Memtag_heap) || Bool(s.Memtag_stack) || + Bool(s.Memtag_globals)) { sanitize.Properties.SanitizerEnabled = true } @@ -872,7 +893,7 @@ func (s *sanitize) flags(ctx ModuleContext, flags Flags) Flags { flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--no-fatal-warnings") } - if (Bool(sanProps.Memtag_heap) || Bool(sanProps.Memtag_stack)) && ctx.binary() { + if (Bool(sanProps.Memtag_heap) || Bool(sanProps.Memtag_stack) || Bool(sanProps.Memtag_globals)) && ctx.binary() { if Bool(sanProps.Diag.Memtag_heap) { flags.Local.LdFlags = append(flags.Local.LdFlags, "-fsanitize-memtag-mode=sync") } else { @@ -995,6 +1016,8 @@ func (s *sanitize) getSanitizerBoolPtr(t SanitizerType) *bool { return s.Properties.SanitizeMutated.Memtag_heap case Memtag_stack: return s.Properties.SanitizeMutated.Memtag_stack + case Memtag_globals: + return s.Properties.SanitizeMutated.Memtag_globals case Fuzzer: return s.Properties.SanitizeMutated.Fuzzer default: @@ -1011,6 +1034,7 @@ func (sanitize *sanitize) isUnsanitizedVariant() bool { !sanitize.isSanitizerEnabled(scs) && !sanitize.isSanitizerEnabled(Memtag_heap) && !sanitize.isSanitizerEnabled(Memtag_stack) && + !sanitize.isSanitizerEnabled(Memtag_globals) && !sanitize.isSanitizerEnabled(Fuzzer) } @@ -1032,10 +1056,12 @@ func (sanitize *sanitize) SetSanitizer(t SanitizerType, b bool) { sanitize.Properties.SanitizeMutated.Address = bPtr // For ASAN variant, we need to disable Memtag_stack sanitize.Properties.SanitizeMutated.Memtag_stack = nil + sanitize.Properties.SanitizeMutated.Memtag_globals = nil case Hwasan: sanitize.Properties.SanitizeMutated.Hwaddress = bPtr // For HWAsan variant, we need to disable Memtag_stack sanitize.Properties.SanitizeMutated.Memtag_stack = nil + sanitize.Properties.SanitizeMutated.Memtag_globals = nil case tsan: sanitize.Properties.SanitizeMutated.Thread = bPtr case intOverflow: @@ -1049,6 +1075,8 @@ func (sanitize *sanitize) SetSanitizer(t SanitizerType, b bool) { case Memtag_stack: sanitize.Properties.SanitizeMutated.Memtag_stack = bPtr // We do not need to disable ASAN or HWASan here, as there is no Memtag_stack variant. + case Memtag_globals: + sanitize.Properties.Sanitize.Memtag_globals = bPtr case Fuzzer: sanitize.Properties.SanitizeMutated.Fuzzer = bPtr default: @@ -1536,6 +1564,13 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { sanitizers = append(sanitizers, "memtag-stack") } + if Bool(sanProps.Memtag_globals) { + sanitizers = append(sanitizers, "memtag-globals") + // TODO(mitchp): For now, enable memtag-heap with memtag-globals because the linker + // isn't new enough (https://reviews.llvm.org/differential/changeset/?ref=4243566). + sanitizers = append(sanitizers, "memtag-heap") + } + if Bool(sanProps.Fuzzer) { sanitizers = append(sanitizers, "fuzzer-no-link") } From 039d8dfb674057bfdafc702c9438a88d8a09c7fd Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 20 Jun 2023 22:40:02 -0700 Subject: [PATCH 32/80] Move android_library resource handling to Bazel's ResourceProcessorBusyBox The R.Java files generated by aapt2 link --no-static-lib-packages cause scaling problems by combining all resources into every package listed in a dependencies' AndroidManifest.xml file. For SystemUI-core this results in 74 R.java files, each with 76k lines, and takes 20 seconds to compile in javac. Both AGP and Bazel have workarounds for this that avoid using the R.java files generated by aapt2, instead generating more efficient R.class files directly based on the R.txt file. Bazel uses the ResourceProcessorBusyBox tool that is already present in our tree to process the resources. Reuse the same tool in Soong to create the R.jar. The more efficient R.class files require modifiying source files that use incorrect packages to refer to resources. Ignore-AOSP-First: merge conflict Bug: 284023594 Test: TestAndroidResourceProcessor Change-Id: I026073b40dabcfdb10e5d7a52e9348205b0e9a66 --- java/aar.go | 239 ++++++++++++++++++++++++++++++------- java/androidmk.go | 2 + java/app.go | 18 ++- java/app_test.go | 269 ++++++++++++++++++++++++++++++++++++++++-- java/base.go | 16 ++- java/config/config.go | 2 + java/java.go | 2 +- 7 files changed, 486 insertions(+), 62 deletions(-) diff --git a/java/aar.go b/java/aar.go index 95b9a67bb..fc039fa36 100644 --- a/java/aar.go +++ b/java/aar.go @@ -88,28 +88,40 @@ type aaptProperties struct { // do not include AndroidManifest from dependent libraries Dont_merge_manifests *bool + // If use_resource_processor is set, use Bazel's resource processor instead of aapt2 to generate R.class files. + // The resource processor produces more optimal R.class files that only list resources in the package of the + // library that provided them, as opposed to aapt2 which produces R.java files for every package containing + // every resource. Using the resource processor can provide significant build time speedups, but requires + // fixing the module to use the correct package to reference each resource, and to avoid having any other + // libraries in the tree that use the same package name. Defaults to false, but will default to true in the + // future. + Use_resource_processor *bool + // true if RRO is enforced for any of the dependent modules RROEnforcedForDependent bool `blueprint:"mutated"` } type aapt struct { - aaptSrcJar android.Path - exportPackage android.Path - manifestPath android.Path - proguardOptionsFile android.Path - rTxt android.Path - extraAaptPackagesFile android.Path - mergedManifestFile android.Path - noticeFile android.OptionalPath - assetPackage android.OptionalPath - isLibrary bool - defaultManifestVersion string - useEmbeddedNativeLibs bool - useEmbeddedDex bool - usesNonSdkApis bool - hasNoCode bool - LoggingParent string - resourceFiles android.Paths + aaptSrcJar android.Path + transitiveAaptRJars android.Paths + transitiveAaptResourcePackages android.Paths + exportPackage android.Path + manifestPath android.Path + proguardOptionsFile android.Path + rTxt android.Path + rJar android.Path + extraAaptPackagesFile android.Path + mergedManifestFile android.Path + noticeFile android.OptionalPath + assetPackage android.OptionalPath + isLibrary bool + defaultManifestVersion string + useEmbeddedNativeLibs bool + useEmbeddedDex bool + usesNonSdkApis bool + hasNoCode bool + LoggingParent string + resourceFiles android.Paths splitNames []string splits []split @@ -139,6 +151,10 @@ func propagateRROEnforcementMutator(ctx android.TopDownMutatorContext) { } } +func (a *aapt) useResourceProcessorBusyBox() bool { + return BoolDefault(a.aaptProperties.Use_resource_processor, false) +} + func (a *aapt) ExportPackage() android.Path { return a.exportPackage } @@ -175,7 +191,6 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext android.SdkConte // Flags specified in Android.bp linkFlags = append(linkFlags, a.aaptProperties.Aaptflags...) - linkFlags = append(linkFlags, "--no-static-lib-packages") linkFlags = append(linkFlags, "--enable-compact-entries") // Find implicit or explicit asset and resource dirs @@ -349,6 +364,19 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon linkFlags = append(linkFlags, "--static-lib") } + if a.isLibrary && a.useResourceProcessorBusyBox() { + // When building an android_library using ResourceProcessorBusyBox the resources are merged into + // package-res.apk with --merge-only, but --no-static-lib-packages is not used so that R.txt only + // contains resources from this library. + linkFlags = append(linkFlags, "--merge-only") + } else { + // When building and app or when building an android_library without ResourceProcessorBusyBox + // --no-static-lib-packages is used to put all the resources into the app. If ResourceProcessorBusyBox + // is used then the app's R.txt will be post-processed along with the R.txt files from dependencies to + // sort resources into the right packages in R.class. + linkFlags = append(linkFlags, "--no-static-lib-packages") + } + packageRes := android.PathForModuleOut(ctx, "package-res.apk") // the subdir "android" is required to be filtered by package names srcJar := android.PathForModuleGen(ctx, "android", "R.srcjar") @@ -356,6 +384,7 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon rTxt := android.PathForModuleOut(ctx, "R.txt") // This file isn't used by Soong, but is generated for exporting extraPackages := android.PathForModuleOut(ctx, "extra_packages") + var transitiveRJars android.Paths var compiledResDirs []android.Paths for _, dir := range resDirs { @@ -375,7 +404,23 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon // of transitiveStaticLibs. transitiveStaticLibs := android.ReversePaths(staticDeps.resPackages()) - compiledOverlay = append(compiledOverlay, transitiveStaticLibs...) + if a.isLibrary && a.useResourceProcessorBusyBox() { + // When building an android_library with ResourceProcessorBusyBox enabled treat static library dependencies + // as imports. The resources from dependencies will not be merged into this module's package-res.apk, and + // instead modules depending on this module will reference package-res.apk from all transitive static + // dependencies. + for _, staticDep := range staticDeps { + linkDeps = append(linkDeps, staticDep.resPackage) + linkFlags = append(linkFlags, "-I "+staticDep.resPackage.String()) + if staticDep.usedResourceProcessor { + transitiveRJars = append(transitiveRJars, staticDep.rJar) + } + } + } else { + // When building an app or building a library without ResourceProcessorBusyBox enabled all static + // dependencies are compiled into this module's package-res.apk as overlays. + compiledOverlay = append(compiledOverlay, transitiveStaticLibs...) + } if len(transitiveStaticLibs) > 0 { // If we are using static android libraries, every source file becomes an overlay. @@ -438,7 +483,16 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon a.assetPackage = android.OptionalPathForPath(assets) } + if a.useResourceProcessorBusyBox() { + rJar := android.PathForModuleOut(ctx, "busybox/R.jar") + resourceProcessorBusyBoxGenerateBinaryR(ctx, rTxt, a.mergedManifestFile, rJar, staticDeps, a.isLibrary) + transitiveRJars = append(transitiveRJars, rJar) + a.rJar = rJar + } + a.aaptSrcJar = srcJar + a.transitiveAaptRJars = transitiveRJars + a.transitiveAaptResourcePackages = staticDeps.resPackages() a.exportPackage = packageRes a.manifestPath = manifestPath a.proguardOptionsFile = proguardOptionsFile @@ -450,7 +504,11 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon resPackage: a.exportPackage, manifest: a.manifestPath, additionalManifests: additionalManifests, + rTxt: a.rTxt, + rJar: a.rJar, assets: a.assetPackage, + + usedResourceProcessor: a.useResourceProcessorBusyBox(), }). Transitive(staticResourcesNodesDepSet).Build() a.rroDirsDepSet = android.NewDepSetBuilder[rroDir](android.TOPOLOGICAL). @@ -462,34 +520,93 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon Transitive(staticManifestsDepSet).Build() } +var resourceProcessorBusyBox = pctx.AndroidStaticRule("resourceProcessorBusyBox", + blueprint.RuleParams{ + Command: "${config.JavaCmd} -cp ${config.ResourceProcessorBusyBox} " + + "com.google.devtools.build.android.ResourceProcessorBusyBox --tool=GENERATE_BINARY_R -- @${out}.args && " + + "if cmp -s ${out}.tmp ${out} ; then rm ${out}.tmp ; else mv ${out}.tmp ${out}; fi", + CommandDeps: []string{"${config.ResourceProcessorBusyBox}"}, + Rspfile: "${out}.args", + RspfileContent: "--primaryRTxt ${rTxt} --primaryManifest ${manifest} --classJarOutput ${out}.tmp ${args}", + Restat: true, + }, "rTxt", "manifest", "args") + +// resourceProcessorBusyBoxGenerateBinaryR converts the R.txt file produced by aapt2 into R.class files +// using Bazel's ResourceProcessorBusyBox tool, which is faster than compiling the R.java files and +// supports producing classes for static dependencies that only include resources from that dependency. +func resourceProcessorBusyBoxGenerateBinaryR(ctx android.ModuleContext, rTxt, manifest android.Path, + rJar android.WritablePath, transitiveDeps transitiveAarDeps, isLibrary bool) { + + var args []string + var deps android.Paths + + if !isLibrary { + // When compiling an app, pass all R.txt and AndroidManifest.xml from transitive static library dependencies + // to ResourceProcessorBusyBox so that it can regenerate R.class files with the final resource IDs for each + // package. + args, deps = transitiveDeps.resourceProcessorDeps() + } else { + // When compiling a library don't pass any dependencies as it only needs to generate an R.class file for this + // library. Pass --finalFields=false so that the R.class file contains non-final fields so they don't get + // inlined into the library before the final IDs are assigned during app compilation. + args = append(args, "--finalFields=false") + } + + deps = append(deps, rTxt, manifest) + + ctx.Build(pctx, android.BuildParams{ + Rule: resourceProcessorBusyBox, + Output: rJar, + Implicits: deps, + Description: "ResourceProcessorBusyBox", + Args: map[string]string{ + "rTxt": rTxt.String(), + "manifest": manifest.String(), + "args": strings.Join(args, " "), + }, + }) +} + type resourcesNode struct { resPackage android.Path manifest android.Path additionalManifests android.Paths + rTxt android.Path + rJar android.Path assets android.OptionalPath + + usedResourceProcessor bool } type transitiveAarDeps []*resourcesNode func (t transitiveAarDeps) resPackages() android.Paths { - var paths android.Paths + paths := make(android.Paths, 0, len(t)) for _, dep := range t { paths = append(paths, dep.resPackage) } - return android.FirstUniquePaths(paths) + return paths } func (t transitiveAarDeps) manifests() android.Paths { - var paths android.Paths + paths := make(android.Paths, 0, len(t)) for _, dep := range t { paths = append(paths, dep.manifest) paths = append(paths, dep.additionalManifests...) } - return android.FirstUniquePaths(paths) + return paths +} + +func (t transitiveAarDeps) resourceProcessorDeps() (args []string, deps android.Paths) { + for _, dep := range t { + args = append(args, "--library="+dep.rTxt.String()+","+dep.manifest.String()) + deps = append(deps, dep.rTxt, dep.manifest) + } + return args, deps } func (t transitiveAarDeps) assets() android.Paths { - var paths android.Paths + paths := make(android.Paths, 0, len(t)) for _, dep := range t { if dep.assets.Valid() { paths = append(paths, dep.assets.Path()) @@ -614,9 +731,12 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) a.stem = proptools.StringDefault(a.overridableDeviceProperties.Stem, ctx.ModuleName()) - ctx.CheckbuildFile(a.proguardOptionsFile) - ctx.CheckbuildFile(a.exportPackage) - ctx.CheckbuildFile(a.aaptSrcJar) + ctx.CheckbuildFile(a.aapt.proguardOptionsFile) + ctx.CheckbuildFile(a.aapt.exportPackage) + ctx.CheckbuildFile(a.aapt.aaptSrcJar) + if a.useResourceProcessorBusyBox() { + ctx.CheckbuildFile(a.aapt.rJar) + } // apps manifests are handled by aapt, don't let Module see them a.properties.Manifest = nil @@ -628,7 +748,22 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) a.Module.extraProguardFlagFiles = append(a.Module.extraProguardFlagFiles, a.proguardOptionsFile) - a.Module.compile(ctx, a.aaptSrcJar) + var extraSrcJars android.Paths + var extraCombinedJars android.Paths + var extraClasspathJars android.Paths + if a.useResourceProcessorBusyBox() { + // When building a library with ResourceProcessorBusyBox enabled ResourceProcessorBusyBox for this + // library and each of the transitive static android_library dependencies has already created an + // R.class file for the appropriate package. Add all of those R.class files to the classpath. + extraClasspathJars = a.transitiveAaptRJars + } else { + // When building a library without ResourceProcessorBusyBox the aapt2 rule creates R.srcjar containing + // R.java files for the library's package and the packages from all transitive static android_library + // dependencies. Compile the srcjar alongside the rest of the sources. + extraSrcJars = android.Paths{a.aapt.aaptSrcJar} + } + + a.Module.compile(ctx, extraSrcJars, extraClasspathJars, extraCombinedJars) a.aarFile = android.PathForModuleOut(ctx, ctx.ModuleName()+".aar") var res android.Paths @@ -730,12 +865,15 @@ type AARImport struct { properties AARImportProperties - classpathFile android.WritablePath - proguardFlags android.WritablePath - exportPackage android.WritablePath - extraAaptPackagesFile android.WritablePath - manifest android.WritablePath - assetsPackage android.WritablePath + classpathFile android.WritablePath + proguardFlags android.WritablePath + exportPackage android.WritablePath + transitiveAaptResourcePackages android.Paths + extraAaptPackagesFile android.WritablePath + manifest android.WritablePath + assetsPackage android.WritablePath + rTxt android.WritablePath + rJar android.WritablePath resourcesNodesDepSet *android.DepSet[*resourcesNode] manifestsDepSet *android.DepSet[android.Path] @@ -904,12 +1042,13 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.classpathFile = extractedAARDir.Join(ctx, "classes-combined.jar") a.proguardFlags = extractedAARDir.Join(ctx, "proguard.txt") a.manifest = extractedAARDir.Join(ctx, "AndroidManifest.xml") + aarRTxt := extractedAARDir.Join(ctx, "R.txt") a.assetsPackage = android.PathForModuleOut(ctx, "assets.zip") ctx.Build(pctx, android.BuildParams{ Rule: unzipAAR, Input: a.aarPath, - Outputs: android.WritablePaths{a.classpathFile, a.proguardFlags, a.manifest, a.assetsPackage}, + Outputs: android.WritablePaths{a.classpathFile, a.proguardFlags, a.manifest, a.assetsPackage, aarRTxt}, Description: "unzip AAR", Args: map[string]string{ "outDir": extractedAARDir.String(), @@ -929,14 +1068,14 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { // the subdir "android" is required to be filtered by package names srcJar := android.PathForModuleGen(ctx, "android", "R.srcjar") proguardOptionsFile := android.PathForModuleGen(ctx, "proguard.options") - rTxt := android.PathForModuleOut(ctx, "R.txt") + a.rTxt = android.PathForModuleOut(ctx, "R.txt") a.extraAaptPackagesFile = android.PathForModuleOut(ctx, "extra_packages") var linkDeps android.Paths linkFlags := []string{ "--static-lib", - "--no-static-lib-packages", + "--merge-only", "--auto-add-overlay", } @@ -949,25 +1088,35 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { _ = staticRRODirsDepSet staticDeps := transitiveAarDeps(staticResourcesNodesDepSet.ToList()) - // AAPT2 overlays are in lowest to highest priority order, reverse the topological order - // of transitiveStaticLibs. - transitiveStaticLibs := android.ReversePaths(staticDeps.resPackages()) - linkDeps = append(linkDeps, sharedLibs...) - linkDeps = append(linkDeps, transitiveStaticLibs...) + linkDeps = append(linkDeps, staticDeps.resPackages()...) linkFlags = append(linkFlags, libFlags...) - overlayRes := append(android.Paths{flata}, transitiveStaticLibs...) + overlayRes := android.Paths{flata} + + // Treat static library dependencies of static libraries as imports. + transitiveStaticLibs := staticDeps.resPackages() + linkDeps = append(linkDeps, transitiveStaticLibs...) + for _, staticLib := range transitiveStaticLibs { + linkFlags = append(linkFlags, "-I "+staticLib.String()) + } transitiveAssets := android.ReverseSliceInPlace(staticDeps.assets()) - aapt2Link(ctx, a.exportPackage, srcJar, proguardOptionsFile, rTxt, a.extraAaptPackagesFile, + aapt2Link(ctx, a.exportPackage, srcJar, proguardOptionsFile, a.rTxt, a.extraAaptPackagesFile, linkFlags, linkDeps, nil, overlayRes, transitiveAssets, nil) + a.rJar = android.PathForModuleOut(ctx, "busybox/R.jar") + resourceProcessorBusyBoxGenerateBinaryR(ctx, a.rTxt, a.manifest, a.rJar, nil, true) + resourcesNodesDepSetBuilder := android.NewDepSetBuilder[*resourcesNode](android.TOPOLOGICAL) resourcesNodesDepSetBuilder.Direct(&resourcesNode{ resPackage: a.exportPackage, manifest: a.manifest, + rTxt: a.rTxt, + rJar: a.rJar, assets: android.OptionalPathForPath(a.assetsPackage), + + usedResourceProcessor: true, }) resourcesNodesDepSetBuilder.Transitive(staticResourcesNodesDepSet) a.resourcesNodesDepSet = resourcesNodesDepSetBuilder.Build() @@ -982,6 +1131,8 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { _ = staticManifestsDepSet a.manifestsDepSet = manifestDepSetBuilder.Build() + a.transitiveAaptResourcePackages = staticDeps.resPackages() + a.collectTransitiveHeaderJars(ctx) ctx.SetProvider(JavaInfoProvider, JavaInfo{ HeaderJars: android.PathsIfNonNil(a.classpathFile), diff --git a/java/androidmk.go b/java/androidmk.go index 784fa29b5..113137609 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -265,6 +265,7 @@ func (prebuilt *AARImport) AndroidMkEntries() []android.AndroidMkEntries { entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.classpathFile) entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.classpathFile) entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", prebuilt.exportPackage) + entries.SetPaths("LOCAL_SOONG_TRANSITIVE_RES_PACKAGES", prebuilt.transitiveAaptResourcePackages) entries.SetPath("LOCAL_SOONG_EXPORT_PROGUARD_FLAGS", prebuilt.proguardFlags) entries.SetPath("LOCAL_SOONG_STATIC_LIBRARY_EXTRA_PACKAGES", prebuilt.extraAaptPackagesFile) entries.SetPath("LOCAL_FULL_MANIFEST_FILE", prebuilt.manifest) @@ -508,6 +509,7 @@ func (a *AndroidLibrary) AndroidMkEntries() []android.AndroidMkEntries { } entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", a.exportPackage) + entries.SetPaths("LOCAL_SOONG_TRANSITIVE_RES_PACKAGES", a.transitiveAaptResourcePackages) entries.SetPath("LOCAL_SOONG_STATIC_LIBRARY_EXTRA_PACKAGES", a.extraAaptPackagesFile) entries.SetPath("LOCAL_FULL_MANIFEST_FILE", a.mergedManifestFile) entries.AddStrings("LOCAL_SOONG_EXPORT_PROGUARD_FLAGS", a.exportedProguardFlagFiles.Strings()...) diff --git a/java/app.go b/java/app.go index 8e4efd2b2..224bc8867 100755 --- a/java/app.go +++ b/java/app.go @@ -521,7 +521,23 @@ func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { a.dexpreopter.preventInstall = a.appProperties.PreventInstall if ctx.ModuleName() != "framework-res" { - a.Module.compile(ctx, a.aaptSrcJar) + var extraSrcJars android.Paths + var extraClasspathJars android.Paths + var extraCombinedJars android.Paths + if a.useResourceProcessorBusyBox() { + // When building an app with ResourceProcessorBusyBox enabled ResourceProcessorBusyBox has already + // created R.class files that provide IDs for resources in busybox/R.jar. Pass that file in the + // classpath when compiling everything else, and add it to the final classes jar. + extraClasspathJars = android.Paths{a.aapt.rJar} + extraCombinedJars = android.Paths{a.aapt.rJar} + } else { + // When building an app without ResourceProcessorBusyBox the aapt2 rule creates R.srcjar containing + // R.java files for the app's package and the packages from all transitive static android_library + // dependencies. Compile the srcjar alongside the rest of the sources. + extraSrcJars = android.Paths{a.aapt.aaptSrcJar} + } + + a.Module.compile(ctx, extraSrcJars, extraClasspathJars, extraCombinedJars) } return a.dexJarFile.PathOrNil() diff --git a/java/app_test.go b/java/app_test.go index c438b6cfa..4627ff6fc 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -722,7 +722,10 @@ func TestAppJavaResources(t *testing.T) { func TestAndroidResourceProcessor(t *testing.T) { testCases := []struct { - name string + name string + appUsesRP bool + directLibUsesRP bool + transitiveLibUsesRP bool dontVerifyApp bool appResources []string @@ -759,7 +762,12 @@ func TestAndroidResourceProcessor(t *testing.T) { transitiveImportImports []string }{ { - name: "legacy", + // Test with all modules set to use_resource_processor: false (except android_library_import modules, + // which always use resource processor). + name: "legacy", + appUsesRP: false, + directLibUsesRP: false, + transitiveLibUsesRP: false, appResources: nil, appOverlays: []string{ @@ -771,7 +779,6 @@ func TestAndroidResourceProcessor(t *testing.T) { "out/soong/.intermediates/direct_import/android_common/package-res.apk", "out/soong/.intermediates/app/android_common/aapt2/app/res/values_strings.arsc.flat", }, - appImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, appSrcJars: []string{"out/soong/.intermediates/app/android_common/gen/android/R.srcjar"}, appClasspath: []string{ @@ -792,7 +799,6 @@ func TestAndroidResourceProcessor(t *testing.T) { "out/soong/.intermediates/transitive_import/android_common/package-res.apk", "out/soong/.intermediates/direct/android_common/aapt2/direct/res/values_strings.arsc.flat", }, - directImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, directSrcJars: []string{"out/soong/.intermediates/direct/android_common/gen/android/R.srcjar"}, directClasspath: []string{ @@ -814,18 +820,256 @@ func TestAndroidResourceProcessor(t *testing.T) { transitiveCombined: nil, directImportResources: nil, - directImportOverlays: []string{ - "out/soong/.intermediates/direct_import/android_common/flat-res/gen_res.flata", + directImportOverlays: []string{"out/soong/.intermediates/direct_import/android_common/flat-res/gen_res.flata"}, + directImportImports: []string{ + "out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk", "out/soong/.intermediates/direct_import_dep/android_common/package-res.apk", }, - directImportImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, transitiveImportResources: nil, - transitiveImportOverlays: []string{ - "out/soong/.intermediates/transitive_import/android_common/flat-res/gen_res.flata", + transitiveImportOverlays: []string{"out/soong/.intermediates/transitive_import/android_common/flat-res/gen_res.flata"}, + transitiveImportImports: []string{ + "out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk", "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", }, - transitiveImportImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, + }, + { + // Test with all modules set to use_resource_processor: true. + name: "resource_processor", + appUsesRP: true, + directLibUsesRP: true, + transitiveLibUsesRP: true, + + appResources: nil, + appOverlays: []string{ + "out/soong/.intermediates/transitive/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import/android_common/package-res.apk", + "out/soong/.intermediates/direct/android_common/package-res.apk", + "out/soong/.intermediates/direct_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/direct_import/android_common/package-res.apk", + "out/soong/.intermediates/app/android_common/aapt2/app/res/values_strings.arsc.flat", + }, + appImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, + appSrcJars: nil, + appClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + "out/soong/.intermediates/app/android_common/busybox/R.jar", + "out/soong/.intermediates/direct/android_common/turbine-combined/direct.jar", + "out/soong/.intermediates/direct_import/android_common/aar/classes-combined.jar", + }, + appCombined: []string{ + "out/soong/.intermediates/app/android_common/busybox/R.jar", + "out/soong/.intermediates/app/android_common/javac/app.jar", + "out/soong/.intermediates/direct/android_common/combined/direct.jar", + "out/soong/.intermediates/direct_import/android_common/aar/classes-combined.jar", + }, + + directResources: nil, + directOverlays: []string{"out/soong/.intermediates/direct/android_common/aapt2/direct/res/values_strings.arsc.flat"}, + directImports: []string{ + "out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/transitive/android_common/package-res.apk", + }, + directSrcJars: nil, + directClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + "out/soong/.intermediates/transitive_import/android_common/busybox/R.jar", + "out/soong/.intermediates/transitive_import_dep/android_common/busybox/R.jar", + "out/soong/.intermediates/transitive/android_common/busybox/R.jar", + "out/soong/.intermediates/direct/android_common/busybox/R.jar", + "out/soong/.intermediates/transitive/android_common/turbine-combined/transitive.jar", + "out/soong/.intermediates/transitive_import/android_common/aar/classes-combined.jar", + }, + directCombined: []string{ + "out/soong/.intermediates/direct/android_common/javac/direct.jar", + "out/soong/.intermediates/transitive/android_common/javac/transitive.jar", + "out/soong/.intermediates/transitive_import/android_common/aar/classes-combined.jar", + }, + + transitiveResources: []string{"out/soong/.intermediates/transitive/android_common/aapt2/transitive/res/values_strings.arsc.flat"}, + transitiveOverlays: nil, + transitiveImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, + transitiveSrcJars: nil, + transitiveClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + "out/soong/.intermediates/transitive/android_common/busybox/R.jar", + }, + transitiveCombined: nil, + + directImportResources: nil, + directImportOverlays: []string{"out/soong/.intermediates/direct_import/android_common/flat-res/gen_res.flata"}, + directImportImports: []string{ + "out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk", + "out/soong/.intermediates/direct_import_dep/android_common/package-res.apk", + }, + + transitiveImportResources: nil, + transitiveImportOverlays: []string{"out/soong/.intermediates/transitive_import/android_common/flat-res/gen_res.flata"}, + transitiveImportImports: []string{ + "out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", + }, + }, { + // Test an app building with resource processor enabled but with dependencies built without + // resource processor. + name: "app_resource_processor", + appUsesRP: true, + directLibUsesRP: false, + transitiveLibUsesRP: false, + + appResources: nil, + appOverlays: []string{ + "out/soong/.intermediates/transitive/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import/android_common/package-res.apk", + "out/soong/.intermediates/direct/android_common/package-res.apk", + "out/soong/.intermediates/direct_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/direct_import/android_common/package-res.apk", + "out/soong/.intermediates/app/android_common/aapt2/app/res/values_strings.arsc.flat", + }, + appImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, + appSrcJars: nil, + appClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + // R.jar has to come before direct.jar + "out/soong/.intermediates/app/android_common/busybox/R.jar", + "out/soong/.intermediates/direct/android_common/turbine-combined/direct.jar", + "out/soong/.intermediates/direct_import/android_common/aar/classes-combined.jar", + }, + appCombined: []string{ + "out/soong/.intermediates/app/android_common/busybox/R.jar", + "out/soong/.intermediates/app/android_common/javac/app.jar", + "out/soong/.intermediates/direct/android_common/combined/direct.jar", + "out/soong/.intermediates/direct_import/android_common/aar/classes-combined.jar", + }, + + dontVerifyDirect: true, + dontVerifyTransitive: true, + dontVerifyDirectImport: true, + dontVerifyTransitiveImport: true, + }, + { + // Test an app building without resource processor enabled but with a dependency built with + // resource processor. + name: "app_dependency_lib_resource_processor", + appUsesRP: false, + directLibUsesRP: true, + transitiveLibUsesRP: false, + + appOverlays: []string{ + "out/soong/.intermediates/transitive/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import/android_common/package-res.apk", + "out/soong/.intermediates/direct/android_common/package-res.apk", + "out/soong/.intermediates/direct_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/direct_import/android_common/package-res.apk", + "out/soong/.intermediates/app/android_common/aapt2/app/res/values_strings.arsc.flat", + }, + appImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, + appSrcJars: []string{"out/soong/.intermediates/app/android_common/gen/android/R.srcjar"}, + appClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + "out/soong/.intermediates/direct/android_common/turbine-combined/direct.jar", + "out/soong/.intermediates/direct_import/android_common/aar/classes-combined.jar", + }, + appCombined: []string{ + "out/soong/.intermediates/app/android_common/javac/app.jar", + "out/soong/.intermediates/direct/android_common/combined/direct.jar", + "out/soong/.intermediates/direct_import/android_common/aar/classes-combined.jar", + }, + + directResources: nil, + directOverlays: []string{"out/soong/.intermediates/direct/android_common/aapt2/direct/res/values_strings.arsc.flat"}, + directImports: []string{ + "out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/transitive/android_common/package-res.apk", + }, + directSrcJars: nil, + directClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + "out/soong/.intermediates/transitive_import/android_common/busybox/R.jar", + "out/soong/.intermediates/transitive_import_dep/android_common/busybox/R.jar", + "out/soong/.intermediates/direct/android_common/busybox/R.jar", + "out/soong/.intermediates/transitive/android_common/turbine-combined/transitive.jar", + "out/soong/.intermediates/transitive_import/android_common/aar/classes-combined.jar", + }, + directCombined: []string{ + "out/soong/.intermediates/direct/android_common/javac/direct.jar", + "out/soong/.intermediates/transitive/android_common/javac/transitive.jar", + "out/soong/.intermediates/transitive_import/android_common/aar/classes-combined.jar", + }, + + dontVerifyTransitive: true, + dontVerifyDirectImport: true, + dontVerifyTransitiveImport: true, + }, + { + // Test a library building without resource processor enabled but with a dependency built with + // resource processor. + name: "lib_dependency_lib_resource_processor", + appUsesRP: false, + directLibUsesRP: false, + transitiveLibUsesRP: true, + + appOverlays: []string{ + "out/soong/.intermediates/transitive/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import/android_common/package-res.apk", + "out/soong/.intermediates/direct/android_common/package-res.apk", + "out/soong/.intermediates/direct_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/direct_import/android_common/package-res.apk", + "out/soong/.intermediates/app/android_common/aapt2/app/res/values_strings.arsc.flat", + }, + appImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, + appSrcJars: []string{"out/soong/.intermediates/app/android_common/gen/android/R.srcjar"}, + appClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + "out/soong/.intermediates/direct/android_common/turbine-combined/direct.jar", + "out/soong/.intermediates/direct_import/android_common/aar/classes-combined.jar", + }, + appCombined: []string{ + "out/soong/.intermediates/app/android_common/javac/app.jar", + "out/soong/.intermediates/direct/android_common/combined/direct.jar", + "out/soong/.intermediates/direct_import/android_common/aar/classes-combined.jar", + }, + + directResources: nil, + directOverlays: []string{ + "out/soong/.intermediates/transitive/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk", + "out/soong/.intermediates/transitive_import/android_common/package-res.apk", + "out/soong/.intermediates/direct/android_common/aapt2/direct/res/values_strings.arsc.flat", + }, + directImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, + directSrcJars: []string{"out/soong/.intermediates/direct/android_common/gen/android/R.srcjar"}, + directClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + "out/soong/.intermediates/transitive/android_common/turbine-combined/transitive.jar", + "out/soong/.intermediates/transitive_import/android_common/aar/classes-combined.jar", + }, + directCombined: []string{ + "out/soong/.intermediates/direct/android_common/javac/direct.jar", + "out/soong/.intermediates/transitive/android_common/javac/transitive.jar", + "out/soong/.intermediates/transitive_import/android_common/aar/classes-combined.jar", + }, + + transitiveResources: []string{"out/soong/.intermediates/transitive/android_common/aapt2/transitive/res/values_strings.arsc.flat"}, + transitiveOverlays: nil, + transitiveImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"}, + transitiveSrcJars: nil, + transitiveClasspath: []string{ + "out/soong/.intermediates/default/java/android_stubs_current/android_common/turbine-combined/android_stubs_current.jar", + "out/soong/.intermediates/transitive/android_common/busybox/R.jar", + }, + transitiveCombined: nil, + + dontVerifyDirectImport: true, + dontVerifyTransitiveImport: true, }, } @@ -839,6 +1083,7 @@ func TestAndroidResourceProcessor(t *testing.T) { resource_dirs: ["app/res"], manifest: "app/AndroidManifest.xml", static_libs: ["direct", "direct_import"], + use_resource_processor: %v, } android_library { @@ -848,6 +1093,7 @@ func TestAndroidResourceProcessor(t *testing.T) { resource_dirs: ["direct/res"], manifest: "direct/AndroidManifest.xml", static_libs: ["transitive", "transitive_import"], + use_resource_processor: %v, } android_library { @@ -856,6 +1102,7 @@ func TestAndroidResourceProcessor(t *testing.T) { srcs: ["transitive/transitive.java"], resource_dirs: ["transitive/res"], manifest: "transitive/AndroidManifest.xml", + use_resource_processor: %v, } android_library_import { @@ -883,7 +1130,7 @@ func TestAndroidResourceProcessor(t *testing.T) { sdk_version: "current", aars: ["transitive_import_dep.aar"], } - `) + `, testCase.appUsesRP, testCase.directLibUsesRP, testCase.transitiveLibUsesRP) fs := android.MockFS{ "app/res/values/strings.xml": nil, diff --git a/java/base.go b/java/base.go index 8db716256..ce94335d6 100644 --- a/java/base.go +++ b/java/base.go @@ -1057,7 +1057,7 @@ func (module *Module) addGeneratedSrcJars(path android.Path) { module.properties.Generated_srcjars = append(module.properties.Generated_srcjars, path) } -func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { +func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspathJars, extraCombinedJars android.Paths) { j.exportAidlIncludeDirs = android.PathsForModuleSrc(ctx, j.deviceProperties.Aidl.Export_include_dirs) deps := j.collectDeps(ctx) @@ -1095,9 +1095,7 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { srcJars := srcFiles.FilterByExt(".srcjar") srcJars = append(srcJars, deps.srcJars...) - if aaptSrcJar != nil { - srcJars = append(srcJars, aaptSrcJar) - } + srcJars = append(srcJars, extraSrcJars...) srcJars = append(srcJars, j.properties.Generated_srcjars...) srcFiles = srcFiles.FilterOutByExt(".srcjar") @@ -1140,6 +1138,11 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { var kotlinJars android.Paths var kotlinHeaderJars android.Paths + // Prepend extraClasspathJars to classpath so that the resource processor R.jar comes before + // any dependencies so that it can override any non-final R classes from dependencies with the + // final R classes from the app. + flags.classpath = append(android.CopyOf(extraClasspathJars), flags.classpath...) + if srcFiles.HasExt(".kt") { // When using kotlin sources turbine is used to generate annotation processor sources, // including for annotation processors that generate API, so we can use turbine for @@ -1233,8 +1236,9 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { // allow for the use of annotation processors that do function correctly // with sharding enabled. See: b/77284273. } + extraJars := append(android.CopyOf(extraCombinedJars), kotlinHeaderJars...) headerJarFileWithoutDepsOrJarjar, j.headerJarFile = - j.compileJavaHeader(ctx, uniqueJavaFiles, srcJars, deps, flags, jarName, kotlinHeaderJars) + j.compileJavaHeader(ctx, uniqueJavaFiles, srcJars, deps, flags, jarName, extraJars) if ctx.Failed() { return } @@ -1385,6 +1389,8 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { jars = append(jars, servicesJar) } + jars = append(android.CopyOf(extraCombinedJars), jars...) + // Combine the classes built from sources, any manifests, and any static libraries into // classes.jar. If there is only one input jar this step will be skipped. var outputFile android.OutputPath diff --git a/java/config/config.go b/java/config/config.go index 195dae16b..83c27d309 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -148,6 +148,8 @@ func init() { pctx.SourcePathVariable("JavaKytheExtractorJar", "prebuilts/build-tools/common/framework/javac_extractor.jar") pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime") + pctx.SourcePathVariable("ResourceProcessorBusyBox", "prebuilts/bazel/common/android_tools/android_tools/all_android_tools_deploy.jar") + pctx.HostBinToolVariable("GenKotlinBuildFileCmd", "gen-kotlin-build-file") pctx.SourcePathVariable("JarArgsCmd", "build/soong/scripts/jar-args.sh") diff --git a/java/java.go b/java/java.go index 6388d13e3..ce14df778 100644 --- a/java/java.go +++ b/java/java.go @@ -692,7 +692,7 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.dexpreopter.uncompressedDex = *j.dexProperties.Uncompress_dex j.classLoaderContexts = j.usesLibrary.classLoaderContextForUsesLibDeps(ctx) } - j.compile(ctx, nil) + j.compile(ctx, nil, nil, nil) // Collect the module directory for IDE info in java/jdeps.go. j.modulePaths = append(j.modulePaths, ctx.ModuleDir()) From 21862285ace046bc9d089984baa898ef21aff4c6 Mon Sep 17 00:00:00 2001 From: Rico Wind Date: Tue, 1 Aug 2023 14:38:36 +0200 Subject: [PATCH 33/80] Enable compact entries for resource shrunken apps For SystemUIGoogle this is a 2.3 MB reduction (17* of resources arsc) and for Traceur it is a 500k reduction (22% of resources.arsc) Bug: 294016857 Test: Existing Ignore-AOSP-First: The resource system support for this is not in AOSP yet Change-Id: I513078406494ad354c255afa0e829cbe10a9b07a --- java/aapt2.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/aapt2.go b/java/aapt2.go index 7845a0b23..abe154ea0 100644 --- a/java/aapt2.go +++ b/java/aapt2.go @@ -256,7 +256,8 @@ func aapt2Link(ctx android.ModuleContext, var aapt2ConvertRule = pctx.AndroidStaticRule("aapt2Convert", blueprint.RuleParams{ - Command: `${config.Aapt2Cmd} convert --output-format $format $in -o $out`, + Command: `${config.Aapt2Cmd} convert --enable-compact-entries ` + + `--output-format $format $in -o $out`, CommandDeps: []string{"${config.Aapt2Cmd}"}, }, "format", ) From 8fcc99cc0946a00ae4d94f720f592db79fbc28ca Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Wed, 21 Jun 2023 15:42:49 +0100 Subject: [PATCH 34/80] Remove reference to the deprecated and disabled Typo issue Metalava no longer looks for common typos. It relies on the code analyzers that run on Gerrit to detect them. Bug: 294300564 Bug: 288072162 Test: m checkapi Change-Id: I08eca8008af39ebe69442db419ebd3d3378aa0de Merged-In: I08eca8008af39ebe69442db419ebd3d3378aa0de (cherry picked from commit c0036491cd6b454a90bde516555a006d20d06ddd) --- java/sdk_library.go | 1 - 1 file changed, 1 deletion(-) diff --git a/java/sdk_library.go b/java/sdk_library.go index 103f1ace7..76f050b73 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -1678,7 +1678,6 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC "MissingPermission", "SdkConstant", "Todo", - "Typo", "UnavailableSymbol", } droidstubsArgs = append(droidstubsArgs, android.JoinWithPrefix(disabledWarnings, "--hide ")) From d7d52fa7ccd3b007ab46eda218e6ef0fdc6a5748 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Wed, 31 May 2023 17:51:33 +0100 Subject: [PATCH 35/80] Explicitly specify `--api-class-resolution api` for metalava Previously, `--api-class-resolution api` was the default if that option was not specified but the other change in this topic is switching it to `--api-class-resolution api:classpath` to match the behavior in AndroidX. This change explicitly specifies the option needed by Soong's use of metalava. Bug: 294300564 Bug: 285140653 Test: m checkapi Merged-In: Ief3c7f9dfdfa946f21cb048a579ec90309fb350f Change-Id: Ief3c7f9dfdfa946f21cb048a579ec90309fb350f (cherry picked from commit 5b7035f850612aecfd67b7099e62ce82659b3041) --- java/droidstubs.go | 4 ++++ java/java.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/java/droidstubs.go b/java/droidstubs.go index 8a521aabb..d577b22cc 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -535,6 +535,10 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi // b/223382732 FlagWithArg("--hide ", "ChangedDefault") + // Force metalava to ignore classes on the classpath when an API file contains missing classes. + // See b/285140653 for more information. + cmd.FlagWithArg("--api-class-resolution ", "api") + return cmd } diff --git a/java/java.go b/java/java.go index ccb150747..dec7810e8 100644 --- a/java/java.go +++ b/java/java.go @@ -1723,6 +1723,10 @@ func metalavaStubCmd(ctx android.ModuleContext, rule *android.RuleBuilder, FlagWithArg("--hide ", "InvalidNullabilityOverride"). FlagWithArg("--hide ", "ChangedDefault") + // Force metalava to ignore classes on the classpath when an API file contains missing classes. + // See b/285140653 for more information. + cmd.FlagWithArg("--api-class-resolution ", "api") + return cmd } From 2ba521e408f58c9e92e4af1c8cfb63afe3e1d2f9 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Thu, 1 Jun 2023 15:28:54 +0100 Subject: [PATCH 36/80] Explicitly specify `--api-overloaded-method-order source` for metalava Previously, `--api-overloaded-method-order source` was the default if that option was not specified but the other change in this topic is switching it to `--api-overloaded-method-order signature` to match the behavior in AndroidX. This change explicitly specifies the option needed by Soong's use of metalava. Bug: 294300564 Bug: 285140653 Test: m checkapi Merged-In: I00e049e0d991e650ab9940a3d495ed2b6e7480f8 Change-Id: I00e049e0d991e650ab9940a3d495ed2b6e7480f8 (cherry picked from commit 7202ffebf994136e2140e64fc0db7fab1d552a14) --- java/droidstubs.go | 4 ++++ java/java.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/java/droidstubs.go b/java/droidstubs.go index d577b22cc..151c94a43 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -539,6 +539,10 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi // See b/285140653 for more information. cmd.FlagWithArg("--api-class-resolution ", "api") + // Force metalava to sort overloaded methods by their order in the source code. + // See b/285312164 for more information. + cmd.FlagWithArg("--api-overloaded-method-order ", "source") + return cmd } diff --git a/java/java.go b/java/java.go index dec7810e8..75c0a5c4b 100644 --- a/java/java.go +++ b/java/java.go @@ -1727,6 +1727,10 @@ func metalavaStubCmd(ctx android.ModuleContext, rule *android.RuleBuilder, // See b/285140653 for more information. cmd.FlagWithArg("--api-class-resolution ", "api") + // Force metalava to sort overloaded methods by their order in the source code. + // See b/285312164 for more information. + cmd.FlagWithArg("--api-overloaded-method-order ", "source") + return cmd } From beb6e9ffdb4e5aaa29e2916273f140f3f5d71926 Mon Sep 17 00:00:00 2001 From: Mark White Date: Thu, 10 Aug 2023 00:07:03 +0000 Subject: [PATCH 37/80] SdkTestCore for non-updatable modules Provides SdkTestCore/test_core_current sdk_version for non-updatable modules that have their test scope dependent on test apis from framework-minus-apex. Ignore-AOSP-First: Change in topic with internal-first projects Bug: 289776578 Test: m checkapi | go test ./java Change-Id: Iba3213e8c34ea75ac9dd8532a95ef62fb5455e6c --- android/sdk_version.go | 12 ++++++++++-- java/sdk.go | 5 +++-- java/sdk_library.go | 6 ++++++ java/sdk_library_test.go | 26 ++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/android/sdk_version.go b/android/sdk_version.go index 1fadda0ca..aafcee79a 100644 --- a/android/sdk_version.go +++ b/android/sdk_version.go @@ -48,6 +48,7 @@ const ( SdkPublic SdkSystem SdkTest + SdkTestFrameworksCore SdkModule SdkSystemServer SdkPrivate @@ -67,6 +68,8 @@ func (k SdkKind) String() string { return "system" case SdkTest: return "test" + case SdkTestFrameworksCore: + return "test_frameworks_core" case SdkCore: return "core" case SdkCorePlatform: @@ -92,6 +95,8 @@ func (k SdkKind) DefaultJavaLibraryName() string { return "android_system_stubs_current" case SdkTest: return "android_test_stubs_current" + case SdkTestFrameworksCore: + return "android_test_frameworks_core_stubs_current" case SdkCore: return "core.current.stubs" case SdkModule: @@ -137,7 +142,7 @@ func (s SdkSpec) Stable() bool { return true case SdkCore, SdkPublic, SdkSystem, SdkModule, SdkSystemServer: return true - case SdkCorePlatform, SdkTest, SdkPrivate: + case SdkCorePlatform, SdkTest, SdkTestFrameworksCore, SdkPrivate: return false default: panic(fmt.Errorf("unknown SdkKind=%v", s.Kind)) @@ -185,7 +190,8 @@ func (s SdkSpec) UsePrebuilt(ctx EarlyModuleContext) bool { return ctx.Config().AlwaysUsePrebuiltSdks() } else if !s.ApiLevel.IsPreview() { // validation check - if s.Kind != SdkPublic && s.Kind != SdkSystem && s.Kind != SdkTest && s.Kind != SdkModule && s.Kind != SdkSystemServer { + if s.Kind != SdkPublic && s.Kind != SdkSystem && s.Kind != SdkTest && + s.Kind != SdkTestFrameworksCore && s.Kind != SdkModule && s.Kind != SdkSystemServer { panic(fmt.Errorf("prebuilt SDK is not not available for SdkKind=%q", s.Kind)) return false } @@ -266,6 +272,8 @@ func SdkSpecFromWithConfig(config Config, str string) SdkSpec { kind = SdkSystem case "test": kind = SdkTest + case "test_frameworks_core": + kind = SdkTestFrameworksCore case "module": kind = SdkModule case "system_server": diff --git a/java/sdk.go b/java/sdk.go index 7c702c405..ddd99bb33 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -76,7 +76,8 @@ func systemModuleKind(sdkKind android.SdkKind, apiLevel android.ApiLevel) androi // Core is by definition what is included in the system module for the public API so should // just use its system modules. systemModuleKind = android.SdkPublic - } else if systemModuleKind == android.SdkSystem || systemModuleKind == android.SdkTest { + } else if systemModuleKind == android.SdkSystem || systemModuleKind == android.SdkTest || + systemModuleKind == android.SdkTestFrameworksCore { // The core system and test APIs are currently the same as the public API so they should use // its system modules. systemModuleKind = android.SdkPublic @@ -192,7 +193,7 @@ func decodeSdkDep(ctx android.EarlyModuleContext, sdkContext android.SdkContext) bootclasspath: corePlatformBootclasspathLibraries(ctx), noFrameworksLibs: true, } - case android.SdkPublic, android.SdkSystem, android.SdkTest: + case android.SdkPublic, android.SdkSystem, android.SdkTest, android.SdkTestFrameworksCore: return toModule(sdkVersion.Kind.DefaultJavaLibraryName(), sdkFrameworkAidlPath(ctx)) case android.SdkCore: return sdkDep{ diff --git a/java/sdk_library.go b/java/sdk_library.go index dbb2f0247..e7c84f6ff 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -442,6 +442,9 @@ type ApiScopeProperties struct { // or the API file. They both have to use the same sdk_version as is used for // compiling the implementation library. Sdk_version *string + + // Extra libs used when compiling stubs for this scope. + Libs []string } type sdkLibraryProperties struct { @@ -1602,6 +1605,7 @@ func (module *SdkLibrary) createStubsLibrary(mctx android.DefaultableHookContext props.Patch_module = module.properties.Patch_module props.Installable = proptools.BoolPtr(false) props.Libs = module.sdkLibraryProperties.Stub_only_libs + props.Libs = append(props.Libs, module.scopeToProperties[apiScope].Libs...) props.Static_libs = module.sdkLibraryProperties.Stub_only_static_libs // The stub-annotations library contains special versions of the annotations // with CLASS retention policy, so that they're kept. @@ -1674,6 +1678,7 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC props.Libs = module.properties.Libs props.Libs = append(props.Libs, module.properties.Static_libs...) props.Libs = append(props.Libs, module.sdkLibraryProperties.Stub_only_libs...) + props.Libs = append(props.Libs, module.scopeToProperties[apiScope].Libs...) props.Aidl.Include_dirs = module.deviceProperties.Aidl.Include_dirs props.Aidl.Local_include_dirs = module.deviceProperties.Aidl.Local_include_dirs props.Java_version = module.properties.Java_version @@ -1805,6 +1810,7 @@ func (module *SdkLibrary) createApiLibrary(mctx android.DefaultableHookContext, props.Api_contributions = apiContributions props.Libs = module.properties.Libs props.Libs = append(props.Libs, module.sdkLibraryProperties.Stub_only_libs...) + props.Libs = append(props.Libs, module.scopeToProperties[apiScope].Libs...) props.Libs = append(props.Libs, "stub-annotations") props.Static_libs = module.sdkLibraryProperties.Stub_only_static_libs props.Full_api_surface_stub = proptools.StringPtr(apiScope.kind.DefaultJavaLibraryName() + ".from-text") diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index c22b9809b..85bd04541 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -1416,6 +1416,32 @@ func TestJavaSdkLibrary_StubOnlyLibs_PassedToDroidstubs(t *testing.T) { android.AssertStringListContains(t, "foo stubs should depend on bar-lib", fooStubsSources.Javadoc.properties.Libs, "bar-lib") } +func TestJavaSdkLibrary_Scope_Libs_PassedToDroidstubs(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForJavaTest, + PrepareForTestWithJavaSdkLibraryFiles, + FixtureWithLastReleaseApis("foo"), + ).RunTestWithBp(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java"], + public: { + enabled: true, + libs: ["bar-lib"], + }, + } + + java_library { + name: "bar-lib", + srcs: ["b.java"], + } + `) + + // The foo.stubs.source should depend on bar-lib + fooStubsSources := result.ModuleForTests("foo.stubs.source", "android_common").Module().(*Droidstubs) + android.AssertStringListContains(t, "foo stubs should depend on bar-lib", fooStubsSources.Javadoc.properties.Libs, "bar-lib") +} + func TestJavaSdkLibrary_ApiLibrary(t *testing.T) { result := android.GroupFixturePreparers( prepareForJavaTest, From 662e2c5e98f9a6833b779a4f8fde2c31f3d07e3a Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Wed, 16 Aug 2023 19:06:32 +0000 Subject: [PATCH 38/80] Move from-text stub java_api_library modules closer to source This change moves java_api_library modules that were previously defined in build/orchestrator closer to where their from-source equivalents are defined. Ignore-AOSP-First: Resolve merge conflict Test: m (default to from-text stub build) Bug: 274805756 (cherry picked from https://android-review.googlesource.com/q/commit:fb5331fd3eff558b5f0751692db38cd77018f1b6) Change-Id: If66b5bba719bb389a69f2353d2218a89b2b96b90 --- java/core-libraries/Android.bp | 54 +++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/java/core-libraries/Android.bp b/java/core-libraries/Android.bp index de9a82d09..4380f4f5c 100644 --- a/java/core-libraries/Android.bp +++ b/java/core-libraries/Android.bp @@ -55,6 +55,19 @@ java_library { ], } +java_api_library { + name: "core.current.stubs.from-text", + api_surface: "core", + api_contributions: [ + "art.module.public.api.stubs.source.api.contribution", + "conscrypt.module.public.api.stubs.source.api.contribution", + "i18n.module.public.api.stubs.source.api.contribution", + ], + libs: [ + "stub-annotations", + ], +} + java_library { name: "core.current.stubs", defaults: [ @@ -153,7 +166,6 @@ java_defaults { system_modules: "none", } - // A stubs target containing the parts of the public SDK & @SystemApi(MODULE_LIBRARIES) API // provided by the core libraries. // @@ -265,6 +277,32 @@ java_library { ], } +java_defaults { + name: "android_core_platform_stubs_current_contributions", + api_surface: "core_platform", + api_contributions: [ + "art.module.public.api.stubs.source.api.contribution", + "art.module.public.api.stubs.source.system.api.contribution", + "art.module.public.api.stubs.source.module_lib.api.contribution", + "conscrypt.module.platform.api.stubs.source.api.contribution", + "i18n.module.public.api.stubs.source.api.contribution", + ], +} + +java_api_library { + name: "legacy.core.platform.api.stubs.from-text", + api_surface: "core_platform", + defaults: [ + "android_core_platform_stubs_current_contributions", + ], + api_contributions: [ + "legacy.i18n.module.platform.api.stubs.source.api.contribution", + ], + libs: [ + "stub-annotations", + ], +} + java_library { name: "legacy.core.platform.api.stubs", visibility: core_platform_visibility, @@ -328,6 +366,20 @@ java_library { ], } +java_api_library { + name: "stable.core.platform.api.stubs.from-text", + api_surface: "core_platform", + defaults: [ + "android_core_platform_stubs_current_contributions", + ], + api_contributions: [ + "stable.i18n.module.platform.api.stubs.source.api.contribution", + ], + libs: [ + "stub-annotations", + ], +} + java_library { name: "stable.core.platform.api.stubs", visibility: core_platform_visibility, From c9b16799b5088d6c101c56ce62d42d3da7ac641c Mon Sep 17 00:00:00 2001 From: Mark White Date: Thu, 10 Aug 2023 00:07:40 +0000 Subject: [PATCH 39/80] java_library support for building headers-only Flag for java_library modules to build just the Turbine headers and skip building an impl jar. Test: m framework-minus-apex-headers Bug: 289776578 Ignore-AOSP-First: Change stacked with changes in internal repos Change-Id: If8b4033462d39c689cba0ed38cc343e4786ea280 --- android/neverallow.go | 8 +++++++ android/neverallow_test.go | 16 ++++++++++++++ java/androidmk.go | 3 +++ java/base.go | 44 ++++++++++++++++++++++++++++++++++++++ java/java.go | 1 + java/java_test.go | 18 ++++++++++++++++ 6 files changed, 90 insertions(+) diff --git a/android/neverallow.go b/android/neverallow.go index 24031ba39..2be6a74f5 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -60,6 +60,7 @@ func init() { AddNeverAllowRules(createBp2BuildRule()) AddNeverAllowRules(createCcStubsRule()) AddNeverAllowRules(createJavaExcludeStaticLibsRule()) + AddNeverAllowRules(createProhibitHeaderOnlyRule()) } // Add a NeverAllow rule to the set of rules to apply. @@ -264,6 +265,13 @@ func createJavaExcludeStaticLibsRule() Rule { Because("exclude_static_libs property is only allowed for java modules defined in build/soong, libcore, and frameworks/base/api") } +func createProhibitHeaderOnlyRule() Rule { + return NeverAllow(). + Without("name", "framework-minus-apex-headers"). + With("headers_only", "true"). + Because("headers_only can only be used for generating framework-minus-apex headers for non-updatable modules") +} + func neverallowMutator(ctx BottomUpMutatorContext) { m, ok := ctx.Module().(Module) if !ok { diff --git a/android/neverallow_test.go b/android/neverallow_test.go index 2a938b811..b2620ef92 100644 --- a/android/neverallow_test.go +++ b/android/neverallow_test.go @@ -361,6 +361,21 @@ var neverallowTests = []struct { `exclude_static_libs property is only allowed for java modules defined in build/soong, libcore, and frameworks/base/api`, }, }, + // Test for only allowing headers_only for framework-minus-apex-headers + { + name: `"headers_only" outside framework-minus-apex-headers modules`, + fs: map[string][]byte{ + "a/b/Android.bp": []byte(` + java_library { + name: "baz", + headers_only: true, + } + `), + }, + expectedErrors: []string{ + `headers_only can only be used for generating framework-minus-apex headers for non-updatable modules`, + }, + }, } var prepareForNeverAllowTest = GroupFixturePreparers( @@ -451,6 +466,7 @@ type mockJavaLibraryProperties struct { Sdk_version *string Uncompress_dex *bool Exclude_static_libs []string + Headers_only *bool } type mockJavaLibraryModule struct { diff --git a/java/androidmk.go b/java/androidmk.go index 82505e9e3..b7e2d2ff3 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -79,6 +79,9 @@ func (library *Library) AndroidMkEntries() []android.AndroidMkEntries { } else if !library.ApexModuleBase.AvailableFor(android.AvailableToPlatform) { // Platform variant. If not available for the platform, we don't need Make module. entriesList = append(entriesList, android.AndroidMkEntries{Disabled: true}) + } else if library.properties.Headers_only { + // If generating headers only then don't expose to Make. + entriesList = append(entriesList, android.AndroidMkEntries{Disabled: true}) } else { entriesList = append(entriesList, android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", diff --git a/java/base.go b/java/base.go index f5eb01c4b..58ba41098 100644 --- a/java/base.go +++ b/java/base.go @@ -192,6 +192,9 @@ type CommonProperties struct { // Additional srcJars tacked in by GeneratedJavaLibraryModule Generated_srcjars []android.Path `android:"mutated"` + + // If true, then only the headers are built and not the implementation jar. + Headers_only bool } // Properties that are specific to device modules. Host module factories should not add these when @@ -574,6 +577,17 @@ func (j *Module) checkPlatformAPI(ctx android.ModuleContext) { } } +func (j *Module) checkHeadersOnly(ctx android.ModuleContext) { + if _, ok := ctx.Module().(android.SdkContext); ok { + headersOnly := proptools.Bool(&j.properties.Headers_only) + installable := proptools.Bool(j.properties.Installable) + + if headersOnly && installable { + ctx.PropertyErrorf("headers_only", "This module has conflicting settings. headers_only is true which, which means this module doesn't generate an implementation jar. However installable is set to true.") + } + } +} + func (j *Module) addHostProperties() { j.AddProperties( &j.properties, @@ -1151,6 +1165,36 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath // final R classes from the app. flags.classpath = append(android.CopyOf(extraClasspathJars), flags.classpath...) + // If compiling headers then compile them and skip the rest + if j.properties.Headers_only { + if srcFiles.HasExt(".kt") { + ctx.ModuleErrorf("Compiling headers_only with .kt not supported") + } + if ctx.Config().IsEnvFalse("TURBINE_ENABLED") || disableTurbine { + ctx.ModuleErrorf("headers_only is enabled but Turbine is disabled.") + } + + _, j.headerJarFile = + j.compileJavaHeader(ctx, uniqueJavaFiles, srcJars, deps, flags, jarName, + extraCombinedJars) + if ctx.Failed() { + return + } + + ctx.SetProvider(JavaInfoProvider, JavaInfo{ + HeaderJars: android.PathsIfNonNil(j.headerJarFile), + TransitiveLibsHeaderJars: j.transitiveLibsHeaderJars, + TransitiveStaticLibsHeaderJars: j.transitiveStaticLibsHeaderJars, + AidlIncludeDirs: j.exportAidlIncludeDirs, + ExportedPlugins: j.exportedPluginJars, + ExportedPluginClasses: j.exportedPluginClasses, + ExportedPluginDisableTurbine: j.exportedDisableTurbine, + }) + + j.outputFile = j.headerJarFile + return + } + if srcFiles.HasExt(".kt") { // When using kotlin sources turbine is used to generate annotation processor sources, // including for annotation processors that generate API, so we can use turbine for diff --git a/java/java.go b/java/java.go index 70aba8e2c..c60226de6 100644 --- a/java/java.go +++ b/java/java.go @@ -692,6 +692,7 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { } j.checkSdkVersions(ctx) + j.checkHeadersOnly(ctx) if ctx.Device() { j.dexpreopter.installPath = j.dexpreopter.getInstallPath( ctx, android.PathForModuleInstall(ctx, "framework", j.Stem()+".jar")) diff --git a/java/java_test.go b/java/java_test.go index 6110e21cb..27933c3bf 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -2370,3 +2370,21 @@ func TestJavaLibraryWithResourcesStem(t *testing.T) { t.Errorf("Module output does not contain expected jar %s", "test.jar") } } + +func TestHeadersOnly(t *testing.T) { + ctx, _ := testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + headers_only: true, + } + `) + + turbine := ctx.ModuleForTests("foo", "android_common").Rule("turbine") + if len(turbine.Inputs) != 1 || turbine.Inputs[0].String() != "a.java" { + t.Errorf(`foo inputs %v != ["a.java"]`, turbine.Inputs) + } + + javac := ctx.ModuleForTests("foo", "android_common").MaybeRule("javac") + android.AssertDeepEquals(t, "javac rule", nil, javac.Rule) +} From 8bf29fd473708efd44a7e7d05e0a64c1d4ddd5a2 Mon Sep 17 00:00:00 2001 From: Mark White Date: Sun, 6 Aug 2023 00:20:47 +0000 Subject: [PATCH 40/80] Created empty framework-location non-updatable module Ignore-AOSP-First: Change in topic with internal-first projects Bug: 289776578 Test: presubmit Change-Id: Iebc6d016b8313759ac1f6f3ab6f4834f5ad7b93e --- android/config.go | 1 + java/legacy_core_platform_api_usage.go | 1 + 2 files changed, 2 insertions(+) diff --git a/android/config.go b/android/config.go index 2e10ae90a..ec117d097 100644 --- a/android/config.go +++ b/android/config.go @@ -667,6 +667,7 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) "framework-devicelock": {}, "framework-graphics": {}, "framework-healthfitness": {}, + "framework-location": {}, "framework-media": {}, "framework-mediaprovider": {}, "framework-ondevicepersonalization": {}, diff --git a/java/legacy_core_platform_api_usage.go b/java/legacy_core_platform_api_usage.go index 6cb549eaf..4be7d0470 100644 --- a/java/legacy_core_platform_api_usage.go +++ b/java/legacy_core_platform_api_usage.go @@ -28,6 +28,7 @@ var legacyCorePlatformApiModules = []string{ "FloralClocks", "framework-jobscheduler", "framework-minus-apex", + "framework-minus-apex-headers", "framework-minus-apex-intdefs", "FrameworksCoreTests", "HelloOslo", From dc5b8d65384341409a960f452cee234d65ce1232 Mon Sep 17 00:00:00 2001 From: Motomu Utsumi Date: Tue, 22 Aug 2023 12:03:16 +0900 Subject: [PATCH 41/80] Add package/modules/Connectivity bpf_headers path Following CLs will move frameworks/libs/net to packages/modules/Connectivity. This CL is a preparation for that. Bug: 296014682 Test: m (cherry picked from https://android-review.googlesource.com/q/commit:44591557f4bf64c36b831152cb75fc68b8cdad59) Merged-In: Ie6483d1e5dfbf6c4745c10d3bb031344368c8c8f Change-Id: Ie6483d1e5dfbf6c4745c10d3bb031344368c8c8f --- bpf/bpf.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bpf/bpf.go b/bpf/bpf.go index 45009c1e7..d135d5f56 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -153,7 +153,9 @@ func (bpf *bpf) GenerateAndroidBuildActions(ctx android.ModuleContext) { // The architecture doesn't matter here, but asm/types.h is included by linux/types.h. "-isystem bionic/libc/kernel/uapi/asm-arm64", "-isystem bionic/libc/kernel/android/uapi", + // TODO(b/296014682): Remove after the bpf_headers is moved to Connectivity "-I frameworks/libs/net/common/native/bpf_headers/include/bpf", + "-I packages/modules/Connectivity/staticlibs/native/bpf_headers/include/bpf", // TODO(b/149785767): only give access to specific file with AID_* constants "-I system/core/libcutils/include", "-I " + ctx.ModuleDir(), From cba810e0ac80f5c32f5974f1ec4f043b3b0ab54e Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Thu, 7 Sep 2023 12:48:13 +0000 Subject: [PATCH 42/80] Convert Android.bp files in sub-directories of tools/metalava Bug: 299428102 Test: m checkapi (cherry picked from https://android-review.googlesource.com/q/commit:85e33c1cade0af2a1f7cd82370985c29c5029aa1) Merged-In: Ia01a394ec80e7d760cfb920993807c4006d8c36e Change-Id: Ia01a394ec80e7d760cfb920993807c4006d8c36e --- android/allowlists/allowlists.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index b9bbcdcd0..5ccffc30d 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -385,7 +385,7 @@ var ( "tools/apifinder": Bp2BuildDefaultTrue, "tools/apksig": Bp2BuildDefaultTrue, "tools/external_updater": Bp2BuildDefaultTrueRecursively, - "tools/metalava": Bp2BuildDefaultTrue, + "tools/metalava": Bp2BuildDefaultTrueRecursively, "tools/platform-compat/java/android/compat": Bp2BuildDefaultTrueRecursively, "tools/tradefederation/prebuilts/filegroups": Bp2BuildDefaultTrueRecursively, } From bff6432d7cd258fb9450de6b22e367ee8dc0305b Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Wed, 9 Aug 2023 12:36:08 +0100 Subject: [PATCH 43/80] Stop using single-hyphen metalava options `-encoding` and `-source` have been deprecated in metalava. `-encoding` actually does nothing apart from check to make sure that the value is `utf-8` (in some case). Metalava always uses `utf-8`. `-source` has been deprecated in favor of `--java-source` as metalava also can consume kotlin and it has its own `--kotlin-source` option. Bug: 295136054 Test: ./gradlew (cherry picked from https://android-review.googlesource.com/q/commit:808211e519ae43517ae7d6ed5348ffb1e3b63c45) Merged-In: I08e6931958f40022d65d417360e32b72a1d70444 Change-Id: I08e6931958f40022d65d417360e32b72a1d70444 --- java/droiddoc.go | 1 - java/droidstubs.go | 3 +-- java/java.go | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/java/droiddoc.go b/java/droiddoc.go index dbe021d00..1f9f773a3 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -694,7 +694,6 @@ func javadocCmd(ctx android.ModuleContext, rule *android.RuleBuilder, srcs andro cmd := rule.Command(). BuiltTool("soong_javac_wrapper").Tool(config.JavadocCmd(ctx)). Flag(config.JavacVmFlags). - FlagWithArg("-encoding ", "UTF-8"). FlagWithRspFileInputList("@", android.PathForModuleOut(ctx, "javadoc.rsp"), srcs). FlagWithInput("@", srcJarList) diff --git a/java/droidstubs.go b/java/droidstubs.go index 151c94a43..1b258835b 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -512,8 +512,7 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi cmd.BuiltTool("metalava").ImplicitTool(ctx.Config().HostJavaToolPath(ctx, "metalava.jar")). Flag(config.JavacVmFlags). Flag("-J--add-opens=java.base/java.util=ALL-UNNAMED"). - FlagWithArg("-encoding ", "UTF-8"). - FlagWithArg("-source ", javaVersion.String()). + FlagWithArg("--java-source ", javaVersion.String()). FlagWithRspFileInputList("@", android.PathForModuleOut(ctx, "metalava.rsp"), srcs). FlagWithInput("@", srcJarList) diff --git a/java/java.go b/java/java.go index 75c0a5c4b..5e640bc97 100644 --- a/java/java.go +++ b/java/java.go @@ -1705,7 +1705,6 @@ func metalavaStubCmd(ctx android.ModuleContext, rule *android.RuleBuilder, cmd.BuiltTool("metalava").ImplicitTool(ctx.Config().HostJavaToolPath(ctx, "metalava.jar")). Flag(config.JavacVmFlags). Flag("-J--add-opens=java.base/java.util=ALL-UNNAMED"). - FlagWithArg("-encoding ", "UTF-8"). FlagWithInputList("--source-files ", srcs, " ") cmd.Flag("--no-banner"). From a6d2ecd2ff31f2ae6e775a0e2ed437c7968e3e09 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Thu, 10 Aug 2023 15:16:35 +0100 Subject: [PATCH 44/80] Replace -bootclasspath and -classpath metalava options with --classpath Metalava has not differentiated between bootclasspath and classpath for years so this change replaces the use of the two deprecated single hyphen options with `--classpath`. Bug: 295136054 Test: m checkapi (cherry picked from https://android-review.googlesource.com/q/commit:f8aaaa13f4fb87cc0f7bd5231f2334499d174109) Merged-In: I3ab20b76a60cab66a27784b7d87a069813d19835 Change-Id: I3ab20b76a60cab66a27784b7d87a069813d19835 --- java/droidstubs.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/java/droidstubs.go b/java/droidstubs.go index 1b258835b..9379b88aa 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -516,12 +516,12 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi FlagWithRspFileInputList("@", android.PathForModuleOut(ctx, "metalava.rsp"), srcs). FlagWithInput("@", srcJarList) - if len(bootclasspath) > 0 { - cmd.FlagWithInputList("-bootclasspath ", bootclasspath.Paths(), ":") - } - - if len(classpath) > 0 { - cmd.FlagWithInputList("-classpath ", classpath.Paths(), ":") + // Metalava does not differentiate between bootclasspath and classpath and has not done so for + // years, so it is unlikely to change any time soon. + combinedPaths := append(([]android.Path)(nil), bootclasspath.Paths()...) + combinedPaths = append(combinedPaths, classpath.Paths()...) + if len(combinedPaths) > 0 { + cmd.FlagWithInputList("--classpath ", combinedPaths, ":") } cmd.Flag("--no-banner"). From 8fa1eb0e161e1e145735b90ad23c53f63c042d39 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Fri, 11 Aug 2023 22:47:31 +0100 Subject: [PATCH 45/80] Pass the current API file to the metalava --use-same-format-as option That will allow the format of a file to be changed by simply modifying the signature format in the current API file (e.g. from `2.0` to `3.0`) and then running `m update-api` and it will just update the current API file (and the corresponding removed API file). Bug: 295577788 Test: m checkapi && m update-api # Also did as described above and modified the # `libcore/api/current.txt` file from `2.0` to `3.0` and ran # `m update-api` and checked that only that file and the # corresponding `removed.text` file was changed. (cherry picked from https://android-review.googlesource.com/q/commit:10a23c29e279ebfef4869cea72a65207e8de0cab) Merged-In: I20c9bc151677502cf09b0c06fb442fd385caccdd Change-Id: I20c9bc151677502cf09b0c06fb442fd385caccdd --- java/droidstubs.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/java/droidstubs.go b/java/droidstubs.go index 9379b88aa..6e43b3233 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -695,6 +695,13 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { cmd.FlagWithArg("--error-message:compatibility:released ", msg) } + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") { + // Pass the current API file into metalava so it can use it as the basis for determining how to + // generate the output signature files (both api and removed). + currentApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Api_file)) + cmd.FlagWithInput("--use-same-format-as ", currentApiFile) + } + if generateStubs { rule.Command(). BuiltTool("soong_zip"). From 3704158c7039519d560a71a453fb54ef912e8df6 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Wed, 13 Sep 2023 00:00:58 +0100 Subject: [PATCH 46/80] Replace --api-overloaded-method--order with --format-defaults Bug: 300052204 Test: m checkapi (cherry picked from https://android-review.googlesource.com/q/commit:504d3565790b0dc5220f9c9c7ae230fad758fed7) Merged-In: I8952a828437872ceebf7f9da4dc297173ed2a90b Change-Id: I8952a828437872ceebf7f9da4dc297173ed2a90b --- java/droidstubs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/droidstubs.go b/java/droidstubs.go index 9379b88aa..faa7f7d61 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -540,7 +540,7 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi // Force metalava to sort overloaded methods by their order in the source code. // See b/285312164 for more information. - cmd.FlagWithArg("--api-overloaded-method-order ", "source") + cmd.FlagWithArg("--format-defaults ", "overloaded-method-order=source") return cmd } From 71a5cd15c6c99a046aa97059ccfecb16dde8ca3b Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Mon, 5 Jun 2023 12:57:55 -0400 Subject: [PATCH 47/80] Implement bp2build converter for fdo_profile Ignore-AOSP-First: ag/24746588, in the same topic, is in an internal repo. This CL will be cherry-picked to AOSP afterward. Test: go test Bug: 277091218 Change-Id: I389d9535ea176991a1faa9beb46352b93363acd2 --- bp2build/Android.bp | 1 + bp2build/cc_library_conversion_test.go | 12 ++-- bp2build/fdo_profile_conversion_test.go | 85 +++++++++++++++++++++++++ cc/bp2build.go | 19 ++---- cc/fdo_profile.go | 51 ++++++++++++++- cc/testing.go | 2 +- 6 files changed, 149 insertions(+), 21 deletions(-) create mode 100644 bp2build/fdo_profile_conversion_test.go diff --git a/bp2build/Android.bp b/bp2build/Android.bp index 161a7ffcf..b321b38cb 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -63,6 +63,7 @@ bootstrap_go_package { "cc_yasm_conversion_test.go", "conversion_test.go", "droiddoc_exported_dir_conversion_test.go", + "fdo_profile_conversion_test.go", "filegroup_conversion_test.go", "genrule_conversion_test.go", "gensrcs_conversion_test.go", diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index b667fe9dc..3957ff767 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -3640,8 +3640,8 @@ cc_library { { description: "cc_library with afdo enabled and existing profile", filesystem: map[string]string{ - "vendor/google_data/pgo_profile/sampling/BUILD": "", - "vendor/google_data/pgo_profile/sampling/foo.afdo": "", + "vendor/google_data/pgo_profile/sampling/Android.bp": "", + "vendor/google_data/pgo_profile/sampling/foo.afdo": "", }, expectedBazelTargets: []string{ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{}), @@ -3653,8 +3653,8 @@ cc_library { { description: "cc_library with afdo enabled and existing profile in AOSP", filesystem: map[string]string{ - "toolchain/pgo-profiles/sampling/BUILD": "", - "toolchain/pgo-profiles/sampling/foo.afdo": "", + "toolchain/pgo-profiles/sampling/Android.bp": "", + "toolchain/pgo-profiles/sampling/foo.afdo": "", }, expectedBazelTargets: []string{ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{}), @@ -3666,8 +3666,8 @@ cc_library { { description: "cc_library with afdo enabled but profile filename doesn't match with module name", filesystem: map[string]string{ - "toolchain/pgo-profiles/sampling/BUILD": "", - "toolchain/pgo-profiles/sampling/bar.afdo": "", + "toolchain/pgo-profiles/sampling/Android.bp": "", + "toolchain/pgo-profiles/sampling/bar.afdo": "", }, expectedBazelTargets: []string{ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{}), diff --git a/bp2build/fdo_profile_conversion_test.go b/bp2build/fdo_profile_conversion_test.go new file mode 100644 index 000000000..4d04283ca --- /dev/null +++ b/bp2build/fdo_profile_conversion_test.go @@ -0,0 +1,85 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "testing" + + "android/soong/android" + "android/soong/cc" +) + +func runFdoProfileTestCase(t *testing.T, tc Bp2buildTestCase) { + t.Helper() + (&tc).ModuleTypeUnderTest = "fdo_profile" + (&tc).ModuleTypeUnderTestFactory = cc.FdoProfileFactory + RunBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, tc) +} + +func TestFdoProfile(t *testing.T) { + testcases := []struct { + name string + bp string + expectedBazelAttrs AttrNameToString + }{ + { + name: "fdo_profile with arch-specific profiles", + bp: ` +fdo_profile { + name: "foo", + arch: { + arm: { + profile: "foo_arm.afdo", + }, + arm64: { + profile: "foo_arm64.afdo", + } + } +}`, + expectedBazelAttrs: AttrNameToString{ + "profile": `select({ + "//build/bazel/platforms/arch:arm": "foo_arm.afdo", + "//build/bazel/platforms/arch:arm64": "foo_arm64.afdo", + "//conditions:default": None, + })`, + }, + }, + { + name: "fdo_profile with arch-agnostic profile", + bp: ` +fdo_profile { + name: "foo", + profile: "foo.afdo", +}`, + expectedBazelAttrs: AttrNameToString{ + "profile": `"foo.afdo"`, + }, + }, + } + + for _, test := range testcases { + t.Run(test.name, func(t *testing.T) { + expectedBazelTargets := []string{ + // TODO(b/276287371): Add device-only restriction back to fdo_profile targets + MakeBazelTargetNoRestrictions("fdo_profile", "foo", test.expectedBazelAttrs), + } + runFdoProfileTestCase(t, Bp2buildTestCase{ + Description: test.name, + Blueprint: test.bp, + ExpectedBazelTargets: expectedBazelTargets, + }) + }) + } +} diff --git a/cc/bp2build.go b/cc/bp2build.go index 039a3cf74..ce8c96d0c 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1000,6 +1000,8 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) if module.afdo != nil && module.afdo.Properties.Afdo { fdoProfileDep := bp2buildFdoProfile(ctx, module) if fdoProfileDep != nil { + // TODO(b/276287371): Only set fdo_profile for android platform + // https://cs.android.com/android/platform/superproject/main/+/main:build/soong/cc/afdo.go;l=105;drc=2dbe160d1af445de32725098570ec594e3944fc5 (&compilerAttrs).fdoProfile.SetValue(*fdoProfileDep) } } @@ -1109,22 +1111,15 @@ func bp2buildFdoProfile( ctx android.Bp2buildMutatorContext, m *Module, ) *bazel.Label { + // TODO(b/267229066): Convert to afdo boolean attribute and let Bazel handles finding + // fdo_profile target from AfdoProfiles product var for _, project := range globalAfdoProfileProjects { - // Ensure handcrafted BUILD file exists in the project - BUILDPath := android.ExistentPathForSource(ctx, project, "BUILD") - if BUILDPath.Valid() { - // We handcraft a BUILD file with fdo_profile targets that use the existing profiles in the project - // This implementation is assuming that every afdo profile in globalAfdoProfileProjects already has - // an associated fdo_profile target declared in the same package. + // Ensure it's a Soong package + bpPath := android.ExistentPathForSource(ctx, project, "Android.bp") + if bpPath.Valid() { // TODO(b/260714900): Handle arch-specific afdo profiles (e.g. `-arm<64>.afdo`) path := android.ExistentPathForSource(ctx, project, m.Name()+".afdo") if path.Valid() { - // FIXME: Some profiles only exist internally and are not released to AOSP. - // When generated BUILD files are checked in, we'll run into merge conflict. - // The cc_library_shared target in AOSP won't have reference to an fdo_profile target because - // the profile doesn't exist. Internally, the same cc_library_shared target will - // have reference to the fdo_profile. - // For more context, see b/258682955#comment2 fdoProfileLabel := "//" + strings.TrimSuffix(project, "/") + ":" + m.Name() return &bazel.Label{ Label: fdoProfileLabel, diff --git a/cc/fdo_profile.go b/cc/fdo_profile.go index 7fbe71940..d61af7e44 100644 --- a/cc/fdo_profile.go +++ b/cc/fdo_profile.go @@ -16,8 +16,10 @@ package cc import ( "android/soong/android" + "android/soong/bazel" "github.com/google/blueprint" + "github.com/google/blueprint/proptools" ) func init() { @@ -25,11 +27,12 @@ func init() { } func RegisterFdoProfileBuildComponents(ctx android.RegistrationContext) { - ctx.RegisterModuleType("fdo_profile", fdoProfileFactory) + ctx.RegisterModuleType("fdo_profile", FdoProfileFactory) } type fdoProfile struct { android.ModuleBase + android.BazelModuleBase properties fdoProfileProperties } @@ -38,6 +41,49 @@ type fdoProfileProperties struct { Profile *string `android:"arch_variant"` } +type bazelFdoProfileAttributes struct { + Profile bazel.StringAttribute +} + +func (fp *fdoProfile) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + var profileAttr bazel.StringAttribute + + archVariantProps := fp.GetArchVariantProperties(ctx, &fdoProfileProperties{}) + for axis, configToProps := range archVariantProps { + for config, _props := range configToProps { + if archProps, ok := _props.(*fdoProfileProperties); ok { + if axis.String() == "arch" || axis.String() == "no_config" { + if archProps.Profile != nil { + profileAttr.SetSelectValue(axis, config, archProps.Profile) + } + } + } + } + } + + // Ideally, cc_library_shared's fdo_profile attr can be a select statement so that we + // don't lift the restriction here. However, in cc_library_shared macro, fdo_profile + // is used as a string, we need to temporarily lift the host restriction until we can + // pass use fdo_profile attr with select statement + // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/cc/cc_library_shared.bzl;l=127;drc=cc01bdfd39857eddbab04ef69ab6db22dcb1858a + // TODO(b/276287371): Drop the restriction override after fdo_profile path is handled properly + var noRestriction bazel.BoolAttribute + noRestriction.SetSelectValue(bazel.NoConfigAxis, "", proptools.BoolPtr(true)) + + ctx.CreateBazelTargetModuleWithRestrictions( + bazel.BazelTargetModuleProperties{ + Rule_class: "fdo_profile", + }, + android.CommonAttributes{ + Name: fp.Name(), + }, + &bazelFdoProfileAttributes{ + Profile: profileAttr, + }, + noRestriction, + ) +} + // FdoProfileInfo is provided by FdoProfileProvider type FdoProfileInfo struct { Path android.Path @@ -77,9 +123,10 @@ func fdoProfileMutator(ctx android.BottomUpMutatorContext) { } } -func fdoProfileFactory() android.Module { +func FdoProfileFactory() android.Module { m := &fdoProfile{} m.AddProperties(&m.properties) android.InitAndroidMultiTargetsArchModule(m, android.DeviceSupported, android.MultilibBoth) + android.InitBazelModule(m) return m } diff --git a/cc/testing.go b/cc/testing.go index 24d6b0f5f..07bee01be 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -671,7 +671,7 @@ var PrepareForTestWithHostMusl = android.GroupFixturePreparers( // PrepareForTestWithFdoProfile registers module types to test with fdo_profile var PrepareForTestWithFdoProfile = android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { ctx.RegisterModuleType("soong_namespace", android.NamespaceFactory) - ctx.RegisterModuleType("fdo_profile", fdoProfileFactory) + ctx.RegisterModuleType("fdo_profile", FdoProfileFactory) }) // TestConfig is the legacy way of creating a test Config for testing cc modules. From a54231425c71f08c771619f12093dbaaf05ae84e Mon Sep 17 00:00:00 2001 From: Alyssa Ketpreechasawat Date: Fri, 6 Oct 2023 03:42:34 +0000 Subject: [PATCH 48/80] Revert "Disable hiddenapi check for building from prebuilts." Revert submission 2761821-suppress-hiddenapi-check Reason for revert: have some typo - break next build Reverted changes: /q/submissionid:2761821-suppress-hiddenapi-check (cherry picked from https://android-review.googlesource.com/q/commit:bb6b813b1de26306e1deb3b3d8842af41f4b5847) Merged-In: I9fce1e1a9389d58928f1eec50c0eaf016f5f63ac Change-Id: I9fce1e1a9389d58928f1eec50c0eaf016f5f63ac --- android/config.go | 13 ++----------- android/variable.go | 2 -- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/android/config.go b/android/config.go index 8f9c9574a..3e1a96266 100644 --- a/android/config.go +++ b/android/config.go @@ -167,8 +167,7 @@ func (c Config) RunningInsideUnitTest() bool { } // DisableHiddenApiChecks returns true if hiddenapi checks have been disabled. -// For 'eng' target variant hiddenapi checks are disabled by default for performance optimisation -// Hiddenapi checks are also disabled when RELEASE_DEFAULT_MODULE_BUILD_FROM_SOURCE is set to false +// For 'eng' target variant hiddenapi checks are disabled by default for performance optimisation, // but can be enabled by setting environment variable ENABLE_HIDDENAPI_FLAGS=true. // For other target variants hiddenapi check are enabled by default but can be disabled by // setting environment variable UNSAFE_DISABLE_HIDDENAPI_FLAGS=true. @@ -177,8 +176,7 @@ func (c Config) RunningInsideUnitTest() bool { func (c Config) DisableHiddenApiChecks() bool { return !c.IsEnvTrue("ENABLE_HIDDENAPI_FLAGS") && (c.IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") || - Bool(c.productVariables.Eng) || - !c.ReleaseDefaultModuleBuildFromSource()) + Bool(c.productVariables.Eng)) } // MaxPageSizeSupported returns the max page size supported by the device. This @@ -223,13 +221,6 @@ func (c Config) ReleaseAconfigFlagDefaultPermission() string { return c.config.productVariables.ReleaseAconfigFlagDefaultPermission } -// The flag indicating behavior for the tree wrt building modules or using prebuilts -// derived from RELEASE_DEFAULT_MODULE_BUILD_FROM_SOURCE -func (c Config) ReleaseDefaultModuleBuildFromSource() bool { - return c.config.productVariables.ReleaseDefaultModuleBuildFromSource == nil || - Bool(c.config.productVariables.ReleaseDefaultModuleBuildFromSource) -} - // A DeviceConfig object represents the configuration for a particular device // being built. For now there will only be one of these, but in the future there // may be multiple devices being built. diff --git a/android/variable.go b/android/variable.go index 006a77f11..8882e80cb 100644 --- a/android/variable.go +++ b/android/variable.go @@ -479,8 +479,6 @@ type ProductVariables struct { ReleaseAconfigFlagDefaultPermission string `json:",omitempty"` - ReleaseDefaultModuleBuildFromSource *bool `json:",omitempty"` - KeepVndk *bool `json:",omitempty"` CheckVendorSeappViolations *bool `json:",omitempty"` From ea769bc5ec15c2503fe52505b64ca9fadfcc85ed Mon Sep 17 00:00:00 2001 From: Zhi Dou Date: Fri, 6 Oct 2023 07:21:52 +0000 Subject: [PATCH 49/80] Add UnsupportedAppUsage to java_aconfig_library dependency Generated flag requires UnsupportedAppUsage annotation to expose the flag to CTS tests. Bug: 301272559 Test: presubit Ignore-AOSP-First: Need to commit with changes in internal main. This change will be cherry picked to AOSP once merged in main Change-Id: I90c87596ca80766ece429ddee1b45723b01d2760 --- aconfig/java_aconfig_library.go | 2 ++ java/testing.go | 1 + 2 files changed, 3 insertions(+) diff --git a/aconfig/java_aconfig_library.go b/aconfig/java_aconfig_library.go index 48cfb7695..79573c787 100644 --- a/aconfig/java_aconfig_library.go +++ b/aconfig/java_aconfig_library.go @@ -58,6 +58,8 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) DepsMutator(module *ja // Add aconfig-annotations-lib as a dependency for the optimization / code stripping annotations module.AddSharedLibrary("aconfig-annotations-lib") + // TODO(b/303773055): Remove the annotation after access issue is resolved. + module.AddSharedLibrary("unsupportedappusage") } func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuildActions(module *java.GeneratedJavaLibraryModule, ctx android.ModuleContext) android.Path { diff --git a/java/testing.go b/java/testing.go index 98bea7f14..0b7e68539 100644 --- a/java/testing.go +++ b/java/testing.go @@ -410,6 +410,7 @@ func gatherRequiredDepsForTest() string { "stub-annotations", "aconfig-annotations-lib", + "unsupportedappusage", } for _, extra := range extraModules { From 97bcce76952170374f0e054731750afb3d9984bc Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Tue, 10 Oct 2023 17:50:25 +0100 Subject: [PATCH 50/80] Retain concrete overrides of abstract methods by default Bug: 299366704 Test: m checkapi (cherry picked from https://android-review.googlesource.com/q/commit:629b9d287d61b44a587629c7c1176fcc2197590d) Merged-In: I3d9fafefe90a98568c16d80c30e4d1d88a22c350 Change-Id: I3d9fafefe90a98568c16d80c30e4d1d88a22c350 --- java/droidstubs.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java/droidstubs.go b/java/droidstubs.go index 521afd0bd..a9e20e03f 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -540,7 +540,9 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi // Force metalava to sort overloaded methods by their order in the source code. // See b/285312164 for more information. - cmd.FlagWithArg("--format-defaults ", "overloaded-method-order=source") + // And add concrete overrides of abstract methods, see b/299366704 for more + // information. + cmd.FlagWithArg("--format-defaults ", "overloaded-method-order=source,add-additional-overrides=yes") return cmd } From e7889927e3956d252e25c7e52bf52d7e1443bde5 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 18 Oct 2023 19:50:47 +0000 Subject: [PATCH 51/80] Move non-vendor modules to the build/soong allowlist Otherwise, builds fail on git_main-without-vendor. Bug: 290816499 Change-Id: I78c8c359779d99d26a7b621f89d3feb4c2cf62d1 Test: Presubmits Ignore-AOSP-First: These modules are not in aosp yet --- genrule/allowlists.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 8552173a3..06869a281 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -109,6 +109,8 @@ var ( "camera-its", "checkIn-service-stub-lite", "chre_atoms_log.h", + "cobalt-registry-binarypb-gen", + "cobalt-registry-validated-java", "com.android.apex.apkrollback.test.pem", "com.android.apex.apkrollback.test.pubkey", "com.android.apex.cts.shim.debug.pem", @@ -150,6 +152,7 @@ var ( "egl_functions_hdr", "emp_ematch.yacc.c", "emp_ematch.yacc.h", + "exercise_type_data_type_config-pb", "fdt_test_tree_empty_memory_range_dtb", "fdt_test_tree_multiple_memory_ranges_dtb", "fdt_test_tree_one_memory_range_dtb", @@ -236,6 +239,10 @@ var ( "pixelstatsatoms.cpp", "pixelstatsatoms.h", "pvmfw_fdt_template_rs", + "pw_log_log_proto_pwpb_h", + "pw_log_log_raw_rpc_h", + "pw_log_log_rpc_pwpb_h", + "pw_tokenizer_proto_options_pwpb_h", "r8retrace-dexdump-sample-app", "r8retrace-run-retrace", "rootcanal_bredr_bb_packets_cxx_gen", From 5051596b119c0d748a6651bd38c17e184ab1610d Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 18 Oct 2023 19:50:47 +0000 Subject: [PATCH 52/80] Move non-vendor modules to the build/soong allowlist Otherwise, builds fail on git_main-without-vendor. Bug: 290816499 Change-Id: I78c8c359779d99d26a7b621f89d3feb4c2cf62d1 Test: Presubmits Ignore-AOSP-First: These modules are not in aosp yet --- genrule/allowlists.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 8552173a3..06869a281 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -109,6 +109,8 @@ var ( "camera-its", "checkIn-service-stub-lite", "chre_atoms_log.h", + "cobalt-registry-binarypb-gen", + "cobalt-registry-validated-java", "com.android.apex.apkrollback.test.pem", "com.android.apex.apkrollback.test.pubkey", "com.android.apex.cts.shim.debug.pem", @@ -150,6 +152,7 @@ var ( "egl_functions_hdr", "emp_ematch.yacc.c", "emp_ematch.yacc.h", + "exercise_type_data_type_config-pb", "fdt_test_tree_empty_memory_range_dtb", "fdt_test_tree_multiple_memory_ranges_dtb", "fdt_test_tree_one_memory_range_dtb", @@ -236,6 +239,10 @@ var ( "pixelstatsatoms.cpp", "pixelstatsatoms.h", "pvmfw_fdt_template_rs", + "pw_log_log_proto_pwpb_h", + "pw_log_log_raw_rpc_h", + "pw_log_log_rpc_pwpb_h", + "pw_tokenizer_proto_options_pwpb_h", "r8retrace-dexdump-sample-app", "r8retrace-run-retrace", "rootcanal_bredr_bb_packets_cxx_gen", From eb51e88205f1e8d187d95bb64128b3aae504d3ce Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 18 Oct 2023 19:50:47 +0000 Subject: [PATCH 53/80] Move non-vendor modules to the build/soong allowlist Otherwise, builds fail on git_main-without-vendor. Bug: 290816499 Change-Id: I78c8c359779d99d26a7b621f89d3feb4c2cf62d1 Merged-In: I78c8c359779d99d26a7b621f89d3feb4c2cf62d1 Test: Presubmits Ignore-AOSP-First: These modules are not in aosp yet --- genrule/allowlists.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 8552173a3..06869a281 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -109,6 +109,8 @@ var ( "camera-its", "checkIn-service-stub-lite", "chre_atoms_log.h", + "cobalt-registry-binarypb-gen", + "cobalt-registry-validated-java", "com.android.apex.apkrollback.test.pem", "com.android.apex.apkrollback.test.pubkey", "com.android.apex.cts.shim.debug.pem", @@ -150,6 +152,7 @@ var ( "egl_functions_hdr", "emp_ematch.yacc.c", "emp_ematch.yacc.h", + "exercise_type_data_type_config-pb", "fdt_test_tree_empty_memory_range_dtb", "fdt_test_tree_multiple_memory_ranges_dtb", "fdt_test_tree_one_memory_range_dtb", @@ -236,6 +239,10 @@ var ( "pixelstatsatoms.cpp", "pixelstatsatoms.h", "pvmfw_fdt_template_rs", + "pw_log_log_proto_pwpb_h", + "pw_log_log_raw_rpc_h", + "pw_log_log_rpc_pwpb_h", + "pw_tokenizer_proto_options_pwpb_h", "r8retrace-dexdump-sample-app", "r8retrace-run-retrace", "rootcanal_bredr_bb_packets_cxx_gen", From 2e196473d07cfe0b9e63de05535847355dbf9d5e Mon Sep 17 00:00:00 2001 From: Sebastian Pickl Date: Tue, 24 Oct 2023 11:18:34 +0000 Subject: [PATCH 54/80] Revert "Default from-text stub generation in build" This reverts commit 182b56b870c52a204055eb32c583828ac0ac0fa4. Reason for revert: breaking boot tests b/307495247, b/307411752 Bug:307495247 (cherry picked from https://android-review.googlesource.com/q/commit:1c4188c9da7b55cdae967e2d2a0a3c9a51c0ad9f) Merged-In: Iea05703b767d2699ca3cf69377eb44b1d21697ad Change-Id: Iea05703b767d2699ca3cf69377eb44b1d21697ad --- android/config.go | 14 +++++++------- cmd/soong_build/main.go | 2 +- ui/build/config.go | 8 ++++---- ui/build/soong.go | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/android/config.go b/android/config.go index dd23d56cd..1deaf3521 100644 --- a/android/config.go +++ b/android/config.go @@ -99,7 +99,7 @@ type CmdArgs struct { UseBazelProxy bool - BuildFromSourceStub bool + BuildFromTextStub bool EnsureAllowlistIntegrity bool } @@ -337,9 +337,9 @@ type config struct { // unix sockets, instead of spawning Bazel as a subprocess. UseBazelProxy bool - // If buildFromSourceStub is true then the Java API stubs are - // built from the source Java files, not the signature text files. - buildFromSourceStub bool + // If buildFromTextStub is true then the Java API stubs are + // built from the signature text files, not the source Java files. + buildFromTextStub bool // If ensureAllowlistIntegrity is true, then the presence of any allowlisted // modules that aren't mixed-built for at least one variant will cause a build @@ -556,7 +556,7 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) MultitreeBuild: cmdArgs.MultitreeBuild, UseBazelProxy: cmdArgs.UseBazelProxy, - buildFromSourceStub: cmdArgs.BuildFromSourceStub, + buildFromTextStub: cmdArgs.BuildFromTextStub, } config.deviceConfig = &deviceConfig{ @@ -2075,11 +2075,11 @@ func (c *config) BuildFromTextStub() bool { // TODO: b/302320354 - Remove the coverage build specific logic once the // robust solution for handling native properties in from-text stub build // is implemented. - return !c.buildFromSourceStub && !c.JavaCoverageEnabled() + return c.buildFromTextStub && !c.JavaCoverageEnabled() } func (c *config) SetBuildFromTextStub(b bool) { - c.buildFromSourceStub = !b + c.buildFromTextStub = b c.productVariables.Build_from_text_stub = boolPtr(b) } diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 53f5d8036..d20847bb2 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -84,7 +84,7 @@ func init() { flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules") flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules") flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses") - flag.BoolVar(&cmdlineArgs.BuildFromSourceStub, "build-from-source-stub", false, "build Java stubs from source files instead of API text files") + flag.BoolVar(&cmdlineArgs.BuildFromTextStub, "build-from-text-stub", false, "build Java stubs from API text files instead of source files") flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built") // Flags that probably shouldn't be flags of soong_build, but we haven't found // the time to remove them yet diff --git a/ui/build/config.go b/ui/build/config.go index eae379f96..5925b283c 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -85,7 +85,7 @@ type configImpl struct { searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces skipMetricsUpload bool buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time - buildFromSourceStub bool + buildFromTextStub bool ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built bazelExitCode int32 // For b runs - necessary for updating NonZeroExit besId string // For b runs, to identify the BuildEventService logs @@ -808,8 +808,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { } else { ctx.Fatalf("unknown option for ninja_weight_source: %s", source) } - } else if arg == "--build-from-source-stub" { - c.buildFromSourceStub = true + } else if arg == "--build-from-text-stub" { + c.buildFromTextStub = true } else if strings.HasPrefix(arg, "--build-command=") { buildCmd := strings.TrimPrefix(arg, "--build-command=") // remove quotations @@ -1157,7 +1157,7 @@ func (c *configImpl) SkipConfig() bool { } func (c *configImpl) BuildFromTextStub() bool { - return !c.buildFromSourceStub + return c.buildFromTextStub } func (c *configImpl) TargetProduct() string { diff --git a/ui/build/soong.go b/ui/build/soong.go index 18eee75ac..ac9bf3a60 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -188,8 +188,8 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuil if pb.config.multitreeBuild { commonArgs = append(commonArgs, "--multitree-build") } - if pb.config.buildFromSourceStub { - commonArgs = append(commonArgs, "--build-from-source-stub") + if pb.config.buildFromTextStub { + commonArgs = append(commonArgs, "--build-from-text-stub") } commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list")) @@ -305,8 +305,8 @@ func bootstrapBlueprint(ctx Context, config Config) { if config.MultitreeBuild() { mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build") } - if config.buildFromSourceStub { - mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-source-stub") + if config.buildFromTextStub { + mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-text-stub") } if config.ensureAllowlistIntegrity { mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--ensure-allowlist-integrity") From 6dd8c8bc18ec27c784ce0141ecccd106e9bff981 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 25 Oct 2023 18:06:37 -0700 Subject: [PATCH 55/80] Remove cobalt registry genrules from denylist go/roboleaf-busy-beavers-sandboxing Bug: 307824623 Test: m cobalt-registry-binarypb-gen cobalt-registry-validated-java Change-Id: Icbb386acf8b1518c74ba16c5508733317c574619 --- genrule/allowlists.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 715ff1e14..048b971db 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -109,8 +109,6 @@ var ( "camera-its", "checkIn-service-stub-lite", "chre_atoms_log.h", - "cobalt-registry-binarypb-gen", - "cobalt-registry-validated-java", "com.android.apex.apkrollback.test.pem", "com.android.apex.apkrollback.test.pubkey", "com.android.apex.cts.shim.debug.pem", From b20fdc217b0bde4cd74bb554fc668e106fbdad04 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 26 Oct 2023 11:00:34 -0700 Subject: [PATCH 56/80] Remove cobalt genrules from denylist These genrules only existed on udc-mainline-prod and main. Ignore-AOSP-First: This change is not on aosp Bug: 307824623 Test: Presubmits Change-Id: I15ce8e04546cd3bfb7c777adc27abef3c4903a08 --- genrule/allowlists.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 715ff1e14..048b971db 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -109,8 +109,6 @@ var ( "camera-its", "checkIn-service-stub-lite", "chre_atoms_log.h", - "cobalt-registry-binarypb-gen", - "cobalt-registry-validated-java", "com.android.apex.apkrollback.test.pem", "com.android.apex.apkrollback.test.pubkey", "com.android.apex.cts.shim.debug.pem", From 90498937ed9ae9735a21087daad9f24fb67de50e Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 26 Oct 2023 12:47:16 -0700 Subject: [PATCH 57/80] Enable sandboxing for exercise_type_data_type_config-pb go/roboleaf-busy-beavers-sandboxing Ignore-AOSP-First: exercise_type_data_type_config-pb only exists on internal Bug: 307824623 Test: m exercise_type_data_type_config-pb Change-Id: I2fb6e8254c124c4c0382b294a2e5a86a50be02b9 --- genrule/allowlists.go | 1 - 1 file changed, 1 deletion(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 715ff1e14..17f3c42d9 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -152,7 +152,6 @@ var ( "egl_functions_hdr", "emp_ematch.yacc.c", "emp_ematch.yacc.h", - "exercise_type_data_type_config-pb", "fdt_test_tree_empty_memory_range_dtb", "fdt_test_tree_multiple_memory_ranges_dtb", "fdt_test_tree_one_memory_range_dtb", From 50005b08c26992eca5af99dbfa79dca0197e5105 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 26 Oct 2023 18:09:06 -0700 Subject: [PATCH 58/80] Enable sandboxing for pigweed gnerules go/roboleaf-busy-beavers-sandboxing Ignore-AOSP-First: these modules only exist on internal Bug: 307824623 Test: m pw_log_log_proto_pwpb_h pw_log_log_raw_rpc_h pw_log_log_rpc_pwpb_h pw_tokenizer_proto_options_pwpb_h Change-Id: I5f4c68657a9440a3db339e7a51a4714c3bc5df55 --- genrule/allowlists.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 715ff1e14..4e7c19a00 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -239,10 +239,6 @@ var ( "pixelstatsatoms.cpp", "pixelstatsatoms.h", "pvmfw_fdt_template_rs", - "pw_log_log_proto_pwpb_h", - "pw_log_log_raw_rpc_h", - "pw_log_log_rpc_pwpb_h", - "pw_tokenizer_proto_options_pwpb_h", "r8retrace-dexdump-sample-app", "r8retrace-run-retrace", "rootcanal_bredr_bb_packets_cxx_gen", From 61aedb8be3e6c379fab7659419d9e757176a4aaa Mon Sep 17 00:00:00 2001 From: Yu Liu Date: Thu, 5 Oct 2023 11:40:06 -0700 Subject: [PATCH 59/80] Change java_test_host to support cov variant. Bug: 279960392 Test: CI and m --skip-soong-tests TARGET_PRODUCT=aosp_x86_64 EMMA_INSTRUMENT=true CLANG_COVERAGE=true NATIVE_COVERAGE_PATHS="external/cronet" mts (cherry picked from https://android-review.googlesource.com/q/commit:d8aa20062aad83b170b0945b71db79e48daa9fe8) Merged-In: I4489ed725aee6097e6e340f5f6d06ecaf1c64222 Change-Id: I4489ed725aee6097e6e340f5f6d06ecaf1c64222 --- java/java.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/java.go b/java/java.go index 5e640bc97..9161b2253 100644 --- a/java/java.go +++ b/java/java.go @@ -1059,6 +1059,10 @@ func (j *JavaTestImport) InstallInTestcases() bool { return true } +func (j *TestHost) IsNativeCoverageNeeded(ctx android.BaseModuleContext) bool { + return ctx.DeviceConfig().NativeCoverageEnabled() +} + func (j *TestHost) addDataDeviceBinsDeps(ctx android.BottomUpMutatorContext) { if len(j.testHostProperties.Data_device_bins_first) > 0 { deviceVariations := ctx.Config().AndroidFirstDeviceTarget.Variations() From d519a66332f8f89044e7e9f28d5fe2cd01c39486 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Tue, 10 Oct 2023 18:36:59 +0000 Subject: [PATCH 60/80] Set download_tmp_dir explicitly to be a directory under the output directory. This directory is where outputs are download first before being moved to their final destination to ensure atomicity. Fixes: 307322068 Test: m # when out is located in a different partition (cherry picked from commit bc06176350d52222c54610c73882fcb84db4940e - AOSP/main) Merged-In: Ic224bf21c07566de00d292d02f1c0f7f727dcb08 Change-Id: Ic224bf21c07566de00d292d02f1c0f7f727dcb08 --- ui/build/config.go | 13 +++++++++++++ ui/build/rbe.go | 15 ++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index bf4aec9b9..31031eb86 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -1370,6 +1370,19 @@ func (c *configImpl) rbeProxyLogsDir() string { return v } } + return c.rbeTmpDir() +} + +func (c *configImpl) rbeDownloadTmpDir() string { + for _, f := range []string{"RBE_download_tmp_dir", "FLAG_download_tmp_dir"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + return c.rbeTmpDir() +} + +func (c *configImpl) rbeTmpDir() string { buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir()) return filepath.Join(buildTmpDir, "rbe") } diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 6479925dd..58c693039 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -55,13 +55,14 @@ func rbeCommand(ctx Context, config Config, rbeCmd string) string { func getRBEVars(ctx Context, config Config) map[string]string { vars := map[string]string{ - "RBE_log_dir": config.rbeProxyLogsDir(), - "RBE_re_proxy": config.rbeReproxy(), - "RBE_exec_root": config.rbeExecRoot(), - "RBE_output_dir": config.rbeProxyLogsDir(), - "RBE_proxy_log_dir": config.rbeProxyLogsDir(), - "RBE_cache_dir": config.rbeCacheDir(), - "RBE_platform": "container-image=" + remoteexec.DefaultImage, + "RBE_log_dir": config.rbeProxyLogsDir(), + "RBE_re_proxy": config.rbeReproxy(), + "RBE_exec_root": config.rbeExecRoot(), + "RBE_output_dir": config.rbeProxyLogsDir(), + "RBE_proxy_log_dir": config.rbeProxyLogsDir(), + "RBE_cache_dir": config.rbeCacheDir(), + "RBE_download_tmp_dir": config.rbeDownloadTmpDir(), + "RBE_platform": "container-image=" + remoteexec.DefaultImage, } if config.StartRBE() { name, err := config.rbeSockAddr(absPath(ctx, config.TempDir())) From db56d5a229ec149088bb4d03e1bf560ae1d68f33 Mon Sep 17 00:00:00 2001 From: Anas Sulaiman Date: Thu, 2 Nov 2023 17:38:35 +0000 Subject: [PATCH 61/80] fix potential encoding issues in remote actions The new docker image contains all en_*.UTF-8 locales to ensure Java (and other) actions produce the correct bytes. Bug: b/300624128 Test: Ran an android build and verified there are no encoding issues in metalava outputs. Change-Id: Icb52f7f073867648a133c857de68175d821ca56c --- remoteexec/remoteexec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remoteexec/remoteexec.go b/remoteexec/remoteexec.go index 860db5568..63d080b65 100644 --- a/remoteexec/remoteexec.go +++ b/remoteexec/remoteexec.go @@ -33,7 +33,7 @@ const ( // DefaultImage is the default container image used for Android remote execution. The // image was built with the Dockerfile at // https://android.googlesource.com/platform/prebuilts/remoteexecution-client/+/refs/heads/master/docker/Dockerfile - DefaultImage = "docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:582efb38f0c229ea39952fff9e132ccbe183e14869b39888010dacf56b360d62" + DefaultImage = "docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:953fed4a6b2501256a0d17f055dc17884ff71b024e50ade773e0b348a6c303e6" // DefaultWrapperPath is the default path to the remote execution wrapper. DefaultWrapperPath = "prebuilts/remoteexecution-client/live/rewrapper" From 7d0783ce22b3d7d369786d4b16067c79c033185b Mon Sep 17 00:00:00 2001 From: Ludovic Barman Date: Wed, 1 Nov 2023 16:35:04 +0000 Subject: [PATCH 62/80] Rust protobuf 2->3: Update build system tldr: remove protobuf2 from Android Rust and upgrade everything to pb3. These commits update: - The grpcio rust crate to use protobuf 3 - The build system to always use the new libprotobuf (pb3) instead of the _deprecated variant (for pb2); they also force-enable libprotobuf everywhere, ignoring the use_protobuf3 flag, removes the tests related to pb2 - The following modules, which required syntax changes: -- packages/modules/Bluetooth -- device/google/cuttlefish -- vendor/auto Bug: 308790516 Test: m rust Ignore-AOSP-First: must be done in main to upgrade cuttlefish+vendor with the rest all at once Change-Id: I1d07c433aca41ccfe2e710821cf930c413dc12d6 --- rust/protobuf.go | 32 ++++------------------------- rust/protobuf_test.go | 48 ------------------------------------------- rust/testing.go | 6 ------ 3 files changed, 4 insertions(+), 82 deletions(-) diff --git a/rust/protobuf.go b/rust/protobuf.go index 2982efdf2..282a33931 100644 --- a/rust/protobuf.go +++ b/rust/protobuf.go @@ -59,6 +59,7 @@ type ProtobufProperties struct { // Use protobuf version 3.x. This will be deleted once we migrate all current users // of protobuf off of 2.x. + // ludovicb@: DEPRECATED, to be removed Use_protobuf3 *bool // List of exported include paths containing proto files for dependent rust_protobuf modules. @@ -77,10 +78,6 @@ type protobufDecorator struct { protoFlags android.ProtoFlags } -func (proto *protobufDecorator) useProtobuf3() bool { - return Bool(proto.Properties.Use_protobuf3) -} - func (proto *protobufDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) android.Path { var protoFlags android.ProtoFlags var grpcProtoFlags android.ProtoFlags @@ -90,12 +87,7 @@ func (proto *protobufDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) protoFiles := android.PathsForModuleSrc(ctx, proto.Properties.Protos) grpcFiles := android.PathsForModuleSrc(ctx, proto.Properties.Grpc_protos) - // For now protobuf2 (the deprecated version) remains the default. This will change in the - // future as we update the various users. - protoPluginPath := ctx.Config().HostToolPath(ctx, "protoc-gen-rust-deprecated") - if proto.useProtobuf3() == true { - protoPluginPath = ctx.Config().HostToolPath(ctx, "protoc-gen-rust") - } + protoPluginPath := ctx.Config().HostToolPath(ctx, "protoc-gen-rust") commonProtoFlags = append(commonProtoFlags, defaultProtobufFlags...) commonProtoFlags = append(commonProtoFlags, proto.Properties.Proto_flags...) @@ -219,13 +211,7 @@ func (proto *protobufDecorator) genModFileContents() string { lines = append( lines, "pub mod empty {", - " pub use protobuf::well_known_types::Empty;", - "}", - "pub mod wrappers {", - " pub use protobuf::well_known_types::{", - " DoubleValue, FloatValue, Int64Value, UInt64Value, Int32Value, UInt32Value,", - " BoolValue, StringValue, BytesValue", - " };", + " pub use protobuf::well_known_types::empty::Empty;", "}") } @@ -238,20 +224,10 @@ func (proto *protobufDecorator) SourceProviderProps() []interface{} { func (proto *protobufDecorator) SourceProviderDeps(ctx DepsContext, deps Deps) Deps { deps = proto.BaseSourceProvider.SourceProviderDeps(ctx, deps) - useProtobuf3 := proto.useProtobuf3() - if useProtobuf3 == true { - deps.Rustlibs = append(deps.Rustlibs, "libprotobuf") - } else { - deps.Rustlibs = append(deps.Rustlibs, "libprotobuf_deprecated") - } + deps.Rustlibs = append(deps.Rustlibs, "libprotobuf") deps.HeaderLibs = append(deps.SharedLibs, proto.Properties.Header_libs...) if len(proto.Properties.Grpc_protos) > 0 { - if useProtobuf3 == true { - ctx.PropertyErrorf("protos", "rust_protobuf with grpc_protos defined must currently use "+ - "`use_protobuf3: false,` in the Android.bp file. This is temporary until the "+ - "grpcio crate is updated to use the current version of the protobuf crate.") - } deps.Rustlibs = append(deps.Rustlibs, "libgrpcio", "libfutures") deps.HeaderLibs = append(deps.HeaderLibs, "libprotobuf-cpp-full") } diff --git a/rust/protobuf_test.go b/rust/protobuf_test.go index 9dca0290f..b375a6463 100644 --- a/rust/protobuf_test.go +++ b/rust/protobuf_test.go @@ -21,54 +21,6 @@ import ( "android/soong/android" ) -func TestRustProtobuf(t *testing.T) { - ctx := testRust(t, ` - rust_protobuf { - name: "librust_proto", - protos: ["buf.proto", "proto.proto"], - crate_name: "rust_proto", - source_stem: "buf", - shared_libs: ["libfoo_shared"], - static_libs: ["libfoo_static"], - } - cc_library_shared { - name: "libfoo_shared", - export_include_dirs: ["shared_include"], - } - cc_library_static { - name: "libfoo_static", - export_include_dirs: ["static_include"], - } - `) - // Check that libprotobuf is added as a dependency. - librust_proto := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_dylib").Module().(*Module) - if !android.InList("libprotobuf_deprecated", librust_proto.Properties.AndroidMkDylibs) { - t.Errorf("libprotobuf_deprecated dependency missing for rust_protobuf (dependency missing from AndroidMkDylibs)") - } - - // Make sure the correct plugin is being used. - librust_proto_out := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_source").Output("buf.rs") - cmd := librust_proto_out.RuleParams.Command - if w := "protoc-gen-rust-deprecated"; !strings.Contains(cmd, w) { - t.Errorf("expected %q in %q", w, cmd) - } - - // Check exported include directories - if w := "-Ishared_include"; !strings.Contains(cmd, w) { - t.Errorf("expected %q in %q", w, cmd) - } - if w := "-Istatic_include"; !strings.Contains(cmd, w) { - t.Errorf("expected %q in %q", w, cmd) - } - - // Check proto.rs, the second protobuf, is listed as an output - librust_proto_outputs := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_source").AllOutputs() - if android.InList("proto.rs", librust_proto_outputs) { - t.Errorf("rust_protobuf is not producing multiple outputs; expected 'proto.rs' in list, got: %#v ", - librust_proto_outputs) - } -} - func TestRustProtobuf3(t *testing.T) { ctx := testRust(t, ` rust_protobuf { diff --git a/rust/testing.go b/rust/testing.go index 3fe751e17..0b34c9732 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -132,12 +132,6 @@ func GatherRequiredDepsForTest() string { srcs: ["foo.rs"], host_supported: true, } - rust_library { - name: "libprotobuf_deprecated", - crate_name: "protobuf", - srcs: ["foo.rs"], - host_supported: true, - } rust_library { name: "libgrpcio", crate_name: "grpcio", From df97bbfcc29436d297ecf6b8f0307cef112213a7 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Fri, 18 Aug 2023 22:43:28 +0000 Subject: [PATCH 63/80] android_app APEX_GLOBAL_MIN_SDK_VERSION_OVERRIDE Currently the product variable that is controlled by APEX_GLOBAL_MIN_SDK_VERSION_OVERRIDE only overrides apexes, but it doesn't apply to android_apps. This commit allows android_apps which set updatable: true in their Android.bp to have their min_sdk_version be overriden by this product variable. Bug: 295311875 Test: go test (cherry picked from https://android-review.googlesource.com/q/commit:0e0d96efa71e7146f2d43ec9c2fbe66226426811) Merged-In: If3cb82a17cae4553b577dd1a4637ee13c3c95302 Change-Id: If3cb82a17cae4553b577dd1a4637ee13c3c95302 --- android/apex.go | 13 +++++++++++++ apex/apex.go | 19 +++---------------- java/app.go | 11 +++++++++++ java/app_test.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 16 deletions(-) diff --git a/android/apex.go b/android/apex.go index 35e1d025a..5a2529734 100644 --- a/android/apex.go +++ b/android/apex.go @@ -918,6 +918,19 @@ func CheckMinSdkVersion(ctx ModuleContext, minSdkVersion ApiLevel, walk WalkPayl }) } +// Construct ApiLevel object from min_sdk_version string value +func MinSdkVersionFromValue(ctx EarlyModuleContext, value string) ApiLevel { + if value == "" { + return NoneApiLevel + } + apiLevel, err := ApiLevelFromUser(ctx, value) + if err != nil { + ctx.PropertyErrorf("min_sdk_version", "%s", err.Error()) + return NoneApiLevel + } + return apiLevel +} + // Implemented by apexBundle. type ApexTestInterface interface { // Return true if the apex bundle is an apex_test diff --git a/apex/apex.go b/apex/apex.go index f7c74440c..fca09440f 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -2943,13 +2943,13 @@ func (a *apexBundle) minSdkVersionValue(ctx android.EarlyModuleContext) string { // Only override the minSdkVersion value on Apexes which already specify // a min_sdk_version (it's optional for non-updatable apexes), and that its // min_sdk_version value is lower than the one to override with. - minApiLevel := minSdkVersionFromValue(ctx, proptools.String(a.properties.Min_sdk_version)) + minApiLevel := android.MinSdkVersionFromValue(ctx, proptools.String(a.properties.Min_sdk_version)) if minApiLevel.IsNone() { return "" } overrideMinSdkValue := ctx.DeviceConfig().ApexGlobalMinSdkVersionOverride() - overrideApiLevel := minSdkVersionFromValue(ctx, overrideMinSdkValue) + overrideApiLevel := android.MinSdkVersionFromValue(ctx, overrideMinSdkValue) if !overrideApiLevel.IsNone() && overrideApiLevel.CompareTo(minApiLevel) > 0 { minApiLevel = overrideApiLevel } @@ -2964,20 +2964,7 @@ func (a *apexBundle) MinSdkVersion(ctx android.EarlyModuleContext) android.ApiLe // Returns apex's min_sdk_version ApiLevel, honoring overrides func (a *apexBundle) minSdkVersion(ctx android.EarlyModuleContext) android.ApiLevel { - return minSdkVersionFromValue(ctx, a.minSdkVersionValue(ctx)) -} - -// Construct ApiLevel object from min_sdk_version string value -func minSdkVersionFromValue(ctx android.EarlyModuleContext, value string) android.ApiLevel { - if value == "" { - return android.NoneApiLevel - } - apiLevel, err := android.ApiLevelFromUser(ctx, value) - if err != nil { - ctx.PropertyErrorf("min_sdk_version", "%s", err.Error()) - return android.NoneApiLevel - } - return apiLevel + return android.MinSdkVersionFromValue(ctx, a.minSdkVersionValue(ctx)) } // Ensures that a lib providing stub isn't statically linked diff --git a/java/app.go b/java/app.go index 8d965571e..254906d2e 100755 --- a/java/app.go +++ b/java/app.go @@ -318,6 +318,17 @@ func (a *AndroidApp) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.generateJavaUsedByApex(ctx) } +func (a *AndroidApp) MinSdkVersion(ctx android.EarlyModuleContext) android.ApiLevel { + defaultMinSdkVersion := a.Module.MinSdkVersion(ctx) + if proptools.Bool(a.appProperties.Updatable) { + overrideApiLevel := android.MinSdkVersionFromValue(ctx, ctx.DeviceConfig().ApexGlobalMinSdkVersionOverride()) + if !overrideApiLevel.IsNone() && overrideApiLevel.CompareTo(defaultMinSdkVersion) > 0 { + return overrideApiLevel + } + } + return defaultMinSdkVersion +} + func (a *AndroidApp) checkAppSdkVersions(ctx android.ModuleContext) { if a.Updatable() { if !a.SdkVersion(ctx).Stable() { diff --git a/java/app_test.go b/java/app_test.go index 746ab75e7..f078021aa 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3668,3 +3668,49 @@ func TestPrivappAllowlistAndroidMk(t *testing.T) { "\\S+soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/bar.xml", ) } + +func TestApexGlobalMinSdkVersionOverride(t *testing.T) { + result := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.ApexGlobalMinSdkVersionOverride = proptools.StringPtr("Tiramisu") + }), + ).RunTestWithBp(t, ` + android_app { + name: "com.android.bar", + srcs: ["a.java"], + sdk_version: "current", + } + android_app { + name: "com.android.foo", + srcs: ["a.java"], + sdk_version: "current", + min_sdk_version: "S", + updatable: true, + } + override_android_app { + name: "com.android.go.foo", + base: "com.android.foo", + } + `) + foo := result.ModuleForTests("com.android.foo", "android_common").Rule("manifestFixer") + fooOverride := result.ModuleForTests("com.android.foo", "android_common_com.android.go.foo").Rule("manifestFixer") + bar := result.ModuleForTests("com.android.bar", "android_common").Rule("manifestFixer") + + android.AssertStringDoesContain(t, + "expected manifest fixer to set com.android.bar minSdkVersion to S", + bar.BuildParams.Args["args"], + "--minSdkVersion S", + ) + android.AssertStringDoesContain(t, + "com.android.foo: expected manifest fixer to set minSdkVersion to T", + foo.BuildParams.Args["args"], + "--minSdkVersion T", + ) + android.AssertStringDoesContain(t, + "com.android.go.foo: expected manifest fixer to set minSdkVersion to T", + fooOverride.BuildParams.Args["args"], + "--minSdkVersion T", + ) + +} From 32eb1338f017df63a4c086df8b055d4abab08906 Mon Sep 17 00:00:00 2001 From: Makoto Onuki Date: Fri, 20 Oct 2023 10:42:47 -0700 Subject: [PATCH 64/80] Add `android_ravenwood_test` build rule Modeled after android_robolectric_test. Bug: 292141694 Test: m nothing --no-skip-soong-tests Test: m hoststubgen ravenwood-runtime tradefed atest && atest-dev HostStubGenTest-framework-test Change-Id: Ibef8953db45b86264c8a827868793b42f8f2d6ab --- android/test_suites.go | 26 +++++- java/Android.bp | 2 + java/ravenwood.go | 183 +++++++++++++++++++++++++++++++++++++++++ java/ravenwood_test.go | 113 +++++++++++++++++++++++++ tradefed/config.go | 1 + 5 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 java/ravenwood.go create mode 100644 java/ravenwood_test.go diff --git a/android/test_suites.go b/android/test_suites.go index 9ded99832..adcc15a6e 100644 --- a/android/test_suites.go +++ b/android/test_suites.go @@ -24,6 +24,7 @@ func testSuiteFilesFactory() Singleton { type testSuiteFiles struct { robolectric WritablePath + ravenwood WritablePath } type TestSuiteModule interface { @@ -47,12 +48,15 @@ func (t *testSuiteFiles) GenerateBuildActions(ctx SingletonContext) { }) t.robolectric = robolectricTestSuite(ctx, files["robolectric-tests"]) - ctx.Phony("robolectric-tests", t.robolectric) + + t.ravenwood = ravenwoodTestSuite(ctx, files["ravenwood-tests"]) + ctx.Phony("ravenwood-tests", t.ravenwood) } func (t *testSuiteFiles) MakeVars(ctx MakeVarsContext) { ctx.DistForGoal("robolectric-tests", t.robolectric) + ctx.DistForGoal("ravenwood-tests", t.ravenwood) } func robolectricTestSuite(ctx SingletonContext, files map[string]InstallPaths) WritablePath { @@ -74,3 +78,23 @@ func robolectricTestSuite(ctx SingletonContext, files map[string]InstallPaths) W return outputFile } + +func ravenwoodTestSuite(ctx SingletonContext, files map[string]InstallPaths) WritablePath { + var installedPaths InstallPaths + for _, module := range SortedKeys(files) { + installedPaths = append(installedPaths, files[module]...) + } + testCasesDir := pathForInstall(ctx, ctx.Config().BuildOS, X86, "testcases") + + outputFile := PathForOutput(ctx, "packaging", "ravenwood-tests.zip") + rule := NewRuleBuilder(pctx, ctx) + rule.Command().BuiltTool("soong_zip"). + FlagWithOutput("-o ", outputFile). + FlagWithArg("-P ", "host/testcases"). + FlagWithArg("-C ", testCasesDir.String()). + FlagWithRspFileInputList("-r ", outputFile.ReplaceExtension(ctx, "rsp"), installedPaths.Paths()). + Flag("-sha256") + rule.Build("ravenwood_tests_zip", "ravenwood-tests.zip") + + return outputFile +} diff --git a/java/Android.bp b/java/Android.bp index 4450c4275..b8d26bf78 100644 --- a/java/Android.bp +++ b/java/Android.bp @@ -66,6 +66,7 @@ bootstrap_go_package { "plugin.go", "prebuilt_apis.go", "proto.go", + "ravenwood.go", "resourceshrinker.go", "robolectric.go", "rro.go", @@ -106,6 +107,7 @@ bootstrap_go_package { "plugin_test.go", "prebuilt_apis_test.go", "proto_test.go", + "ravenwood_test.go", "resourceshrinker_test.go", "rro_test.go", "sdk_test.go", diff --git a/java/ravenwood.go b/java/ravenwood.go new file mode 100644 index 000000000..a03d8fb17 --- /dev/null +++ b/java/ravenwood.go @@ -0,0 +1,183 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package java + +import ( + "android/soong/android" + "android/soong/tradefed" + "github.com/google/blueprint/proptools" +) + +func init() { + RegisterRavenwoodBuildComponents(android.InitRegistrationContext) +} + +func RegisterRavenwoodBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("android_ravenwood_test", ravenwoodTestFactory) + ctx.RegisterModuleType("android_ravenwood_libgroup", ravenwoodLibgroupFactory) +} + +var ravenwoodTag = dependencyTag{name: "ravenwood"} + +const ravenwoodUtilsName = "ravenwood-utils" +const ravenwoodRuntimeName = "ravenwood-runtime" + +type ravenwoodTest struct { + Library + + testProperties testProperties + testConfig android.Path + + forceOSType android.OsType + forceArchType android.ArchType +} + +func ravenwoodTestFactory() android.Module { + module := &ravenwoodTest{} + + module.addHostAndDeviceProperties() + module.AddProperties(&module.testProperties) + + module.Module.dexpreopter.isTest = true + module.Module.linter.properties.Lint.Test = proptools.BoolPtr(true) + + module.testProperties.Test_suites = []string{"ravenwood-tests"} + module.testProperties.Test_options.Unit_test = proptools.BoolPtr(false) + + InitJavaModule(module, android.DeviceSupported) + android.InitDefaultableModule(module) + + return module +} + +func (r *ravenwoodTest) InstallInTestcases() bool { return true } +func (r *ravenwoodTest) InstallForceOS() (*android.OsType, *android.ArchType) { + return &r.forceOSType, &r.forceArchType +} +func (r *ravenwoodTest) TestSuites() []string { + return r.testProperties.Test_suites +} + +func (r *ravenwoodTest) DepsMutator(ctx android.BottomUpMutatorContext) { + r.Library.DepsMutator(ctx) + + // Generically depend on the runtime so that it's installed together with us + ctx.AddVariationDependencies(nil, ravenwoodTag, ravenwoodRuntimeName) + + // Directly depend on any utils so that we link against them + utils := ctx.AddVariationDependencies(nil, ravenwoodTag, ravenwoodUtilsName)[0] + if utils != nil { + for _, lib := range utils.(*ravenwoodLibgroup).ravenwoodLibgroupProperties.Libs { + ctx.AddVariationDependencies(nil, libTag, lib) + } + } +} + +func (r *ravenwoodTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { + r.forceOSType = ctx.Config().BuildOS + r.forceArchType = ctx.Config().BuildArch + + r.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ + TestConfigProp: r.testProperties.Test_config, + TestConfigTemplateProp: r.testProperties.Test_config_template, + TestSuites: r.testProperties.Test_suites, + AutoGenConfig: r.testProperties.Auto_gen_config, + DeviceTemplate: "${RavenwoodTestConfigTemplate}", + HostTemplate: "${RavenwoodTestConfigTemplate}", + }) + + r.Library.GenerateAndroidBuildActions(ctx) + + // Start by depending on all files installed by dependancies + var installDeps android.Paths + for _, dep := range ctx.GetDirectDepsWithTag(ravenwoodTag) { + for _, installFile := range dep.FilesToInstall() { + installDeps = append(installDeps, installFile) + } + } + + // Also depend on our config + installPath := android.PathForModuleInstall(ctx, r.BaseModuleName()) + installConfig := ctx.InstallFile(installPath, ctx.ModuleName()+".config", r.testConfig) + installDeps = append(installDeps, installConfig) + + // Finally install our JAR with all dependencies + ctx.InstallFile(installPath, ctx.ModuleName()+".jar", r.outputFile, installDeps...) +} + +func (r *ravenwoodTest) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := r.Library.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", "ravenwood-tests") + if r.testConfig != nil { + entries.SetPath("LOCAL_FULL_TEST_CONFIG", r.testConfig) + } + }) + return entriesList +} + +type ravenwoodLibgroupProperties struct { + Libs []string +} + +type ravenwoodLibgroup struct { + android.ModuleBase + + ravenwoodLibgroupProperties ravenwoodLibgroupProperties + + forceOSType android.OsType + forceArchType android.ArchType +} + +func ravenwoodLibgroupFactory() android.Module { + module := &ravenwoodLibgroup{} + module.AddProperties(&module.ravenwoodLibgroupProperties) + + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + return module +} + +func (r *ravenwoodLibgroup) InstallInTestcases() bool { return true } +func (r *ravenwoodLibgroup) InstallForceOS() (*android.OsType, *android.ArchType) { + return &r.forceOSType, &r.forceArchType +} +func (r *ravenwoodLibgroup) TestSuites() []string { + return []string{"ravenwood-tests"} +} + +func (r *ravenwoodLibgroup) DepsMutator(ctx android.BottomUpMutatorContext) { + // Always depends on our underlying libs + for _, lib := range r.ravenwoodLibgroupProperties.Libs { + ctx.AddVariationDependencies(nil, ravenwoodTag, lib) + } +} + +func (r *ravenwoodLibgroup) GenerateAndroidBuildActions(ctx android.ModuleContext) { + r.forceOSType = ctx.Config().BuildOS + r.forceArchType = ctx.Config().BuildArch + + // Install our runtime into expected location for packaging + installPath := android.PathForModuleInstall(ctx, r.BaseModuleName()) + for _, lib := range r.ravenwoodLibgroupProperties.Libs { + libModule := ctx.GetDirectDepWithTag(lib, ravenwoodTag) + libJar := android.OutputFileForModule(ctx, libModule, "") + ctx.InstallFile(installPath, lib+".jar", libJar) + } + + // Normal build should perform install steps + ctx.Phony(r.BaseModuleName(), android.PathForPhony(ctx, r.BaseModuleName()+"-install")) +} diff --git a/java/ravenwood_test.go b/java/ravenwood_test.go new file mode 100644 index 000000000..4ae1d7cf5 --- /dev/null +++ b/java/ravenwood_test.go @@ -0,0 +1,113 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "testing" + + "android/soong/android" +) + +var prepareRavenwoodRuntime = android.GroupFixturePreparers( + android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + RegisterRavenwoodBuildComponents(ctx) + }), + android.FixtureAddTextFile("ravenwood/Android.bp", ` + java_library_static { + name: "framework-minus-apex.ravenwood", + srcs: ["Framework.java"], + } + java_library_static { + name: "framework-services.ravenwood", + srcs: ["Services.java"], + } + java_library_static { + name: "framework-rules.ravenwood", + srcs: ["Rules.java"], + } + android_ravenwood_libgroup { + name: "ravenwood-runtime", + libs: [ + "framework-minus-apex.ravenwood", + "framework-services.ravenwood", + ], + } + android_ravenwood_libgroup { + name: "ravenwood-utils", + libs: [ + "framework-rules.ravenwood", + ], + } + `), +) + +var installPathPrefix = "out/soong/host/linux-x86/testcases" + +func TestRavenwoodRuntime(t *testing.T) { + ctx := android.GroupFixturePreparers( + PrepareForIntegrationTestWithJava, + prepareRavenwoodRuntime, + ).RunTest(t) + + // Verify that our runtime depends on underlying libs + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-runtime", "android_common", "framework-minus-apex.ravenwood") + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-runtime", "android_common", "framework-services.ravenwood") + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-utils", "android_common", "framework-rules.ravenwood") + + // Verify that we've emitted artifacts in expected location + runtime := ctx.ModuleForTests("ravenwood-runtime", "android_common") + runtime.Output(installPathPrefix + "/ravenwood-runtime/framework-minus-apex.ravenwood.jar") + runtime.Output(installPathPrefix + "/ravenwood-runtime/framework-services.ravenwood.jar") + utils := ctx.ModuleForTests("ravenwood-utils", "android_common") + utils.Output(installPathPrefix + "/ravenwood-utils/framework-rules.ravenwood.jar") +} + +func TestRavenwoodTest(t *testing.T) { + ctx := android.GroupFixturePreparers( + PrepareForIntegrationTestWithJava, + prepareRavenwoodRuntime, + ).RunTestWithBp(t, ` + android_ravenwood_test { + name: "ravenwood-test", + srcs: ["Test.java"], + sdk_version: "test_current", + } + `) + + // Verify that our test depends on underlying libs + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-test", "android_common", "ravenwood-buildtime") + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-test", "android_common", "ravenwood-utils") + + module := ctx.ModuleForTests("ravenwood-test", "android_common") + classpath := module.Rule("javac").Args["classpath"] + + // Verify that we're linking against test_current + android.AssertStringDoesContain(t, "classpath", classpath, "android_test_stubs_current.jar") + // Verify that we're linking against utils + android.AssertStringDoesContain(t, "classpath", classpath, "framework-rules.ravenwood.jar") + // Verify that we're *NOT* linking against runtime + android.AssertStringDoesNotContain(t, "classpath", classpath, "framework-minus-apex.ravenwood.jar") + android.AssertStringDoesNotContain(t, "classpath", classpath, "framework-services.ravenwood.jar") + + // Verify that we've emitted test artifacts in expected location + outputJar := module.Output(installPathPrefix + "/ravenwood-test/ravenwood-test.jar") + module.Output(installPathPrefix + "/ravenwood-test/ravenwood-test.config") + + // Verify that we're going to install underlying libs + orderOnly := outputJar.OrderOnly.Strings() + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-runtime/framework-minus-apex.ravenwood.jar") + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-runtime/framework-services.ravenwood.jar") + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-utils/framework-rules.ravenwood.jar") +} diff --git a/tradefed/config.go b/tradefed/config.go index 326a00666..b0150348b 100644 --- a/tradefed/config.go +++ b/tradefed/config.go @@ -33,6 +33,7 @@ func init() { pctx.SourcePathVariable("NativeTestConfigTemplate", "build/make/core/native_test_config_template.xml") pctx.SourcePathVariable("PythonBinaryHostMoblyTestConfigTemplate", "build/make/core/python_binary_host_mobly_test_config_template.xml") pctx.SourcePathVariable("PythonBinaryHostTestConfigTemplate", "build/make/core/python_binary_host_test_config_template.xml") + pctx.SourcePathVariable("RavenwoodTestConfigTemplate", "build/make/core/ravenwood_test_config_template.xml") pctx.SourcePathVariable("RustDeviceTestConfigTemplate", "build/make/core/rust_device_test_config_template.xml") pctx.SourcePathVariable("RustHostTestConfigTemplate", "build/make/core/rust_host_test_config_template.xml") pctx.SourcePathVariable("RustDeviceBenchmarkConfigTemplate", "build/make/core/rust_device_benchmark_config_template.xml") From 28216dbe8d69a52bc6d1f17ee8be5f556b53923e Mon Sep 17 00:00:00 2001 From: Ludovic Barman Date: Wed, 1 Nov 2023 16:35:04 +0000 Subject: [PATCH 65/80] Rust protobuf 2->3: Update build system tldr: remove protobuf2 from Android Rust and upgrade everything to pb3. These commits update: - The grpcio rust crate to use protobuf 3 - The build system to always use the new libprotobuf (pb3) instead of the _deprecated variant (for pb2); they also force-enable libprotobuf everywhere, ignoring the use_protobuf3 flag, removes the tests related to pb2 - The following modules, which required syntax changes: -- packages/modules/Bluetooth -- device/google/cuttlefish -- vendor/auto Bug: 308790516 Test: m rust Ignore-AOSP-First: must be done in main to upgrade cuttlefish+vendor with the rest all at once (cherry picked from https://googleplex-android-review.googlesource.com/q/commit:7d0783ce22b3d7d369786d4b16067c79c033185b) Merged-In: I1d07c433aca41ccfe2e710821cf930c413dc12d6 Change-Id: I1d07c433aca41ccfe2e710821cf930c413dc12d6 --- rust/protobuf.go | 32 ++++------------------------- rust/protobuf_test.go | 48 ------------------------------------------- rust/testing.go | 6 ------ 3 files changed, 4 insertions(+), 82 deletions(-) diff --git a/rust/protobuf.go b/rust/protobuf.go index 2982efdf2..282a33931 100644 --- a/rust/protobuf.go +++ b/rust/protobuf.go @@ -59,6 +59,7 @@ type ProtobufProperties struct { // Use protobuf version 3.x. This will be deleted once we migrate all current users // of protobuf off of 2.x. + // ludovicb@: DEPRECATED, to be removed Use_protobuf3 *bool // List of exported include paths containing proto files for dependent rust_protobuf modules. @@ -77,10 +78,6 @@ type protobufDecorator struct { protoFlags android.ProtoFlags } -func (proto *protobufDecorator) useProtobuf3() bool { - return Bool(proto.Properties.Use_protobuf3) -} - func (proto *protobufDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) android.Path { var protoFlags android.ProtoFlags var grpcProtoFlags android.ProtoFlags @@ -90,12 +87,7 @@ func (proto *protobufDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) protoFiles := android.PathsForModuleSrc(ctx, proto.Properties.Protos) grpcFiles := android.PathsForModuleSrc(ctx, proto.Properties.Grpc_protos) - // For now protobuf2 (the deprecated version) remains the default. This will change in the - // future as we update the various users. - protoPluginPath := ctx.Config().HostToolPath(ctx, "protoc-gen-rust-deprecated") - if proto.useProtobuf3() == true { - protoPluginPath = ctx.Config().HostToolPath(ctx, "protoc-gen-rust") - } + protoPluginPath := ctx.Config().HostToolPath(ctx, "protoc-gen-rust") commonProtoFlags = append(commonProtoFlags, defaultProtobufFlags...) commonProtoFlags = append(commonProtoFlags, proto.Properties.Proto_flags...) @@ -219,13 +211,7 @@ func (proto *protobufDecorator) genModFileContents() string { lines = append( lines, "pub mod empty {", - " pub use protobuf::well_known_types::Empty;", - "}", - "pub mod wrappers {", - " pub use protobuf::well_known_types::{", - " DoubleValue, FloatValue, Int64Value, UInt64Value, Int32Value, UInt32Value,", - " BoolValue, StringValue, BytesValue", - " };", + " pub use protobuf::well_known_types::empty::Empty;", "}") } @@ -238,20 +224,10 @@ func (proto *protobufDecorator) SourceProviderProps() []interface{} { func (proto *protobufDecorator) SourceProviderDeps(ctx DepsContext, deps Deps) Deps { deps = proto.BaseSourceProvider.SourceProviderDeps(ctx, deps) - useProtobuf3 := proto.useProtobuf3() - if useProtobuf3 == true { - deps.Rustlibs = append(deps.Rustlibs, "libprotobuf") - } else { - deps.Rustlibs = append(deps.Rustlibs, "libprotobuf_deprecated") - } + deps.Rustlibs = append(deps.Rustlibs, "libprotobuf") deps.HeaderLibs = append(deps.SharedLibs, proto.Properties.Header_libs...) if len(proto.Properties.Grpc_protos) > 0 { - if useProtobuf3 == true { - ctx.PropertyErrorf("protos", "rust_protobuf with grpc_protos defined must currently use "+ - "`use_protobuf3: false,` in the Android.bp file. This is temporary until the "+ - "grpcio crate is updated to use the current version of the protobuf crate.") - } deps.Rustlibs = append(deps.Rustlibs, "libgrpcio", "libfutures") deps.HeaderLibs = append(deps.HeaderLibs, "libprotobuf-cpp-full") } diff --git a/rust/protobuf_test.go b/rust/protobuf_test.go index 9dca0290f..b375a6463 100644 --- a/rust/protobuf_test.go +++ b/rust/protobuf_test.go @@ -21,54 +21,6 @@ import ( "android/soong/android" ) -func TestRustProtobuf(t *testing.T) { - ctx := testRust(t, ` - rust_protobuf { - name: "librust_proto", - protos: ["buf.proto", "proto.proto"], - crate_name: "rust_proto", - source_stem: "buf", - shared_libs: ["libfoo_shared"], - static_libs: ["libfoo_static"], - } - cc_library_shared { - name: "libfoo_shared", - export_include_dirs: ["shared_include"], - } - cc_library_static { - name: "libfoo_static", - export_include_dirs: ["static_include"], - } - `) - // Check that libprotobuf is added as a dependency. - librust_proto := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_dylib").Module().(*Module) - if !android.InList("libprotobuf_deprecated", librust_proto.Properties.AndroidMkDylibs) { - t.Errorf("libprotobuf_deprecated dependency missing for rust_protobuf (dependency missing from AndroidMkDylibs)") - } - - // Make sure the correct plugin is being used. - librust_proto_out := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_source").Output("buf.rs") - cmd := librust_proto_out.RuleParams.Command - if w := "protoc-gen-rust-deprecated"; !strings.Contains(cmd, w) { - t.Errorf("expected %q in %q", w, cmd) - } - - // Check exported include directories - if w := "-Ishared_include"; !strings.Contains(cmd, w) { - t.Errorf("expected %q in %q", w, cmd) - } - if w := "-Istatic_include"; !strings.Contains(cmd, w) { - t.Errorf("expected %q in %q", w, cmd) - } - - // Check proto.rs, the second protobuf, is listed as an output - librust_proto_outputs := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_source").AllOutputs() - if android.InList("proto.rs", librust_proto_outputs) { - t.Errorf("rust_protobuf is not producing multiple outputs; expected 'proto.rs' in list, got: %#v ", - librust_proto_outputs) - } -} - func TestRustProtobuf3(t *testing.T) { ctx := testRust(t, ` rust_protobuf { diff --git a/rust/testing.go b/rust/testing.go index 3fe751e17..0b34c9732 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -132,12 +132,6 @@ func GatherRequiredDepsForTest() string { srcs: ["foo.rs"], host_supported: true, } - rust_library { - name: "libprotobuf_deprecated", - crate_name: "protobuf", - srcs: ["foo.rs"], - host_supported: true, - } rust_library { name: "libgrpcio", crate_name: "grpcio", From a45bf7f254028b8c4d1bf1518493aefce8e9edb0 Mon Sep 17 00:00:00 2001 From: Jeff Sharkey Date: Mon, 6 Nov 2023 13:56:38 -0700 Subject: [PATCH 66/80] Fix tests that are broken on darwin builds. Bug: 292141694 Test: m nothing --no-skip-soong-tests Change-Id: Ie9a2f8961a8c43bbe1388471fc0d7aadc1456c57 --- java/ravenwood_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/java/ravenwood_test.go b/java/ravenwood_test.go index 4ae1d7cf5..2443ea8e7 100644 --- a/java/ravenwood_test.go +++ b/java/ravenwood_test.go @@ -15,6 +15,7 @@ package java import ( + "runtime" "testing" "android/soong/android" @@ -56,6 +57,10 @@ var prepareRavenwoodRuntime = android.GroupFixturePreparers( var installPathPrefix = "out/soong/host/linux-x86/testcases" func TestRavenwoodRuntime(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires linux") + } + ctx := android.GroupFixturePreparers( PrepareForIntegrationTestWithJava, prepareRavenwoodRuntime, @@ -75,6 +80,10 @@ func TestRavenwoodRuntime(t *testing.T) { } func TestRavenwoodTest(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires linux") + } + ctx := android.GroupFixturePreparers( PrepareForIntegrationTestWithJava, prepareRavenwoodRuntime, From 856cedb2c0ba56e736dc993fb7e716a97471c052 Mon Sep 17 00:00:00 2001 From: Anton Hansson Date: Thu, 2 Nov 2023 15:18:09 +0000 Subject: [PATCH 67/80] Add a new flag to stop disabling lint errors java_sdk_library has historically suppressed a few legitimate lint errors. Make it possible to remove these suppressions via a flag. Ignore-AOSP-First: in internal topic, will cp after Bug: 306806877 Test: checkapi Change-Id: I10577dacbd7ea8063c6e1e02f3b6c60a2bee5094 --- java/sdk_library.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/java/sdk_library.go b/java/sdk_library.go index 1de89727f..11fde8c6a 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -618,6 +618,10 @@ type sdkLibraryProperties struct { Api_lint struct { // Enable api linting. Enabled *bool + + // If API lint is enabled, this flag controls whether a set of legitimate lint errors + // are turned off. The default is true. + Legacy_errors_allowed *bool } // TODO: determines whether to create HTML doc or not @@ -1747,13 +1751,16 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC android.JoinWithPrefix(module.sdkLibraryProperties.Hidden_api_packages, " --hide-package ")) } droidstubsArgs = append(droidstubsArgs, module.sdkLibraryProperties.Droiddoc_options...) - disabledWarnings := []string{ - "BroadcastBehavior", - "DeprecationMismatch", - "HiddenSuperclass", - "MissingPermission", - "SdkConstant", - "Todo", + disabledWarnings := []string{"HiddenSuperclass"} + if proptools.BoolDefault(module.sdkLibraryProperties.Api_lint.Legacy_errors_allowed, true) { + disabledWarnings = append(disabledWarnings, + "BroadcastBehavior", + "DeprecationMismatch", + "HiddenSuperclass", + "MissingPermission", + "SdkConstant", + "Todo", + ) } droidstubsArgs = append(droidstubsArgs, android.JoinWithPrefix(disabledWarnings, "--hide ")) From 8b9661f3656a96716414ca6af95f1c7ad0a25dde Mon Sep 17 00:00:00 2001 From: Anton Hansson Date: Tue, 7 Nov 2023 10:11:18 +0000 Subject: [PATCH 68/80] Remove duplicated hidden warning HiddenSuperclass is already in disabledWarnings. Test: presubmit Change-Id: I0ddd145a7b39d2367b27164fa65266fc0042631c --- java/sdk_library.go | 1 - 1 file changed, 1 deletion(-) diff --git a/java/sdk_library.go b/java/sdk_library.go index 11fde8c6a..ea451743b 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -1756,7 +1756,6 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC disabledWarnings = append(disabledWarnings, "BroadcastBehavior", "DeprecationMismatch", - "HiddenSuperclass", "MissingPermission", "SdkConstant", "Todo", From b3a9fcf6649697535ab6b49b7f0a620adda6c0c3 Mon Sep 17 00:00:00 2001 From: Anton Hansson Date: Wed, 8 Nov 2023 13:21:04 +0000 Subject: [PATCH 69/80] Re-enable the bannedThrow check It's been disabled previously due to pre-existing failures. Baseline those and remove the disabling. Bug: 222738070 Test: m checkapi Ignore-AOSP-First: This error is not being re-enabled in AOSP yet Change-Id: Ia4f91f1470a5bce332df1896b81550276c4b174d --- java/config/droidstubs.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/java/config/droidstubs.go b/java/config/droidstubs.go index b7aab5a98..1d7a2d22e 100644 --- a/java/config/droidstubs.go +++ b/java/config/droidstubs.go @@ -50,8 +50,6 @@ var ( "--hide HiddenTypedefConstant", "--hide SuperfluousPrefix", "--hide AnnotationExtraction", - // b/222738070 - "--hide BannedThrow", } MetalavaAnnotationsWarningsFlags = strings.Join(metalavaAnnotationsWarningsFlags, " ") From d972a088d934cb94b36ca59651fc6c42e5169b22 Mon Sep 17 00:00:00 2001 From: Anton Hansson Date: Wed, 8 Nov 2023 18:23:38 +0000 Subject: [PATCH 70/80] Re-enable the InvalidNullabilityOverride check It's been disabled previously due to pre-existing failures. Baseline those and remove the disabling. Ignore-AOSP-First: This error is not being re-enabled in AOSP yet Bug: 222738070 Test: m checkapi Change-Id: I3382152186d7d8f63da7c1cc95d3ef962f422642 --- java/config/droidstubs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/java/config/droidstubs.go b/java/config/droidstubs.go index 1d7a2d22e..45c6f0f31 100644 --- a/java/config/droidstubs.go +++ b/java/config/droidstubs.go @@ -23,7 +23,6 @@ var ( "--format=v2", "--repeat-errors-max 10", "--hide UnresolvedImport", - "--hide InvalidNullabilityOverride", // Force metalava to ignore classes on the classpath when an API file contains missing classes. // See b/285140653 for more information. From a6ae274e3b405e0d1a57aedb009cfe8cb51b86f2 Mon Sep 17 00:00:00 2001 From: Anton Hansson Date: Thu, 9 Nov 2023 19:26:25 +0000 Subject: [PATCH 71/80] Re-enable the AnnotationExtraction error This prevents errors in typedef definitions. Re-enable it. Ignore-AOSP-First: This error is not being re-enabled in AOSP yet Bug: 309971481 Test: m checkapi Change-Id: If9f0e580cda62d0021e82b1831746b3d15d1bfbd --- java/config/droidstubs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/java/config/droidstubs.go b/java/config/droidstubs.go index 1d7a2d22e..8a526ff59 100644 --- a/java/config/droidstubs.go +++ b/java/config/droidstubs.go @@ -49,7 +49,6 @@ var ( // TODO(tnorbye): find owners to fix these warnings when annotation was enabled. "--hide HiddenTypedefConstant", "--hide SuperfluousPrefix", - "--hide AnnotationExtraction", } MetalavaAnnotationsWarningsFlags = strings.Join(metalavaAnnotationsWarningsFlags, " ") From 190b1aacc828d103b6aae6348d0c9db2a17004e4 Mon Sep 17 00:00:00 2001 From: Makoto Onuki Date: Fri, 10 Nov 2023 15:42:00 -0800 Subject: [PATCH 72/80] Allow Ravenwood to cross variant boundaries Bug: 310268946 Bug: 292141694 Test: atest RavenwoodMockitoTest Change-Id: Ia395d32bad57007ec477a6410dcc4201853c8790 --- android/neverallow.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/android/neverallow.go b/android/neverallow.go index 2be6a74f5..f721b945c 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -169,6 +169,8 @@ func createJavaDeviceForHostRules() []Rule { "external/kotlinx.coroutines", "external/robolectric-shadows", "external/robolectric", + "frameworks/base/ravenwood", + "frameworks/base/tools/hoststubgen", "frameworks/layoutlib", } From 5fcd83edaa4290b6beb38d4baf7ffacb1c41f4fd Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 15 Nov 2023 12:29:33 -0800 Subject: [PATCH 73/80] Change deps of ctx.Install* from Paths to InstallPaths Installed files should only depend on other installed files, change the deps arguments of the ctx.Install* methods from Paths to InstallPaths. Bug: 311428265 Test: builds Ignore-AOSP-First: resolving conflict Change-Id: I1ebef60a943bdbe907744cc43aa985371ac56d32 --- android/module_context.go | 20 ++++++++++---------- apex/apex.go | 2 +- apex/builder.go | 2 +- apex/prebuilt.go | 2 +- java/app.go | 2 +- java/java.go | 4 ++-- java/ravenwood.go | 3 ++- java/robolectric.go | 2 +- java/tradefed.go | 4 ++-- 9 files changed, 21 insertions(+), 20 deletions(-) diff --git a/android/module_context.go b/android/module_context.go index 0b4d28c54..a0a410428 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -114,7 +114,7 @@ type ModuleContext interface { // installed file will be returned by PackagingSpecs() on this module or by // TransitivePackagingSpecs() on modules that depend on this module through dependency tags // for which IsInstallDepNeeded returns true. - InstallExecutable(installPath InstallPath, name string, srcPath Path, deps ...Path) InstallPath + InstallExecutable(installPath InstallPath, name string, srcPath Path, deps ...InstallPath) InstallPath // InstallFile creates a rule to copy srcPath to name in the installPath directory, // with the given additional dependencies. @@ -123,7 +123,7 @@ type ModuleContext interface { // installed file will be returned by PackagingSpecs() on this module or by // TransitivePackagingSpecs() on modules that depend on this module through dependency tags // for which IsInstallDepNeeded returns true. - InstallFile(installPath InstallPath, name string, srcPath Path, deps ...Path) InstallPath + InstallFile(installPath InstallPath, name string, srcPath Path, deps ...InstallPath) InstallPath // InstallFileWithExtraFilesZip creates a rule to copy srcPath to name in the installPath // directory, and also unzip a zip file containing extra files to install into the same @@ -133,7 +133,7 @@ type ModuleContext interface { // installed file will be returned by PackagingSpecs() on this module or by // TransitivePackagingSpecs() on modules that depend on this module through dependency tags // for which IsInstallDepNeeded returns true. - InstallFileWithExtraFilesZip(installPath InstallPath, name string, srcPath Path, extraZip Path, deps ...Path) InstallPath + InstallFileWithExtraFilesZip(installPath InstallPath, name string, srcPath Path, extraZip Path, deps ...InstallPath) InstallPath // InstallSymlink creates a rule to create a symlink from src srcPath to name in the installPath // directory. @@ -451,17 +451,17 @@ func (m *moduleContext) skipInstall() bool { } func (m *moduleContext) InstallFile(installPath InstallPath, name string, srcPath Path, - deps ...Path) InstallPath { + deps ...InstallPath) InstallPath { return m.installFile(installPath, name, srcPath, deps, false, nil) } func (m *moduleContext) InstallExecutable(installPath InstallPath, name string, srcPath Path, - deps ...Path) InstallPath { + deps ...InstallPath) InstallPath { return m.installFile(installPath, name, srcPath, deps, true, nil) } func (m *moduleContext) InstallFileWithExtraFilesZip(installPath InstallPath, name string, srcPath Path, - extraZip Path, deps ...Path) InstallPath { + extraZip Path, deps ...InstallPath) InstallPath { return m.installFile(installPath, name, srcPath, deps, false, &extraFilesZip{ zip: extraZip, dir: installPath, @@ -487,23 +487,23 @@ func (m *moduleContext) packageFile(fullInstallPath InstallPath, srcPath Path, e return spec } -func (m *moduleContext) installFile(installPath InstallPath, name string, srcPath Path, deps []Path, +func (m *moduleContext) installFile(installPath InstallPath, name string, srcPath Path, deps []InstallPath, executable bool, extraZip *extraFilesZip) InstallPath { fullInstallPath := installPath.Join(m, name) m.module.base().hooks.runInstallHooks(m, srcPath, fullInstallPath, false) if !m.skipInstall() { - deps = append(deps, InstallPaths(m.module.base().installFilesDepSet.ToList()).Paths()...) + deps = append(deps, InstallPaths(m.module.base().installFilesDepSet.ToList())...) var implicitDeps, orderOnlyDeps Paths if m.Host() { // Installed host modules might be used during the build, depend directly on their // dependencies so their timestamp is updated whenever their dependency is updated - implicitDeps = deps + implicitDeps = InstallPaths(deps).Paths() } else { - orderOnlyDeps = deps + orderOnlyDeps = InstallPaths(deps).Paths() } if m.Config().KatiEnabled() { diff --git a/apex/apex.go b/apex/apex.go index 4c0230539..ecc794b5b 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1890,7 +1890,7 @@ func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) { installSuffix = imageCapexSuffix } a.installedFile = ctx.InstallFile(a.installDir, a.Name()+installSuffix, a.outputFile, - a.compatSymlinks.Paths()...) + a.compatSymlinks...) // filesInfo in mixed mode must retrieve all information about the apex's // contents completely from the Starlark providers. It should never rely on diff --git a/apex/builder.go b/apex/builder.go index ba4df5f6b..909a4792a 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -956,7 +956,7 @@ func (a *apexBundle) buildApex(ctx android.ModuleContext) { // Install to $OUT/soong/{target,host}/.../apex. a.installedFile = ctx.InstallFile(a.installDir, a.Name()+installSuffix, a.outputFile, - a.compatSymlinks.Paths()...) + a.compatSymlinks...) // installed-files.txt is dist'ed a.installedFilesFile = a.buildInstalledFilesFile(ctx, a.outputFile, imageDir) diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 7a9d23e54..7d339d54e 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -794,7 +794,7 @@ func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) { } if p.installable() { - p.installedFile = ctx.InstallFile(p.installDir, p.installFilename, p.inputApex, p.compatSymlinks.Paths()...) + p.installedFile = ctx.InstallFile(p.installDir, p.installFilename, p.inputApex, p.compatSymlinks...) p.provenanceMetaDataFile = provenance.GenerateArtifactProvenanceMetaData(ctx, p.inputApex, p.installedFile) } } diff --git a/java/app.go b/java/app.go index 41848ce0b..883b950e8 100755 --- a/java/app.go +++ b/java/app.go @@ -858,7 +858,7 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { ctx.InstallFile(allowlistInstallPath, allowlistInstallFilename, a.privAppAllowlist.Path()) } - var extraInstalledPaths android.Paths + var extraInstalledPaths android.InstallPaths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) diff --git a/java/java.go b/java/java.go index 500ae37c7..bb9357cc7 100644 --- a/java/java.go +++ b/java/java.go @@ -641,7 +641,7 @@ type Library struct { exportedProguardFlagFiles android.Paths - InstallMixin func(ctx android.ModuleContext, installPath android.Path) (extraInstallDeps android.Paths) + InstallMixin func(ctx android.ModuleContext, installPath android.Path) (extraInstallDeps android.InstallPaths) } var _ android.ApexModule = (*Library)(nil) @@ -722,7 +722,7 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { exclusivelyForApex := !apexInfo.IsForPlatform() if (Bool(j.properties.Installable) || ctx.Host()) && !exclusivelyForApex { - var extraInstallDeps android.Paths + var extraInstallDeps android.InstallPaths if j.InstallMixin != nil { extraInstallDeps = j.InstallMixin(ctx, j.outputFile) } diff --git a/java/ravenwood.go b/java/ravenwood.go index a03d8fb17..85f64a86c 100644 --- a/java/ravenwood.go +++ b/java/ravenwood.go @@ -16,6 +16,7 @@ package java import ( "android/soong/android" "android/soong/tradefed" + "github.com/google/blueprint/proptools" ) @@ -100,7 +101,7 @@ func (r *ravenwoodTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { r.Library.GenerateAndroidBuildActions(ctx) // Start by depending on all files installed by dependancies - var installDeps android.Paths + var installDeps android.InstallPaths for _, dep := range ctx.GetDirectDepsWithTag(ravenwoodTag) { for _, installFile := range dep.FilesToInstall() { installDeps = append(installDeps, installFile) diff --git a/java/robolectric.go b/java/robolectric.go index a8e6bfaff..a66b31007 100644 --- a/java/robolectric.go +++ b/java/robolectric.go @@ -226,7 +226,7 @@ func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) } installPath := android.PathForModuleInstall(ctx, r.BaseModuleName()) - var installDeps android.Paths + var installDeps android.InstallPaths if r.manifest != nil { r.data = append(r.data, r.manifest) diff --git a/java/tradefed.go b/java/tradefed.go index ebbdec13d..349b327e8 100644 --- a/java/tradefed.go +++ b/java/tradefed.go @@ -30,8 +30,8 @@ func tradefedJavaLibraryFactory() android.Module { return module } -func tradefedJavaLibraryInstall(ctx android.ModuleContext, path android.Path) android.Paths { +func tradefedJavaLibraryInstall(ctx android.ModuleContext, path android.Path) android.InstallPaths { installedPath := ctx.InstallFile(android.PathForModuleInstall(ctx, "tradefed"), ctx.ModuleName()+".jar", path) - return android.Paths{installedPath} + return android.InstallPaths{installedPath} } From a966a35e94297c31e8fabef3f9fc19231f12116f Mon Sep 17 00:00:00 2001 From: Roshan Pius Date: Mon, 13 Nov 2023 17:16:35 -0800 Subject: [PATCH 74/80] soong(nfc): Create framework-nfc sdk library Using `framework-location` as reference. Ignore-AOSP-First: Submitting separately to avoid merge conflicts. See aosp/2828288 Bug: 303286040 Test: Device boots up after flashing Test: atest CtsNfcTestCases Change-Id: I309e76165f5563147dde5e581316ee4ac463fbc4 --- android/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/android/config.go b/android/config.go index 8aeca0fdd..68c33f4c2 100644 --- a/android/config.go +++ b/android/config.go @@ -696,6 +696,7 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) "framework-location": {}, "framework-media": {}, "framework-mediaprovider": {}, + "framework-nfc": {}, "framework-ondevicepersonalization": {}, "framework-permission": {}, "framework-permission-s": {}, From 815536fb6435d2aeef6b939443b6cbe672ec6df1 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 26 Oct 2023 12:47:16 -0700 Subject: [PATCH 75/80] Enable sandboxing for exercise_type_data_type_config-pb go/roboleaf-busy-beavers-sandboxing Ignore-AOSP-First: exercise_type_data_type_config-pb only exists on internal Bug: 307824623 Test: m exercise_type_data_type_config-pb Change-Id: I2fb6e8254c124c4c0382b294a2e5a86a50be02b9 (cherry picked from commit 90498937ed9ae9735a21087daad9f24fb67de50e) --- genrule/allowlists.go | 1 - 1 file changed, 1 deletion(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index adf27862f..cf7a8e8e3 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -57,7 +57,6 @@ var ( "deqp_spvtools_update_build_version", "emp_ematch.yacc.c", "emp_ematch.yacc.h", - "exercise_type_data_type_config-pb", "fdt_test_tree_empty_memory_range_dtb", "fdt_test_tree_multiple_memory_ranges_dtb", "fdt_test_tree_one_memory_range_dtb", From cf829793bc18653e3dce16180c4356f71c0dd507 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 26 Oct 2023 18:09:06 -0700 Subject: [PATCH 76/80] Enable sandboxing for pigweed gnerules go/roboleaf-busy-beavers-sandboxing Ignore-AOSP-First: these modules only exist on internal Bug: 307824623 Test: m pw_log_log_proto_pwpb_h pw_log_log_raw_rpc_h pw_log_log_rpc_pwpb_h pw_tokenizer_proto_options_pwpb_h Change-Id: I5f4c68657a9440a3db339e7a51a4714c3bc5df55 (cherry picked from commit 50005b08c26992eca5af99dbfa79dca0197e5105) --- genrule/allowlists.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 284a3415c..297976879 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -53,10 +53,6 @@ var ( "libc_musl_sysroot_bits", "libcore-non-cts-tests-txt", "pvmfw_fdt_template_rs", - "pw_log_log_proto_pwpb_h", - "pw_log_log_raw_rpc_h", - "pw_log_log_rpc_pwpb_h", - "pw_tokenizer_proto_options_pwpb_h", "r8retrace-dexdump-sample-app", "r8retrace-run-retrace", "seller-frontend-service-stub-lite", From 8d6ba4367d853f5d7227ab7ccc9b0512b1191417 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Fri, 24 Nov 2023 15:55:39 +0000 Subject: [PATCH 77/80] Allow module SDK snapshot build to hide flagged APIs Temporary workaround to allow module SDK snapshot builds to hide flagged APIs until such time as development moves to `main` and the build in `main` can handle that properly. Bug: 299570421 Test: UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true vendor/google/build/mainline_modules_sdks.sh # Check that flagged APIs are excluded from everything but # for-latest-build. Change-Id: I5b133b98aab91b2ad5ba717dd663c60a5ed90c9d Merged-In: I4967376c0236bad729398af80fa59b48dbab5f21 --- java/droidstubs.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/java/droidstubs.go b/java/droidstubs.go index a9e20e03f..eaf388740 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -544,6 +544,15 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi // information. cmd.FlagWithArg("--format-defaults ", "overloaded-method-order=source,add-additional-overrides=yes") + // If requested hide the flagged APIs from the output of metalava. This + // should be implemented in the module SDK snapshot code by depending on + // special metalava rules that hide the flagged APIs but that will take + // lots of work. In the meantime, this is a temporary workaround that + // can and will only be used when building module SDK snapshots. + if ctx.Config().GetenvWithDefault("SOONG_SDK_SNAPSHOT_HIDE_FLAGGED_APIS", "false") == "true" { + cmd.Flag("--hide-annotation android.annotation.FlaggedApi") + } + return cmd } From 20c2010030612da735e0416c03a3ca884f1ef6ac Mon Sep 17 00:00:00 2001 From: Rico Wind Date: Thu, 23 Nov 2023 08:41:28 +0100 Subject: [PATCH 78/80] Reapply "Use R8 for resource shrinking" This reverts commit a9fd59a7f28df05f9d75bda7d661241e4d1a3f6a. We are moving the resource shinking pipeline into r8 (gennerally, not just for platform) This disables the usage of the resource shrinker cli from cmd-line tools There are no changes in this cl compared to the original land, the fix was done in R8 (to use the same compression for res folder entries as in the original) Bug: 308710394 Bug: 309078004 Test: Existing, validated that resource table on SystemUI was byte<>byte equal, validated uncompression Ignore-AOSP-First: Merge does not apply cleanly Change-Id: Ib8a6fb128084e994325b975c46a036cb41494654 --- java/Android.bp | 2 -- java/app.go | 24 ++++++++++++---- java/app_builder.go | 8 +----- java/dex.go | 39 ++++++++++++++++++-------- java/resourceshrinker.go | 45 ----------------------------- java/resourceshrinker_test.go | 53 ----------------------------------- 6 files changed, 46 insertions(+), 125 deletions(-) delete mode 100644 java/resourceshrinker.go delete mode 100644 java/resourceshrinker_test.go diff --git a/java/Android.bp b/java/Android.bp index fe9bac1e4..1dc560268 100644 --- a/java/Android.bp +++ b/java/Android.bp @@ -68,7 +68,6 @@ bootstrap_go_package { "prebuilt_apis.go", "proto.go", "ravenwood.go", - "resourceshrinker.go", "robolectric.go", "rro.go", "sdk.go", @@ -109,7 +108,6 @@ bootstrap_go_package { "prebuilt_apis_test.go", "proto_test.go", "ravenwood_test.go", - "resourceshrinker_test.go", "rro_test.go", "sdk_test.go", "sdk_library_test.go", diff --git a/java/app.go b/java/app.go index 6d7411d47..6ec6bb273 100755 --- a/java/app.go +++ b/java/app.go @@ -544,7 +544,7 @@ func (a *AndroidApp) installPath(ctx android.ModuleContext) android.InstallPath return android.PathForModuleInstall(ctx, installDir, a.installApkName+".apk") } -func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { +func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) (android.Path, android.Path) { a.dexpreopter.installPath = a.installPath(ctx) a.dexpreopter.isApp = true if a.dexProperties.Uncompress_dex == nil { @@ -557,7 +557,15 @@ func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { a.dexpreopter.manifestFile = a.mergedManifestFile a.dexpreopter.preventInstall = a.appProperties.PreventInstall + var packageResources = a.exportPackage + if ctx.ModuleName() != "framework-res" { + if Bool(a.dexProperties.Optimize.Shrink_resources) { + protoFile := android.PathForModuleOut(ctx, packageResources.Base()+".proto.apk") + aapt2Convert(ctx, protoFile, packageResources, "proto") + a.dexer.resourcesInput = android.OptionalPathForPath(protoFile) + } + var extraSrcJars android.Paths var extraClasspathJars android.Paths var extraCombinedJars android.Paths @@ -575,9 +583,14 @@ func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { } a.Module.compile(ctx, extraSrcJars, extraClasspathJars, extraCombinedJars) + if Bool(a.dexProperties.Optimize.Shrink_resources) { + binaryResources := android.PathForModuleOut(ctx, packageResources.Base()+".binary.out.apk") + aapt2Convert(ctx, binaryResources, a.dexer.resourcesOutput.Path(), "binary") + packageResources = binaryResources + } } - return a.dexJarFile.PathOrNil() + return a.dexJarFile.PathOrNil(), packageResources } func (a *AndroidApp) jniBuildActions(jniLibs []jniLib, prebuiltJniPackages android.Paths, ctx android.ModuleContext) android.WritablePath { @@ -762,7 +775,6 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // Process all building blocks, from AAPT to certificates. a.aaptBuildActions(ctx) - // The decision to enforce checks is made before adding implicit SDK libraries. a.usesLibrary.freezeEnforceUsesLibraries() @@ -788,7 +800,7 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { a.linter.resources = a.aapt.resourceFiles a.linter.buildModuleReportZip = ctx.Config().UnbundledBuildApps() - dexJarFile := a.dexBuildActions(ctx) + dexJarFile, packageResources := a.dexBuildActions(ctx) jniLibs, prebuiltJniPackages, certificates := collectAppDeps(ctx, a, a.shouldEmbedJnis(ctx), !Bool(a.appProperties.Jni_uses_platform_apis)) jniJarFile := a.jniBuildActions(jniLibs, prebuiltJniPackages, ctx) @@ -812,7 +824,7 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { } rotationMinSdkVersion := String(a.overridableAppProperties.RotationMinSdkVersion) - CreateAndSignAppPackage(ctx, packageFile, a.exportPackage, jniJarFile, dexJarFile, certificates, apkDeps, v4SignatureFile, lineageFile, rotationMinSdkVersion, Bool(a.dexProperties.Optimize.Shrink_resources)) + CreateAndSignAppPackage(ctx, packageFile, packageResources, jniJarFile, dexJarFile, certificates, apkDeps, v4SignatureFile, lineageFile, rotationMinSdkVersion) a.outputFile = packageFile if v4SigningRequested { a.extraOutputFiles = append(a.extraOutputFiles, v4SignatureFile) @@ -841,7 +853,7 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { if v4SigningRequested { v4SignatureFile = android.PathForModuleOut(ctx, a.installApkName+"_"+split.suffix+".apk.idsig") } - CreateAndSignAppPackage(ctx, packageFile, split.path, nil, nil, certificates, apkDeps, v4SignatureFile, lineageFile, rotationMinSdkVersion, false) + CreateAndSignAppPackage(ctx, packageFile, split.path, nil, nil, certificates, apkDeps, v4SignatureFile, lineageFile, rotationMinSdkVersion) a.extraOutputFiles = append(a.extraOutputFiles, packageFile) if v4SigningRequested { a.extraOutputFiles = append(a.extraOutputFiles, v4SignatureFile) diff --git a/java/app_builder.go b/java/app_builder.go index d397ff7f5..943ce317b 100644 --- a/java/app_builder.go +++ b/java/app_builder.go @@ -52,7 +52,7 @@ var combineApk = pctx.AndroidStaticRule("combineApk", }) func CreateAndSignAppPackage(ctx android.ModuleContext, outputFile android.WritablePath, - packageFile, jniJarFile, dexJarFile android.Path, certificates []Certificate, deps android.Paths, v4SignatureFile android.WritablePath, lineageFile android.Path, rotationMinSdkVersion string, shrinkResources bool) { + packageFile, jniJarFile, dexJarFile android.Path, certificates []Certificate, deps android.Paths, v4SignatureFile android.WritablePath, lineageFile android.Path, rotationMinSdkVersion string) { unsignedApkName := strings.TrimSuffix(outputFile.Base(), ".apk") + "-unsigned.apk" unsignedApk := android.PathForModuleOut(ctx, unsignedApkName) @@ -71,12 +71,6 @@ func CreateAndSignAppPackage(ctx android.ModuleContext, outputFile android.Writa Output: unsignedApk, Implicits: deps, }) - - if shrinkResources { - shrunkenApk := android.PathForModuleOut(ctx, "resource-shrunken", unsignedApk.Base()) - ShrinkResources(ctx, unsignedApk, shrunkenApk) - unsignedApk = shrunkenApk - } SignAppPackage(ctx, outputFile, unsignedApk, certificates, v4SignatureFile, lineageFile, rotationMinSdkVersion) } diff --git a/java/dex.go b/java/dex.go index aa017834d..ee28643e9 100644 --- a/java/dex.go +++ b/java/dex.go @@ -95,6 +95,8 @@ type dexer struct { proguardDictionary android.OptionalPath proguardConfiguration android.OptionalPath proguardUsageZip android.OptionalPath + resourcesInput android.OptionalPath + resourcesOutput android.OptionalPath providesTransitiveHeaderJars } @@ -161,7 +163,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", "$r8Template": &remoteexec.REParams{ Labels: map[string]string{"type": "compile", "compiler": "r8"}, Inputs: []string{"$implicits", "${config.R8Jar}"}, - OutputFiles: []string{"${outUsage}", "${outConfig}", "${outDict}"}, + OutputFiles: []string{"${outUsage}", "${outConfig}", "${outDict}", "${resourcesOutput}"}, ExecStrategy: "${config.RER8ExecStrategy}", ToolchainInputs: []string{"${config.JavaCmd}"}, Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, @@ -181,7 +183,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, }, }, []string{"outDir", "outDict", "outConfig", "outUsage", "outUsageZip", "outUsageDir", - "r8Flags", "zipFlags", "mergeZipsFlags"}, []string{"implicits"}) + "r8Flags", "zipFlags", "mergeZipsFlags", "resourcesOutput"}, []string{"implicits"}) func (d *dexer) dexCommonFlags(ctx android.ModuleContext, dexParams *compileDexParams) (flags []string, deps android.Paths) { @@ -350,6 +352,12 @@ func (d *dexer) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8Fl r8Flags = append(r8Flags, "-ignorewarnings") } + if d.resourcesInput.Valid() { + r8Flags = append(r8Flags, "--resource-input", d.resourcesInput.Path().String()) + r8Deps = append(r8Deps, d.resourcesInput.Path()) + r8Flags = append(r8Flags, "--resource-output", d.resourcesOutput.Path().String()) + } + return r8Flags, r8Deps } @@ -391,6 +399,8 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam android.ModuleNameWithPossibleOverride(ctx), "unused.txt") proguardUsageZip := android.PathForModuleOut(ctx, "proguard_usage.zip") d.proguardUsageZip = android.OptionalPathForPath(proguardUsageZip) + resourcesOutput := android.PathForModuleOut(ctx, "package-res-shrunken.apk") + d.resourcesOutput = android.OptionalPathForPath(resourcesOutput) r8Flags, r8Deps := d.r8Flags(ctx, dexParams.flags) r8Deps = append(r8Deps, commonDeps...) rule := r8 @@ -409,17 +419,22 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam rule = r8RE args["implicits"] = strings.Join(r8Deps.Strings(), ",") } + implicitOutputs := android.WritablePaths{ + proguardDictionary, + proguardUsageZip, + proguardConfiguration} + if d.resourcesInput.Valid() { + implicitOutputs = append(implicitOutputs, resourcesOutput) + args["resourcesOutput"] = resourcesOutput.String() + } ctx.Build(pctx, android.BuildParams{ - Rule: rule, - Description: "r8", - Output: javalibJar, - ImplicitOutputs: android.WritablePaths{ - proguardDictionary, - proguardUsageZip, - proguardConfiguration}, - Input: dexParams.classesJar, - Implicits: r8Deps, - Args: args, + Rule: rule, + Description: "r8", + Output: javalibJar, + ImplicitOutputs: implicitOutputs, + Input: dexParams.classesJar, + Implicits: r8Deps, + Args: args, }) } else { d8Flags, d8Deps := d8Flags(dexParams.flags) diff --git a/java/resourceshrinker.go b/java/resourceshrinker.go deleted file mode 100644 index af13aa3cb..000000000 --- a/java/resourceshrinker.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2022 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "android/soong/android" - - "github.com/google/blueprint" -) - -var shrinkResources = pctx.AndroidStaticRule("shrinkResources", - blueprint.RuleParams{ - // Note that we suppress stdout to avoid successful log confirmations. - Command: `RESOURCESHRINKER_OPTS=-Dcom.android.tools.r8.dexContainerExperiment ` + - `${config.ResourceShrinkerCmd} --output $out --input $in --raw_resources $raw_resources >/dev/null`, - CommandDeps: []string{"${config.ResourceShrinkerCmd}"}, - }, "raw_resources") - -func ShrinkResources(ctx android.ModuleContext, apk android.Path, outputFile android.WritablePath) { - protoFile := android.PathForModuleOut(ctx, apk.Base()+".proto.apk") - aapt2Convert(ctx, protoFile, apk, "proto") - strictModeFile := android.PathForSource(ctx, "prebuilts/cmdline-tools/shrinker.xml") - protoOut := android.PathForModuleOut(ctx, apk.Base()+".proto.out.apk") - ctx.Build(pctx, android.BuildParams{ - Rule: shrinkResources, - Input: protoFile, - Output: protoOut, - Args: map[string]string{ - "raw_resources": strictModeFile.String(), - }, - }) - aapt2Convert(ctx, outputFile, protoOut, "binary") -} diff --git a/java/resourceshrinker_test.go b/java/resourceshrinker_test.go deleted file mode 100644 index 3bbf11670..000000000 --- a/java/resourceshrinker_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2022 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "testing" - - "android/soong/android" -) - -func TestShrinkResourcesArgs(t *testing.T) { - result := android.GroupFixturePreparers( - PrepareForTestWithJavaDefaultModules, - ).RunTestWithBp(t, ` - android_app { - name: "app_shrink", - platform_apis: true, - optimize: { - shrink_resources: true, - } - } - - android_app { - name: "app_no_shrink", - platform_apis: true, - optimize: { - shrink_resources: false, - } - } - `) - - appShrink := result.ModuleForTests("app_shrink", "android_common") - appShrinkResources := appShrink.Rule("shrinkResources") - android.AssertStringDoesContain(t, "expected shrinker.xml in app_shrink resource shrinker flags", - appShrinkResources.Args["raw_resources"], "shrinker.xml") - - appNoShrink := result.ModuleForTests("app_no_shrink", "android_common") - if appNoShrink.MaybeRule("shrinkResources").Rule != nil { - t.Errorf("unexpected shrinkResources rule for app_no_shrink") - } -} From 640380699f0eab0c445a59a757bb670954d8f1ed Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Tue, 28 Nov 2023 16:16:50 +0000 Subject: [PATCH 79/80] Move special next handling of hidden APIs to java_sdk_library Previously, the logic to add the Metalava command line option `--hide-annotation android.annotation.FlaggedApi` when `SOONG_SDK_SNAPSHOT_HIDE_FLAGGED_APIS=true` was done in `droidstubs.go`. It worked but it turns out that we need to pass more options to Metalava in that situation because while we need to hide most flagged APIs we need to release some flagged APIs whose features are ready. They cannot have the `@FlaggedApi` annotation removed because that would cause the APIs to be released in the next QPR which is not what we want. So, in addition to the above option we will need to pass additional options to Metalava to tell it to keep some flagged APIs. Rather than hard code that in the build, requiring build changes for every new SDK extension release this change uses the argument expansion mechanism in Metalava (treating `@` as if the contents of `` were passed as command line options) to delegate that to ":sdkext-released-flagged-apis". The easiest way to do that was to move the logic to `sdk_library.go` as `droidstub` would do the work of tracking the file dependencies correctly. It also reduces the affect modules to `java_sdk_library` modules rather than all `droidstubs` modules. Bug: 313398274 Test: UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true vendor/google/build/mainline_modules_sdks.sh --build-release=next Change-Id: Ib1cf22f8ef74abc135215537e9311936372eaac7 Merged-In: I4967376c0236bad729398af80fa59b48dbab5f21 --- java/droidstubs.go | 9 --------- java/sdk_library.go | 20 +++++++++++++++++++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/java/droidstubs.go b/java/droidstubs.go index eaf388740..a9e20e03f 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -544,15 +544,6 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi // information. cmd.FlagWithArg("--format-defaults ", "overloaded-method-order=source,add-additional-overrides=yes") - // If requested hide the flagged APIs from the output of metalava. This - // should be implemented in the module SDK snapshot code by depending on - // special metalava rules that hide the flagged APIs but that will take - // lots of work. In the meantime, this is a temporary workaround that - // can and will only be used when building module SDK snapshots. - if ctx.Config().GetenvWithDefault("SOONG_SDK_SNAPSHOT_HIDE_FLAGGED_APIS", "false") == "true" { - cmd.Flag("--hide-annotation android.annotation.FlaggedApi") - } - return cmd } diff --git a/java/sdk_library.go b/java/sdk_library.go index 76f050b73..853792d20 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -1682,6 +1682,24 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC } droidstubsArgs = append(droidstubsArgs, android.JoinWithPrefix(disabledWarnings, "--hide ")) + droidstubsFiles := append([]string{}, module.sdkLibraryProperties.Droiddoc_option_files...) + + // If requested hide the flagged APIs from the output of metalava. This + // should be implemented in the module SDK snapshot code by depending on + // special metalava rules that hide the flagged APIs but that will take + // lots of work. In the meantime, this is a temporary workaround that + // can and will only be used when building module SDK snapshots. This + // delegates the decision as to what options are passed to metalava to + // the "sdkext-released-flagged-apis" module by using metalava's support + // for expanding an argument of the form "@" to the list of + // arguments found in . + if mctx.Config().GetenvWithDefault("SOONG_SDK_SNAPSHOT_HIDE_FLAGGED_APIS", "false") == "true" { + metalavaHideFlaggedApisSource := ":sdkext-released-flagged-apis" + droidstubsArgs = append(droidstubsArgs, + fmt.Sprintf("@$(location %s)", metalavaHideFlaggedApisSource)) + droidstubsFiles = append(droidstubsFiles, metalavaHideFlaggedApisSource) + } + // Output Javadoc comments for public scope. if apiScope == apiScopePublic { props.Output_javadoc_comments = proptools.BoolPtr(true) @@ -1689,7 +1707,7 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC // Add in scope specific arguments. droidstubsArgs = append(droidstubsArgs, scopeSpecificDroidstubsArgs...) - props.Arg_files = module.sdkLibraryProperties.Droiddoc_option_files + props.Arg_files = droidstubsFiles props.Args = proptools.StringPtr(strings.Join(droidstubsArgs, " ")) // List of APIs identified from the provided source files are created. They are later From 7a19abf260a9790fd107e3fb1e7a28495f21db78 Mon Sep 17 00:00:00 2001 From: Jeff Sharkey Date: Thu, 30 Nov 2023 09:29:50 -0700 Subject: [PATCH 80/80] Include Ravenwood tests in `general-tests`. EngProd has indicated that we need to be included in that suite to support TEST_MAPPING files in their infrastructure. Bug: 308854804 Test: TH Change-Id: I763b033f0f5f275091db45ab62df6af48dcddc66 --- java/ravenwood.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/java/ravenwood.go b/java/ravenwood.go index 85f64a86c..e36200307 100644 --- a/java/ravenwood.go +++ b/java/ravenwood.go @@ -53,7 +53,10 @@ func ravenwoodTestFactory() android.Module { module.Module.dexpreopter.isTest = true module.Module.linter.properties.Lint.Test = proptools.BoolPtr(true) - module.testProperties.Test_suites = []string{"ravenwood-tests"} + module.testProperties.Test_suites = []string{ + "general-tests", + "ravenwood-tests", + } module.testProperties.Test_options.Unit_test = proptools.BoolPtr(false) InitJavaModule(module, android.DeviceSupported) @@ -123,7 +126,8 @@ func (r *ravenwoodTest) AndroidMkEntries() []android.AndroidMkEntries { entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) - entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", "ravenwood-tests") + entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", + "general-tests", "ravenwood-tests") if r.testConfig != nil { entries.SetPath("LOCAL_FULL_TEST_CONFIG", r.testConfig) } @@ -157,7 +161,10 @@ func (r *ravenwoodLibgroup) InstallForceOS() (*android.OsType, *android.ArchType return &r.forceOSType, &r.forceArchType } func (r *ravenwoodLibgroup) TestSuites() []string { - return []string{"ravenwood-tests"} + return []string{ + "general-tests", + "ravenwood-tests", + } } func (r *ravenwoodLibgroup) DepsMutator(ctx android.BottomUpMutatorContext) {