From 9f9c0a22faedef806fd3c35e047e1d5a0e313718 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Tue, 29 Nov 2022 11:19:37 -0500 Subject: [PATCH] transitive Java deps for r8 The -libraryjars argument to r8 was missing transitive dependencies, and so complained when there were classes used in the program JAR which were not provided via libraries. This CL propagates transitive dependencies to the r8 command to reduce the warnings that are generated for missing classes. Bug: 242088131 Change-Id: Ifad7bc7c5d406e3de8d98ea963c97e88c84b45a1 --- java/aar.go | 5 ++ java/app.go | 3 + java/base.go | 49 +++++++++++++ java/dex.go | 34 ++++++++- java/dex_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++- java/java.go | 11 ++- java/kotlin_test.go | 14 ++++ 7 files changed, 283 insertions(+), 5 deletions(-) diff --git a/java/aar.go b/java/aar.go index f4a2ff287..082669357 100644 --- a/java/aar.go +++ b/java/aar.go @@ -651,6 +651,8 @@ type AARImport struct { // Functionality common to Module and Import. embeddableInModuleAndImport + providesTransitiveHeaderJars + properties AARImportProperties classpathFile android.WritablePath @@ -897,8 +899,11 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.assetsPackage = mergedAssets } + a.collectTransitiveHeaderJars(ctx) ctx.SetProvider(JavaInfoProvider, JavaInfo{ HeaderJars: android.PathsIfNonNil(a.classpathFile), + TransitiveLibsHeaderJars: a.transitiveLibsHeaderJars, + TransitiveStaticLibsHeaderJars: a.transitiveStaticLibsHeaderJars, ImplementationAndResourcesJars: android.PathsIfNonNil(a.classpathFile), ImplementationJars: android.PathsIfNonNil(a.classpathFile), }) diff --git a/java/app.go b/java/app.go index 3c5760b58..7eff4c355 100755 --- a/java/app.go +++ b/java/app.go @@ -1310,6 +1310,9 @@ func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, addCompatDeps boo ctx.AddVariationDependencies(nil, usesLibCompat28OptTag, dexpreopt.OptionalCompatUsesLibs28...) ctx.AddVariationDependencies(nil, usesLibCompat30OptTag, dexpreopt.OptionalCompatUsesLibs30...) } + } else { + ctx.AddVariationDependencies(nil, r8LibraryJarTag, u.usesLibraryProperties.Uses_libs...) + ctx.AddVariationDependencies(nil, r8LibraryJarTag, u.presentOptionalUsesLibs(ctx)...) } } diff --git a/java/base.go b/java/base.go index 84fda37cb..cce06a4e9 100644 --- a/java/base.go +++ b/java/base.go @@ -1583,6 +1583,8 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { ctx.SetProvider(JavaInfoProvider, JavaInfo{ HeaderJars: android.PathsIfNonNil(j.headerJarFile), + TransitiveLibsHeaderJars: j.transitiveLibsHeaderJars, + TransitiveStaticLibsHeaderJars: j.transitiveStaticLibsHeaderJars, ImplementationAndResourcesJars: android.PathsIfNonNil(j.implementationAndResourcesJar), ImplementationJars: android.PathsIfNonNil(j.implementationJarFile), ResourceJars: android.PathsIfNonNil(j.resourceJar), @@ -1719,6 +1721,52 @@ func (j *Module) instrument(ctx android.ModuleContext, flags javaBuilderFlags, return instrumentedJar } +type providesTransitiveHeaderJars struct { + // set of header jars for all transitive libs deps + transitiveLibsHeaderJars *android.DepSet + // set of header jars for all transitive static libs deps + transitiveStaticLibsHeaderJars *android.DepSet +} + +func (j *providesTransitiveHeaderJars) TransitiveLibsHeaderJars() *android.DepSet { + return j.transitiveLibsHeaderJars +} + +func (j *providesTransitiveHeaderJars) TransitiveStaticLibsHeaderJars() *android.DepSet { + return j.transitiveStaticLibsHeaderJars +} + +func (j *providesTransitiveHeaderJars) collectTransitiveHeaderJars(ctx android.ModuleContext) { + directLibs := android.Paths{} + directStaticLibs := android.Paths{} + transitiveLibs := []*android.DepSet{} + transitiveStaticLibs := []*android.DepSet{} + ctx.VisitDirectDeps(func(module android.Module) { + // don't add deps of the prebuilt version of the same library + if ctx.ModuleName() == android.RemoveOptionalPrebuiltPrefix(module.Name()) { + return + } + + dep := ctx.OtherModuleProvider(module, JavaInfoProvider).(JavaInfo) + if dep.TransitiveLibsHeaderJars != nil { + transitiveLibs = append(transitiveLibs, dep.TransitiveLibsHeaderJars) + } + if dep.TransitiveStaticLibsHeaderJars != nil { + transitiveStaticLibs = append(transitiveStaticLibs, dep.TransitiveStaticLibsHeaderJars) + } + + tag := ctx.OtherModuleDependencyTag(module) + _, isUsesLibDep := tag.(usesLibraryDependencyTag) + if tag == libTag || tag == r8LibraryJarTag || isUsesLibDep { + directLibs = append(directLibs, dep.HeaderJars...) + } else if tag == staticLibTag { + directStaticLibs = append(directStaticLibs, dep.HeaderJars...) + } + }) + j.transitiveLibsHeaderJars = android.NewDepSet(android.POSTORDER, directLibs, transitiveLibs) + j.transitiveStaticLibsHeaderJars = android.NewDepSet(android.POSTORDER, directStaticLibs, transitiveStaticLibs) +} + func (j *Module) HeaderJars() android.Paths { if j.headerJarFile == nil { return nil @@ -1947,6 +1995,7 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { sdkLinkType, _ := j.getSdkLinkType(ctx, ctx.ModuleName()) + j.collectTransitiveHeaderJars(ctx) ctx.VisitDirectDeps(func(module android.Module) { otherName := ctx.OtherModuleName(module) tag := ctx.OtherModuleDependencyTag(module) diff --git a/java/dex.go b/java/dex.go index 40ee99d3b..971da925e 100644 --- a/java/dex.go +++ b/java/dex.go @@ -22,6 +22,7 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" + "android/soong/java/config" "android/soong/remoteexec" ) @@ -90,6 +91,8 @@ type dexer struct { extraProguardFlagFiles android.Paths proguardDictionary android.OptionalPath proguardUsageZip android.OptionalPath + + providesTransitiveHeaderJars } func (d *dexer) effectiveOptimizeEnabled() bool { @@ -249,12 +252,37 @@ func (d *dexer) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8Fl }) r8Flags = append(r8Flags, proguardRaiseDeps.FormJavaClassPath("-libraryjars")) - r8Flags = append(r8Flags, flags.bootClasspath.FormJavaClassPath("-libraryjars")) - r8Flags = append(r8Flags, flags.dexClasspath.FormJavaClassPath("-libraryjars")) - r8Deps = append(r8Deps, proguardRaiseDeps...) + r8Flags = append(r8Flags, flags.bootClasspath.FormJavaClassPath("-libraryjars")) r8Deps = append(r8Deps, flags.bootClasspath...) + r8Flags = append(r8Flags, flags.dexClasspath.FormJavaClassPath("-libraryjars")) r8Deps = append(r8Deps, flags.dexClasspath...) + r8Flags = append(r8Flags, flags.processorPath.FormJavaClassPath("-libraryjars")) + r8Deps = append(r8Deps, flags.processorPath...) + + errorProneClasspath := classpath(android.PathsForSource(ctx, config.ErrorProneClasspath)) + r8Flags = append(r8Flags, errorProneClasspath.FormJavaClassPath("-libraryjars")) + r8Deps = append(r8Deps, errorProneClasspath...) + + transitiveStaticLibsLookupMap := map[android.Path]bool{} + if d.transitiveStaticLibsHeaderJars != nil { + for _, jar := range d.transitiveStaticLibsHeaderJars.ToList() { + transitiveStaticLibsLookupMap[jar] = true + } + } + transitiveHeaderJars := android.Paths{} + if d.transitiveLibsHeaderJars != nil { + for _, jar := range d.transitiveLibsHeaderJars.ToList() { + if _, ok := transitiveStaticLibsLookupMap[jar]; ok { + // don't include a lib if it is already packaged in the current JAR as a static lib + continue + } + transitiveHeaderJars = append(transitiveHeaderJars, jar) + } + } + transitiveClasspath := classpath(transitiveHeaderJars) + r8Flags = append(r8Flags, transitiveClasspath.FormJavaClassPath("-libraryjars")) + r8Deps = append(r8Deps, transitiveClasspath...) flagFiles := android.Paths{ android.PathForSource(ctx, "build/make/core/proguard.flags"), diff --git a/java/dex_test.go b/java/dex_test.go index fc6cd0f3f..dc85f9e34 100644 --- a/java/dex_test.go +++ b/java/dex_test.go @@ -18,6 +18,8 @@ import ( "testing" "android/soong/android" + + "github.com/google/blueprint/proptools" ) func TestR8(t *testing.T) { @@ -74,7 +76,7 @@ func TestR8(t *testing.T) { android.AssertStringDoesContain(t, "expected lib header jar in app r8 classpath", appR8.Args["r8Flags"], libHeader.String()) - android.AssertStringDoesNotContain(t, "expected no static_lib header jar in app javac classpath", + android.AssertStringDoesNotContain(t, "expected no static_lib header jar in app r8 classpath", appR8.Args["r8Flags"], staticLibHeader.String()) android.AssertStringDoesContain(t, "expected -ignorewarnings in app r8 flags", appR8.Args["r8Flags"], "-ignorewarnings") @@ -86,6 +88,174 @@ func TestR8(t *testing.T) { corePlatformAppR8.Args["r8Flags"], "--android-platform-build") } +func TestR8TransitiveDeps(t *testing.T) { + bp := ` + override_android_app { + name: "override_app", + base: "app", + } + + android_app { + name: "app", + srcs: ["foo.java"], + libs: [ + "lib", + "uses_libs_dep_import", + ], + static_libs: [ + "static_lib", + "repeated_dep", + ], + platform_apis: true, + } + + java_library { + name: "static_lib", + srcs: ["foo.java"], + } + + java_library { + name: "lib", + libs: [ + "transitive_lib", + "repeated_dep", + "prebuilt_lib", + ], + static_libs: ["transitive_static_lib"], + srcs: ["foo.java"], + } + + java_library { + name: "repeated_dep", + srcs: ["foo.java"], + } + + java_library { + name: "transitive_static_lib", + srcs: ["foo.java"], + } + + java_library { + name: "transitive_lib", + srcs: ["foo.java"], + libs: ["transitive_lib_2"], + } + + java_library { + name: "transitive_lib_2", + srcs: ["foo.java"], + } + + java_import { + name: "lib", + jars: ["lib.jar"], + } + + java_library { + name: "uses_lib", + srcs: ["foo.java"], + } + + java_library { + name: "optional_uses_lib", + srcs: ["foo.java"], + } + + android_library { + name: "uses_libs_dep", + uses_libs: ["uses_lib"], + optional_uses_libs: ["optional_uses_lib"], + } + + android_library_import { + name: "uses_libs_dep_import", + aars: ["aar.aar"], + static_libs: ["uses_libs_dep"], + } + ` + + testcases := []struct { + name string + unbundled bool + }{ + { + name: "non-unbundled build", + unbundled: false, + }, + { + name: "unbundled build", + unbundled: true, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + fixturePreparer := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd + if tc.unbundled { + fixturePreparer = android.GroupFixturePreparers( + fixturePreparer, + android.FixtureModifyProductVariables( + func(variables android.FixtureProductVariables) { + variables.Unbundled_build = proptools.BoolPtr(true) + }, + ), + ) + } + result := fixturePreparer.RunTestWithBp(t, bp) + + getHeaderJar := func(name string) android.Path { + mod := result.ModuleForTests(name, "android_common") + return mod.Output("turbine-combined/" + name + ".jar").Output + } + + appR8 := result.ModuleForTests("app", "android_common").Rule("r8") + overrideAppR8 := result.ModuleForTests("app", "android_common_override_app").Rule("r8") + appHeader := getHeaderJar("app") + overrideAppHeader := result.ModuleForTests("app", "android_common_override_app").Output("turbine-combined/app.jar").Output + libHeader := getHeaderJar("lib") + transitiveLibHeader := getHeaderJar("transitive_lib") + transitiveLib2Header := getHeaderJar("transitive_lib_2") + staticLibHeader := getHeaderJar("static_lib") + transitiveStaticLibHeader := getHeaderJar("transitive_static_lib") + repeatedDepHeader := getHeaderJar("repeated_dep") + usesLibHeader := getHeaderJar("uses_lib") + optionalUsesLibHeader := getHeaderJar("optional_uses_lib") + prebuiltLibHeader := result.ModuleForTests("prebuilt_lib", "android_common").Output("combined/lib.jar").Output + + for _, rule := range []android.TestingBuildParams{appR8, overrideAppR8} { + android.AssertStringDoesNotContain(t, "expected no app header jar in app r8 classpath", + rule.Args["r8Flags"], appHeader.String()) + android.AssertStringDoesNotContain(t, "expected no override_app header jar in app r8 classpath", + rule.Args["r8Flags"], overrideAppHeader.String()) + android.AssertStringDoesContain(t, "expected transitive lib header jar in app r8 classpath", + rule.Args["r8Flags"], transitiveLibHeader.String()) + android.AssertStringDoesContain(t, "expected transitive lib ^2 header jar in app r8 classpath", + rule.Args["r8Flags"], transitiveLib2Header.String()) + android.AssertStringDoesContain(t, "expected lib header jar in app r8 classpath", + rule.Args["r8Flags"], libHeader.String()) + android.AssertStringDoesContain(t, "expected uses_lib header jar in app r8 classpath", + rule.Args["r8Flags"], usesLibHeader.String()) + android.AssertStringDoesContain(t, "expected optional_uses_lib header jar in app r8 classpath", + rule.Args["r8Flags"], optionalUsesLibHeader.String()) + android.AssertStringDoesNotContain(t, "expected no static_lib header jar in app r8 classpath", + rule.Args["r8Flags"], staticLibHeader.String()) + android.AssertStringDoesNotContain(t, "expected no transitive static_lib header jar in app r8 classpath", + rule.Args["r8Flags"], transitiveStaticLibHeader.String()) + // we shouldn't list this dep because it is already included as static_libs in the app + android.AssertStringDoesNotContain(t, "expected no repeated_dep header jar in app r8 classpath", + rule.Args["r8Flags"], repeatedDepHeader.String()) + // skip a prebuilt transitive dep if the source is also a transitive dep + android.AssertStringDoesNotContain(t, "expected no prebuilt header jar in app r8 classpath", + rule.Args["r8Flags"], prebuiltLibHeader.String()) + android.AssertStringDoesContain(t, "expected -ignorewarnings in app r8 flags", + rule.Args["r8Flags"], "-ignorewarnings") + android.AssertStringDoesContain(t, "expected --android-platform-build in app r8 flags", + rule.Args["r8Flags"], "--android-platform-build") + } + }) + } +} + func TestR8Flags(t *testing.T) { result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` android_app { diff --git a/java/java.go b/java/java.go index 3b0ad8d9f..10d60f096 100644 --- a/java/java.go +++ b/java/java.go @@ -230,6 +230,12 @@ type JavaInfo struct { // against this module. If empty, ImplementationJars should be used instead. HeaderJars android.Paths + // set of header jars for all transitive libs deps + TransitiveLibsHeaderJars *android.DepSet + + // set of header jars for all transitive static libs deps + TransitiveStaticLibsHeaderJars *android.DepSet + // ImplementationAndResourceJars is a list of jars that contain the implementations of classes // in the module as well as any resources included in the module. ImplementationAndResourcesJars android.Paths @@ -380,6 +386,7 @@ var ( instrumentationForTag = dependencyTag{name: "instrumentation_for"} extraLintCheckTag = dependencyTag{name: "extra-lint-check", toolchain: true} jniLibTag = dependencyTag{name: "jnilib", runtimeLinked: true} + r8LibraryJarTag = dependencyTag{name: "r8-libraryjar", runtimeLinked: true} syspropPublicStubDepTag = dependencyTag{name: "sysprop public stub"} jniInstallTag = installDependencyTag{name: "jni install"} binaryInstallTag = installDependencyTag{name: "binary install"} @@ -1926,9 +1933,9 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { var flags javaBuilderFlags + j.collectTransitiveHeaderJars(ctx) ctx.VisitDirectDeps(func(module android.Module) { tag := ctx.OtherModuleDependencyTag(module) - if ctx.OtherModuleHasProvider(module, JavaInfoProvider) { dep := ctx.OtherModuleProvider(module, JavaInfoProvider).(JavaInfo) switch tag { @@ -2018,6 +2025,8 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.SetProvider(JavaInfoProvider, JavaInfo{ HeaderJars: android.PathsIfNonNil(j.combinedClasspathFile), + TransitiveLibsHeaderJars: j.transitiveLibsHeaderJars, + TransitiveStaticLibsHeaderJars: j.transitiveStaticLibsHeaderJars, ImplementationAndResourcesJars: android.PathsIfNonNil(j.combinedClasspathFile), ImplementationJars: android.PathsIfNonNil(j.combinedClasspathFile), AidlIncludeDirs: j.exportAidlIncludeDirs, diff --git a/java/kotlin_test.go b/java/kotlin_test.go index 491ce2939..933fc5187 100644 --- a/java/kotlin_test.go +++ b/java/kotlin_test.go @@ -44,6 +44,10 @@ func TestKotlin(t *testing.T) { kotlinStdlib := ctx.ModuleForTests("kotlin-stdlib", "android_common"). Output("turbine-combined/kotlin-stdlib.jar").Output + kotlinStdlibJdk7 := ctx.ModuleForTests("kotlin-stdlib-jdk7", "android_common"). + Output("turbine-combined/kotlin-stdlib-jdk7.jar").Output + kotlinStdlibJdk8 := ctx.ModuleForTests("kotlin-stdlib-jdk8", "android_common"). + Output("turbine-combined/kotlin-stdlib-jdk8.jar").Output kotlinAnnotations := ctx.ModuleForTests("kotlin-annotations", "android_common"). Output("turbine-combined/kotlin-annotations.jar").Output @@ -79,6 +83,16 @@ func TestKotlin(t *testing.T) { fooJar.Inputs.Strings(), kotlinStdlib.String()) } + if !inList(kotlinStdlibJdk7.String(), fooJar.Inputs.Strings()) { + t.Errorf("foo jar inputs %v does not contain %v", + fooJar.Inputs.Strings(), kotlinStdlibJdk7.String()) + } + + if !inList(kotlinStdlibJdk8.String(), fooJar.Inputs.Strings()) { + t.Errorf("foo jar inputs %v does not contain %v", + fooJar.Inputs.Strings(), kotlinStdlibJdk8.String()) + } + if !inList(kotlinAnnotations.String(), fooJar.Inputs.Strings()) { t.Errorf("foo jar inputs %v does not contain %v", fooJar.Inputs.Strings(), kotlinAnnotations.String())