From 2d37b641c5d0338c911d01a101992deb1b1b0113 Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Tue, 14 Mar 2023 16:11:38 +0000 Subject: [PATCH] Populate apexBundle#filesInfo using bazel info. This CL adds a few things: 1) Populate the filesInfo struct with cquery'd information from an apex's ApexMkInfo provider. This filesInfo is then used in apex/androidmk.go to generate Make modules (soong_cc_rust_prebuilt.mk), which are then used in packaging to generate zip files of symbols in $PRODUCT_OUT. 2) Make a list of dicts of primitives JSON-encodable. 3) Tests. Bug: 271423316 Bug: 271423062 Test: presubmits Change-Id: Iaa34f51044de310510e580d9cf1fe60bbef801c1 --- android/bazel_handler.go | 10 ++- apex/androidmk.go | 42 +++++++---- apex/apex.go | 54 +++++++++++++- apex/bp2build_test.go | 140 ++++++++++++++++++++++++++++++++++- bazel/cquery/request_type.go | 4 +- 5 files changed, 228 insertions(+), 22 deletions(-) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index e7b84e304..66832d5e7 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -1022,7 +1022,15 @@ def json_encode(input): fail("unsupported value '%s' of type '%s'" % (p, type(p))) def encode_list(list): - return "[%s]" % ", ".join([encode_primitive(item) for item in list]) + items = [] + for item in list: + if type(item) == "dict": + # support encoding dict of primitive keys and values. not list currently, because calling encode_list again is recursive. + kv_pairs = [("%s: %s" % (encode_primitive(k), encode_primitive(v))) for (k, v) in item.items()] + items.append("{ %s }" % ", ".join(kv_pairs)) + else: + items.append(encode_primitive(item)) + return "[%s]" % ", ".join(items) def encode_list_or_primitive(v): return encode_list(v) if type(v) == "list" else encode_primitive(v) diff --git a/apex/androidmk.go b/apex/androidmk.go index 7f0362109..684833de4 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -63,15 +63,17 @@ func (class apexFileClass) nameInMake() string { } // Return the full module name for a dependency module, which appends the apex module name unless re-using a system lib. -func (a *apexBundle) fullModuleName(apexBundleName string, fi *apexFile) string { - linkToSystemLib := a.linkToSystemLib && fi.transitiveDep && fi.availableToPlatform() - +func (a *apexBundle) fullModuleName(apexBundleName string, linkToSystemLib bool, fi *apexFile) string { if linkToSystemLib { return fi.androidMkModuleName } return fi.androidMkModuleName + "." + apexBundleName + a.suffix } +// androidMkForFiles generates Make definitions for the contents of an +// apexBundle (apexBundle#filesInfo). The filesInfo structure can either be +// populated by Soong for unconverted APEXes, or Bazel in mixed mode. Use +// apexFile#isBazelPrebuilt to differentiate. func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir string, apexAndroidMkData android.AndroidMkData) []string { @@ -95,8 +97,7 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st for _, fi := range a.filesInfo { linkToSystemLib := a.linkToSystemLib && fi.transitiveDep && fi.availableToPlatform() - - moduleName := a.fullModuleName(apexBundleName, &fi) + moduleName := a.fullModuleName(apexBundleName, linkToSystemLib, &fi) // This name will be added to LOCAL_REQUIRED_MODULES of the APEX. We need to be // arch-specific otherwise we will end up installing both ABIs even when only @@ -124,6 +125,7 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) } fmt.Fprintln(w, "LOCAL_MODULE :=", moduleName) + if fi.module != nil && fi.module.Owner() != "" { fmt.Fprintln(w, "LOCAL_MODULE_OWNER :=", fi.module.Owner()) } @@ -161,6 +163,7 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", fi.builtFile.String()) fmt.Fprintln(w, "LOCAL_MODULE_CLASS :=", fi.class.nameInMake()) if fi.module != nil { + // This apexFile's module comes from Soong archStr := fi.module.Target().Arch.ArchType.String() host := false switch fi.module.Target().Os.Class { @@ -188,6 +191,9 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st fmt.Fprintln(w, "LOCAL_MODULE_HOST_OS :=", makeOs) fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true") } + } else if fi.isBazelPrebuilt && fi.arch != "" { + // This apexFile comes from Bazel + fmt.Fprintln(w, "LOCAL_MODULE_TARGET_ARCH :=", fi.arch) } if fi.jacocoReportClassesFile != nil { fmt.Fprintln(w, "LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR :=", fi.jacocoReportClassesFile.String()) @@ -231,17 +237,21 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_android_app_set.mk") case nativeSharedLib, nativeExecutable, nativeTest: fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", fi.stem()) - if ccMod, ok := fi.module.(*cc.Module); ok { - if ccMod.UnstrippedOutputFile() != nil { - fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", ccMod.UnstrippedOutputFile().String()) - } - ccMod.AndroidMkWriteAdditionalDependenciesForSourceAbiDiff(w) - if ccMod.CoverageOutputFile().Valid() { - fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", ccMod.CoverageOutputFile().String()) - } - } else if rustMod, ok := fi.module.(*rust.Module); ok { - if rustMod.UnstrippedOutputFile() != nil { - fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", rustMod.UnstrippedOutputFile().String()) + if fi.isBazelPrebuilt { + fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", fi.unstrippedBuiltFile) + } else { + if ccMod, ok := fi.module.(*cc.Module); ok { + if ccMod.UnstrippedOutputFile() != nil { + fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", ccMod.UnstrippedOutputFile().String()) + } + ccMod.AndroidMkWriteAdditionalDependenciesForSourceAbiDiff(w) + if ccMod.CoverageOutputFile().Valid() { + fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", ccMod.CoverageOutputFile().String()) + } + } else if rustMod, ok := fi.module.(*rust.Module); ok { + if rustMod.UnstrippedOutputFile() != nil { + fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", rustMod.UnstrippedOutputFile().String()) + } } } fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_cc_rust_prebuilt.mk") diff --git a/apex/apex.go b/apex/apex.go index f50687613..b2ca6c480 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -509,6 +509,21 @@ const ( shBinary ) +var ( + classes = map[string]apexFileClass{ + "app": app, + "appSet": appSet, + "etc": etc, + "goBinary": goBinary, + "javaSharedLib": javaSharedLib, + "nativeExecutable": nativeExecutable, + "nativeSharedLib": nativeSharedLib, + "nativeTest": nativeTest, + "pyBinary": pyBinary, + "shBinary": shBinary, + } +) + // apexFile represents a file in an APEX bundle. This is created during the first half of // GenerateAndroidBuildActions by traversing the dependencies of the APEX. Then in the second half // of the function, this is used to create commands that copies the files into a staging directory, @@ -543,6 +558,10 @@ type apexFile struct { multilib string + isBazelPrebuilt bool + unstrippedBuiltFile android.Path + arch string + // TODO(jiyong): remove this module android.Module } @@ -1710,6 +1729,7 @@ func apexFileForGoBinary(ctx android.BaseModuleContext, depName string, gb boots // NB: Since go binaries are static we don't need the module for anything here, which is // good since the go tool is a blueprint.Module not an android.Module like we would // normally use. + // return newApexFile(ctx, fileToCopy, depName, dirInApex, goBinary, nil) } @@ -2003,13 +2023,41 @@ func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) { panic(fmt.Errorf("internal error: unexpected apex_type for the ProcessBazelQueryResponse: %v", a.properties.ApexType)) } - // filesInfo is not set in mixed mode, because all information about the - // apex's contents should completely come from the Starlark providers. + // filesInfo in mixed mode must retrieve all information about the apex's + // contents completely from the Starlark providers. It should never rely on + // Android.bp information, as they might not exist for fully migrated + // dependencies. // // Prevent accidental writes to filesInfo in the earlier parts Soong by // asserting it to be nil. if a.filesInfo != nil { - panic(fmt.Errorf("internal error: filesInfo must be nil for an apex handled by Bazel.")) + panic( + fmt.Errorf("internal error: filesInfo must be nil for an apex handled by Bazel. " + + "Did something else set filesInfo before this line of code?")) + } + for _, f := range outputs.PayloadFilesInfo { + fileInfo := apexFile{ + isBazelPrebuilt: true, + + builtFile: android.PathForBazelOut(ctx, f["built_file"]), + unstrippedBuiltFile: android.PathForBazelOut(ctx, f["unstripped_built_file"]), + androidMkModuleName: f["make_module_name"], + installDir: f["install_dir"], + class: classes[f["class"]], + customStem: f["basename"], + moduleDir: f["package"], + } + + arch := f["arch"] + fileInfo.arch = arch + if len(arch) > 0 { + fileInfo.multilib = "lib32" + if strings.HasSuffix(arch, "64") { + fileInfo.multilib = "lib64" + } + } + + a.filesInfo = append(a.filesInfo, fileInfo) } } diff --git a/apex/bp2build_test.go b/apex/bp2build_test.go index 2f2b61e64..2a0f6e9e2 100644 --- a/apex/bp2build_test.go +++ b/apex/bp2build_test.go @@ -25,6 +25,7 @@ func TestApexImageInMixedBuilds(t *testing.T) { apex_key{ name: "foo_key", } + apex { name: "foo", key: "foo_key", @@ -59,6 +60,16 @@ apex { ProvidesLibs: []string{"a", "b"}, // ApexMkInfo Starlark provider + PayloadFilesInfo: []map[string]string{ + { + "built_file": "bazel-out/adbd", + "install_dir": "bin", + "class": "nativeExecutable", + "make_module_name": "adbd", + "basename": "adbd", + "package": "foo", + }, + }, MakeModulesToInstall: []string{"c"}, // d deliberately omitted }, }, @@ -68,10 +79,12 @@ apex { m := result.ModuleForTests("foo", "android_common_foo_image").Module() ab, ok := m.(*apexBundle) + if !ok { t.Fatalf("Expected module to be an apexBundle, was not") } + // TODO: refactor to android.AssertStringEquals if w, g := "out/bazel/execroot/__main__/public_key", ab.publicKeyFile.String(); w != g { t.Errorf("Expected public key %q, got %q", w, g) } @@ -120,11 +133,136 @@ apex { if len(ab.makeModulesToInstall) != 1 && ab.makeModulesToInstall[0] != "c" { t.Errorf("Expected makeModulesToInstall slice to only contain 'c', got %q", ab.makeModulesToInstall) } - if w := "LOCAL_REQUIRED_MODULES := c"; !strings.Contains(data, w) { + if w := "LOCAL_REQUIRED_MODULES := adbd.foo c"; !strings.Contains(data, w) { t.Errorf("Expected %q in androidmk data, but did not find it in %q", w, data) } } +func TestApexImageCreatesFilesInfoForMake(t *testing.T) { + bp := ` +apex_key{ + name: "foo_key", +} + +apex { + name: "foo", + key: "foo_key", + updatable: true, + min_sdk_version: "31", + file_contexts: ":myapex-file_contexts", + bazel_module: { label: "//:foo" }, +}` + + outputBaseDir := "out/bazel" + result := android.GroupFixturePreparers( + prepareForApexTest, + android.FixtureModifyConfig(func(config android.Config) { + config.BazelContext = android.MockBazelContext{ + OutputBaseDir: outputBaseDir, + LabelToApexInfo: map[string]cquery.ApexInfo{ + "//:foo": { + // ApexInfo Starlark provider. Necessary for the test. + SignedOutput: "signed_out.apex", + BundleKeyInfo: []string{"public_key", "private_key"}, + ContainerKeyInfo: []string{"container_cert", "container_private"}, + + // ApexMkInfo Starlark provider + PayloadFilesInfo: []map[string]string{ + { + "arch": "arm64", + "basename": "libcrypto.so", + "built_file": "bazel-out/64/libcrypto.so", + "class": "nativeSharedLib", + "install_dir": "lib64", + "make_module_name": "libcrypto", + "package": "foo/bar", + "unstripped_built_file": "bazel-out/64/unstripped_libcrypto.so", + }, + { + "arch": "arm", + "basename": "libcrypto.so", + "built_file": "bazel-out/32/libcrypto.so", + "class": "nativeSharedLib", + "install_dir": "lib", + "make_module_name": "libcrypto", + "package": "foo/bar", + }, + { + "arch": "arm64", + "basename": "adbd", + "built_file": "bazel-out/adbd", + "class": "nativeExecutable", + "install_dir": "bin", + "make_module_name": "adbd", + "package": "foo", + }, + }, + }, + }, + } + }), + ).RunTestWithBp(t, bp) + + m := result.ModuleForTests("foo", "android_common_foo_image").Module() + ab, ok := m.(*apexBundle) + + if !ok { + t.Fatalf("Expected module to be an apexBundle, was not") + } + + expectedFilesInfo := []apexFile{ + { + androidMkModuleName: "libcrypto", + builtFile: android.PathForTesting("out/bazel/execroot/__main__/bazel-out/64/libcrypto.so"), + class: nativeSharedLib, + customStem: "libcrypto.so", + installDir: "lib64", + moduleDir: "foo/bar", + arch: "arm64", + unstrippedBuiltFile: android.PathForTesting("out/bazel/execroot/__main__/bazel-out/64/unstripped_libcrypto.so"), + }, + { + androidMkModuleName: "libcrypto", + builtFile: android.PathForTesting("out/bazel/execroot/__main__/bazel-out/32/libcrypto.so"), + class: nativeSharedLib, + customStem: "libcrypto.so", + installDir: "lib", + moduleDir: "foo/bar", + arch: "arm", + }, + { + androidMkModuleName: "adbd", + builtFile: android.PathForTesting("out/bazel/execroot/__main__/bazel-out/adbd"), + class: nativeExecutable, + customStem: "adbd", + installDir: "bin", + moduleDir: "foo", + arch: "arm64", + }, + } + + if len(ab.filesInfo) != len(expectedFilesInfo) { + t.Errorf("Expected %d entries in ab.filesInfo, but got %d", len(ab.filesInfo), len(expectedFilesInfo)) + } + + for idx, f := range ab.filesInfo { + expected := expectedFilesInfo[idx] + android.AssertSame(t, "different class", expected.class, f.class) + android.AssertStringEquals(t, "different built file", expected.builtFile.String(), f.builtFile.String()) + android.AssertStringEquals(t, "different custom stem", expected.customStem, f.customStem) + android.AssertStringEquals(t, "different install dir", expected.installDir, f.installDir) + android.AssertStringEquals(t, "different make module name", expected.androidMkModuleName, f.androidMkModuleName) + android.AssertStringEquals(t, "different moduleDir", expected.moduleDir, f.moduleDir) + android.AssertStringEquals(t, "different arch", expected.arch, f.arch) + if expected.unstrippedBuiltFile != nil { + if f.unstrippedBuiltFile == nil { + t.Errorf("expected an unstripped built file path.") + } + android.AssertStringEquals(t, "different unstripped built file", expected.unstrippedBuiltFile.String(), f.unstrippedBuiltFile.String()) + } + } +} + func TestCompressedApexImageInMixedBuilds(t *testing.T) { bp := ` apex_key{ diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go index 0c8247a34..cf649a47e 100644 --- a/bazel/cquery/request_type.go +++ b/bazel/cquery/request_type.go @@ -281,6 +281,7 @@ return json_encode({ "bundle_file": info.base_with_config_zip.path, "installed_files": info.installed_files.path, "make_modules_to_install": mk_info.make_modules_to_install, + "files_info": mk_info.files_info, "tidy_files": [t for t in tidy_files], })` } @@ -303,7 +304,8 @@ type ApexInfo struct { TidyFiles []string `json:"tidy_files"` // From the ApexMkInfo provider - MakeModulesToInstall []string `json:"make_modules_to_install"` + MakeModulesToInstall []string `json:"make_modules_to_install"` + PayloadFilesInfo []map[string]string `json:"files_info"` } // ParseResult returns a value obtained by parsing the result of the request's Starlark function.