Populate apexBundle#filesInfo using bazel info.

This CL adds a few things:

1) Populate the filesInfo struct with cquery'd information from an
apex's ApexMkInfo provider. This filesInfo is then used in
apex/androidmk.go to generate Make modules (soong_cc_rust_prebuilt.mk),
which are then used in packaging to generate zip files of symbols in $PRODUCT_OUT.
2) Make a list of dicts of primitives JSON-encodable.
3) Tests.

Bug: 271423316
Bug: 271423062
Test: presubmits
Change-Id: Iaa34f51044de310510e580d9cf1fe60bbef801c1

android/bazel_handler.go

@@ -1022,7 +1022,15 @@ def json_encode(input):
     fail("unsupported value '%s' of type '%s'" % (p, type(p)))
 
   def encode_list(list):
-    return "[%s]" % ", ".join([encode_primitive(item) for item in list])
+    items = []
+    for item in list:
+        if type(item) == "dict":
+            # support encoding dict of primitive keys and values. not list currently, because calling encode_list again is recursive.
+            kv_pairs = [("%s: %s" % (encode_primitive(k), encode_primitive(v))) for (k, v) in item.items()]
+            items.append("{ %s }" % ", ".join(kv_pairs))
+        else:
+            items.append(encode_primitive(item))
+    return "[%s]" % ", ".join(items)
 
   def encode_list_or_primitive(v):
     return encode_list(v) if type(v) == "list" else encode_primitive(v)

apex/androidmk.go

@@ -63,15 +63,17 @@ func (class apexFileClass) nameInMake() string {
 }
 
 // Return the full module name for a dependency module, which appends the apex module name unless re-using a system lib.
-func (a *apexBundle) fullModuleName(apexBundleName string, fi *apexFile) string {
+func (a *apexBundle) fullModuleName(apexBundleName string, linkToSystemLib bool, fi *apexFile) string {
-	linkToSystemLib := a.linkToSystemLib && fi.transitiveDep && fi.availableToPlatform()
-
 	if linkToSystemLib {
 		return fi.androidMkModuleName
 	}
 	return fi.androidMkModuleName + "." + apexBundleName + a.suffix
 }
 
+// androidMkForFiles generates Make definitions for the contents of an
+// apexBundle (apexBundle#filesInfo).  The filesInfo structure can either be
+// populated by Soong for unconverted APEXes, or Bazel in mixed mode. Use
+// apexFile#isBazelPrebuilt to differentiate.
 func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir string,
 	apexAndroidMkData android.AndroidMkData) []string {
 
@@ -95,8 +97,7 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st
 
 	for _, fi := range a.filesInfo {
 		linkToSystemLib := a.linkToSystemLib && fi.transitiveDep && fi.availableToPlatform()
-
+		moduleName := a.fullModuleName(apexBundleName, linkToSystemLib, &fi)
-		moduleName := a.fullModuleName(apexBundleName, &fi)
 
 		// This name will be added to LOCAL_REQUIRED_MODULES of the APEX. We need to be
 		// arch-specific otherwise we will end up installing both ABIs even when only
@@ -124,6 +125,7 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st
 			fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir)
 		}
 		fmt.Fprintln(w, "LOCAL_MODULE :=", moduleName)
+
 		if fi.module != nil && fi.module.Owner() != "" {
 			fmt.Fprintln(w, "LOCAL_MODULE_OWNER :=", fi.module.Owner())
 		}
@@ -161,6 +163,7 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st
 		fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", fi.builtFile.String())
 		fmt.Fprintln(w, "LOCAL_MODULE_CLASS :=", fi.class.nameInMake())
 		if fi.module != nil {
+			// This apexFile's module comes from Soong
 			archStr := fi.module.Target().Arch.ArchType.String()
 			host := false
 			switch fi.module.Target().Os.Class {
@@ -188,6 +191,9 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st
 				fmt.Fprintln(w, "LOCAL_MODULE_HOST_OS :=", makeOs)
 				fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true")
 			}
+		} else if fi.isBazelPrebuilt && fi.arch != "" {
+			// This apexFile comes from Bazel
+			fmt.Fprintln(w, "LOCAL_MODULE_TARGET_ARCH :=", fi.arch)
 		}
 		if fi.jacocoReportClassesFile != nil {
 			fmt.Fprintln(w, "LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR :=", fi.jacocoReportClassesFile.String())
@@ -231,17 +237,21 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir st
 			fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_android_app_set.mk")
 		case nativeSharedLib, nativeExecutable, nativeTest:
 			fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", fi.stem())
-			if ccMod, ok := fi.module.(*cc.Module); ok {
+			if fi.isBazelPrebuilt {
-				if ccMod.UnstrippedOutputFile() != nil {
+				fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", fi.unstrippedBuiltFile)
-					fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", ccMod.UnstrippedOutputFile().String())
+			} else {
-				}
+				if ccMod, ok := fi.module.(*cc.Module); ok {
-				ccMod.AndroidMkWriteAdditionalDependenciesForSourceAbiDiff(w)
+					if ccMod.UnstrippedOutputFile() != nil {
-				if ccMod.CoverageOutputFile().Valid() {
+						fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", ccMod.UnstrippedOutputFile().String())
-					fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", ccMod.CoverageOutputFile().String())
+					}
-				}
+					ccMod.AndroidMkWriteAdditionalDependenciesForSourceAbiDiff(w)
-			} else if rustMod, ok := fi.module.(*rust.Module); ok {
+					if ccMod.CoverageOutputFile().Valid() {
-				if rustMod.UnstrippedOutputFile() != nil {
+						fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", ccMod.CoverageOutputFile().String())
-					fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", rustMod.UnstrippedOutputFile().String())
+					}
+				} else if rustMod, ok := fi.module.(*rust.Module); ok {
+					if rustMod.UnstrippedOutputFile() != nil {
+						fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", rustMod.UnstrippedOutputFile().String())
+					}
 				}
 			}
 			fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_cc_rust_prebuilt.mk")

apex/apex.go

@@ -509,6 +509,21 @@ const (
 	shBinary
 )
 
+var (
+	classes = map[string]apexFileClass{
+		"app":              app,
+		"appSet":           appSet,
+		"etc":              etc,
+		"goBinary":         goBinary,
+		"javaSharedLib":    javaSharedLib,
+		"nativeExecutable": nativeExecutable,
+		"nativeSharedLib":  nativeSharedLib,
+		"nativeTest":       nativeTest,
+		"pyBinary":         pyBinary,
+		"shBinary":         shBinary,
+	}
+)
+
 // apexFile represents a file in an APEX bundle. This is created during the first half of
 // GenerateAndroidBuildActions by traversing the dependencies of the APEX. Then in the second half
 // of the function, this is used to create commands that copies the files into a staging directory,
@@ -543,6 +558,10 @@ type apexFile struct {
 
 	multilib string
 
+	isBazelPrebuilt     bool
+	unstrippedBuiltFile android.Path
+	arch                string
+
 	// TODO(jiyong): remove this
 	module android.Module
 }
@@ -1710,6 +1729,7 @@ func apexFileForGoBinary(ctx android.BaseModuleContext, depName string, gb boots
 	// NB: Since go binaries are static we don't need the module for anything here, which is
 	// good since the go tool is a blueprint.Module not an android.Module like we would
 	// normally use.
+	//
 	return newApexFile(ctx, fileToCopy, depName, dirInApex, goBinary, nil)
 }
 
@@ -2003,13 +2023,41 @@ func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) {
 		panic(fmt.Errorf("internal error: unexpected apex_type for the ProcessBazelQueryResponse: %v", a.properties.ApexType))
 	}
 
-	// filesInfo is not set in mixed mode, because all information about the
+	// filesInfo in mixed mode must retrieve all information about the apex's
-	// apex's contents should completely come from the Starlark providers.
+	// contents completely from the Starlark providers. It should never rely on
+	// Android.bp information, as they might not exist for fully migrated
+	// dependencies.
 	//
 	// Prevent accidental writes to filesInfo in the earlier parts Soong by
 	// asserting it to be nil.
 	if a.filesInfo != nil {
-		panic(fmt.Errorf("internal error: filesInfo must be nil for an apex handled by Bazel."))
+		panic(
+			fmt.Errorf("internal error: filesInfo must be nil for an apex handled by Bazel. " +
+				"Did something else set filesInfo before this line of code?"))
+	}
+	for _, f := range outputs.PayloadFilesInfo {
+		fileInfo := apexFile{
+			isBazelPrebuilt: true,
+
+			builtFile:           android.PathForBazelOut(ctx, f["built_file"]),
+			unstrippedBuiltFile: android.PathForBazelOut(ctx, f["unstripped_built_file"]),
+			androidMkModuleName: f["make_module_name"],
+			installDir:          f["install_dir"],
+			class:               classes[f["class"]],
+			customStem:          f["basename"],
+			moduleDir:           f["package"],
+		}
+
+		arch := f["arch"]
+		fileInfo.arch = arch
+		if len(arch) > 0 {
+			fileInfo.multilib = "lib32"
+			if strings.HasSuffix(arch, "64") {
+				fileInfo.multilib = "lib64"
+			}
+		}
+
+		a.filesInfo = append(a.filesInfo, fileInfo)
 	}
 }
 

apex/bp2build_test.go

@@ -25,6 +25,7 @@ func TestApexImageInMixedBuilds(t *testing.T) {
 apex_key{
 	name: "foo_key",
 }
+
 apex {
 	name: "foo",
 	key: "foo_key",
@@ -59,6 +60,16 @@ apex {
 						ProvidesLibs: []string{"a", "b"},
 
 						// ApexMkInfo Starlark provider
+						PayloadFilesInfo: []map[string]string{
+							{
+								"built_file":       "bazel-out/adbd",
+								"install_dir":      "bin",
+								"class":            "nativeExecutable",
+								"make_module_name": "adbd",
+								"basename":         "adbd",
+								"package":          "foo",
+							},
+						},
 						MakeModulesToInstall: []string{"c"}, // d deliberately omitted
 					},
 				},
@@ -68,10 +79,12 @@ apex {
 
 	m := result.ModuleForTests("foo", "android_common_foo_image").Module()
 	ab, ok := m.(*apexBundle)
+
 	if !ok {
 		t.Fatalf("Expected module to be an apexBundle, was not")
 	}
 
+	// TODO: refactor to android.AssertStringEquals
 	if w, g := "out/bazel/execroot/__main__/public_key", ab.publicKeyFile.String(); w != g {
 		t.Errorf("Expected public key %q, got %q", w, g)
 	}
@@ -120,11 +133,136 @@ apex {
 	if len(ab.makeModulesToInstall) != 1 && ab.makeModulesToInstall[0] != "c" {
 		t.Errorf("Expected makeModulesToInstall slice to only contain 'c', got %q", ab.makeModulesToInstall)
 	}
-	if w := "LOCAL_REQUIRED_MODULES := c"; !strings.Contains(data, w) {
+	if w := "LOCAL_REQUIRED_MODULES := adbd.foo c"; !strings.Contains(data, w) {
 		t.Errorf("Expected %q in androidmk data, but did not find it in %q", w, data)
 	}
 }
 
+func TestApexImageCreatesFilesInfoForMake(t *testing.T) {
+	bp := `
+apex_key{
+	name: "foo_key",
+}
+
+apex {
+	name: "foo",
+	key: "foo_key",
+	updatable: true,
+	min_sdk_version: "31",
+	file_contexts: ":myapex-file_contexts",
+	bazel_module: { label: "//:foo" },
+}`
+
+	outputBaseDir := "out/bazel"
+	result := android.GroupFixturePreparers(
+		prepareForApexTest,
+		android.FixtureModifyConfig(func(config android.Config) {
+			config.BazelContext = android.MockBazelContext{
+				OutputBaseDir: outputBaseDir,
+				LabelToApexInfo: map[string]cquery.ApexInfo{
+					"//:foo": {
+						// ApexInfo Starlark provider. Necessary for the test.
+						SignedOutput:     "signed_out.apex",
+						BundleKeyInfo:    []string{"public_key", "private_key"},
+						ContainerKeyInfo: []string{"container_cert", "container_private"},
+
+						// ApexMkInfo Starlark provider
+						PayloadFilesInfo: []map[string]string{
+							{
+								"arch":                  "arm64",
+								"basename":              "libcrypto.so",
+								"built_file":            "bazel-out/64/libcrypto.so",
+								"class":                 "nativeSharedLib",
+								"install_dir":           "lib64",
+								"make_module_name":      "libcrypto",
+								"package":               "foo/bar",
+								"unstripped_built_file": "bazel-out/64/unstripped_libcrypto.so",
+							},
+							{
+								"arch":             "arm",
+								"basename":         "libcrypto.so",
+								"built_file":       "bazel-out/32/libcrypto.so",
+								"class":            "nativeSharedLib",
+								"install_dir":      "lib",
+								"make_module_name": "libcrypto",
+								"package":          "foo/bar",
+							},
+							{
+								"arch":             "arm64",
+								"basename":         "adbd",
+								"built_file":       "bazel-out/adbd",
+								"class":            "nativeExecutable",
+								"install_dir":      "bin",
+								"make_module_name": "adbd",
+								"package":          "foo",
+							},
+						},
+					},
+				},
+			}
+		}),
+	).RunTestWithBp(t, bp)
+
+	m := result.ModuleForTests("foo", "android_common_foo_image").Module()
+	ab, ok := m.(*apexBundle)
+
+	if !ok {
+		t.Fatalf("Expected module to be an apexBundle, was not")
+	}
+
+	expectedFilesInfo := []apexFile{
+		{
+			androidMkModuleName: "libcrypto",
+			builtFile:           android.PathForTesting("out/bazel/execroot/__main__/bazel-out/64/libcrypto.so"),
+			class:               nativeSharedLib,
+			customStem:          "libcrypto.so",
+			installDir:          "lib64",
+			moduleDir:           "foo/bar",
+			arch:                "arm64",
+			unstrippedBuiltFile: android.PathForTesting("out/bazel/execroot/__main__/bazel-out/64/unstripped_libcrypto.so"),
+		},
+		{
+			androidMkModuleName: "libcrypto",
+			builtFile:           android.PathForTesting("out/bazel/execroot/__main__/bazel-out/32/libcrypto.so"),
+			class:               nativeSharedLib,
+			customStem:          "libcrypto.so",
+			installDir:          "lib",
+			moduleDir:           "foo/bar",
+			arch:                "arm",
+		},
+		{
+			androidMkModuleName: "adbd",
+			builtFile:           android.PathForTesting("out/bazel/execroot/__main__/bazel-out/adbd"),
+			class:               nativeExecutable,
+			customStem:          "adbd",
+			installDir:          "bin",
+			moduleDir:           "foo",
+			arch:                "arm64",
+		},
+	}
+
+	if len(ab.filesInfo) != len(expectedFilesInfo) {
+		t.Errorf("Expected %d entries in ab.filesInfo, but got %d", len(ab.filesInfo), len(expectedFilesInfo))
+	}
+
+	for idx, f := range ab.filesInfo {
+		expected := expectedFilesInfo[idx]
+		android.AssertSame(t, "different class", expected.class, f.class)
+		android.AssertStringEquals(t, "different built file", expected.builtFile.String(), f.builtFile.String())
+		android.AssertStringEquals(t, "different custom stem", expected.customStem, f.customStem)
+		android.AssertStringEquals(t, "different install dir", expected.installDir, f.installDir)
+		android.AssertStringEquals(t, "different make module name", expected.androidMkModuleName, f.androidMkModuleName)
+		android.AssertStringEquals(t, "different moduleDir", expected.moduleDir, f.moduleDir)
+		android.AssertStringEquals(t, "different arch", expected.arch, f.arch)
+		if expected.unstrippedBuiltFile != nil {
+			if f.unstrippedBuiltFile == nil {
+				t.Errorf("expected an unstripped built file path.")
+			}
+			android.AssertStringEquals(t, "different unstripped built file", expected.unstrippedBuiltFile.String(), f.unstrippedBuiltFile.String())
+		}
+	}
+}
+
 func TestCompressedApexImageInMixedBuilds(t *testing.T) {
 	bp := `
 apex_key{

bazel/cquery/request_type.go

@@ -281,6 +281,7 @@ return json_encode({
     "bundle_file": info.base_with_config_zip.path,
     "installed_files": info.installed_files.path,
     "make_modules_to_install": mk_info.make_modules_to_install,
+    "files_info": mk_info.files_info,
     "tidy_files": [t for t in tidy_files],
 })`
 }
@@ -303,7 +304,8 @@ type ApexInfo struct {
 	TidyFiles              []string `json:"tidy_files"`
 
 	// From the ApexMkInfo provider
-	MakeModulesToInstall []string `json:"make_modules_to_install"`
+	MakeModulesToInstall []string            `json:"make_modules_to_install"`
+	PayloadFilesInfo     []map[string]string `json:"files_info"`
 }
 
 // ParseResult returns a value obtained by parsing the result of the request's Starlark function.