Generate android_certificate_directory

Previously, partners were required to add an
android_certificate_directory filegroup in their certificate
directories, and allowlist that BUILD file. Now, we generate the
filegroup automatically.

We're using a different name, generated_android_certificate_directory,
to avoid conflicts with already-checked-in filegroups.

Bug: 285777389
Test: b test //build/bazel/rules/apex/...
Change-Id: Ib1bde487acd79d58368faf0aad02ded0bcdaceb4

bp2build/bp2build.go

@@ -82,15 +82,25 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics {
 		os.Exit(1)
 	}
 	var bp2buildFiles []BazelFile
+	productConfig, err := createProductConfigFiles(ctx, res.metrics)
 	ctx.Context().EventHandler.Do("CreateBazelFile", func() {
-		bp2buildFiles = CreateBazelFiles(nil, res.buildFileToTargets, ctx.mode)
+		allTargets := make(map[string]BazelTargets)
+		for k, v := range res.buildFileToTargets {
+			allTargets[k] = append(allTargets[k], v...)
+		}
+		for k, v := range productConfig.bp2buildTargets {
+			allTargets[k] = append(allTargets[k], v...)
+		}
+		bp2buildFiles = CreateBazelFiles(nil, allTargets, ctx.mode)
 	})
-	injectionFiles, additionalBp2buildFiles, err := CreateSoongInjectionDirFiles(ctx, res.metrics)
+	bp2buildFiles = append(bp2buildFiles, productConfig.bp2buildFiles...)
+	injectionFiles, err := createSoongInjectionDirFiles(ctx, res.metrics)
 	if err != nil {
 		fmt.Printf("%s\n", err.Error())
 		os.Exit(1)
 	}
-	bp2buildFiles = append(bp2buildFiles, additionalBp2buildFiles...)
+	injectionFiles = append(injectionFiles, productConfig.injectionFiles...)
+
 	writeFiles(ctx, bp2buildDir, bp2buildFiles)
 	// Delete files under the bp2build root which weren't just written. An
 	// alternative would have been to delete the whole directory and write these
@@ -109,26 +119,6 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics {
 	return &res.metrics
 }
 
-// Wrapper function that will be responsible for all files in soong_injection directory
-// This includes
-// 1. config value(s) that are hardcoded in Soong
-// 2. product_config variables
-func CreateSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) ([]BazelFile, []BazelFile, error) {
-	var ret []BazelFile
-
-	productConfigInjectionFiles, productConfigBp2BuildDirFiles, err := CreateProductConfigFiles(ctx, metrics)
-	if err != nil {
-		return nil, nil, err
-	}
-	ret = append(ret, productConfigInjectionFiles...)
-	injectionFiles, err := soongInjectionFiles(ctx.Config(), metrics)
-	if err != nil {
-		return nil, nil, err
-	}
-	ret = append(injectionFiles, ret...)
-	return ret, productConfigBp2BuildDirFiles, nil
-}
-
 // Get the output directory and create it if it doesn't exist.
 func getOrCreateOutputDir(outputDir android.OutputPath, ctx android.PathContext, dir string) android.OutputPath {
 	dirPath := outputDir.Join(ctx, dir)

bp2build/bp2build_product_config.go

@@ -12,14 +12,19 @@ import (
 	"android/soong/android/soongconfig"
 	"android/soong/starlark_import"
 
-	"github.com/google/blueprint"
 	"github.com/google/blueprint/proptools"
 	"go.starlark.net/starlark"
 )
 
-func CreateProductConfigFiles(
+type createProductConfigFilesResult struct {
+	injectionFiles  []BazelFile
+	bp2buildFiles   []BazelFile
+	bp2buildTargets map[string]BazelTargets
+}
+
+func createProductConfigFiles(
 	ctx *CodegenContext,
-	metrics CodegenMetrics) ([]BazelFile, []BazelFile, error) {
+	metrics CodegenMetrics) (createProductConfigFilesResult, error) {
 	cfg := &ctx.config
 	targetProduct := "unknown"
 	if cfg.HasDeviceProduct() {
@@ -32,29 +37,22 @@ func CreateProductConfigFiles(
 		targetBuildVariant = "userdebug"
 	}
 
+	var res createProductConfigFilesResult
+
 	productVariablesFileName := cfg.ProductVariablesFileName
 	if !strings.HasPrefix(productVariablesFileName, "/") {
 		productVariablesFileName = filepath.Join(ctx.topDir, productVariablesFileName)
 	}
 	productVariablesBytes, err := os.ReadFile(productVariablesFileName)
 	if err != nil {
-		return nil, nil, err
+		return res, err
 	}
 	productVariables := android.ProductVariables{}
 	err = json.Unmarshal(productVariablesBytes, &productVariables)
 	if err != nil {
-		return nil, nil, err
+		return res, err
 	}
 
-	// Visit all modules to determine the list of ndk libraries
-	// This list will be used to add additional flags for cc stub generation
-	ndkLibsStringFormatted := []string{}
-	ctx.Context().VisitAllModules(func(m blueprint.Module) {
-		if ctx.Context().ModuleType(m) == "ndk_library" {
-			ndkLibsStringFormatted = append(ndkLibsStringFormatted, fmt.Sprintf(`"%s"`, m.Name())) // name will be `"libc.ndk"`
-		}
-	})
-
 	// TODO(b/249685973): the name is product_config_platforms because product_config
 	// was already used for other files. Deduplicate them.
 	currentProductFolder := fmt.Sprintf("product_config_platforms/products/%s-%s", targetProduct, targetBuildVariant)
@@ -64,25 +62,36 @@ func CreateProductConfigFiles(
 		"{VARIANT}", targetBuildVariant,
 		"{PRODUCT_FOLDER}", currentProductFolder)
 
-	platformMappingContent, err := platformMappingContent(
-		productReplacer.Replace("@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}"),
-		&productVariables,
-		ctx.Config().Bp2buildSoongConfigDefinitions,
-		metrics.convertedModulePathMap)
-	if err != nil {
-		return nil, nil, err
-	}
-
 	productsForTestingMap, err := starlark_import.GetStarlarkValue[map[string]map[string]starlark.Value]("products_for_testing")
 	if err != nil {
-		return nil, nil, err
+		return res, err
 	}
 	productsForTesting := android.SortedKeys(productsForTestingMap)
 	for i := range productsForTesting {
 		productsForTesting[i] = fmt.Sprintf("  \"@//build/bazel/tests/products:%s\",", productsForTesting[i])
 	}
 
-	injectionDirFiles := []BazelFile{
+	productLabelsToVariables := make(map[string]*android.ProductVariables)
+	productLabelsToVariables[productReplacer.Replace("@soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}")] = &productVariables
+	for product, productVariablesStarlark := range productsForTestingMap {
+		productVariables, err := starlarkMapToProductVariables(productVariablesStarlark)
+		if err != nil {
+			return res, err
+		}
+		productLabelsToVariables["@//build/bazel/tests/products:"+product] = &productVariables
+	}
+
+	res.bp2buildTargets = createTargets(productLabelsToVariables)
+
+	platformMappingContent, err := platformMappingContent(
+		productLabelsToVariables,
+		ctx.Config().Bp2buildSoongConfigDefinitions,
+		metrics.convertedModulePathMap)
+	if err != nil {
+		return res, err
+	}
+
+	res.injectionFiles = []BazelFile{
 		newFile(
 			currentProductFolder,
 			"soong.variables.bzl",
@@ -164,30 +173,21 @@ build --host_platform @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_lin
 			productReplacer.Replace(`
 build --host_platform @soong_injection//{PRODUCT_FOLDER}:{PRODUCT}-{VARIANT}_darwin_x86_64
 `)),
-		newFile(
-			"cc_toolchain",
-			"ndk_libs.bzl",
-			fmt.Sprintf("ndk_libs = [%v]", strings.Join(ndkLibsStringFormatted, ", ")),
-		),
 	}
-	bp2buildDirFiles := []BazelFile{
+	res.bp2buildFiles = []BazelFile{
 		newFile(
 			"",
 			"platform_mappings",
 			platformMappingContent),
 	}
-	return injectionDirFiles, bp2buildDirFiles, nil
+
+	return res, nil
 }
 
 func platformMappingContent(
-	mainProductLabel string,
+	productLabelToVariables map[string]*android.ProductVariables,
-	mainProductVariables *android.ProductVariables,
 	soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions,
 	convertedModulePathMap map[string]string) (string, error) {
-	productsForTesting, err := starlark_import.GetStarlarkValue[map[string]map[string]starlark.Value]("products_for_testing")
-	if err != nil {
-		return "", err
-	}
 	var result strings.Builder
 
 	mergedConvertedModulePathMap := make(map[string]string)
@@ -203,13 +203,8 @@ func platformMappingContent(
 	}
 
 	result.WriteString("platforms:\n")
-	platformMappingSingleProduct(mainProductLabel, mainProductVariables, soongConfigDefinitions, mergedConvertedModulePathMap, &result)
+	for productLabel, productVariables := range productLabelToVariables {
-	for product, productVariablesStarlark := range productsForTesting {
+		platformMappingSingleProduct(productLabel, productVariables, soongConfigDefinitions, mergedConvertedModulePathMap, &result)
-		productVariables, err := starlarkMapToProductVariables(productVariablesStarlark)
-		if err != nil {
-			return "", err
-		}
-		platformMappingSingleProduct("@//build/bazel/tests/products:"+product, &productVariables, soongConfigDefinitions, mergedConvertedModulePathMap, &result)
 	}
 	return result.String(), nil
 }
@@ -248,7 +243,7 @@ func platformMappingSingleProduct(
 
 	defaultAppCertificateFilegroup := "//build/bazel/utils:empty_filegroup"
 	if proptools.String(productVariables.DefaultAppCertificate) != "" {
-		defaultAppCertificateFilegroup = "@//" + filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) + ":android_certificate_directory"
+		defaultAppCertificateFilegroup = "@//" + filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) + ":generated_android_certificate_directory"
 	}
 
 	for _, suffix := range bazelPlatformSuffixes {
@@ -419,3 +414,33 @@ func starlarkMapToProductVariables(in map[string]starlark.Value) (android.Produc
 
 	return result, nil
 }
+
+func createTargets(productLabelsToVariables map[string]*android.ProductVariables) map[string]BazelTargets {
+	res := make(map[string]BazelTargets)
+	var allDefaultAppCertificateDirs []string
+	for _, productVariables := range productLabelsToVariables {
+		if proptools.String(productVariables.DefaultAppCertificate) != "" {
+			d := filepath.Dir(proptools.String(productVariables.DefaultAppCertificate))
+			if !android.InList(d, allDefaultAppCertificateDirs) {
+				allDefaultAppCertificateDirs = append(allDefaultAppCertificateDirs, d)
+			}
+		}
+	}
+	for _, dir := range allDefaultAppCertificateDirs {
+		content := fmt.Sprintf(ruleTargetTemplate, "filegroup", "generated_android_certificate_directory", propsToAttributes(map[string]string{
+			"srcs": `glob([
+        "*.pk8",
+        "*.pem",
+        "*.avbpubkey",
+    ])`,
+			"visibility": `["//visibility:public"]`,
+		}))
+		res[dir] = append(res[dir], BazelTarget{
+			name:        "generated_android_certificate_directory",
+			packageName: dir,
+			content:     content,
+			ruleClass:   "filegroup",
+		})
+	}
+	return res
+}

bp2build/conversion.go

@@ -15,6 +15,7 @@ import (
 	rust_config "android/soong/rust/config"
 	"android/soong/starlark_fmt"
 
+	"github.com/google/blueprint"
 	"github.com/google/blueprint/proptools"
 )
 
@@ -24,16 +25,28 @@ type BazelFile struct {
 	Contents string
 }
 
-// PRIVATE: Use CreateSoongInjectionDirFiles instead
+// createSoongInjectionDirFiles returns most of the files to write to the soong_injection directory.
-func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) ([]BazelFile, error) {
+// Some other files also come from CreateProductConfigFiles
+func createSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) ([]BazelFile, error) {
+	cfg := ctx.Config()
 	var files []BazelFile
 
 	files = append(files, newFile("android", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package.
 	files = append(files, newFile("android", "constants.bzl", android.BazelCcToolchainVars(cfg)))
 
+	// Visit all modules to determine the list of ndk libraries
+	// This list will be used to add additional flags for cc stub generation
+	ndkLibsStringFormatted := []string{}
+	ctx.Context().VisitAllModules(func(m blueprint.Module) {
+		if ctx.Context().ModuleType(m) == "ndk_library" {
+			ndkLibsStringFormatted = append(ndkLibsStringFormatted, fmt.Sprintf(`"%s"`, m.Name())) // name will be `"libc.ndk"`
+		}
+	})
+
 	files = append(files, newFile("cc_toolchain", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package.
 	files = append(files, newFile("cc_toolchain", "config_constants.bzl", cc_config.BazelCcToolchainVars(cfg)))
 	files = append(files, newFile("cc_toolchain", "sanitizer_constants.bzl", cc.BazelCcSanitizerToolchainVars(cfg)))
+	files = append(files, newFile("cc_toolchain", "ndk_libs.bzl", fmt.Sprintf("ndk_libs = [%v]", strings.Join(ndkLibsStringFormatted, ", "))))
 
 	files = append(files, newFile("java_toolchain", GeneratedBuildFileName, "")) // Creates a //java_toolchain package.
 	files = append(files, newFile("java_toolchain", "constants.bzl", java_config.BazelJavaToolchainVars(cfg)))

bp2build/conversion_test.go

@@ -83,7 +83,8 @@ func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) {
 
 func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) {
 	testConfig := android.TestConfig("", make(map[string]string), "", make(map[string][]byte))
-	files, err := soongInjectionFiles(testConfig, CreateCodegenMetrics())
+	codegenCtx := NewCodegenContext(testConfig, android.NewTestContext(testConfig).Context, Bp2Build, "")
+	files, err := createSoongInjectionDirFiles(codegenCtx, CreateCodegenMetrics())
 	if err != nil {
 		t.Error(err)
 	}
@@ -104,6 +105,10 @@ func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) {
 			dir:      "cc_toolchain",
 			basename: "config_constants.bzl",
 		},
+		{
+			dir:      "cc_toolchain",
+			basename: "ndk_libs.bzl",
+		},
 		{
 			dir:      "cc_toolchain",
 			basename: "sanitizer_constants.bzl",
@@ -182,15 +187,45 @@ func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) {
 		},
 	}
 
-	if len(files) != len(expectedFilePaths) {
+	less := func(a bazelFilepath, b bazelFilepath) bool {
-		t.Errorf("Expected %d file, got %d", len(expectedFilePaths), len(files))
+		return a.dir+"/"+a.basename < b.dir+"/"+b.basename
 	}
 
-	for i := range files {
+	fileToFilepath := func(a BazelFile) bazelFilepath {
-		actualFile, expectedFile := files[i], expectedFilePaths[i]
+		return bazelFilepath{basename: a.Basename, dir: a.Dir}
+	}
 
-		if actualFile.Dir != expectedFile.dir || actualFile.Basename != expectedFile.basename {
+	sort.Slice(expectedFilePaths, func(i, j int) bool {
-			t.Errorf("Did not find expected file %s/%s", actualFile.Dir, actualFile.Basename)
+		return less(expectedFilePaths[i], expectedFilePaths[j])
+	})
+	sort.Slice(files, func(i, j int) bool {
+		return less(fileToFilepath(files[i]), fileToFilepath(files[j]))
+	})
+
+	i := 0
+	j := 0
+	for i < len(expectedFilePaths) && j < len(files) {
+		expectedFile, actualFile := expectedFilePaths[i], files[j]
+
+		if actualFile.Dir == expectedFile.dir && actualFile.Basename == expectedFile.basename {
+			i++
+			j++
+		} else if less(expectedFile, fileToFilepath(actualFile)) {
+			t.Errorf("Did not find expected file %s/%s", expectedFile.dir, expectedFile.basename)
+			i++
+		} else {
+			t.Errorf("Found unexpected file %s/%s", actualFile.Dir, actualFile.Basename)
+			j++
 		}
 	}
+	for i < len(expectedFilePaths) {
+		expectedFile := expectedFilePaths[i]
+		t.Errorf("Did not find expected file %s/%s", expectedFile.dir, expectedFile.basename)
+		i++
+	}
+	for j < len(files) {
+		actualFile := files[j]
+		t.Errorf("Found unexpected file %s/%s", actualFile.Dir, actualFile.Basename)
+		j++
+	}
 }