StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove infrastructure to run bp2build

Browse Source 
Bug: 315353489
Test: m blueprint_tests
Change-Id: Idcf6377d389b94c39e4e6ff4b8efa8a9f9e78b17
This commit is contained in:
Colin Cross
2023-12-07 16:54:51 -08:00
parent 8ff105860d
commit b63d7b3af7
32 changed files with 62 additions and 9310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bp2build/Android.bp
View File

@@ -7,16 +7,11 @@ bootstrap_go_package {
pkgPath: "android/soong/bp2build",
srcs: [
"androidbp_to_build_templates.go",
"bp2build.go",
"bp2build_product_config.go",
"build_conversion.go",
"bzl_conversion.go",
"configurability.go",
"constants.go",
"conversion.go",
"metrics.go",
"symlink_forest.go",
"testing.go",
],
deps: [
"blueprint-bootstrap",
@@ -41,7 +36,6 @@ bootstrap_go_package {
],
testSrcs: [
"conversion_test.go",
"performance_test.go",
],
pluginFor: [
"soong_build",

145
bp2build/bp2build.go
View File

@@ -1,145 +0,0 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"fmt"
"os"
"path/filepath"
"strings"
"android/soong/android"
"android/soong/bazel"
"android/soong/shared"
"android/soong/starlark_import"
)
func deleteFilesExcept(ctx *CodegenContext, rootOutputPath android.OutputPath, except []BazelFile) {
// Delete files that should no longer be present.
bp2buildDirAbs := shared.JoinPath(ctx.topDir, rootOutputPath.String())
filesToDelete := make(map[string]struct{})
err := filepath.Walk(bp2buildDirAbs,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
relPath, err := filepath.Rel(bp2buildDirAbs, path)
if err != nil {
return err
}
filesToDelete[relPath] = struct{}{}
}
return nil
})
if err != nil {
fmt.Printf("ERROR reading %s: %s", bp2buildDirAbs, err)
os.Exit(1)
}
for _, bazelFile := range except {
filePath := filepath.Join(bazelFile.Dir, bazelFile.Basename)
delete(filesToDelete, filePath)
}
for f, _ := range filesToDelete {
absPath := shared.JoinPath(bp2buildDirAbs, f)
if err := os.RemoveAll(absPath); err != nil {
fmt.Printf("ERROR deleting %s: %s", absPath, err)
os.Exit(1)
}
}
}
// Codegen is the backend of bp2build. The code generator is responsible for
// writing .bzl files that are equivalent to Android.bp files that are capable
// of being built with Bazel.
func Codegen(ctx *CodegenContext) *CodegenMetrics {
ctx.Context().BeginEvent("Codegen")
defer ctx.Context().EndEvent("Codegen")
// This directory stores BUILD files that could be eventually checked-in.
bp2buildDir := android.PathForOutput(ctx, "bp2build")
res, errs := GenerateBazelTargets(ctx, true)
if len(errs) > 0 {
errMsgs := make([]string, len(errs))
for i, err := range errs {
errMsgs[i] = fmt.Sprintf("%q", err)
}
fmt.Printf("ERROR: Encountered %d error(s): \nERROR: %s", len(errs), strings.Join(errMsgs, "\n"))
os.Exit(1)
}
var bp2buildFiles []BazelFile
productConfig, err := createProductConfigFiles(ctx, res.moduleNameToPartition, res.metrics.convertedModulePathMap)
ctx.Context().EventHandler.Do("CreateBazelFile", func() {
allTargets := make(map[string]BazelTargets)
for k, v := range res.buildFileToTargets {
allTargets[k] = append(allTargets[k], v...)
}
for k, v := range productConfig.bp2buildTargets {
allTargets[k] = append(allTargets[k], v...)
}
bp2buildFiles = CreateBazelFiles(nil, allTargets, ctx.mode)
})
bp2buildFiles = append(bp2buildFiles, productConfig.bp2buildFiles...)
injectionFiles, err := createSoongInjectionDirFiles(ctx, res.metrics)
if err != nil {
fmt.Printf("%s\n", err.Error())
os.Exit(1)
}
injectionFiles = append(injectionFiles, productConfig.injectionFiles...)
writeFiles(ctx, bp2buildDir, bp2buildFiles)
// Delete files under the bp2build root which weren't just written. An
// alternative would have been to delete the whole directory and write these
// files. However, this would regenerate files which were otherwise unchanged
// since the last bp2build run, which would have negative incremental
// performance implications.
deleteFilesExcept(ctx, bp2buildDir, bp2buildFiles)
writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles)
starlarkDeps, err := starlark_import.GetNinjaDeps()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(1)
}
ctx.AddNinjaFileDeps(starlarkDeps...)
return &res.metrics
}
// Get the output directory and create it if it doesn't exist.
func getOrCreateOutputDir(outputDir android.OutputPath, ctx android.PathContext, dir string) android.OutputPath {
dirPath := outputDir.Join(ctx, dir)
if err := android.CreateOutputDirIfNonexistent(dirPath, os.ModePerm); err != nil {
fmt.Printf("ERROR: path %s: %s", dirPath, err.Error())
}
return dirPath
}
// writeFiles materializes a list of BazelFile rooted at outputDir.
func writeFiles(ctx android.PathContext, outputDir android.OutputPath, files []BazelFile) {
for _, f := range files {
p := getOrCreateOutputDir(outputDir, ctx, f.Dir).Join(ctx, f.Basename)
if err := writeFile(p, f.Contents); err != nil {
panic(fmt.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err))
}
}
}
func writeFile(pathToFile android.OutputPath, content string) error {
// These files are made editable to allow users to modify and iterate on them
// in the source tree.
return android.WriteFileToOutputDir(pathToFile, []byte(content), 0644)
}

885
bp2build/bp2build_product_config.go
View File

@@ -1,885 +0,0 @@
package bp2build
import (
"encoding/json"
"fmt"
"path/filepath"
"reflect"
"sort"
"strings"
"android/soong/android"
"android/soong/android/soongconfig"
"android/soong/starlark_import"
"github.com/google/blueprint/proptools"
"go.starlark.net/starlark"
)
type createProductConfigFilesResult struct {
injectionFiles []BazelFile
bp2buildFiles []BazelFile
bp2buildTargets map[string]BazelTargets
}
type bazelLabel struct {
repo string
pkg string
target string
}
const releaseAconfigValueSetsName = "release_aconfig_value_sets"
func (l *bazelLabel) Less(other *bazelLabel) bool {
if l.repo < other.repo {
return true
}
if l.repo > other.repo {
return false
}
if l.pkg < other.pkg {
return true
}
if l.pkg > other.pkg {
return false
}
return l.target < other.target
}
func (l *bazelLabel) String() string {
return fmt.Sprintf("@%s//%s:%s", l.repo, l.pkg, l.target)
}
func createProductConfigFiles(
ctx *CodegenContext,
moduleNameToPartition map[string]string,
convertedModulePathMap map[string]string) (createProductConfigFilesResult, error) {
cfg := &ctx.config
targetProduct := "unknown"
if cfg.HasDeviceProduct() {
targetProduct = cfg.DeviceProduct()
}
targetBuildVariant := "user"
if cfg.Eng() {
targetBuildVariant = "eng"
} else if cfg.Debuggable() {
targetBuildVariant = "userdebug"
}
var res createProductConfigFilesResult
productVariables := ctx.Config().ProductVariables()
// TODO(b/306243251): For some reason, using the real value of native_coverage makes some select
// statements ambiguous
productVariables.Native_coverage = nil
productVariablesBytes, err := json.Marshal(productVariables)
if err != nil {
return res, err
}
currentProductFolder := fmt.Sprintf("build/bazel/products/%s", targetProduct)
if len(productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory) > 0 {
currentProductFolder = fmt.Sprintf("%s%s", productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory, targetProduct)
}
productReplacer := strings.NewReplacer(
"{PRODUCT}", targetProduct,
"{VARIANT}", targetBuildVariant,
"{PRODUCT_FOLDER}", currentProductFolder)
productsForTestingMap, err := starlark_import.GetStarlarkValue[map[string]map[string]starlark.Value]("products_for_testing")
if err != nil {
return res, err
}
productsForTesting := android.SortedKeys(productsForTestingMap)
for i := range productsForTesting {
productsForTesting[i] = fmt.Sprintf(" \"@//build/bazel/tests/products:%s\",", productsForTesting[i])
}
productLabelsToVariables := make(map[bazelLabel]*android.ProductVariables)
productLabelsToVariables[bazelLabel{
repo: "",
pkg: currentProductFolder,
target: targetProduct,
}] = &productVariables
for product, productVariablesStarlark := range productsForTestingMap {
productVariables, err := starlarkMapToProductVariables(productVariablesStarlark)
if err != nil {
return res, err
}
productLabelsToVariables[bazelLabel{
repo: "",
pkg: "build/bazel/tests/products",
target: product,
}] = &productVariables
}
res.bp2buildTargets = make(map[string]BazelTargets)
res.bp2buildTargets[currentProductFolder] = append(res.bp2buildTargets[currentProductFolder], BazelTarget{
name: productReplacer.Replace("{PRODUCT}"),
packageName: currentProductFolder,
content: productReplacer.Replace(`android_product(
name = "{PRODUCT}",
soong_variables = _soong_variables,
)`),
ruleClass: "android_product",
loads: []BazelLoad{
{
file: ":soong.variables.bzl",
symbols: []BazelLoadSymbol{{
symbol: "variables",
alias: "_soong_variables",
}},
},
{
file: "//build/bazel/product_config:android_product.bzl",
symbols: []BazelLoadSymbol{{symbol: "android_product"}},
},
},
})
createTargets(ctx, productLabelsToVariables, moduleNameToPartition, convertedModulePathMap, res.bp2buildTargets)
platformMappingContent, err := platformMappingContent(
productLabelsToVariables,
ctx.Config().Bp2buildSoongConfigDefinitions,
convertedModulePathMap)
if err != nil {
return res, err
}
res.injectionFiles = []BazelFile{
newFile(
"product_config_platforms",
"BUILD.bazel",
productReplacer.Replace(`
package(default_visibility = [
"@//build/bazel/product_config:__subpackages__",
"@soong_injection//product_config_platforms:__subpackages__",
])
load("@//{PRODUCT_FOLDER}:soong.variables.bzl", _soong_variables = "variables")
load("@//build/bazel/product_config:android_product.bzl", "android_product")
# Bazel will qualify its outputs by the platform name. When switching between products, this
# means that soong-built files that depend on bazel-built files will suddenly get different
# dependency files, because the path changes, and they will be rebuilt. In order to avoid this
# extra rebuilding, make mixed builds always use a single platform so that the bazel artifacts
# are always under the same path.
android_product(
name = "mixed_builds_product",
soong_variables = _soong_variables,
extra_constraints = ["@//build/bazel/platforms:mixed_builds"],
)
`)),
newFile(
"product_config_platforms",
"product_labels.bzl",
productReplacer.Replace(`
# This file keeps a list of all the products in the android source tree, because they're
# discovered as part of a preprocessing step before bazel runs.
# TODO: When we start generating the platforms for more than just the
# currently lunched product, they should all be listed here
product_labels = [
"@soong_injection//product_config_platforms:mixed_builds_product",
"@//{PRODUCT_FOLDER}:{PRODUCT}",
`)+strings.Join(productsForTesting, "\n")+"\n]\n"),
newFile(
"product_config_platforms",
"common.bazelrc",
productReplacer.Replace(`
build --platform_mappings=platform_mappings
build --platforms @//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64
build --//build/bazel/product_config:target_build_variant={VARIANT}
build:android --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}
build:linux_x86 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86
build:linux_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64
build:linux_bionic_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_bionic_x86_64
build:linux_musl_x86 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_musl_x86
build:linux_musl_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_musl_x86_64
`)),
newFile(
"product_config_platforms",
"linux.bazelrc",
productReplacer.Replace(`
build --host_platform @//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64
`)),
newFile(
"product_config_platforms",
"darwin.bazelrc",
productReplacer.Replace(`
build --host_platform @//{PRODUCT_FOLDER}:{PRODUCT}_darwin_x86_64
`)),
}
res.bp2buildFiles = []BazelFile{
newFile(
"",
"platform_mappings",
platformMappingContent),
newFile(
currentProductFolder,
"soong.variables.bzl",
`variables = json.decode("""`+strings.ReplaceAll(string(productVariablesBytes), "\\", "\\\\")+`""")`),
}
return res, nil
}
func platformMappingContent(
productLabelToVariables map[bazelLabel]*android.ProductVariables,
soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions,
convertedModulePathMap map[string]string) (string, error) {
var result strings.Builder
mergedConvertedModulePathMap := make(map[string]string)
for k, v := range convertedModulePathMap {
mergedConvertedModulePathMap[k] = v
}
additionalModuleNamesToPackages, err := starlark_import.GetStarlarkValue[map[string]string]("additional_module_names_to_packages")
if err != nil {
return "", err
}
for k, v := range additionalModuleNamesToPackages {
mergedConvertedModulePathMap[k] = v
}
productLabels := make([]bazelLabel, 0, len(productLabelToVariables))
for k := range productLabelToVariables {
productLabels = append(productLabels, k)
}
sort.Slice(productLabels, func(i, j int) bool {
return productLabels[i].Less(&productLabels[j])
})
result.WriteString("platforms:\n")
for _, productLabel := range productLabels {
platformMappingSingleProduct(productLabel, productLabelToVariables[productLabel], soongConfigDefinitions, mergedConvertedModulePathMap, &result)
}
return result.String(), nil
}
var bazelPlatformSuffixes = []string{
"",
"_darwin_arm64",
"_darwin_x86_64",
"_linux_bionic_arm64",
"_linux_bionic_x86_64",
"_linux_musl_x86",
"_linux_musl_x86_64",
"_linux_x86",
"_linux_x86_64",
"_windows_x86",
"_windows_x86_64",
}
func platformMappingSingleProduct(
label bazelLabel,
productVariables *android.ProductVariables,
soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions,
convertedModulePathMap map[string]string,
result *strings.Builder) {
platform_sdk_version := -1
if productVariables.Platform_sdk_version != nil {
platform_sdk_version = *productVariables.Platform_sdk_version
}
defaultAppCertificateFilegroup := "//build/bazel/utils:empty_filegroup"
if proptools.String(productVariables.DefaultAppCertificate) != "" {
defaultAppCertificateFilegroup = "@//" + filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) + ":generated_android_certificate_directory"
}
// TODO: b/301598690 - commas can't be escaped in a string-list passed in a platform mapping,
// so commas are switched for ":" here, and must be back-substituted into commas
// wherever the AAPTCharacteristics product config variable is used.
AAPTConfig := []string{}
for _, conf := range productVariables.AAPTConfig {
AAPTConfig = append(AAPTConfig, strings.Replace(conf, ",", ":", -1))
}
for _, suffix := range bazelPlatformSuffixes {
result.WriteString(" ")
result.WriteString(label.String())
result.WriteString(suffix)
result.WriteString("\n")
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_characteristics=%s\n", proptools.String(productVariables.AAPTCharacteristics)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_config=%s\n", strings.Join(AAPTConfig, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_preferred_config=%s\n", proptools.String(productVariables.AAPTPreferredConfig)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:always_use_prebuilt_sdks=%t\n", proptools.Bool(productVariables.Always_use_prebuilt_sdks)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:arc=%t\n", proptools.Bool(productVariables.Arc)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:apex_global_min_sdk_version_override=%s\n", proptools.String(productVariables.ApexGlobalMinSdkVersionOverride)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:binder32bit=%t\n", proptools.Bool(productVariables.Binder32bit)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_from_text_stub=%t\n", proptools.Bool(productVariables.Build_from_text_stub)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_broken_incorrect_partition_images=%t\n", productVariables.BuildBrokenIncorrectPartitionImages))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_id=%s\n", proptools.String(productVariables.BuildId)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_version_tags=%s\n", strings.Join(productVariables.BuildVersionTags, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:cfi_exclude_paths=%s\n", strings.Join(productVariables.CFIExcludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:cfi_include_paths=%s\n", strings.Join(productVariables.CFIIncludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:compressed_apex=%t\n", proptools.Bool(productVariables.CompressedApex)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:default_app_certificate=%s\n", proptools.String(productVariables.DefaultAppCertificate)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:default_app_certificate_filegroup=%s\n", defaultAppCertificateFilegroup))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_abi=%s\n", strings.Join(productVariables.DeviceAbi, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_max_page_size_supported=%s\n", proptools.String(productVariables.DeviceMaxPageSizeSupported)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_name=%s\n", proptools.String(productVariables.DeviceName)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_no_bionic_page_size_macro=%t\n", proptools.Bool(productVariables.DeviceNoBionicPageSizeMacro)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_product=%s\n", proptools.String(productVariables.DeviceProduct)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_platform=%s\n", label.String()))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:enable_cfi=%t\n", proptools.BoolDefault(productVariables.EnableCFI, true)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:enforce_vintf_manifest=%t\n", proptools.Bool(productVariables.Enforce_vintf_manifest)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_not_svelte=%t\n", proptools.Bool(productVariables.Malloc_not_svelte)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_pattern_fill_contents=%t\n", proptools.Bool(productVariables.Malloc_pattern_fill_contents)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_zero_contents=%t\n", proptools.Bool(productVariables.Malloc_zero_contents)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_exclude_paths=%s\n", strings.Join(productVariables.MemtagHeapExcludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_async_include_paths=%s\n", strings.Join(productVariables.MemtagHeapAsyncIncludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_sync_include_paths=%s\n", strings.Join(productVariables.MemtagHeapSyncIncludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:manifest_package_name_overrides=%s\n", strings.Join(productVariables.ManifestPackageNameOverrides, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:native_coverage=%t\n", proptools.Bool(productVariables.Native_coverage)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_sdk_final=%t\n", proptools.Bool(productVariables.Platform_sdk_final)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_security_patch=%s\n", proptools.String(productVariables.Platform_security_patch)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_version_last_stable=%s\n", proptools.String(productVariables.Platform_version_last_stable)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_version_name=%s\n", proptools.String(productVariables.Platform_version_name)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:product_brand=%s\n", productVariables.ProductBrand))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:product_manufacturer=%s\n", productVariables.ProductManufacturer))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_aconfig_flag_default_permission=%s\n", productVariables.ReleaseAconfigFlagDefaultPermission))
releaseAconfigValueSets := "//build/bazel/product_config:empty_aconfig_value_sets"
if len(productVariables.ReleaseAconfigValueSets) > 0 {
releaseAconfigValueSets = "@//" + label.pkg + ":" + releaseAconfigValueSetsName + "_" + label.target
}
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_aconfig_value_sets=%s\n", releaseAconfigValueSets))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_version=%s\n", productVariables.ReleaseVersion))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_sdk_version=%d\n", platform_sdk_version))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:safestack=%t\n", proptools.Bool(productVariables.Safestack)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:treble_linker_namespaces=%t\n", proptools.Bool(productVariables.Treble_linker_namespaces)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:tidy_checks=%s\n", proptools.String(productVariables.TidyChecks)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:uml=%t\n", proptools.Bool(productVariables.Uml)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:unbundled_build=%t\n", proptools.Bool(productVariables.Unbundled_build)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:unbundled_build_apps=%s\n", strings.Join(productVariables.Unbundled_build_apps, ",")))
for _, override := range productVariables.CertificateOverrides {
parts := strings.SplitN(override, ":", 2)
if apexPath, ok := convertedModulePathMap[parts[0]]; ok {
if overrideCertPath, ok := convertedModulePathMap[parts[1]]; ok {
result.WriteString(fmt.Sprintf(" --%s:%s_certificate_override=%s:%s\n", apexPath, parts[0], overrideCertPath, parts[1]))
}
}
}
for _, namespace := range android.SortedKeys(productVariables.VendorVars) {
for _, variable := range android.SortedKeys(productVariables.VendorVars[namespace]) {
value := productVariables.VendorVars[namespace][variable]
key := namespace + "__" + variable
_, hasBool := soongConfigDefinitions.BoolVars[key]
_, hasString := soongConfigDefinitions.StringVars[key]
_, hasValue := soongConfigDefinitions.ValueVars[key]
if !hasBool && !hasString && !hasValue {
// Not all soong config variables are defined in Android.bp files. For example,
// prebuilt_bootclasspath_fragment uses soong config variables in a nonstandard
// way, that causes them to be present in the soong.variables file but not
// defined in an Android.bp file. There's also nothing stopping you from setting
// a variable in make that doesn't exist in soong. We only generate build
// settings for the ones that exist in soong, so skip all others.
continue
}
if hasBool && hasString || hasBool && hasValue || hasString && hasValue {
panic(fmt.Sprintf("Soong config variable %s:%s appears to be of multiple types. bool? %t, string? %t, value? %t", namespace, variable, hasBool, hasString, hasValue))
}
if hasBool {
// Logic copied from soongConfig.Bool()
value = strings.ToLower(value)
if value == "1" || value == "y" || value == "yes" || value == "on" || value == "true" {
value = "true"
} else {
value = "false"
}
}
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config/soong_config_variables:%s=%s\n", strings.ToLower(key), value))
}
}
}
}
func starlarkMapToProductVariables(in map[string]starlark.Value) (android.ProductVariables, error) {
result := android.ProductVariables{}
productVarsReflect := reflect.ValueOf(&result).Elem()
for i := 0; i < productVarsReflect.NumField(); i++ {
field := productVarsReflect.Field(i)
fieldType := productVarsReflect.Type().Field(i)
name := fieldType.Name
if name == "BootJars" || name == "ApexBootJars" || name == "VendorSnapshotModules" ||
name == "RecoverySnapshotModules" {
// These variables have more complicated types, and we don't need them right now
continue
}
if _, ok := in[name]; ok {
if name == "VendorVars" {
vendorVars, err := starlark_import.Unmarshal[map[string]map[string]string](in[name])
if err != nil {
return result, err
}
field.Set(reflect.ValueOf(vendorVars))
continue
}
switch field.Type().Kind() {
case reflect.Bool:
val, err := starlark_import.Unmarshal[bool](in[name])
if err != nil {
return result, err
}
field.SetBool(val)
case reflect.String:
val, err := starlark_import.Unmarshal[string](in[name])
if err != nil {
return result, err
}
field.SetString(val)
case reflect.Slice:
if field.Type().Elem().Kind() != reflect.String {
return result, fmt.Errorf("slices of types other than strings are unimplemented")
}
val, err := starlark_import.UnmarshalReflect(in[name], field.Type())
if err != nil {
return result, err
}
field.Set(val)
case reflect.Pointer:
switch field.Type().Elem().Kind() {
case reflect.Bool:
val, err := starlark_import.UnmarshalNoneable[bool](in[name])
if err != nil {
return result, err
}
field.Set(reflect.ValueOf(val))
case reflect.String:
val, err := starlark_import.UnmarshalNoneable[string](in[name])
if err != nil {
return result, err
}
field.Set(reflect.ValueOf(val))
case reflect.Int:
val, err := starlark_import.UnmarshalNoneable[int](in[name])
if err != nil {
return result, err
}
field.Set(reflect.ValueOf(val))
default:
return result, fmt.Errorf("pointers of types other than strings/bools are unimplemented: %s", field.Type().Elem().Kind().String())
}
default:
return result, fmt.Errorf("unimplemented type: %s", field.Type().String())
}
}
}
result.Native_coverage = proptools.BoolPtr(
proptools.Bool(result.GcovCoverage) ||
proptools.Bool(result.ClangCoverage))
return result, nil
}
func createTargets(
ctx *CodegenContext,
productLabelsToVariables map[bazelLabel]*android.ProductVariables,
moduleNameToPartition map[string]string,
convertedModulePathMap map[string]string,
res map[string]BazelTargets) {
createGeneratedAndroidCertificateDirectories(productLabelsToVariables, res)
createAvbKeyFilegroups(productLabelsToVariables, res)
createReleaseAconfigValueSetsFilegroup(productLabelsToVariables, res)
for label, variables := range productLabelsToVariables {
createSystemPartition(ctx, label, &variables.PartitionVarsForBazelMigrationOnlyDoNotUse, moduleNameToPartition, convertedModulePathMap, res)
}
}
func createGeneratedAndroidCertificateDirectories(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) {
var allDefaultAppCertificateDirs []string
for _, productVariables := range productLabelsToVariables {
if proptools.String(productVariables.DefaultAppCertificate) != "" {
d := filepath.Dir(proptools.String(productVariables.DefaultAppCertificate))
if !android.InList(d, allDefaultAppCertificateDirs) {
allDefaultAppCertificateDirs = append(allDefaultAppCertificateDirs, d)
}
}
}
for _, dir := range allDefaultAppCertificateDirs {
content := `filegroup(
name = "generated_android_certificate_directory",
srcs = glob([
"*.pk8",
"*.pem",
"*.avbpubkey",
]),
visibility = ["//visibility:public"],
)`
targets[dir] = append(targets[dir], BazelTarget{
name: "generated_android_certificate_directory",
packageName: dir,
content: content,
ruleClass: "filegroup",
})
}
}
func createReleaseAconfigValueSetsFilegroup(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) {
for label, productVariables := range productLabelsToVariables {
if len(productVariables.ReleaseAconfigValueSets) > 0 {
key := label.target
dir := label.pkg
var value_sets strings.Builder
for _, value_set := range productVariables.ReleaseAconfigValueSets {
value_sets.WriteString(" \"" + value_set + "\",\n")
}
name := releaseAconfigValueSetsName + "_" + key
content := "aconfig_value_sets(\n" +
" name = \"" + name + "\",\n" +
" value_sets = [\n" +
value_sets.String() +
" ],\n" +
" visibility = [\"//visibility:public\"],\n" +
")"
targets[dir] = append(targets[dir], BazelTarget{
name: name,
packageName: dir,
content: content,
ruleClass: "aconfig_value_sets",
loads: []BazelLoad{{
file: "//build/bazel/rules/aconfig:aconfig_value_sets.bzl",
symbols: []BazelLoadSymbol{{
symbol: "aconfig_value_sets",
}},
}},
})
}
}
}
func createAvbKeyFilegroups(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) {
var allAvbKeys []string
for _, productVariables := range productLabelsToVariables {
for _, partitionVariables := range productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.PartitionQualifiedVariables {
if partitionVariables.BoardAvbKeyPath != "" {
if !android.InList(partitionVariables.BoardAvbKeyPath, allAvbKeys) {
allAvbKeys = append(allAvbKeys, partitionVariables.BoardAvbKeyPath)
}
}
}
}
for _, key := range allAvbKeys {
dir := filepath.Dir(key)
name := filepath.Base(key)
content := fmt.Sprintf(`filegroup(
name = "%s_filegroup",
srcs = ["%s"],
visibility = ["//visibility:public"],
)`, name, name)
targets[dir] = append(targets[dir], BazelTarget{
name: name + "_filegroup",
packageName: dir,
content: content,
ruleClass: "filegroup",
})
}
}
func createSystemPartition(
ctx *CodegenContext,
platformLabel bazelLabel,
variables *android.PartitionVariables,
moduleNameToPartition map[string]string,
convertedModulePathMap map[string]string,
targets map[string]BazelTargets) {
if !variables.PartitionQualifiedVariables["system"].BuildingImage {
return
}
qualifiedVariables := variables.PartitionQualifiedVariables["system"]
imageProps := generateImagePropDictionary(variables, "system")
imageProps["skip_fsck"] = "true"
var properties strings.Builder
for _, prop := range android.SortedKeys(imageProps) {
properties.WriteString(prop)
properties.WriteRune('=')
properties.WriteString(imageProps[prop])
properties.WriteRune('\n')
}
var extraProperties strings.Builder
if variables.BoardAvbEnable {
extraProperties.WriteString(" avb_enable = True,\n")
extraProperties.WriteString(fmt.Sprintf(" avb_add_hashtree_footer_args = %q,\n", qualifiedVariables.BoardAvbAddHashtreeFooterArgs))
keypath := qualifiedVariables.BoardAvbKeyPath
if keypath != "" {
extraProperties.WriteString(fmt.Sprintf(" avb_key = \"//%s:%s\",\n", filepath.Dir(keypath), filepath.Base(keypath)+"_filegroup"))
extraProperties.WriteString(fmt.Sprintf(" avb_algorithm = %q,\n", qualifiedVariables.BoardAvbAlgorithm))
extraProperties.WriteString(fmt.Sprintf(" avb_rollback_index = %s,\n", qualifiedVariables.BoardAvbRollbackIndex))
extraProperties.WriteString(fmt.Sprintf(" avb_rollback_index_location = %s,\n", qualifiedVariables.BoardAvbRollbackIndexLocation))
}
}
var deps []string
for _, mod := range variables.ProductPackages {
if path, ok := convertedModulePathMap[mod]; ok && ctx.Config().BazelContext.IsModuleNameAllowed(mod, false) {
if partition, ok := moduleNameToPartition[mod]; ok && partition == "system" {
if path == "//." {
path = "//"
}
deps = append(deps, fmt.Sprintf(" \"%s:%s\",\n", path, mod))
}
}
}
if len(deps) > 0 {
sort.Strings(deps)
extraProperties.WriteString(" deps = [\n")
for _, dep := range deps {
extraProperties.WriteString(dep)
}
extraProperties.WriteString(" ],\n")
}
targets[platformLabel.pkg] = append(targets[platformLabel.pkg], BazelTarget{
name: "system_image",
packageName: platformLabel.pkg,
content: fmt.Sprintf(`partition(
name = "system_image",
base_staging_dir = "//build/bazel/bazel_sandwich:system_staging_dir",
base_staging_dir_file_list = "//build/bazel/bazel_sandwich:system_staging_dir_file_list",
root_dir = "//build/bazel/bazel_sandwich:root_staging_dir",
selinux_file_contexts = "//build/bazel/bazel_sandwich:selinux_file_contexts",
image_properties = """
%s
""",
%s
type = "system",
)`, properties.String(), extraProperties.String()),
ruleClass: "partition",
loads: []BazelLoad{{
file: "//build/bazel/rules/partitions:partition.bzl",
symbols: []BazelLoadSymbol{{
symbol: "partition",
}},
}},
}, BazelTarget{
name: "system_image_test",
packageName: platformLabel.pkg,
content: `partition_diff_test(
name = "system_image_test",
partition1 = "//build/bazel/bazel_sandwich:make_system_image",
partition2 = ":system_image",
)`,
ruleClass: "partition_diff_test",
loads: []BazelLoad{{
file: "//build/bazel/rules/partitions/diff:partition_diff.bzl",
symbols: []BazelLoadSymbol{{
symbol: "partition_diff_test",
}},
}},
}, BazelTarget{
name: "run_system_image_test",
packageName: platformLabel.pkg,
content: `run_test_in_build(
name = "run_system_image_test",
test = ":system_image_test",
)`,
ruleClass: "run_test_in_build",
loads: []BazelLoad{{
file: "//build/bazel/bazel_sandwich:run_test_in_build.bzl",
symbols: []BazelLoadSymbol{{
symbol: "run_test_in_build",
}},
}},
})
}
var allPartitionTypes = []string{
"system",
"vendor",
"cache",
"userdata",
"product",
"system_ext",
"oem",
"odm",
"vendor_dlkm",
"odm_dlkm",
"system_dlkm",
}
// An equivalent of make's generate-image-prop-dictionary function
func generateImagePropDictionary(variables *android.PartitionVariables, partitionType string) map[string]string {
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
ret := map[string]string{}
if partitionType == "system" {
if len(variables.PartitionQualifiedVariables["system_other"].BoardPartitionSize) > 0 {
ret["system_other_size"] = variables.PartitionQualifiedVariables["system_other"].BoardPartitionSize
}
if len(partitionQualifiedVariables.ProductHeadroom) > 0 {
ret["system_headroom"] = partitionQualifiedVariables.ProductHeadroom
}
addCommonRoFlagsToImageProps(variables, partitionType, ret)
}
// TODO: other partition-specific logic
if variables.TargetUserimagesUseExt2 {
ret["fs_type"] = "ext2"
} else if variables.TargetUserimagesUseExt3 {
ret["fs_type"] = "ext3"
} else if variables.TargetUserimagesUseExt4 {
ret["fs_type"] = "ext4"
}
if !variables.TargetUserimagesSparseExtDisabled {
ret["extfs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseErofsDisabled {
ret["erofs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseSquashfsDisabled {
ret["squashfs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseF2fsDisabled {
ret["f2fs_sparse_flag"] = "-S"
}
erofsCompressor := variables.BoardErofsCompressor
if len(erofsCompressor) == 0 && hasErofsPartition(variables) {
if len(variables.BoardErofsUseLegacyCompression) > 0 {
erofsCompressor = "lz4"
} else {
erofsCompressor = "lz4hc,9"
}
}
if len(erofsCompressor) > 0 {
ret["erofs_default_compressor"] = erofsCompressor
}
if len(variables.BoardErofsCompressorHints) > 0 {
ret["erofs_default_compress_hints"] = variables.BoardErofsCompressorHints
}
if len(variables.BoardErofsCompressorHints) > 0 {
ret["erofs_default_compress_hints"] = variables.BoardErofsCompressorHints
}
if len(variables.BoardErofsPclusterSize) > 0 {
ret["erofs_pcluster_size"] = variables.BoardErofsPclusterSize
}
if len(variables.BoardErofsShareDupBlocks) > 0 {
ret["erofs_share_dup_blocks"] = variables.BoardErofsShareDupBlocks
}
if len(variables.BoardErofsUseLegacyCompression) > 0 {
ret["erofs_use_legacy_compression"] = variables.BoardErofsUseLegacyCompression
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
if len(variables.BoardFlashLogicalBlockSize) > 0 {
ret["flash_logical_block_size"] = variables.BoardFlashLogicalBlockSize
}
if len(variables.BoardFlashEraseBlockSize) > 0 {
ret["flash_erase_block_size"] = variables.BoardFlashEraseBlockSize
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
for _, partitionType := range allPartitionTypes {
if qualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]; ok && len(qualifiedVariables.ProductVerityPartition) > 0 {
ret[partitionType+"_verity_block_device"] = qualifiedVariables.ProductVerityPartition
}
}
// TODO: Vboot
// TODO: AVB
if variables.BoardUsesRecoveryAsBoot {
ret["recovery_as_boot"] = "true"
}
if variables.ProductUseDynamicPartitionSize {
ret["use_dynamic_partition_size"] = "true"
}
if variables.CopyImagesForTargetFilesZip {
ret["use_fixed_timestamp"] = "true"
}
return ret
}
// Soong equivalent of make's add-common-ro-flags-to-image-props
func addCommonRoFlagsToImageProps(variables *android.PartitionVariables, partitionType string, ret map[string]string) {
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
if len(partitionQualifiedVariables.BoardErofsCompressor) > 0 {
ret[partitionType+"_erofs_compressor"] = partitionQualifiedVariables.BoardErofsCompressor
}
if len(partitionQualifiedVariables.BoardErofsCompressHints) > 0 {
ret[partitionType+"_erofs_compress_hints"] = partitionQualifiedVariables.BoardErofsCompressHints
}
if len(partitionQualifiedVariables.BoardErofsPclusterSize) > 0 {
ret[partitionType+"_erofs_pcluster_size"] = partitionQualifiedVariables.BoardErofsPclusterSize
}
if len(partitionQualifiedVariables.BoardExtfsRsvPct) > 0 {
ret[partitionType+"_extfs_rsv_pct"] = partitionQualifiedVariables.BoardExtfsRsvPct
}
if len(partitionQualifiedVariables.BoardF2fsSloadCompressFlags) > 0 {
ret[partitionType+"_f2fs_sldc_flags"] = partitionQualifiedVariables.BoardF2fsSloadCompressFlags
}
if len(partitionQualifiedVariables.BoardFileSystemCompress) > 0 {
ret[partitionType+"_f2fs_compress"] = partitionQualifiedVariables.BoardFileSystemCompress
}
if len(partitionQualifiedVariables.BoardFileSystemType) > 0 {
ret[partitionType+"_fs_type"] = partitionQualifiedVariables.BoardFileSystemType
}
if len(partitionQualifiedVariables.BoardJournalSize) > 0 {
ret[partitionType+"_journal_size"] = partitionQualifiedVariables.BoardJournalSize
}
if len(partitionQualifiedVariables.BoardPartitionReservedSize) > 0 {
ret[partitionType+"_reserved_size"] = partitionQualifiedVariables.BoardPartitionReservedSize
}
if len(partitionQualifiedVariables.BoardPartitionSize) > 0 {
ret[partitionType+"_size"] = partitionQualifiedVariables.BoardPartitionSize
}
if len(partitionQualifiedVariables.BoardSquashfsBlockSize) > 0 {
ret[partitionType+"_squashfs_block_size"] = partitionQualifiedVariables.BoardSquashfsBlockSize
}
if len(partitionQualifiedVariables.BoardSquashfsCompressor) > 0 {
ret[partitionType+"_squashfs_compressor"] = partitionQualifiedVariables.BoardSquashfsCompressor
}
if len(partitionQualifiedVariables.BoardSquashfsCompressorOpt) > 0 {
ret[partitionType+"_squashfs_compressor_opt"] = partitionQualifiedVariables.BoardSquashfsCompressorOpt
}
if len(partitionQualifiedVariables.BoardSquashfsDisable4kAlign) > 0 {
ret[partitionType+"_squashfs_disable_4k_align"] = partitionQualifiedVariables.BoardSquashfsDisable4kAlign
}
if len(partitionQualifiedVariables.BoardPartitionSize) == 0 && len(partitionQualifiedVariables.BoardPartitionReservedSize) == 0 && len(partitionQualifiedVariables.ProductHeadroom) == 0 {
ret[partitionType+"_disable_sparse"] = "true"
}
addCommonFlagsToImageProps(variables, partitionType, ret)
}
func hasErofsPartition(variables *android.PartitionVariables) bool {
return variables.PartitionQualifiedVariables["product"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system_ext"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["odm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["vendor"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["vendor_dlkm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["odm_dlkm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system_dlkm"].BoardFileSystemType == "erofs"
}
// Soong equivalent of make's add-common-flags-to-image-props
func addCommonFlagsToImageProps(variables *android.PartitionVariables, partitionType string, ret map[string]string) {
// The selinux_fc will be handled separately
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
ret["building_"+partitionType+"_image"] = boolToMakeString(partitionQualifiedVariables.BuildingImage)
}
func boolToMakeString(b bool) string {
if b {
return "true"
}
return ""
}

605
bp2build/build_conversion.go
View File

@@ -28,10 +28,7 @@ import (
"android/soong/android"
"android/soong/bazel"
"android/soong/starlark_fmt"
"android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/proptools"
)
@@ -201,18 +198,13 @@ func (ctx *CodegenContext) Mode() CodegenMode {
type CodegenMode int
const (
// Bp2Build - generate BUILD files with targets buildable by Bazel directly.
//
// This mode is used for the Soong->Bazel build definition conversion.
Bp2Build CodegenMode = iota
// QueryView - generate BUILD files with targets representing fully mutated
// Soong modules, representing the fully configured Soong module graph with
// variants and dependency edges.
//
// This mode is used for discovering and introspecting the existing Soong
// module graph.
QueryView
QueryView CodegenMode = iota
)
type unconvertedDepsMode int
@@ -227,8 +219,6 @@ const (
func (mode CodegenMode) String() string {
switch mode {
case Bp2Build:
return "Bp2Build"
case QueryView:
return "QueryView"
default:
@@ -256,9 +246,6 @@ func (ctx *CodegenContext) Context() *android.Context { return ctx.context }
// writing BUILD files in the output directory.
func NewCodegenContext(config android.Config, context *android.Context, mode CodegenMode, topDir string) *CodegenContext {
var unconvertedDeps unconvertedDepsMode
if config.IsEnvTrue("BP2BUILD_ERROR_UNCONVERTED") {
unconvertedDeps = errorModulesUnconvertedDeps
}
return &CodegenContext{
context: context,
config: config,
@@ -281,526 +268,30 @@ func propsToAttributes(props map[string]string) string {
type conversionResults struct {
buildFileToTargets map[string]BazelTargets
moduleNameToPartition map[string]string
metrics CodegenMetrics
}
func (r conversionResults) BuildDirToTargets() map[string]BazelTargets {
return r.buildFileToTargets
}
// struct to store state of b bazel targets (e.g. go targets which do not implement android.Module)
// this implements bp2buildModule interface and is passed to generateBazelTargets
type bTarget struct {
targetName string
targetPackage string
bazelRuleClass string
bazelRuleLoadLocation string
bazelAttributes []interface{}
}
var _ bp2buildModule = (*bTarget)(nil)
func (b bTarget) TargetName() string {
return b.targetName
}
func (b bTarget) TargetPackage() string {
return b.targetPackage
}
func (b bTarget) BazelRuleClass() string {
return b.bazelRuleClass
}
func (b bTarget) BazelRuleLoadLocation() string {
return b.bazelRuleLoadLocation
}
func (b bTarget) BazelAttributes() []interface{} {
return b.bazelAttributes
}
// Creates a target_compatible_with entry that is *not* compatible with android
func targetNotCompatibleWithAndroid() bazel.LabelListAttribute {
ret := bazel.LabelListAttribute{}
ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsAndroid,
bazel.MakeLabelList(
[]bazel.Label{
bazel.Label{
Label: "@platforms//:incompatible",
},
},
),
)
return ret
}
// helper function to return labels for srcs used in bootstrap_go_package and bootstrap_go_binary
// this function has the following limitations which make it unsuitable for widespread use
// - wildcard patterns in srcs
// This is ok for go since build/blueprint does not support it.
//
// Prefer to use `BazelLabelForModuleSrc` instead
func goSrcLabels(cfg android.Config, moduleDir string, srcs []string, linuxSrcs, darwinSrcs []string) bazel.LabelListAttribute {
labels := func(srcs []string) bazel.LabelList {
ret := []bazel.Label{}
for _, src := range srcs {
srcLabel := bazel.Label{
Label: src,
}
ret = append(ret, srcLabel)
}
// Respect package boundaries
return android.TransformSubpackagePaths(
cfg,
moduleDir,
bazel.MakeLabelList(ret),
)
}
ret := bazel.LabelListAttribute{}
// common
ret.SetSelectValue(bazel.NoConfigAxis, "", labels(srcs))
// linux
ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsLinux, labels(linuxSrcs))
// darwin
ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsDarwin, labels(darwinSrcs))
return ret
}
func goDepLabels(deps []string, goModulesMap nameToGoLibraryModule) bazel.LabelListAttribute {
labels := []bazel.Label{}
for _, dep := range deps {
moduleDir := goModulesMap[dep].Dir
if moduleDir == "." {
moduleDir = ""
}
label := bazel.Label{
Label: fmt.Sprintf("//%s:%s", moduleDir, dep),
}
labels = append(labels, label)
}
return bazel.MakeLabelListAttribute(bazel.MakeLabelList(labels))
}
// attributes common to blueprint_go_binary and bootstap_go_package
type goAttributes struct {
Importpath bazel.StringAttribute
Srcs bazel.LabelListAttribute
Deps bazel.LabelListAttribute
Data bazel.LabelListAttribute
Target_compatible_with bazel.LabelListAttribute
// attributes for the dynamically generated go_test target
Embed bazel.LabelListAttribute
}
type goTestProperties struct {
name string
dir string
testSrcs []string
linuxTestSrcs []string
darwinTestSrcs []string
testData []string
// Name of the target that should be compiled together with the test
embedName string
}
// Creates a go_test target for bootstrap_go_package / blueprint_go_binary
func generateBazelTargetsGoTest(ctx *android.Context, goModulesMap nameToGoLibraryModule, gp goTestProperties) (BazelTarget, error) {
ca := android.CommonAttributes{
Name: gp.name,
}
ga := goAttributes{
Srcs: goSrcLabels(ctx.Config(), gp.dir, gp.testSrcs, gp.linuxTestSrcs, gp.darwinTestSrcs),
Data: goSrcLabels(ctx.Config(), gp.dir, gp.testData, []string{}, []string{}),
Embed: bazel.MakeLabelListAttribute(
bazel.MakeLabelList(
[]bazel.Label{bazel.Label{Label: ":" + gp.embedName}},
),
),
Target_compatible_with: targetNotCompatibleWithAndroid(),
}
libTest := bTarget{
targetName: gp.name,
targetPackage: gp.dir,
bazelRuleClass: "go_test",
bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
bazelAttributes: []interface{}{&ca, &ga},
}
return generateBazelTarget(ctx, libTest)
}
// TODO - b/288491147: testSrcs of certain bootstrap_go_package/blueprint_go_binary are not hermetic and depend on
// testdata checked into the filesystem.
// Denylist the generation of go_test targets for these Soong modules.
// The go_library/go_binary will still be generated, since those are hermitic.
var (
goTestsDenylist = []string{
"android-archive-zip",
"bazel_notice_gen",
"blueprint-bootstrap-bpdoc",
"blueprint-microfactory",
"blueprint-pathtools",
"bssl_ar",
"compliance_checkmetadata",
"compliance_checkshare",
"compliance_dumpgraph",
"compliance_dumpresolutions",
"compliance_listshare",
"compliance-module",
"compliancenotice_bom",
"compliancenotice_shippedlibs",
"compliance_rtrace",
"compliance_sbom",
"golang-protobuf-internal-fuzz-jsonfuzz",
"golang-protobuf-internal-fuzz-textfuzz",
"golang-protobuf-internal-fuzz-wirefuzz",
"htmlnotice",
"protoc-gen-go",
"rbcrun-module",
"spdx-tools-builder",
"spdx-tools-builder2v1",
"spdx-tools-builder2v2",
"spdx-tools-builder2v3",
"spdx-tools-idsearcher",
"spdx-tools-spdx-json",
"spdx-tools-utils",
"soong-ui-build",
"textnotice",
"xmlnotice",
}
)
func testOfGoPackageIsIncompatible(g *bootstrap.GoPackage) bool {
return android.InList(g.Name(), goTestsDenylist) ||
// Denylist tests of soong_build
// Theses tests have a guard that prevent usage outside a test environment
// The guard (`ensureTestOnly`) looks for a `-test` in os.Args, which is present in soong's gotestrunner, but missing in `b test`
g.IsPluginFor("soong_build") ||
// soong-android is a dep of soong_build
// This dependency is created by soong_build by listing it in its deps explicitly in Android.bp, and not via `plugin_for` in `soong-android`
g.Name() == "soong-android"
}
func testOfGoBinaryIsIncompatible(g *bootstrap.GoBinary) bool {
return android.InList(g.Name(), goTestsDenylist)
}
func generateBazelTargetsGoPackage(ctx *android.Context, g *bootstrap.GoPackage, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) {
ca := android.CommonAttributes{
Name: g.Name(),
}
// For this bootstrap_go_package dep chain,
// A --> B --> C ( ---> depends on)
// Soong provides the convenience of only listing B as deps of A even if a src file of A imports C
// Bazel OTOH
// 1. requires C to be listed in `deps` expllicity.
// 2. does not require C to be listed if src of A does not import C
//
// bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps
transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap)
ga := goAttributes{
Importpath: bazel.StringAttribute{
Value: proptools.StringPtr(g.GoPkgPath()),
},
Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()),
Deps: goDepLabels(
android.FirstUniqueStrings(transitiveDeps),
goModulesMap,
),
Target_compatible_with: targetNotCompatibleWithAndroid(),
}
lib := bTarget{
targetName: g.Name(),
targetPackage: ctx.ModuleDir(g),
bazelRuleClass: "go_library",
bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
bazelAttributes: []interface{}{&ca, &ga},
}
retTargets := []BazelTarget{}
var retErrs []error
if libTarget, err := generateBazelTarget(ctx, lib); err == nil {
retTargets = append(retTargets, libTarget)
} else {
retErrs = []error{err}
}
// If the library contains test srcs, create an additional go_test target
if !testOfGoPackageIsIncompatible(g) && (len(g.TestSrcs()) > 0 || len(g.LinuxTestSrcs()) > 0 || len(g.DarwinTestSrcs()) > 0) {
gp := goTestProperties{
name: g.Name() + "-test",
dir: ctx.ModuleDir(g),
testSrcs: g.TestSrcs(),
linuxTestSrcs: g.LinuxTestSrcs(),
darwinTestSrcs: g.DarwinTestSrcs(),
testData: g.TestData(),
embedName: g.Name(), // embed the source go_library in the test so that its .go files are included in the compilation unit
}
if libTestTarget, err := generateBazelTargetsGoTest(ctx, goModulesMap, gp); err == nil {
retTargets = append(retTargets, libTestTarget)
} else {
retErrs = append(retErrs, err)
}
}
return retTargets, retErrs
}
type goLibraryModule struct {
Dir string
Deps []string
}
type buildConversionMetadata struct {
nameToGoLibraryModule nameToGoLibraryModule
ndkHeaders []blueprint.Module
}
type nameToGoLibraryModule map[string]goLibraryModule
// Visit each module in the graph, and collect metadata about the build graph
// If a module is of type `bootstrap_go_package`, return a map containing metadata like its dir and deps
// If a module is of type `ndk_headers`, add it to a list and return the list
func createBuildConversionMetadata(ctx *android.Context) buildConversionMetadata {
goMap := nameToGoLibraryModule{}
ndkHeaders := []blueprint.Module{}
ctx.VisitAllModules(func(m blueprint.Module) {
moduleType := ctx.ModuleType(m)
// We do not need to store information about blueprint_go_binary since it does not have any rdeps
if moduleType == "bootstrap_go_package" {
goMap[m.Name()] = goLibraryModule{
Dir: ctx.ModuleDir(m),
Deps: m.(*bootstrap.GoPackage).Deps(),
}
} else if moduleType == "ndk_headers" || moduleType == "versioned_ndk_headers" {
ndkHeaders = append(ndkHeaders, m)
}
})
return buildConversionMetadata{
nameToGoLibraryModule: goMap,
ndkHeaders: ndkHeaders,
}
}
// Returns the deps in the transitive closure of a go target
func transitiveGoDeps(directDeps []string, goModulesMap nameToGoLibraryModule) []string {
allDeps := directDeps
i := 0
for i < len(allDeps) {
curr := allDeps[i]
allDeps = append(allDeps, goModulesMap[curr].Deps...)
i += 1
}
allDeps = android.SortedUniqueStrings(allDeps)
return allDeps
}
func generateBazelTargetsGoBinary(ctx *android.Context, g *bootstrap.GoBinary, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) {
ca := android.CommonAttributes{
Name: g.Name(),
}
retTargets := []BazelTarget{}
var retErrs []error
// For this bootstrap_go_package dep chain,
// A --> B --> C ( ---> depends on)
// Soong provides the convenience of only listing B as deps of A even if a src file of A imports C
// Bazel OTOH
// 1. requires C to be listed in `deps` expllicity.
// 2. does not require C to be listed if src of A does not import C
//
// bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps
transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap)
goSource := ""
// If the library contains test srcs, create an additional go_test target
// The go_test target will embed a go_source containining the source .go files it tests
if !testOfGoBinaryIsIncompatible(g) && (len(g.TestSrcs()) > 0 || len(g.LinuxTestSrcs()) > 0 || len(g.DarwinTestSrcs()) > 0) {
// Create a go_source containing the source .go files of go_library
// This target will be an `embed` of the go_binary and go_test
goSource = g.Name() + "-source"
ca := android.CommonAttributes{
Name: goSource,
}
ga := goAttributes{
Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()),
Deps: goDepLabels(transitiveDeps, goModulesMap),
Target_compatible_with: targetNotCompatibleWithAndroid(),
}
libTestSource := bTarget{
targetName: goSource,
targetPackage: ctx.ModuleDir(g),
bazelRuleClass: "go_source",
bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
bazelAttributes: []interface{}{&ca, &ga},
}
if libSourceTarget, err := generateBazelTarget(ctx, libTestSource); err == nil {
retTargets = append(retTargets, libSourceTarget)
} else {
retErrs = append(retErrs, err)
}
// Create a go_test target
gp := goTestProperties{
name: g.Name() + "-test",
dir: ctx.ModuleDir(g),
testSrcs: g.TestSrcs(),
linuxTestSrcs: g.LinuxTestSrcs(),
darwinTestSrcs: g.DarwinTestSrcs(),
testData: g.TestData(),
// embed the go_source in the test
embedName: g.Name() + "-source",
}
if libTestTarget, err := generateBazelTargetsGoTest(ctx, goModulesMap, gp); err == nil {
retTargets = append(retTargets, libTestTarget)
} else {
retErrs = append(retErrs, err)
}
}
// Create a go_binary target
ga := goAttributes{
Deps: goDepLabels(transitiveDeps, goModulesMap),
Target_compatible_with: targetNotCompatibleWithAndroid(),
}
// If the binary has testSrcs, embed the common `go_source`
if goSource != "" {
ga.Embed = bazel.MakeLabelListAttribute(
bazel.MakeLabelList(
[]bazel.Label{bazel.Label{Label: ":" + goSource}},
),
)
} else {
ga.Srcs = goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs())
}
bin := bTarget{
targetName: g.Name(),
targetPackage: ctx.ModuleDir(g),
bazelRuleClass: "go_binary",
bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
bazelAttributes: []interface{}{&ca, &ga},
}
if binTarget, err := generateBazelTarget(ctx, bin); err == nil {
retTargets = append(retTargets, binTarget)
} else {
retErrs = []error{err}
}
return retTargets, retErrs
}
func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (conversionResults, []error) {
ctx.Context().BeginEvent("GenerateBazelTargets")
defer ctx.Context().EndEvent("GenerateBazelTargets")
buildFileToTargets := make(map[string]BazelTargets)
// Simple metrics tracking for bp2build
metrics := CreateCodegenMetrics()
dirs := make(map[string]bool)
moduleNameToPartition := make(map[string]string)
var errs []error
// Visit go libraries in a pre-run and store its state in a map
// The time complexity remains O(N), and this does not add significant wall time.
meta := createBuildConversionMetadata(ctx.Context())
nameToGoLibMap := meta.nameToGoLibraryModule
ndkHeaders := meta.ndkHeaders
bpCtx := ctx.Context()
bpCtx.VisitAllModules(func(m blueprint.Module) {
dir := bpCtx.ModuleDir(m)
moduleType := bpCtx.ModuleType(m)
dirs[dir] = true
var targets []BazelTarget
var targetErrs []error
switch ctx.Mode() {
case Bp2Build:
if aModule, ok := m.(android.Module); ok {
reason := aModule.GetUnconvertedReason()
if reason != nil {
// If this module was force-enabled, cause an error.
if _, ok := ctx.Config().BazelModulesForceEnabledByFlag()[m.Name()]; ok && m.Name() != "" {
err := fmt.Errorf("Force Enabled Module %s not converted", m.Name())
errs = append(errs, err)
}
// Log the module isn't to be converted by bp2build.
// TODO: b/291598248 - Log handcrafted modules differently than other unconverted modules.
metrics.AddUnconvertedModule(m, moduleType, dir, *reason)
return
}
if len(aModule.Bp2buildTargets()) == 0 {
panic(fmt.Errorf("illegal bp2build invariant: module '%s' was neither converted nor marked unconvertible", aModule.Name()))
}
// Handle modules converted to generated targets.
targets, targetErrs = generateBazelTargets(bpCtx, aModule)
errs = append(errs, targetErrs...)
for _, t := range targets {
// A module can potentially generate more than 1 Bazel
// target, each of a different rule class.
metrics.IncrementRuleClassCount(t.ruleClass)
}
// record the partition
moduleNameToPartition[android.RemoveOptionalPrebuiltPrefix(aModule.Name())] = aModule.GetPartitionForBp2build()
// Log the module.
metrics.AddConvertedModule(aModule, moduleType, dir)
// Handle modules with unconverted deps. By default, emit a warning.
if unconvertedDeps := aModule.GetUnconvertedBp2buildDeps(); len(unconvertedDeps) > 0 {
msg := fmt.Sprintf("%s %s:%s depends on unconverted modules: %s",
moduleType, bpCtx.ModuleDir(m), m.Name(), strings.Join(unconvertedDeps, ", "))
switch ctx.unconvertedDepMode {
case warnUnconvertedDeps:
metrics.moduleWithUnconvertedDepsMsgs = append(metrics.moduleWithUnconvertedDepsMsgs, msg)
case errorModulesUnconvertedDeps:
errs = append(errs, fmt.Errorf(msg))
return
}
}
if unconvertedDeps := aModule.GetMissingBp2buildDeps(); len(unconvertedDeps) > 0 {
msg := fmt.Sprintf("%s %s:%s depends on missing modules: %s",
moduleType, bpCtx.ModuleDir(m), m.Name(), strings.Join(unconvertedDeps, ", "))
switch ctx.unconvertedDepMode {
case warnUnconvertedDeps:
metrics.moduleWithMissingDepsMsgs = append(metrics.moduleWithMissingDepsMsgs, msg)
case errorModulesUnconvertedDeps:
errs = append(errs, fmt.Errorf(msg))
return
}
}
} else if glib, ok := m.(*bootstrap.GoPackage); ok {
targets, targetErrs = generateBazelTargetsGoPackage(bpCtx, glib, nameToGoLibMap)
errs = append(errs, targetErrs...)
metrics.IncrementRuleClassCount("bootstrap_go_package")
metrics.AddConvertedModule(glib, "bootstrap_go_package", dir)
} else if gbin, ok := m.(*bootstrap.GoBinary); ok {
targets, targetErrs = generateBazelTargetsGoBinary(bpCtx, gbin, nameToGoLibMap)
errs = append(errs, targetErrs...)
metrics.IncrementRuleClassCount("blueprint_go_binary")
metrics.AddConvertedModule(gbin, "blueprint_go_binary", dir)
} else {
metrics.AddUnconvertedModule(m, moduleType, dir, android.UnconvertedReason{
ReasonType: int(bp2build_metrics_proto.UnconvertedReasonType_TYPE_UNSUPPORTED),
})
return
}
case QueryView:
// Blocklist certain module types from being generated.
if canonicalizeModuleType(bpCtx.ModuleType(m)) == "package" {
@@ -824,37 +315,6 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
}
})
// Create an ndk_sysroot target that has a dependency edge on every target corresponding to Soong's ndk_headers
// This root target will provide headers to sdk variants of jni libraries
if ctx.Mode() == Bp2Build {
var depLabels bazel.LabelList
for _, ndkHeader := range ndkHeaders {
depLabel := bazel.Label{
Label: "//" + bpCtx.ModuleDir(ndkHeader) + ":" + ndkHeader.Name(),
}
depLabels.Add(&depLabel)
}
a := struct {
Deps bazel.LabelListAttribute
}{
Deps: bazel.MakeLabelListAttribute(bazel.UniqueSortedBazelLabelList(depLabels)),
}
ndkSysroot := bTarget{
targetName: "ndk_sysroot",
targetPackage: "build/bazel/rules/cc", // The location is subject to change, use build/bazel for now
bazelRuleClass: "cc_library_headers",
bazelRuleLoadLocation: "//build/bazel/rules/cc:cc_library_headers.bzl",
bazelAttributes: []interface{}{&a},
}
if t, err := generateBazelTarget(bpCtx, ndkSysroot); err == nil {
dir := ndkSysroot.targetPackage
buildFileToTargets[dir] = append(buildFileToTargets[dir], t)
} else {
errs = append(errs, err)
}
}
if len(errs) > 0 {
return conversionResults{}, errs
}
@@ -883,72 +343,9 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
return conversionResults{
buildFileToTargets: buildFileToTargets,
moduleNameToPartition: moduleNameToPartition,
metrics: metrics,
}, errs
}
func generateBazelTargets(ctx bpToBuildContext, m android.Module) ([]BazelTarget, []error) {
var targets []BazelTarget
var errs []error
for _, m := range m.Bp2buildTargets() {
target, err := generateBazelTarget(ctx, m)
if err != nil {
errs = append(errs, err)
return targets, errs
}
targets = append(targets, target)
}
return targets, errs
}
type bp2buildModule interface {
TargetName() string
TargetPackage() string
BazelRuleClass() string
BazelRuleLoadLocation() string
BazelAttributes() []interface{}
}
func generateBazelTarget(ctx bpToBuildContext, m bp2buildModule) (BazelTarget, error) {
ruleClass := m.BazelRuleClass()
bzlLoadLocation := m.BazelRuleLoadLocation()
// extract the bazel attributes from the module.
attrs := m.BazelAttributes()
props, err := extractModuleProperties(attrs, true)
if err != nil {
return BazelTarget{}, err
}
// name is handled in a special manner
delete(props.Attrs, "name")
// Return the Bazel target with rule class and attributes, ready to be
// code-generated.
attributes := propsToAttributes(props.Attrs)
var content string
targetName := m.TargetName()
if targetName != "" {
content = fmt.Sprintf(ruleTargetTemplate, ruleClass, targetName, attributes)
} else {
content = fmt.Sprintf(unnamedRuleTargetTemplate, ruleClass, attributes)
}
var loads []BazelLoad
if bzlLoadLocation != "" {
loads = append(loads, BazelLoad{
file: bzlLoadLocation,
symbols: []BazelLoadSymbol{{symbol: ruleClass}},
})
}
return BazelTarget{
name: targetName,
packageName: m.TargetPackage(),
ruleClass: ruleClass,
loads: loads,
content: content,
}, nil
}
// Convert a module and its deps and props into a Bazel macro/rule
// representation in the BUILD file.
func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) (BazelTarget, error) {

101
bp2build/conversion.go
View File

@@ -1,15 +1,10 @@
package bp2build
import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
"android/soong/android"
"android/soong/starlark_fmt"
"github.com/google/blueprint/proptools"
)
@@ -19,79 +14,6 @@ type BazelFile struct {
Contents string
}
// createSoongInjectionDirFiles returns most of the files to write to the soong_injection directory.
// Some other files also come from CreateProductConfigFiles
func createSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) ([]BazelFile, error) {
cfg := ctx.Config()
var files []BazelFile
files = append(files, newFile("android", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package.
files = append(files, newFile("android", "constants.bzl", android.BazelCcToolchainVars(cfg)))
if buf, err := json.MarshalIndent(metrics.convertedModuleWithType, "", " "); err != nil {
return []BazelFile{}, err
} else {
files = append(files, newFile("metrics", "converted_modules.json", string(buf)))
}
convertedModulePathMap, err := json.MarshalIndent(metrics.convertedModulePathMap, "", "\t")
if err != nil {
panic(err)
}
files = append(files, newFile("metrics", GeneratedBuildFileName, "")) // Creates a //metrics package.
files = append(files, newFile("metrics", "converted_modules_path_map.json", string(convertedModulePathMap)))
files = append(files, newFile("metrics", "converted_modules_path_map.bzl", "modules = "+strings.ReplaceAll(string(convertedModulePathMap), "\\", "\\\\")))
files = append(files, newFile("product_config", "soong_config_variables.bzl", cfg.Bp2buildSoongConfigDefinitions.String()))
files = append(files, newFile("product_config", "arch_configuration.bzl", android.StarlarkArchConfigurations()))
apiLevelsMap, err := android.GetApiLevelsMap(cfg)
if err != nil {
return nil, err
}
apiLevelsContent, err := json.Marshal(apiLevelsMap)
if err != nil {
return nil, err
}
files = append(files, newFile("api_levels", GeneratedBuildFileName, `exports_files(["api_levels.json"])`))
// TODO(b/269691302) value of apiLevelsContent is product variable dependent and should be avoided for soong injection
files = append(files, newFile("api_levels", "api_levels.json", string(apiLevelsContent)))
files = append(files, newFile("api_levels", "platform_versions.bzl", platformVersionContents(cfg)))
files = append(files, newFile("allowlists", GeneratedBuildFileName, ""))
// TODO(b/262781701): Create an alternate soong_build entrypoint for writing out these files only when requested
files = append(files, newFile("allowlists", "mixed_build_prod_allowlist.txt", strings.Join(android.GetBazelEnabledModules(android.BazelProdMode), "\n")+"\n"))
files = append(files, newFile("allowlists", "mixed_build_staging_allowlist.txt", strings.Join(android.GetBazelEnabledModules(android.BazelStagingMode), "\n")+"\n"))
return files, nil
}
func platformVersionContents(cfg android.Config) string {
// Despite these coming from cfg.productVariables, they are actually hardcoded in global
// makefiles, not set in individual product config makesfiles, so they're safe to just export
// and load() directly.
platformVersionActiveCodenames := make([]string, 0, len(cfg.PlatformVersionActiveCodenames()))
for _, codename := range cfg.PlatformVersionActiveCodenames() {
platformVersionActiveCodenames = append(platformVersionActiveCodenames, fmt.Sprintf("%q", codename))
}
platformSdkVersion := "None"
if cfg.RawPlatformSdkVersion() != nil {
platformSdkVersion = strconv.Itoa(*cfg.RawPlatformSdkVersion())
}
return fmt.Sprintf(`
platform_versions = struct(
platform_sdk_final = %s,
platform_sdk_version = %s,
platform_sdk_codename = %q,
platform_version_active_codenames = [%s],
)
`, starlark_fmt.PrintBool(cfg.PlatformSdkFinal()), platformSdkVersion, cfg.PlatformSdkCodename(), strings.Join(platformVersionActiveCodenames, ", "))
}
func CreateBazelFiles(ruleShims map[string]RuleShim, buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile {
var files []BazelFile
@@ -125,20 +47,7 @@ func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode)
targets.sort()
var content string
if mode == Bp2Build {
content = `# READ THIS FIRST:
# This file was automatically generated by bp2build for the Bazel migration project.
# Feel free to edit or test it, but do *not* check it into your version control system.
`
content += targets.LoadStatements()
content += "\n\n"
// Get package rule from the handcrafted BUILD file, otherwise emit the default one.
prText := "package(default_visibility = [\"//visibility:public\"])\n"
if pr := targets.packageRule(); pr != nil {
prText = pr.content
}
content += prText
} else if mode == QueryView {
if mode == QueryView {
content = soongModuleLoad
}
if content != "" {
@@ -161,14 +70,6 @@ func newFile(dir, basename, content string) BazelFile {
const (
bazelRulesSubDir = "build/bazel/queryview_rules"
// additional files:
// * workspace file
// * base BUILD file
// * rules BUILD file
// * rules providers.bzl file
// * rules soong_module.bzl file
numAdditionalFiles = 5
)
var (

231
bp2build/metrics.go
View File

@@ -1,231 +0,0 @@
package bp2build
import (
"fmt"
"os"
"path/filepath"
"strings"
"android/soong/android"
"android/soong/shared"
"android/soong/ui/metrics/bp2build_metrics_proto"
"google.golang.org/protobuf/proto"
"github.com/google/blueprint"
)
type moduleInfo struct {
Name string `json:"name"`
Type string `json:"type"`
}
// CodegenMetrics represents information about the Blueprint-to-BUILD
// conversion process.
// Use CreateCodegenMetrics() to get a properly initialized instance
type CodegenMetrics struct {
serialized *bp2build_metrics_proto.Bp2BuildMetrics
// List of modules with unconverted deps
// NOTE: NOT in the .proto
moduleWithUnconvertedDepsMsgs []string
// List of modules with missing deps
// NOTE: NOT in the .proto
moduleWithMissingDepsMsgs []string
// Map of converted modules and paths to call
// NOTE: NOT in the .proto
convertedModulePathMap map[string]string
// Name and type of converted modules
convertedModuleWithType []moduleInfo
}
func CreateCodegenMetrics() CodegenMetrics {
return CodegenMetrics{
serialized: &bp2build_metrics_proto.Bp2BuildMetrics{
RuleClassCount: make(map[string]uint64),
ConvertedModuleTypeCount: make(map[string]uint64),
TotalModuleTypeCount: make(map[string]uint64),
UnconvertedModules: make(map[string]*bp2build_metrics_proto.UnconvertedReason),
},
convertedModulePathMap: make(map[string]string),
}
}
// Serialize returns the protoized version of CodegenMetrics: bp2build_metrics_proto.Bp2BuildMetrics
func (metrics *CodegenMetrics) Serialize() *bp2build_metrics_proto.Bp2BuildMetrics {
return metrics.serialized
}
// Print the codegen metrics to stdout.
func (metrics *CodegenMetrics) Print() {
generatedTargetCount := uint64(0)
for _, ruleClass := range android.SortedKeys(metrics.serialized.RuleClassCount) {
count := metrics.serialized.RuleClassCount[ruleClass]
fmt.Printf("[bp2build] %s: %d targets\n", ruleClass, count)
generatedTargetCount += count
}
fmt.Printf(
`[bp2build] Converted %d Android.bp modules to %d total generated BUILD targets. Included %d handcrafted BUILD targets. There are %d total Android.bp modules.
%d converted modules have unconverted deps:
%s
%d converted modules have missing deps:
%s
`,
metrics.serialized.GeneratedModuleCount,
generatedTargetCount,
metrics.serialized.HandCraftedModuleCount,
metrics.TotalModuleCount(),
len(metrics.moduleWithUnconvertedDepsMsgs),
strings.Join(metrics.moduleWithUnconvertedDepsMsgs, "\n\t"),
len(metrics.moduleWithMissingDepsMsgs),
strings.Join(metrics.moduleWithMissingDepsMsgs, "\n\t"),
)
}
const bp2buildMetricsFilename = "bp2build_metrics.pb"
// fail prints $PWD to stderr, followed by the given printf string and args (vals),
// then the given alert, and then exits with 1 for failure
func fail(err error, alertFmt string, vals ...interface{}) {
cwd, wderr := os.Getwd()
if wderr != nil {
cwd = "FAILED TO GET $PWD: " + wderr.Error()
}
fmt.Fprintf(os.Stderr, "\nIn "+cwd+":\n"+alertFmt+"\n"+err.Error()+"\n", vals...)
os.Exit(1)
}
// Write the bp2build-protoized codegen metrics into the given directory
func (metrics *CodegenMetrics) Write(dir string) {
if _, err := os.Stat(dir); os.IsNotExist(err) {
// The metrics dir doesn't already exist, so create it (and parents)
if err := os.MkdirAll(dir, 0755); err != nil { // rx for all; w for user
fail(err, "Failed to `mkdir -p` %s", dir)
}
} else if err != nil {
fail(err, "Failed to `stat` %s", dir)
}
metricsFile := filepath.Join(dir, bp2buildMetricsFilename)
if err := metrics.dump(metricsFile); err != nil {
fail(err, "Error outputting %s", metricsFile)
}
if _, err := os.Stat(metricsFile); err != nil {
if os.IsNotExist(err) {
fail(err, "MISSING BP2BUILD METRICS OUTPUT: %s", metricsFile)
} else {
fail(err, "FAILED TO `stat` BP2BUILD METRICS OUTPUT: %s", metricsFile)
}
}
}
// ReadCodegenMetrics loads CodegenMetrics from `dir`
// returns a nil pointer if the file doesn't exist
func ReadCodegenMetrics(dir string) *CodegenMetrics {
metricsFile := filepath.Join(dir, bp2buildMetricsFilename)
if _, err := os.Stat(metricsFile); err != nil {
if os.IsNotExist(err) {
return nil
} else {
fail(err, "FAILED TO `stat` BP2BUILD METRICS OUTPUT: %s", metricsFile)
panic("unreachable after fail")
}
}
if buf, err := os.ReadFile(metricsFile); err != nil {
fail(err, "FAILED TO READ BP2BUILD METRICS OUTPUT: %s", metricsFile)
panic("unreachable after fail")
} else {
bp2BuildMetrics := bp2build_metrics_proto.Bp2BuildMetrics{
RuleClassCount: make(map[string]uint64),
ConvertedModuleTypeCount: make(map[string]uint64),
TotalModuleTypeCount: make(map[string]uint64),
}
if err := proto.Unmarshal(buf, &bp2BuildMetrics); err != nil {
fail(err, "FAILED TO PARSE BP2BUILD METRICS OUTPUT: %s", metricsFile)
}
return &CodegenMetrics{
serialized: &bp2BuildMetrics,
convertedModulePathMap: make(map[string]string),
}
}
}
func (metrics *CodegenMetrics) IncrementRuleClassCount(ruleClass string) {
metrics.serialized.RuleClassCount[ruleClass] += 1
}
func (metrics *CodegenMetrics) AddEvent(event *bp2build_metrics_proto.Event) {
metrics.serialized.Events = append(metrics.serialized.Events, event)
}
func (metrics *CodegenMetrics) SetSymlinkCount(n uint64) {
if m := metrics.serialized.WorkspaceSymlinkCount; m != 0 {
fmt.Fprintf(os.Stderr, "unexpected non-zero workspaceSymlinkCount of %d", m)
}
metrics.serialized.WorkspaceSymlinkCount = n
}
func (metrics *CodegenMetrics) SetMkDirCount(n uint64) {
if m := metrics.serialized.WorkspaceMkDirCount; m != 0 {
fmt.Fprintf(os.Stderr, "unexpected non-zero workspaceDirCount of %d", m)
}
metrics.serialized.WorkspaceMkDirCount = n
}
func (metrics *CodegenMetrics) TotalModuleCount() uint64 {
return metrics.serialized.HandCraftedModuleCount +
metrics.serialized.GeneratedModuleCount +
metrics.serialized.UnconvertedModuleCount
}
// Dump serializes the metrics to the given filename
func (metrics *CodegenMetrics) dump(filename string) (err error) {
ser := metrics.Serialize()
return shared.Save(ser, filename)
}
type ConversionType int
const (
Generated ConversionType = iota
Handcrafted
)
func (metrics *CodegenMetrics) AddConvertedModule(m blueprint.Module, moduleType string, dir string) {
//a package module has empty name
if moduleType == "package" {
return
}
// Undo prebuilt_ module name prefix modifications
moduleName := android.RemoveOptionalPrebuiltPrefix(m.Name())
metrics.serialized.ConvertedModules = append(metrics.serialized.ConvertedModules, moduleName)
metrics.convertedModuleWithType = append(metrics.convertedModuleWithType, moduleInfo{
moduleName,
moduleType,
})
metrics.convertedModulePathMap[moduleName] = "//" + dir
metrics.serialized.ConvertedModuleTypeCount[moduleType] += 1
metrics.serialized.TotalModuleTypeCount[moduleType] += 1
metrics.serialized.GeneratedModuleCount += 1
}
func (metrics *CodegenMetrics) AddUnconvertedModule(m blueprint.Module, moduleType string, dir string,
reason android.UnconvertedReason) {
//a package module has empty name
if moduleType == "package" {
return
}
// Undo prebuilt_ module name prefix modifications
moduleName := android.RemoveOptionalPrebuiltPrefix(m.Name())
metrics.serialized.UnconvertedModules[moduleName] = &bp2build_metrics_proto.UnconvertedReason{
Type: bp2build_metrics_proto.UnconvertedReasonType(reason.ReasonType),
Detail: reason.Detail,
}
metrics.serialized.UnconvertedModuleCount += 1
metrics.serialized.TotalModuleTypeCount[moduleType] += 1
if reason.ReasonType == int(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE) {
metrics.serialized.HandCraftedModuleCount += 1
}
}

219
bp2build/performance_test.go
View File

@@ -1,219 +0,0 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
// to run the benchmarks in this file, you must run go test with the -bench.
// The benchmarked portion will run for the specified time (can be set via -benchtime)
// This can mean if you are benchmarking a faster portion of a larger operation, it will take
// longer.
// If you are seeing a small number of iterations for a specific run, the data is less reliable, to
// run for longer, set -benchtime to a larger value.
import (
"fmt"
"math"
"strings"
"testing"
"android/soong/android"
)
const (
performance_test_dir = "."
)
func genCustomModule(i int, convert bool) string {
var conversionString string
if convert {
conversionString = `bazel_module: { bp2build_available: true },`
}
return fmt.Sprintf(`
custom {
name: "arch_paths_%[1]d",
string_list_prop: ["\t", "\n"],
string_prop: "a\t\n\r",
arch_paths: ["outer", ":outer_dep_%[1]d"],
arch: {
x86: {
arch_paths: ["abc", ":x86_dep_%[1]d"],
},
x86_64: {
arch_paths: ["64bit"],
arch_paths_exclude: ["outer"],
},
},
%[2]s
}
custom {
name: "outer_dep_%[1]d",
%[2]s
}
custom {
name: "x86_dep_%[1]d",
%[2]s
}
`, i, conversionString)
}
func genCustomModuleBp(pctConverted float64) string {
modules := 100
bp := make([]string, 0, modules)
toConvert := int(math.Round(float64(modules) * pctConverted))
for i := 0; i < modules; i++ {
bp = append(bp, genCustomModule(i, i < toConvert))
}
return strings.Join(bp, "\n\n")
}
type testConfig struct {
config android.Config
ctx *android.TestContext
codegenCtx *CodegenContext
}
func (tc testConfig) parse() []error {
_, errs := tc.ctx.ParseFileList(performance_test_dir, []string{"Android.bp"})
return errs
}
func (tc testConfig) resolveDependencies() []error {
_, errs := tc.ctx.ResolveDependencies(tc.config)
return errs
}
func (tc testConfig) convert() {
generateBazelTargetsForDir(tc.codegenCtx, performance_test_dir)
}
func setup(builddir string, tcSize float64) testConfig {
config := android.TestConfig(buildDir, nil, genCustomModuleBp(tcSize), nil)
ctx := android.NewTestContext(config)
registerCustomModuleForBp2buildConversion(ctx)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
return testConfig{
config,
ctx,
codegenCtx,
}
}
var pctToConvert = []float64{0.0, 0.01, 0.05, 0.10, 0.25, 0.5, 0.75, 1.0}
// This is not intended to test performance, but to verify performance infra continues to work
func TestConvertManyModulesFull(t *testing.T) {
for _, tcSize := range pctToConvert {
t.Run(fmt.Sprintf("pctConverted %f", tcSize), func(t *testing.T) {
testConfig := setup(buildDir, tcSize)
errs := testConfig.parse()
if len(errs) > 0 {
t.Fatalf("Unexpected errors: %s", errs)
}
errs = testConfig.resolveDependencies()
if len(errs) > 0 {
t.Fatalf("Unexpected errors: %s", errs)
}
testConfig.convert()
})
}
}
func BenchmarkManyModulesFull(b *testing.B) {
for _, tcSize := range pctToConvert {
b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) {
for n := 0; n < b.N; n++ {
b.StopTimer()
testConfig := setup(buildDir, tcSize)
b.StartTimer()
errs := testConfig.parse()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
errs = testConfig.resolveDependencies()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
testConfig.convert()
b.StopTimer()
}
})
}
}
func BenchmarkManyModulesResolveDependencies(b *testing.B) {
for _, tcSize := range pctToConvert {
b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) {
for n := 0; n < b.N; n++ {
b.StopTimer()
// setup we don't want to measure
testConfig := setup(buildDir, tcSize)
errs := testConfig.parse()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
b.StartTimer()
errs = testConfig.resolveDependencies()
b.StopTimer()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
testConfig.convert()
}
})
}
}
func BenchmarkManyModulesGenerateBazelTargetsForDir(b *testing.B) {
for _, tcSize := range pctToConvert {
b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) {
for n := 0; n < b.N; n++ {
b.StopTimer()
// setup we don't want to measure
testConfig := setup(buildDir, tcSize)
errs := testConfig.parse()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
errs = testConfig.resolveDependencies()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
b.StartTimer()
testConfig.convert()
b.StopTimer()
}
})
}
}

511
bp2build/symlink_forest.go
View File

@@ -1,511 +0,0 @@
// Copyright 2022 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"sync"
"sync/atomic"
"android/soong/shared"
"github.com/google/blueprint/pathtools"
)
// A tree structure that describes what to do at each directory in the created
// symlink tree. Currently, it is used to enumerate which files/directories
// should be excluded from symlinking. Each instance of "node" represents a file
// or a directory. If excluded is true, then that file/directory should be
// excluded from symlinking. Otherwise, the node is not excluded, but one of its
// descendants is (otherwise the node in question would not exist)
type instructionsNode struct {
name string
excluded bool // If false, this is just an intermediate node
children map[string]*instructionsNode
}
type symlinkForestContext struct {
verbose bool
topdir string // $TOPDIR
// State
wg sync.WaitGroup
depCh chan string
mkdirCount atomic.Uint64
symlinkCount atomic.Uint64
}
// Ensures that the node for the given path exists in the tree and returns it.
func ensureNodeExists(root *instructionsNode, path string) *instructionsNode {
if path == "" {
return root
}
if path[len(path)-1] == '/' {
path = path[:len(path)-1] // filepath.Split() leaves a trailing slash
}
dir, base := filepath.Split(path)
// First compute the parent node...
dn := ensureNodeExists(root, dir)
// then create the requested node as its direct child, if needed.
if child, ok := dn.children[base]; ok {
return child
} else {
dn.children[base] = &instructionsNode{base, false, make(map[string]*instructionsNode)}
return dn.children[base]
}
}
// Turns a list of paths to be excluded into a tree
func instructionsFromExcludePathList(paths []string) *instructionsNode {
result := &instructionsNode{"", false, make(map[string]*instructionsNode)}
for _, p := range paths {
ensureNodeExists(result, p).excluded = true
}
return result
}
func mergeBuildFiles(output string, srcBuildFile string, generatedBuildFile string, verbose bool) error {
srcBuildFileContent, err := os.ReadFile(srcBuildFile)
if err != nil {
return err
}
generatedBuildFileContent, err := os.ReadFile(generatedBuildFile)
if err != nil {
return err
}
// There can't be a package() call in both the source and generated BUILD files.
// bp2build will generate a package() call for licensing information, but if
// there's no licensing information, it will still generate a package() call
// that just sets default_visibility=public. If the handcrafted build file
// also has a package() call, we'll allow it to override the bp2build
// generated one if it doesn't have any licensing information. If the bp2build
// one has licensing information and the handcrafted one exists, we'll leave
// them both in for bazel to throw an error.
packageRegex := regexp.MustCompile(`(?m)^package\s*\(`)
packageDefaultVisibilityRegex := regexp.MustCompile(`(?m)^package\s*\(\s*default_visibility\s*=\s*\[\s*"//visibility:public",?\s*]\s*\)`)
if packageRegex.Find(srcBuildFileContent) != nil {
if verbose && packageDefaultVisibilityRegex.Find(generatedBuildFileContent) != nil {
fmt.Fprintf(os.Stderr, "Both '%s' and '%s' have a package() target, removing the first one\n",
generatedBuildFile, srcBuildFile)
}
generatedBuildFileContent = packageDefaultVisibilityRegex.ReplaceAll(generatedBuildFileContent, []byte{})
}
newContents := generatedBuildFileContent
if newContents[len(newContents)-1] != '\n' {
newContents = append(newContents, '\n')
}
newContents = append(newContents, srcBuildFileContent...)
// Say you run bp2build 4 times:
// - The first time there's only an Android.bp file. bp2build will convert it to a build file
// under out/soong/bp2build, then symlink from the forest to that generated file
// - Then you add a handcrafted BUILD file in the same directory. bp2build will merge this with
// the generated one, and write the result to the output file in the forest. But the output
// file was a symlink to out/soong/bp2build from the previous step! So we erroneously update
// the file in out/soong/bp2build instead. So far this doesn't cause any problems...
// - You run a 3rd bp2build with no relevant changes. Everything continues to work.
// - You then add a comment to the handcrafted BUILD file. This causes a merge with the
// generated file again. But since we wrote to the generated file in step 2, the generated
// file has an old copy of the handcrafted file in it! This probably causes duplicate bazel
// targets.
// To solve this, if we see that the output file is a symlink from a previous build, remove it.
stat, err := os.Lstat(output)
if err != nil && !os.IsNotExist(err) {
return err
} else if err == nil {
if stat.Mode()&os.ModeSymlink == os.ModeSymlink {
if verbose {
fmt.Fprintf(os.Stderr, "Removing symlink so that we can replace it with a merged file: %s\n", output)
}
err = os.Remove(output)
if err != nil {
return err
}
}
}
return pathtools.WriteFileIfChanged(output, newContents, 0666)
}
// Calls readdir() and returns it as a map from the basename of the files in dir
// to os.FileInfo.
func readdirToMap(dir string) map[string]os.FileInfo {
entryList, err := ioutil.ReadDir(dir)
result := make(map[string]os.FileInfo)
if err != nil {
if os.IsNotExist(err) {
// It's okay if a directory doesn't exist; it just means that one of the
// trees to be merged contains parts the other doesn't
return result
} else {
fmt.Fprintf(os.Stderr, "Cannot readdir '%s': %s\n", dir, err)
os.Exit(1)
}
}
for _, fi := range entryList {
result[fi.Name()] = fi
}
return result
}
// Creates a symbolic link at dst pointing to src
func symlinkIntoForest(topdir, dst, src string) uint64 {
srcPath := shared.JoinPath(topdir, src)
dstPath := shared.JoinPath(topdir, dst)
// Check whether a symlink already exists.
if dstInfo, err := os.Lstat(dstPath); err != nil {
if !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Failed to lstat '%s': %s", dst, err)
os.Exit(1)
}
} else {
if dstInfo.Mode()&os.ModeSymlink != 0 {
// Assume that the link's target is correct, i.e. no manual tampering.
// E.g. OUT_DIR could have been previously used with a different source tree check-out!
return 0
} else {
if err := os.RemoveAll(dstPath); err != nil {
fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", dst, err)
os.Exit(1)
}
}
}
// Create symlink.
if err := os.Symlink(srcPath, dstPath); err != nil {
fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err)
os.Exit(1)
}
return 1
}
func isDir(path string, fi os.FileInfo) bool {
if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
return fi.IsDir()
}
fi2, statErr := os.Stat(path)
if statErr == nil {
return fi2.IsDir()
}
// Check if this is a dangling symlink. If so, treat it like a file, not a dir.
_, lstatErr := os.Lstat(path)
if lstatErr != nil {
fmt.Fprintf(os.Stderr, "Cannot stat or lstat '%s': %s\n%s\n", path, statErr, lstatErr)
os.Exit(1)
}
return false
}
// Returns the mtime of the soong_build binary to determine whether we should
// force symlink_forest to re-execute
func getSoongBuildMTime() (int64, error) {
binaryPath, err := os.Executable()
if err != nil {
return 0, err
}
info, err := os.Stat(binaryPath)
if err != nil {
return 0, err
}
return info.ModTime().UnixMilli(), nil
}
// cleanSymlinkForest will remove the whole symlink forest directory
func cleanSymlinkForest(topdir, forest string) error {
return os.RemoveAll(shared.JoinPath(topdir, forest))
}
// This returns whether symlink forest should clean and replant symlinks.
// It compares the mtime of this executable with the mtime of the last-run
// soong_build binary. If they differ, then we should clean and replant.
func shouldCleanSymlinkForest(topdir string, forest string, soongBuildMTime int64) (bool, error) {
mtimeFilePath := shared.JoinPath(topdir, forest, "soong_build_mtime")
mtimeFileContents, err := os.ReadFile(mtimeFilePath)
if err != nil {
if os.IsNotExist(err) {
// This is likely the first time this has run with this functionality - clean away!
return true, nil
} else {
return false, err
}
}
return strconv.FormatInt(soongBuildMTime, 10) != string(mtimeFileContents), nil
}
func writeSoongBuildMTimeFile(topdir, forest string, mtime int64) error {
mtimeFilePath := shared.JoinPath(topdir, forest, "soong_build_mtime")
contents := []byte(strconv.FormatInt(mtime, 10))
return os.WriteFile(mtimeFilePath, contents, 0666)
}
// Recursively plants a symlink forest at forestDir. The symlink tree will
// contain every file in buildFilesDir and srcDir excluding the files in
// instructions. Collects every directory encountered during the traversal of
// srcDir .
func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *instructionsNode, forestDir string, buildFilesDir string, srcDir string) {
defer context.wg.Done()
if instructions != nil && instructions.excluded {
// Excluded paths are skipped at the level of the non-excluded parent.
fmt.Fprintf(os.Stderr, "may not specify a root-level exclude directory '%s'", srcDir)
os.Exit(1)
}
// We don't add buildFilesDir here because the bp2build files marker files is
// already a dependency which covers it. If we ever wanted to turn this into
// a generic symlink forest creation tool, we'd need to add it, too.
context.depCh <- srcDir
srcDirMap := readdirToMap(shared.JoinPath(context.topdir, srcDir))
buildFilesMap := readdirToMap(shared.JoinPath(context.topdir, buildFilesDir))
renamingBuildFile := false
if _, ok := srcDirMap["BUILD"]; ok {
if _, ok := srcDirMap["BUILD.bazel"]; !ok {
if _, ok := buildFilesMap["BUILD.bazel"]; ok {
renamingBuildFile = true
srcDirMap["BUILD.bazel"] = srcDirMap["BUILD"]
delete(srcDirMap, "BUILD")
if instructions != nil {
if _, ok := instructions.children["BUILD"]; ok {
instructions.children["BUILD.bazel"] = instructions.children["BUILD"]
delete(instructions.children, "BUILD")
}
}
}
}
}
allEntries := make([]string, 0, len(srcDirMap)+len(buildFilesMap))
for n := range srcDirMap {
allEntries = append(allEntries, n)
}
for n := range buildFilesMap {
if _, ok := srcDirMap[n]; !ok {
allEntries = append(allEntries, n)
}
}
// Tests read the error messages generated, so ensure their order is deterministic
sort.Strings(allEntries)
fullForestPath := shared.JoinPath(context.topdir, forestDir)
createForestDir := false
if fi, err := os.Lstat(fullForestPath); err != nil {
if os.IsNotExist(err) {
createForestDir = true
} else {
fmt.Fprintf(os.Stderr, "Could not read info for '%s': %s\n", forestDir, err)
}
} else if fi.Mode()&os.ModeDir == 0 {
if err := os.RemoveAll(fullForestPath); err != nil {
fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", forestDir, err)
os.Exit(1)
}
createForestDir = true
}
if createForestDir {
if err := os.MkdirAll(fullForestPath, 0777); err != nil {
fmt.Fprintf(os.Stderr, "Could not mkdir '%s': %s\n", forestDir, err)
os.Exit(1)
}
context.mkdirCount.Add(1)
}
// Start with a list of items that already exist in the forest, and remove
// each element as it is processed in allEntries. Any remaining items in
// forestMapForDeletion must be removed. (This handles files which were
// removed since the previous forest generation).
forestMapForDeletion := readdirToMap(shared.JoinPath(context.topdir, forestDir))
for _, f := range allEntries {
if f[0] == '.' {
continue // Ignore dotfiles
}
delete(forestMapForDeletion, f)
// todo add deletionCount metric
// The full paths of children in the input trees and in the output tree
forestChild := shared.JoinPath(forestDir, f)
srcChild := shared.JoinPath(srcDir, f)
if f == "BUILD.bazel" && renamingBuildFile {
srcChild = shared.JoinPath(srcDir, "BUILD")
}
buildFilesChild := shared.JoinPath(buildFilesDir, f)
// Descend in the instruction tree if it exists
var instructionsChild *instructionsNode
if instructions != nil {
instructionsChild = instructions.children[f]
}
srcChildEntry, sExists := srcDirMap[f]
buildFilesChildEntry, bExists := buildFilesMap[f]
if instructionsChild != nil && instructionsChild.excluded {
if bExists {
context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild))
}
continue
}
sDir := sExists && isDir(shared.JoinPath(context.topdir, srcChild), srcChildEntry)
bDir := bExists && isDir(shared.JoinPath(context.topdir, buildFilesChild), buildFilesChildEntry)
if !sExists {
if bDir && instructionsChild != nil {
// Not in the source tree, but we have to exclude something from under
// this subtree, so descend
context.wg.Add(1)
go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
} else {
// Not in the source tree, symlink BUILD file
context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild))
}
} else if !bExists {
if sDir && instructionsChild != nil {
// Not in the build file tree, but we have to exclude something from
// under this subtree, so descend
context.wg.Add(1)
go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
} else {
// Not in the build file tree, symlink source tree, carry on
context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, srcChild))
}
} else if sDir && bDir {
// Both are directories. Descend.
context.wg.Add(1)
go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
} else if !sDir && !bDir {
// Neither is a directory. Merge them.
srcBuildFile := shared.JoinPath(context.topdir, srcChild)
generatedBuildFile := shared.JoinPath(context.topdir, buildFilesChild)
// The Android.bp file that codegen used to produce `buildFilesChild` is
// already a dependency, we can ignore `buildFilesChild`.
context.depCh <- srcChild
if err := mergeBuildFiles(shared.JoinPath(context.topdir, forestChild), srcBuildFile, generatedBuildFile, context.verbose); err != nil {
fmt.Fprintf(os.Stderr, "Error merging %s and %s: %s",
srcBuildFile, generatedBuildFile, err)
os.Exit(1)
}
} else {
// Both exist and one is a file. This is an error.
fmt.Fprintf(os.Stderr,
"Conflict in workspace symlink tree creation: both '%s' and '%s' exist and exactly one is a directory\n",
srcChild, buildFilesChild)
os.Exit(1)
}
}
// Remove all files in the forest that exist in neither the source
// tree nor the build files tree. (This handles files which were removed
// since the previous forest generation).
for f := range forestMapForDeletion {
var instructionsChild *instructionsNode
if instructions != nil {
instructionsChild = instructions.children[f]
}
if instructionsChild != nil && instructionsChild.excluded {
// This directory may be excluded because bazel writes to it under the
// forest root. Thus this path is intentionally left alone.
continue
}
forestChild := shared.JoinPath(context.topdir, forestDir, f)
if err := os.RemoveAll(forestChild); err != nil {
fmt.Fprintf(os.Stderr, "Failed to remove '%s/%s': %s", forestDir, f, err)
os.Exit(1)
}
}
}
// PlantSymlinkForest Creates a symlink forest by merging the directory tree at "buildFiles" and
// "srcDir" while excluding paths listed in "exclude". Returns the set of paths
// under srcDir on which readdir() had to be called to produce the symlink
// forest.
func PlantSymlinkForest(verbose bool, topdir string, forest string, buildFiles string, exclude []string) (deps []string, mkdirCount, symlinkCount uint64) {
context := &symlinkForestContext{
verbose: verbose,
topdir: topdir,
depCh: make(chan string),
mkdirCount: atomic.Uint64{},
symlinkCount: atomic.Uint64{},
}
// Check whether soong_build has been modified since the last run
soongBuildMTime, err := getSoongBuildMTime()
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
shouldClean, err := shouldCleanSymlinkForest(topdir, forest, soongBuildMTime)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
} else if shouldClean {
err = cleanSymlinkForest(topdir, forest)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
instructions := instructionsFromExcludePathList(exclude)
go func() {
context.wg.Add(1)
plantSymlinkForestRecursive(context, instructions, forest, buildFiles, ".")
context.wg.Wait()
close(context.depCh)
}()
for dep := range context.depCh {
deps = append(deps, dep)
}
err = writeSoongBuildMTimeFile(topdir, forest, soongBuildMTime)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
return deps, context.mkdirCount.Load(), context.symlinkCount.Load()
}

785
bp2build/testing.go
View File

@@ -1,785 +0,0 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
/*
For shareable/common bp2build testing functionality and dumping ground for
specific-but-shared functionality among tests in package
*/
import (
"fmt"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
"android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint/proptools"
"android/soong/android"
"android/soong/android/allowlists"
"android/soong/bazel"
)
var (
buildDir string
)
var labelRegex = regexp.MustCompile(`^//([^: ]+):([^ ]+)$`)
var simpleModuleNameRegex = regexp.MustCompile(`^[^: /]+$`)
func checkError(t *testing.T, errs []error, expectedErr error) bool {
t.Helper()
if len(errs) != 1 {
return false
}
if strings.Contains(errs[0].Error(), expectedErr.Error()) {
return true
}
return false
}
func errored(t *testing.T, tc Bp2buildTestCase, errs []error) bool {
t.Helper()
if tc.ExpectedErr != nil {
// Rely on checkErrors, as this test case is expected to have an error.
return false
}
if len(errs) > 0 {
for _, err := range errs {
t.Errorf("%s: %s", tc.Description, err)
}
return true
}
// All good, continue execution.
return false
}
func RunBp2BuildTestCaseSimple(t *testing.T, tc Bp2buildTestCase) {
t.Helper()
RunBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, tc)
}
type Bp2buildTestCase struct {
Description string
ModuleTypeUnderTest string
ModuleTypeUnderTestFactory android.ModuleFactory
// Text to add to the toplevel, root Android.bp file. If Dir is not set, all
// ExpectedBazelTargets are assumed to be generated by this file.
Blueprint string
// ExpectedBazelTargets compares the BazelTargets generated in `Dir` (if not empty).
// Otherwise, it checks the BazelTargets generated by `Blueprint` in the root directory.
ExpectedBazelTargets []string
// ExpectedConvertedModules asserts that modules in this list are labeled as "converted
// by bp2build" in the metrics reported by bp2build.
ExpectedConvertedModules []string
// ExpectedHandcraftedModules asserts that modules in this list are labeled as "handcrafted
// in build files" in the metrics reported by bp2build. Such modules are either explicitly
// defined in a BUILD file (by name), or registered as "otherwise implicitly handled"
// by bp2build (for example, by macros owned by other modules).
ExpectedHandcraftedModules []string
// AlreadyExistingBuildContents, if non-empty, simulates an already-present source BUILD file
// in the directory under test. The BUILD file has the given contents. This BUILD file
// will also be treated as "BUILD file to keep" by the simulated bp2build environment.
AlreadyExistingBuildContents string
// StubbedBuildDefinitions, if non-empty, adds stub definitions to already-present source
// BUILD files for each bazel label given. The BUILD files with these stub definitions
// are added to the BUILD file given in AlreadyExistingBuildContents.
// Labels may be of the form //pkg/to:target_name (which would be defined in pkg/to/BUILD.bazel)
// or `target_name` (which would be defined in ./BUILD.bazel).
StubbedBuildDefinitions []string
Filesystem map[string]string
// Dir sets the directory which will be compared against the targets in ExpectedBazelTargets.
// This should used in conjunction with the Filesystem property to check for targets
// generated from a directory that is not the root.
// If not set, all ExpectedBazelTargets are assumed to be generated by the text in the
// Blueprint property.
Dir string
// An error with a string contained within the string of the expected error
ExpectedErr error
UnconvertedDepsMode unconvertedDepsMode
// For every directory listed here, the BUILD file for that directory will
// be merged with the generated BUILD file. This allows custom BUILD targets
// to be used in tests, or use BUILD files to draw package boundaries.
KeepBuildFileForDirs []string
// An extra FixturePreparer to use when running the test. If you need multiple extra
// FixturePreparers, use android.GroupFixturePreparers()
ExtraFixturePreparer android.FixturePreparer
// If bp2build_product_config.go should run as part of the test.
RunBp2buildProductConfig bool
}
func RunBp2BuildTestCase(t *testing.T, registerModuleTypes func(ctx android.RegistrationContext), tc Bp2buildTestCase) {
t.Helper()
preparers := []android.FixturePreparer{
android.FixtureRegisterWithContext(registerModuleTypes),
}
if tc.ExtraFixturePreparer != nil {
preparers = append(preparers, tc.ExtraFixturePreparer)
}
preparers = append(preparers, android.FixtureSetTestRunner(&bazelTestRunner{generateProductConfigTargets: tc.RunBp2buildProductConfig}))
bp2buildSetup := android.GroupFixturePreparers(
preparers...,
)
runBp2BuildTestCaseWithSetup(t, bp2buildSetup, tc)
}
func runBp2BuildTestCaseWithSetup(t *testing.T, extraPreparer android.FixturePreparer, tc Bp2buildTestCase) {
t.Helper()
if tc.Filesystem == nil {
tc.Filesystem = map[string]string{}
}
checkDir := "."
if tc.Dir != "" {
checkDir = tc.Dir
}
keepExistingBuildDirs := tc.KeepBuildFileForDirs
buildFilesToParse := []string{}
if len(tc.StubbedBuildDefinitions) > 0 {
for _, buildDef := range tc.StubbedBuildDefinitions {
globalLabelMatch := labelRegex.FindStringSubmatch(buildDef)
var dir, targetName string
if len(globalLabelMatch) > 0 {
dir = globalLabelMatch[1]
targetName = globalLabelMatch[2]
} else {
if !simpleModuleNameRegex.MatchString(buildDef) {
t.Errorf("Stubbed build definition '%s' must be either a simple module name or of global target syntax (//foo/bar:baz).", buildDef)
return
}
dir = "."
targetName = buildDef
}
buildFilePath := filepath.Join(dir, "BUILD")
tc.Filesystem[buildFilePath] +=
MakeBazelTarget(
"bp2build_test_stub",
targetName,
AttrNameToString{})
keepExistingBuildDirs = append(keepExistingBuildDirs, dir)
buildFilesToParse = append(buildFilesToParse, buildFilePath)
}
}
if len(tc.AlreadyExistingBuildContents) > 0 {
buildFilePath := filepath.Join(checkDir, "BUILD")
tc.Filesystem[buildFilePath] += tc.AlreadyExistingBuildContents
keepExistingBuildDirs = append(keepExistingBuildDirs, checkDir)
buildFilesToParse = append(buildFilesToParse, buildFilePath)
}
filesystem := make(map[string][]byte)
for f, content := range tc.Filesystem {
filesystem[f] = []byte(content)
}
preparers := []android.FixturePreparer{
extraPreparer,
android.FixtureMergeMockFs(filesystem),
android.FixtureWithRootAndroidBp(tc.Blueprint),
android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) {
ctx.RegisterModuleType(tc.ModuleTypeUnderTest, tc.ModuleTypeUnderTestFactory)
}),
android.FixtureModifyContextWithMockFs(func(ctx *android.TestContext) {
// A default configuration for tests to not have to specify bp2build_available on top level
// targets.
bp2buildConfig := android.NewBp2BuildAllowlist().SetDefaultConfig(
allowlists.Bp2BuildConfig{
android.Bp2BuildTopLevel: allowlists.Bp2BuildDefaultTrueRecursively,
},
)
for _, f := range keepExistingBuildDirs {
bp2buildConfig.SetKeepExistingBuildFile(map[string]bool{
f: /*recursive=*/ false,
})
}
ctx.RegisterBp2BuildConfig(bp2buildConfig)
// This setting is added to bp2build invocations. It prevents bp2build
// from cloning modules to their original state after mutators run. This
// would lose some data intentionally set by these mutators.
ctx.SkipCloneModulesAfterMutators = true
err := ctx.RegisterExistingBazelTargets(".", buildFilesToParse)
if err != nil {
t.Errorf("error parsing build files in test setup: %s", err)
}
}),
android.FixtureModifyEnv(func(env map[string]string) {
if tc.UnconvertedDepsMode == errorModulesUnconvertedDeps {
env["BP2BUILD_ERROR_UNCONVERTED"] = "true"
}
}),
}
preparer := android.GroupFixturePreparers(preparers...)
if tc.ExpectedErr != nil {
pattern := "\\Q" + tc.ExpectedErr.Error() + "\\E"
preparer = preparer.ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern(pattern))
}
result := preparer.RunTestWithCustomResult(t).(*BazelTestResult)
if len(result.Errs) > 0 {
return
}
expectedTargets := map[string][]string{
checkDir: tc.ExpectedBazelTargets,
}
result.CompareAllBazelTargets(t, tc, expectedTargets, true)
}
// bazelTestRunner customizes the test fixture mechanism to run tests of the bp2build build mode.
type bazelTestRunner struct {
generateProductConfigTargets bool
}
func (b *bazelTestRunner) FinalPreparer(result *android.TestResult) android.CustomTestResult {
ctx := result.TestContext
ctx.RegisterForBazelConversion()
return &BazelTestResult{TestResult: result}
}
func (b *bazelTestRunner) PostParseProcessor(result android.CustomTestResult) {
bazelResult := result.(*BazelTestResult)
ctx := bazelResult.TestContext
config := bazelResult.Config
_, errs := ctx.ResolveDependencies(config)
if bazelResult.CollateErrs(errs) {
return
}
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
res, errs := GenerateBazelTargets(codegenCtx, false)
if bazelResult.CollateErrs(errs) {
return
}
if b.generateProductConfigTargets {
productConfig, err := createProductConfigFiles(codegenCtx, res.moduleNameToPartition, res.metrics.convertedModulePathMap)
if err != nil {
bazelResult.CollateErrs([]error{err})
return
}
for k, v := range productConfig.bp2buildTargets {
res.buildFileToTargets[k] = append(res.buildFileToTargets[k], v...)
}
}
// Store additional data for access by tests.
bazelResult.conversionResults = res
}
// BazelTestResult is a wrapper around android.TestResult to provide type safe access to the bazel
// specific data stored by the bazelTestRunner.
type BazelTestResult struct {
*android.TestResult
// The result returned by the GenerateBazelTargets function.
conversionResults
}
// CompareAllBazelTargets compares the BazelTargets produced by the test for all the directories
// with the supplied set of expected targets.
//
// If ignoreUnexpected=false then this enforces an exact match where every BazelTarget produced must
// have a corresponding expected BazelTarget.
//
// If ignoreUnexpected=true then it will ignore directories for which there are no expected targets.
func (b BazelTestResult) CompareAllBazelTargets(t *testing.T, tc Bp2buildTestCase, expectedTargets map[string][]string, ignoreUnexpected bool) {
t.Helper()
actualTargets := b.buildFileToTargets
// Generate the sorted set of directories to check.
dirsToCheck := android.SortedKeys(expectedTargets)
if !ignoreUnexpected {
// This needs to perform an exact match so add the directories in which targets were
// produced to the list of directories to check.
dirsToCheck = append(dirsToCheck, android.SortedKeys(actualTargets)...)
dirsToCheck = android.SortedUniqueStrings(dirsToCheck)
}
for _, dir := range dirsToCheck {
expected := expectedTargets[dir]
actual := actualTargets[dir]
if expected == nil {
if actual != nil {
t.Errorf("did not expect any bazel modules in %q but found %d", dir, len(actual))
}
} else if actual == nil {
expectedCount := len(expected)
if expectedCount > 0 {
t.Errorf("expected %d bazel modules in %q but did not find any", expectedCount, dir)
}
} else {
b.CompareBazelTargets(t, tc.Description, expected, actual)
}
}
for _, module := range tc.ExpectedConvertedModules {
if _, found := b.metrics.convertedModulePathMap[module]; !found {
t.Errorf("expected %s to be generated by bp2build, but was not. Map of converted modules: %s", module, b.metrics.convertedModulePathMap)
}
}
for _, module := range tc.ExpectedHandcraftedModules {
if reason, found := b.metrics.serialized.UnconvertedModules[module]; !found {
t.Errorf("expected %s to be marked 'unconverted' by bp2build, but was not found. Full list: %s",
module, b.metrics.serialized.UnconvertedModules)
} else {
if reason.Type != bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE {
t.Errorf("expected %s to be marked 'handcrafted' by bp2build, but was disabled for another reason: %s", module, reason)
}
}
}
}
func (b BazelTestResult) CompareBazelTargets(t *testing.T, description string, expectedContents []string, actualTargets BazelTargets) {
t.Helper()
if actualCount, expectedCount := len(actualTargets), len(expectedContents); actualCount != expectedCount {
t.Errorf("%s: Expected %d bazel target (%s), got %d (%s)",
description, expectedCount, expectedContents, actualCount, actualTargets)
} else {
sort.SliceStable(actualTargets, func(i, j int) bool {
return actualTargets[i].name < actualTargets[j].name
})
sort.SliceStable(expectedContents, func(i, j int) bool {
return getTargetName(expectedContents[i]) < getTargetName(expectedContents[j])
})
for i, actualTarget := range actualTargets {
if w, g := expectedContents[i], actualTarget.content; w != g {
t.Errorf(
"%s[%d]: Expected generated Bazel target to be `%s`, got `%s`",
description, i, w, g)
}
}
}
}
type nestedProps struct {
Nested_prop *string
}
type EmbeddedProps struct {
Embedded_prop *string
}
type OtherEmbeddedProps struct {
Other_embedded_prop *string
}
type customProps struct {
EmbeddedProps
*OtherEmbeddedProps
Bool_prop bool
Bool_ptr_prop *bool
// Ensure that properties tagged `blueprint:mutated` are omitted
Int_prop int `blueprint:"mutated"`
Int64_ptr_prop *int64
String_prop string
String_literal_prop *string `android:"arch_variant"`
String_ptr_prop *string
String_list_prop []string
Nested_props nestedProps
Nested_props_ptr *nestedProps
Arch_paths []string `android:"path,arch_variant"`
Arch_paths_exclude []string `android:"path,arch_variant"`
// Prop used to indicate this conversion should be 1 module -> multiple targets
One_to_many_prop *bool
// Prop used to simulate an unsupported property in bp2build conversion. If this
// is true, this module should be treated as "unconvertible" via bp2build.
Does_not_convert_to_bazel *bool
Api *string // File describing the APIs of this module
Test_config_setting *bool // Used to test generation of config_setting targets
Dir *string // Dir in which the Bazel Target will be created
}
type customModule struct {
android.ModuleBase
android.BazelModuleBase
props customProps
}
// OutputFiles is needed because some instances of this module use dist with a
// tag property which requires the module implements OutputFileProducer.
func (m *customModule) OutputFiles(tag string) (android.Paths, error) {
return android.PathsForTesting("path" + tag), nil
}
func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// nothing for now.
}
func customModuleFactoryBase() android.Module {
module := &customModule{}
module.AddProperties(&module.props)
android.InitBazelModule(module)
return module
}
func customModuleFactoryHostAndDevice() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibBoth)
return m
}
func customModuleFactoryDeviceSupported() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.DeviceSupported, android.MultilibBoth)
return m
}
func customModuleFactoryHostSupported() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.HostSupported, android.MultilibBoth)
return m
}
func customModuleFactoryHostAndDeviceDefault() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.HostAndDeviceDefault, android.MultilibBoth)
return m
}
func customModuleFactoryNeitherHostNorDeviceSupported() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.NeitherHostNorDeviceSupported, android.MultilibBoth)
return m
}
type testProps struct {
Test_prop struct {
Test_string_prop string
}
}
type customTestModule struct {
android.ModuleBase
props customProps
test_props testProps
}
func (m *customTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// nothing for now.
}
func customTestModuleFactoryBase() android.Module {
m := &customTestModule{}
m.AddProperties(&m.props)
m.AddProperties(&m.test_props)
return m
}
func customTestModuleFactory() android.Module {
m := customTestModuleFactoryBase()
android.InitAndroidModule(m)
return m
}
type customDefaultsModule struct {
android.ModuleBase
android.DefaultsModuleBase
}
func customDefaultsModuleFactoryBase() android.DefaultsModule {
module := &customDefaultsModule{}
module.AddProperties(&customProps{})
return module
}
func customDefaultsModuleFactoryBasic() android.Module {
return customDefaultsModuleFactoryBase()
}
func customDefaultsModuleFactory() android.Module {
m := customDefaultsModuleFactoryBase()
android.InitDefaultsModule(m)
return m
}
type EmbeddedAttr struct {
Embedded_attr *string
}
type OtherEmbeddedAttr struct {
Other_embedded_attr *string
}
type customBazelModuleAttributes struct {
EmbeddedAttr
*OtherEmbeddedAttr
String_literal_prop bazel.StringAttribute
String_ptr_prop *string
String_list_prop []string
Arch_paths bazel.LabelListAttribute
Api bazel.LabelAttribute
}
func (m *customModule) dir() *string {
return m.props.Dir
}
func (m *customModule) ConvertWithBp2build(ctx android.Bp2buildMutatorContext) {
if p := m.props.Does_not_convert_to_bazel; p != nil && *p {
ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_PROPERTY_UNSUPPORTED, "")
return
}
if p := m.props.One_to_many_prop; p != nil && *p {
customBp2buildOneToMany(ctx, m)
return
}
paths := bazel.LabelListAttribute{}
strAttr := bazel.StringAttribute{}
for axis, configToProps := range m.GetArchVariantProperties(ctx, &customProps{}) {
for config, props := range configToProps {
if custProps, ok := props.(*customProps); ok {
if custProps.Arch_paths != nil {
paths.SetSelectValue(axis, config, android.BazelLabelForModuleSrcExcludes(ctx, custProps.Arch_paths, custProps.Arch_paths_exclude))
}
if custProps.String_literal_prop != nil {
strAttr.SetSelectValue(axis, config, custProps.String_literal_prop)
}
}
}
}
productVariableProps, errs := android.ProductVariableProperties(ctx, ctx.Module())
for _, err := range errs {
ctx.ModuleErrorf("ProductVariableProperties error: %s", err)
}
if props, ok := productVariableProps["String_literal_prop"]; ok {
for c, p := range props {
if val, ok := p.(*string); ok {
strAttr.SetSelectValue(c.ConfigurationAxis(), c.SelectKey(), val)
}
}
}
paths.ResolveExcludes()
attrs := &customBazelModuleAttributes{
String_literal_prop: strAttr,
String_ptr_prop: m.props.String_ptr_prop,
String_list_prop: m.props.String_list_prop,
Arch_paths: paths,
}
attrs.Embedded_attr = m.props.Embedded_prop
if m.props.OtherEmbeddedProps != nil {
attrs.OtherEmbeddedAttr = &OtherEmbeddedAttr{Other_embedded_attr: m.props.OtherEmbeddedProps.Other_embedded_prop}
}
props := bazel.BazelTargetModuleProperties{
Rule_class: "custom",
}
ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name(), Dir: m.dir()}, attrs)
if proptools.Bool(m.props.Test_config_setting) {
m.createConfigSetting(ctx)
}
}
func (m *customModule) createConfigSetting(ctx android.Bp2buildMutatorContext) {
csa := bazel.ConfigSettingAttributes{
Flag_values: bazel.StringMapAttribute{
"//build/bazel/rules/my_string_setting": m.Name(),
},
}
ca := android.CommonAttributes{
Name: m.Name() + "_config_setting",
}
ctx.CreateBazelConfigSetting(
csa,
ca,
ctx.ModuleDir(),
)
}
// A bp2build mutator that uses load statements and creates a 1:M mapping from
// module to target.
func customBp2buildOneToMany(ctx android.Bp2buildMutatorContext, m *customModule) {
baseName := m.Name()
attrs := &customBazelModuleAttributes{}
myLibraryProps := bazel.BazelTargetModuleProperties{
Rule_class: "my_library",
Bzl_load_location: "//build/bazel/rules:rules.bzl",
}
ctx.CreateBazelTargetModule(myLibraryProps, android.CommonAttributes{Name: baseName}, attrs)
protoLibraryProps := bazel.BazelTargetModuleProperties{
Rule_class: "proto_library",
Bzl_load_location: "//build/bazel/rules:proto.bzl",
}
ctx.CreateBazelTargetModule(protoLibraryProps, android.CommonAttributes{Name: baseName + "_proto_library_deps"}, attrs)
myProtoLibraryProps := bazel.BazelTargetModuleProperties{
Rule_class: "my_proto_library",
Bzl_load_location: "//build/bazel/rules:proto.bzl",
}
ctx.CreateBazelTargetModule(myProtoLibraryProps, android.CommonAttributes{Name: baseName + "_my_proto_library_deps"}, attrs)
}
// Helper method for tests to easily access the targets in a dir.
func generateBazelTargetsForDir(codegenCtx *CodegenContext, dir string) (BazelTargets, []error) {
// TODO: Set generateFilegroups to true and/or remove the generateFilegroups argument completely
res, err := GenerateBazelTargets(codegenCtx, false)
if err != nil {
return BazelTargets{}, err
}
return res.buildFileToTargets[dir], err
}
func registerCustomModuleForBp2buildConversion(ctx *android.TestContext) {
ctx.RegisterModuleType("custom", customModuleFactoryHostAndDevice)
ctx.RegisterForBazelConversion()
}
func simpleModule(typ, name string) string {
return fmt.Sprintf(`
%s {
name: "%s",
}`, typ, name)
}
type AttrNameToString map[string]string
func (a AttrNameToString) clone() AttrNameToString {
newAttrs := make(AttrNameToString, len(a))
for k, v := range a {
newAttrs[k] = v
}
return newAttrs
}
// makeBazelTargetNoRestrictions returns bazel target build file definition that can be host or
// device specific, or independent of host/device.
func makeBazelTargetHostOrDevice(typ, name string, attrs AttrNameToString, hod android.HostOrDeviceSupported) string {
if _, ok := attrs["target_compatible_with"]; !ok {
switch hod {
case android.HostSupported:
attrs["target_compatible_with"] = `select({
"//build/bazel_common_rules/platforms/os:android": ["@platforms//:incompatible"],
"//conditions:default": [],
})`
case android.DeviceSupported:
attrs["target_compatible_with"] = `["//build/bazel_common_rules/platforms/os:android"]`
}
}
attrStrings := make([]string, 0, len(attrs)+1)
if name != "" {
attrStrings = append(attrStrings, fmt.Sprintf(` name = "%s",`, name))
}
for _, k := range android.SortedKeys(attrs) {
attrStrings = append(attrStrings, fmt.Sprintf(" %s = %s,", k, attrs[k]))
}
return fmt.Sprintf(`%s(
%s
)`, typ, strings.Join(attrStrings, "\n"))
}
// MakeBazelTargetNoRestrictions returns bazel target build file definition that does not add a
// target_compatible_with. This is useful for module types like filegroup and genrule that arch not
// arch variant
func MakeBazelTargetNoRestrictions(typ, name string, attrs AttrNameToString) string {
return makeBazelTargetHostOrDevice(typ, name, attrs, android.HostAndDeviceDefault)
}
// makeBazelTargetNoRestrictions returns bazel target build file definition that is device specific
// as this is the most common default in Soong.
func MakeBazelTarget(typ, name string, attrs AttrNameToString) string {
return makeBazelTargetHostOrDevice(typ, name, attrs, android.DeviceSupported)
}
type ExpectedRuleTarget struct {
Rule string
Name string
Attrs AttrNameToString
Hod android.HostOrDeviceSupported
}
func (ebr ExpectedRuleTarget) String() string {
return makeBazelTargetHostOrDevice(ebr.Rule, ebr.Name, ebr.Attrs, ebr.Hod)
}
func makeCcStubSuiteTargets(name string, attrs AttrNameToString) string {
if _, hasStubs := attrs["stubs_symbol_file"]; !hasStubs {
return ""
}
STUB_SUITE_ATTRS := map[string]string{
"api_surface": "api_surface",
"stubs_symbol_file": "symbol_file",
"stubs_versions": "versions",
"soname": "soname",
"source_library_label": "source_library_label",
}
stubSuiteAttrs := AttrNameToString{}
for key, _ := range attrs {
if _, stubSuiteAttr := STUB_SUITE_ATTRS[key]; stubSuiteAttr {
stubSuiteAttrs[STUB_SUITE_ATTRS[key]] = attrs[key]
} else {
panic(fmt.Sprintf("unused cc_stub_suite attr %q\n", key))
}
}
return MakeBazelTarget("cc_stub_suite", name+"_stub_libs", stubSuiteAttrs)
}
func MakeNeverlinkDuplicateTarget(moduleType string, name string) string {
return MakeNeverlinkDuplicateTargetWithAttrs(moduleType, name, AttrNameToString{
"sdk_version": `"current"`, // use as default
})
}
func MakeNeverlinkDuplicateTargetWithAttrs(moduleType string, name string, extraAttrs AttrNameToString) string {
attrs := extraAttrs
attrs["neverlink"] = `True`
attrs["exports"] = `[":` + name + `"]`
return MakeBazelTarget(moduleType, name+"-neverlink", attrs)
}
func getTargetName(targetContent string) string {
data := strings.Split(targetContent, "name = \"")
if len(data) < 2 {
return ""
} else {
endIndex := strings.Index(data[1], "\"")
return data[1][:endIndex]
}
}