StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Remove infrastructure to run bp2build" into main

Browse Source
This commit is contained in:
Treehugger Robot
2023-12-12 20:03:58 +00:00
committed by Gerrit Code Review
parent c1fb159206 b63d7b3af7
commit fa6995f75e
32 changed files with 62 additions and 9310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
android/Android.bp
View File

@@ -12,14 +12,11 @@ bootstrap_go_package {
"sbox_proto", "sbox_proto",
"soong", "soong",
"soong-android-soongconfig", "soong-android-soongconfig",
"soong-bazel",
"soong-cquery",
"soong-remoteexec", "soong-remoteexec",
"soong-response", "soong-response",
"soong-shared", "soong-shared",
"soong-starlark", "soong-starlark",
"soong-starlark-format", "soong-starlark-format",
"soong-ui-bp2build_metrics_proto",
"soong-ui-metrics_proto", "soong-ui-metrics_proto",
"soong-android-allowlists", "soong-android-allowlists",
@@ -38,9 +35,6 @@ bootstrap_go_package {
"arch.go", "arch.go",
"arch_list.go", "arch_list.go",
"base_module_context.go", "base_module_context.go",
"bazel.go",
"bazel_handler.go",
"bazel_paths.go",
"buildinfo_prop.go", "buildinfo_prop.go",
"config.go", "config.go",
"test_config.go", "test_config.go",
@@ -106,9 +100,6 @@ bootstrap_go_package {
"androidmk_test.go", "androidmk_test.go",
"apex_test.go", "apex_test.go",
"arch_test.go", "arch_test.go",
"bazel_handler_test.go",
"bazel_paths_test.go",
"bazel_test.go",
"config_test.go", "config_test.go",
"config_bp2build_test.go", "config_bp2build_test.go",
"configured_jars_test.go", "configured_jars_test.go",

3
android/apex_contributions.go
View File

@@ -111,9 +111,6 @@ func (a *allApexContributions) SetPrebuiltSelectionInfoProvider(ctx BaseModuleCo
} }
} }
if ctx.Config().Bp2buildMode() { // Skip bp2build
return
}
p := PrebuiltSelectionInfoMap{} p := PrebuiltSelectionInfoMap{}
ctx.VisitDirectDepsWithTag(acDepTag, func(child Module) { ctx.VisitDirectDepsWithTag(acDepTag, func(child Module) {
if m, ok := child.(*apexContributions); ok { if m, ok := child.(*apexContributions); ok {

4
android/arch.go
View File

@@ -425,7 +425,7 @@ func osMutator(bpctx blueprint.BottomUpMutatorContext) {
// blueprint.BottomUpMutatorContext because android.BottomUpMutatorContext // blueprint.BottomUpMutatorContext because android.BottomUpMutatorContext
// filters out non-Soong modules. Now that we've handled them, create a // filters out non-Soong modules. Now that we've handled them, create a
// normal android.BottomUpMutatorContext. // normal android.BottomUpMutatorContext.
mctx := bottomUpMutatorContextFactory(bpctx, module, false, false) mctx := bottomUpMutatorContextFactory(bpctx, module, false)
base := module.base() base := module.base()
@@ -570,7 +570,7 @@ func archMutator(bpctx blueprint.BottomUpMutatorContext) {
// blueprint.BottomUpMutatorContext because android.BottomUpMutatorContext // blueprint.BottomUpMutatorContext because android.BottomUpMutatorContext
// filters out non-Soong modules. Now that we've handled them, create a // filters out non-Soong modules. Now that we've handled them, create a
// normal android.BottomUpMutatorContext. // normal android.BottomUpMutatorContext.
mctx := bottomUpMutatorContextFactory(bpctx, module, false, false) mctx := bottomUpMutatorContextFactory(bpctx, module, false)
base := module.base() base := module.base()

45
android/base_module_context.go
View File

@@ -112,8 +112,6 @@ type BaseModuleContext interface {
// the first DependencyTag. // the first DependencyTag.
GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag)
ModuleFromName(name string) (blueprint.Module, bool)
// VisitDirectDepsBlueprint calls visit for each direct dependency. If there are multiple // VisitDirectDepsBlueprint calls visit for each direct dependency. If there are multiple
// direct dependencies on the same module visit will be called multiple times on that module // direct dependencies on the same module visit will be called multiple times on that module
// and OtherModuleDependencyTag will return a different tag for each. // and OtherModuleDependencyTag will return a different tag for each.
@@ -209,12 +207,6 @@ type BaseModuleContext interface {
// Calling this function prevents adding new dependencies. // Calling this function prevents adding new dependencies.
getMissingDependencies() []string getMissingDependencies() []string
// AddUnconvertedBp2buildDep stores module name of a direct dependency that was not converted via bp2build
AddUnconvertedBp2buildDep(dep string)
// AddMissingBp2buildDep stores the module name of a direct dependency that was not found.
AddMissingBp2buildDep(dep string)
Target() Target Target() Target
TargetPrimary() bool TargetPrimary() bool
@@ -243,12 +235,8 @@ type baseModuleContext struct {
strictVisitDeps bool // If true, enforce that all dependencies are enabled strictVisitDeps bool // If true, enforce that all dependencies are enabled
bazelConversionMode bool
} }
func (b *baseModuleContext) isBazelConversionMode() bool {
return b.bazelConversionMode
}
func (b *baseModuleContext) OtherModuleName(m blueprint.Module) string { func (b *baseModuleContext) OtherModuleName(m blueprint.Module) string {
return b.bp.OtherModuleName(m) return b.bp.OtherModuleName(m)
} }
@@ -296,18 +284,6 @@ func (b *baseModuleContext) blueprintBaseModuleContext() blueprint.BaseModuleCon
return b.bp return b.bp
} }
// AddUnconvertedBp2buildDep stores module name of a dependency that was not converted to Bazel.
func (b *baseModuleContext) AddUnconvertedBp2buildDep(dep string) {
unconvertedDeps := &b.Module().base().commonProperties.BazelConversionStatus.UnconvertedDeps
*unconvertedDeps = append(*unconvertedDeps, dep)
}
// AddMissingBp2buildDep stores module name of a dependency that was not found in a Android.bp file.
func (b *baseModuleContext) AddMissingBp2buildDep(dep string) {
missingDeps := &b.Module().base().commonProperties.BazelConversionStatus.MissingDeps
*missingDeps = append(*missingDeps, dep)
}
func (b *baseModuleContext) AddMissingDependencies(deps []string) { func (b *baseModuleContext) AddMissingDependencies(deps []string) {
if deps != nil { if deps != nil {
missingDeps := &b.Module().base().commonProperties.MissingDeps missingDeps := &b.Module().base().commonProperties.MissingDeps
@@ -435,27 +411,6 @@ func (b *baseModuleContext) GetDirectDep(name string) (blueprint.Module, bluepri
return b.getDirectDepFirstTag(name) return b.getDirectDepFirstTag(name)
} }
func (b *baseModuleContext) ModuleFromName(name string) (blueprint.Module, bool) {
if !b.isBazelConversionMode() {
panic("cannot call ModuleFromName if not in bazel conversion mode")
}
var m blueprint.Module
var ok bool
if moduleName, _ := SrcIsModuleWithTag(name); moduleName != "" {
m, ok = b.bp.ModuleFromName(moduleName)
} else {
m, ok = b.bp.ModuleFromName(name)
}
if !ok {
return m, ok
}
// If this module is not preferred, tried to get the prebuilt version instead
if a, aOk := m.(Module); aOk && !IsModulePrebuilt(a) && !IsModulePreferred(a) {
return b.ModuleFromName("prebuilt_" + name)
}
return m, ok
}
func (b *baseModuleContext) VisitDirectDepsBlueprint(visit func(blueprint.Module)) { func (b *baseModuleContext) VisitDirectDepsBlueprint(visit func(blueprint.Module)) {
b.bp.VisitDirectDeps(visit) b.bp.VisitDirectDeps(visit)
} }

784
android/bazel.go
View File

@@ -1,784 +0,0 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"bufio"
"errors"
"fmt"
"strings"
"android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/proptools"
"android/soong/android/allowlists"
)
const (
// A sentinel value to be used as a key in Bp2BuildConfig for modules with
// no package path. This is also the module dir for top level Android.bp
// modules.
Bp2BuildTopLevel = "."
)
type MixedBuildEnabledStatus int
const (
// This module can be mixed_built.
MixedBuildEnabled = iota
// There is a technical incompatibility preventing this module from being
// bazel-analyzed. Note: the module might also be incompatible.
TechnicalIncompatibility
// This module cannot be mixed_built due to some incompatibility with it
// that is not a platform incompatibility. Example: the module-type is not
// enabled, or is not bp2build-converted.
ModuleIncompatibility
// Missing dependencies. We can't query Bazel for modules if it has missing dependencies, there
// will be failures.
ModuleMissingDeps
)
// FileGroupAsLibrary describes a filegroup module that is converted to some library
// such as aidl_library or proto_library.
type FileGroupAsLibrary interface {
ShouldConvertToAidlLibrary(ctx BazelConversionPathContext) bool
ShouldConvertToProtoLibrary(ctx BazelConversionPathContext) bool
GetAidlLibraryLabel(ctx BazelConversionPathContext) string
GetProtoLibraryLabel(ctx BazelConversionPathContext) string
}
type BazelConversionStatus struct {
// Information about _all_ bp2build targets generated by this module. Multiple targets are
// supported as Soong handles some things within a single target that we may choose to split into
// multiple targets, e.g. renderscript, protos, yacc within a cc module.
Bp2buildInfo []bp2buildInfo `blueprint:"mutated"`
// UnconvertedBp2buildDep stores the module names of direct dependency that were not converted to
// Bazel
UnconvertedDeps []string `blueprint:"mutated"`
// MissingBp2buildDep stores the module names of direct dependency that were not found
MissingDeps []string `blueprint:"mutated"`
// If non-nil, indicates that the module could not be converted successfully
// with bp2build. This will describe the reason the module could not be converted.
UnconvertedReason *UnconvertedReason
// The Partition this module will be installed on.
// TODO(b/306200980) Investigate how to handle modules that are installed in multiple
// partitions.
Partition string `blueprint:"mutated"`
}
// The reason a module could not be converted to a BUILD target via bp2build.
// This should match bp2build_metrics_proto.UnconvertedReason, but omits private
// proto-related fields that prevent copying this struct.
type UnconvertedReason struct {
// Should correspond to a valid value in bp2build_metrics_proto.UnconvertedReasonType.
// A raw int is used here instead, because blueprint logic requires that all transitive
// fields of module definitions be primitives.
ReasonType int
Detail string
}
type BazelModuleProperties struct {
// The label of the Bazel target replacing this Soong module. When run in conversion mode, this
// will import the handcrafted build target into the autogenerated file. Note: this may result in
// a conflict due to duplicate targets if bp2build_available is also set.
Label *string
// If true, bp2build will generate the converted Bazel target for this module. Note: this may
// cause a conflict due to the duplicate targets if label is also set.
//
// This is a bool pointer to support tristates: true, false, not set.
//
// To opt in a module, set bazel_module: { bp2build_available: true }
// To opt out a module, set bazel_module: { bp2build_available: false }
// To defer the default setting for the directory, do not set the value.
Bp2build_available *bool
// CanConvertToBazel is set via InitBazelModule to indicate that a module type can be converted to
// Bazel with Bp2build.
CanConvertToBazel bool `blueprint:"mutated"`
}
// Properties contains common module properties for Bazel migration purposes.
type properties struct {
// In "Bazel mixed build" mode, this represents the Bazel target replacing
// this Soong module.
Bazel_module BazelModuleProperties
}
// namespacedVariableProperties is a map from a string representing a Soong
// config variable namespace, like "android" or "vendor_name" to a slice of
// pointer to a struct containing a single field called Soong_config_variables
// whose value mirrors the structure in the Blueprint file.
type namespacedVariableProperties map[string][]interface{}
// BazelModuleBase contains the property structs with metadata for modules which can be converted to
// Bazel.
type BazelModuleBase struct {
bazelProperties properties
// namespacedVariableProperties is used for soong_config_module_type support
// in bp2build. Soong config modules allow users to set module properties
// based on custom product variables defined in Android.bp files. These
// variables are namespaced to prevent clobbering, especially when set from
// Makefiles.
namespacedVariableProperties namespacedVariableProperties
// baseModuleType is set when this module was created from a module type
// defined by a soong_config_module_type. Every soong_config_module_type
// "wraps" another module type, e.g. a soong_config_module_type can wrap a
// cc_defaults to a custom_cc_defaults, or cc_binary to a custom_cc_binary.
// This baseModuleType is set to the wrapped module type.
baseModuleType string
}
// Bazelable is specifies the interface for modules that can be converted to Bazel.
type Bazelable interface {
bazelProps() *properties
HasHandcraftedLabel() bool
HandcraftedLabel() string
GetBazelLabel(ctx BazelConversionPathContext, module blueprint.Module) string
ShouldConvertWithBp2build(ctx ShouldConvertWithBazelContext) bool
shouldConvertWithBp2build(shouldConvertModuleContext, shouldConvertParams) bool
// ConvertWithBp2build either converts the module to a Bazel build target or
// declares the module as unconvertible (for logging and metrics).
// Modules must implement this function to be bp2build convertible. The function
// must either create at least one Bazel target module (using ctx.CreateBazelTargetModule or
// its related functions), or declare itself unconvertible using ctx.MarkBp2buildUnconvertible.
ConvertWithBp2build(ctx Bp2buildMutatorContext)
// namespacedVariableProps is a map from a soong config variable namespace
// (e.g. acme, android) to a map of interfaces{}, which are really
// reflect.Struct pointers, representing the value of the
// soong_config_variables property of a module. The struct pointer is the
// one with the single member called Soong_config_variables, which itself is
// a struct containing fields for each supported feature in that namespace.
//
// The reason for using a slice of interface{} is to support defaults
// propagation of the struct pointers.
namespacedVariableProps() namespacedVariableProperties
setNamespacedVariableProps(props namespacedVariableProperties)
BaseModuleType() string
SetBaseModuleType(baseModuleType string)
}
// ApiProvider is implemented by modules that contribute to an API surface
type ApiProvider interface {
ConvertWithApiBp2build(ctx TopDownMutatorContext)
}
// MixedBuildBuildable is an interface that module types should implement in order
// to be "handled by Bazel" in a mixed build.
type MixedBuildBuildable interface {
// IsMixedBuildSupported returns true if and only if this module should be
// "handled by Bazel" in a mixed build.
// This "escape hatch" allows modules with corner-case scenarios to opt out
// of being built with Bazel.
IsMixedBuildSupported(ctx BaseModuleContext) bool
// QueueBazelCall invokes request-queueing functions on the BazelContext
// so that these requests are handled when Bazel's cquery is invoked.
QueueBazelCall(ctx BaseModuleContext)
// ProcessBazelQueryResponse uses Bazel information (obtained from the BazelContext)
// to set module fields and providers to propagate this module's metadata upstream.
// This effectively "bridges the gap" between Bazel and Soong in a mixed build.
// Soong modules depending on this module should be oblivious to the fact that
// this module was handled by Bazel.
ProcessBazelQueryResponse(ctx ModuleContext)
}
// BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules.
type BazelModule interface {
Module
Bazelable
}
// InitBazelModule is a wrapper function that decorates a BazelModule with Bazel-conversion
// properties.
func InitBazelModule(module BazelModule) {
module.AddProperties(module.bazelProps())
module.bazelProps().Bazel_module.CanConvertToBazel = true
}
// BazelHandcraftedHook is a load hook to possibly register the current module as
// a "handcrafted" Bazel target of a given name. If the current module should be
// registered in this way, the hook function should return the target name. If
// it should not be registered in this way, this function should return the empty string.
type BazelHandcraftedHook func(ctx LoadHookContext) string
// AddBazelHandcraftedHook adds a load hook to (maybe) mark the given module so that
// it is treated by bp2build as if it has a handcrafted Bazel target.
func AddBazelHandcraftedHook(module BazelModule, hook BazelHandcraftedHook) {
AddLoadHook(module, func(ctx LoadHookContext) {
var targetName string = hook(ctx)
if len(targetName) > 0 {
moduleDir := ctx.ModuleDir()
if moduleDir == Bp2BuildTopLevel {
moduleDir = ""
}
label := fmt.Sprintf("//%s:%s", moduleDir, targetName)
module.bazelProps().Bazel_module.Label = &label
}
})
}
// bazelProps returns the Bazel properties for the given BazelModuleBase.
func (b *BazelModuleBase) bazelProps() *properties {
return &b.bazelProperties
}
func (b *BazelModuleBase) namespacedVariableProps() namespacedVariableProperties {
return b.namespacedVariableProperties
}
func (b *BazelModuleBase) setNamespacedVariableProps(props namespacedVariableProperties) {
b.namespacedVariableProperties = props
}
func (b *BazelModuleBase) BaseModuleType() string {
return b.baseModuleType
}
func (b *BazelModuleBase) SetBaseModuleType(baseModuleType string) {
b.baseModuleType = baseModuleType
}
// HasHandcraftedLabel returns whether this module has a handcrafted Bazel label.
func (b *BazelModuleBase) HasHandcraftedLabel() bool {
return b.bazelProperties.Bazel_module.Label != nil
}
// HandcraftedLabel returns the handcrafted label for this module, or empty string if there is none
func (b *BazelModuleBase) HandcraftedLabel() string {
return proptools.String(b.bazelProperties.Bazel_module.Label)
}
// GetBazelLabel returns the Bazel label for the given BazelModuleBase.
func (b *BazelModuleBase) GetBazelLabel(ctx BazelConversionPathContext, module blueprint.Module) string {
if b.HasHandcraftedLabel() {
return b.HandcraftedLabel()
}
if b.ShouldConvertWithBp2build(ctx) {
return bp2buildModuleLabel(ctx, module)
}
panic(fmt.Errorf("requested non-existent label for module %s", module.Name()))
}
type Bp2BuildConversionAllowlist struct {
// Configure modules in these directories to enable bp2build_available: true or false by default.
defaultConfig allowlists.Bp2BuildConfig
// Keep any existing BUILD files (and do not generate new BUILD files) for these directories
// in the synthetic Bazel workspace.
keepExistingBuildFile map[string]bool
// Per-module allowlist to always opt modules into both bp2build and Bazel Dev Mode mixed
// builds. These modules are usually in directories with many other modules that are not ready
// for conversion.
//
// A module can either be in this list or its directory allowlisted entirely
// in bp2buildDefaultConfig, but not both at the same time.
moduleAlwaysConvert map[string]bool
// Per-module-type allowlist to always opt modules in to both bp2build and
// Bazel Dev Mode mixed builds when they have the same type as one listed.
moduleTypeAlwaysConvert map[string]bool
// Per-module denylist to always opt modules out of bp2build conversion.
moduleDoNotConvert map[string]bool
}
// NewBp2BuildAllowlist creates a new, empty Bp2BuildConversionAllowlist
// which can be populated using builder pattern Set* methods
func NewBp2BuildAllowlist() Bp2BuildConversionAllowlist {
return Bp2BuildConversionAllowlist{
allowlists.Bp2BuildConfig{},
map[string]bool{},
map[string]bool{},
map[string]bool{},
map[string]bool{},
}
}
// SetDefaultConfig copies the entries from defaultConfig into the allowlist
func (a Bp2BuildConversionAllowlist) SetDefaultConfig(defaultConfig allowlists.Bp2BuildConfig) Bp2BuildConversionAllowlist {
if a.defaultConfig == nil {
a.defaultConfig = allowlists.Bp2BuildConfig{}
}
for k, v := range defaultConfig {
a.defaultConfig[k] = v
}
return a
}
// SetKeepExistingBuildFile copies the entries from keepExistingBuildFile into the allowlist
func (a Bp2BuildConversionAllowlist) SetKeepExistingBuildFile(keepExistingBuildFile map[string]bool) Bp2BuildConversionAllowlist {
if a.keepExistingBuildFile == nil {
a.keepExistingBuildFile = map[string]bool{}
}
for k, v := range keepExistingBuildFile {
a.keepExistingBuildFile[k] = v
}
return a
}
// SetModuleAlwaysConvertList copies the entries from moduleAlwaysConvert into the allowlist
func (a Bp2BuildConversionAllowlist) SetModuleAlwaysConvertList(moduleAlwaysConvert []string) Bp2BuildConversionAllowlist {
if a.moduleAlwaysConvert == nil {
a.moduleAlwaysConvert = map[string]bool{}
}
for _, m := range moduleAlwaysConvert {
a.moduleAlwaysConvert[m] = true
}
return a
}
// SetModuleTypeAlwaysConvertList copies the entries from moduleTypeAlwaysConvert into the allowlist
func (a Bp2BuildConversionAllowlist) SetModuleTypeAlwaysConvertList(moduleTypeAlwaysConvert []string) Bp2BuildConversionAllowlist {
if a.moduleTypeAlwaysConvert == nil {
a.moduleTypeAlwaysConvert = map[string]bool{}
}
for _, m := range moduleTypeAlwaysConvert {
a.moduleTypeAlwaysConvert[m] = true
}
return a
}
// SetModuleDoNotConvertList copies the entries from moduleDoNotConvert into the allowlist
func (a Bp2BuildConversionAllowlist) SetModuleDoNotConvertList(moduleDoNotConvert []string) Bp2BuildConversionAllowlist {
if a.moduleDoNotConvert == nil {
a.moduleDoNotConvert = map[string]bool{}
}
for _, m := range moduleDoNotConvert {
a.moduleDoNotConvert[m] = true
}
return a
}
// ShouldKeepExistingBuildFileForDir returns whether an existing BUILD file should be
// added to the build symlink forest based on the current global configuration.
func (a Bp2BuildConversionAllowlist) ShouldKeepExistingBuildFileForDir(dir string) bool {
if _, ok := a.keepExistingBuildFile[dir]; ok {
// Exact dir match
return true
}
var i int
// Check if subtree match
for {
j := strings.Index(dir[i:], "/")
if j == -1 {
return false //default
}
prefix := dir[0 : i+j]
i = i + j + 1 // skip the "/"
if recursive, ok := a.keepExistingBuildFile[prefix]; ok && recursive {
return true
}
}
}
var bp2BuildAllowListKey = NewOnceKey("Bp2BuildAllowlist")
var bp2buildAllowlist OncePer
func GetBp2BuildAllowList() Bp2BuildConversionAllowlist {
return bp2buildAllowlist.Once(bp2BuildAllowListKey, func() interface{} {
return NewBp2BuildAllowlist().SetDefaultConfig(allowlists.Bp2buildDefaultConfig).
SetKeepExistingBuildFile(allowlists.Bp2buildKeepExistingBuildFile).
SetModuleAlwaysConvertList(allowlists.Bp2buildModuleAlwaysConvertList).
SetModuleTypeAlwaysConvertList(allowlists.Bp2buildModuleTypeAlwaysConvertList).
SetModuleDoNotConvertList(allowlists.Bp2buildModuleDoNotConvertList)
}).(Bp2BuildConversionAllowlist)
}
// MixedBuildsEnabled returns a MixedBuildEnabledStatus regarding whether
// a module is ready to be replaced by a converted or handcrafted Bazel target.
// As a side effect, calling this method will also log whether this module is
// mixed build enabled for metrics reporting.
func MixedBuildsEnabled(ctx BaseModuleContext) MixedBuildEnabledStatus {
platformIncompatible := isPlatformIncompatible(ctx.Os(), ctx.Arch().ArchType)
if platformIncompatible {
ctx.Config().LogMixedBuild(ctx, false)
return TechnicalIncompatibility
}
if ctx.Config().AllowMissingDependencies() {
missingDeps := ctx.getMissingDependencies()
// If there are missing dependencies, querying Bazel will fail. Soong instead fails at execution
// time, not loading/analysis. disable mixed builds and fall back to Soong to maintain that
// behavior.
if len(missingDeps) > 0 {
ctx.Config().LogMixedBuild(ctx, false)
return ModuleMissingDeps
}
}
module := ctx.Module()
apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo)
withinApex := !apexInfo.IsForPlatform()
mixedBuildEnabled := ctx.Config().IsMixedBuildsEnabled() &&
module.Enabled() &&
convertedToBazel(ctx, module) &&
ctx.Config().BazelContext.IsModuleNameAllowed(module.Name(), withinApex)
ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled)
if mixedBuildEnabled {
return MixedBuildEnabled
}
return ModuleIncompatibility
}
func isGoModule(module blueprint.Module) bool {
if _, ok := module.(*bootstrap.GoPackage); ok {
return true
}
if _, ok := module.(*bootstrap.GoBinary); ok {
return true
}
return false
}
// ConvertedToBazel returns whether this module has been converted (with bp2build or manually) to Bazel.
func convertedToBazel(ctx BazelConversionContext, module blueprint.Module) bool {
// Special-case bootstrap_go_package and bootstrap_go_binary
// These do not implement Bazelable, but have been converted
if isGoModule(module) {
return true
}
b, ok := module.(Bazelable)
if !ok {
return false
}
return b.HasHandcraftedLabel() || b.shouldConvertWithBp2build(ctx, shouldConvertParams{
module: module,
moduleDir: ctx.OtherModuleDir(module),
moduleName: ctx.OtherModuleName(module),
moduleType: ctx.OtherModuleType(module),
})
}
type ShouldConvertWithBazelContext interface {
ModuleErrorf(format string, args ...interface{})
Module() Module
Config() Config
ModuleType() string
ModuleName() string
ModuleDir() string
}
// ShouldConvertWithBp2build returns whether the given BazelModuleBase should be converted with bp2build
func (b *BazelModuleBase) ShouldConvertWithBp2build(ctx ShouldConvertWithBazelContext) bool {
return b.shouldConvertWithBp2build(ctx, shouldConvertParams{
module: ctx.Module(),
moduleDir: ctx.ModuleDir(),
moduleName: ctx.ModuleName(),
moduleType: ctx.ModuleType(),
})
}
type bazelOtherModuleContext interface {
ModuleErrorf(format string, args ...interface{})
Config() Config
OtherModuleType(m blueprint.Module) string
OtherModuleName(m blueprint.Module) string
OtherModuleDir(m blueprint.Module) string
}
func isPlatformIncompatible(osType OsType, arch ArchType) bool {
return osType == Windows || // Windows toolchains are not currently supported.
osType == LinuxBionic || // Linux Bionic toolchains are not currently supported.
osType == LinuxMusl || // Linux musl toolchains are not currently supported (b/259266326).
arch == Riscv64 // TODO(b/262192655) Riscv64 toolchains are not currently supported.
}
type shouldConvertModuleContext interface {
ModuleErrorf(format string, args ...interface{})
Config() Config
}
type shouldConvertParams struct {
module blueprint.Module
moduleType string
moduleDir string
moduleName string
}
func (b *BazelModuleBase) shouldConvertWithBp2build(ctx shouldConvertModuleContext, p shouldConvertParams) bool {
if !b.bazelProps().Bazel_module.CanConvertToBazel {
return false
}
module := p.module
propValue := b.bazelProperties.Bazel_module.Bp2build_available
packagePath := moduleDirWithPossibleOverride(ctx, module, p.moduleDir)
// Modules in unit tests which are enabled in the allowlist by type or name
// trigger this conditional because unit tests run under the "." package path
isTestModule := packagePath == Bp2BuildTopLevel && proptools.BoolDefault(propValue, false)
if isTestModule {
return true
}
moduleName := moduleNameWithPossibleOverride(ctx, module, p.moduleName)
// use "prebuilt_" + original module name as the java_import(_host) module name,
// to avoid the failure that a normal module and a prebuilt module with
// the same name are both allowlisted. This cannot be applied to all the *_import
// module types. For example, android_library_import has to use original module
// name here otherwise the *-nodeps targets cannot be handled correctly.
// TODO(b/304385140): remove this special casing
if p.moduleType == "java_import" || p.moduleType == "java_import_host" {
moduleName = module.Name()
}
allowlist := ctx.Config().Bp2buildPackageConfig
moduleNameAllowed := allowlist.moduleAlwaysConvert[moduleName]
moduleTypeAllowed := allowlist.moduleTypeAlwaysConvert[p.moduleType]
allowlistConvert := moduleNameAllowed || moduleTypeAllowed
if moduleNameAllowed && moduleTypeAllowed {
ctx.ModuleErrorf("A module %q of type %q cannot be in moduleAlwaysConvert and also be in moduleTypeAlwaysConvert", moduleName, p.moduleType)
return false
}
if allowlist.moduleDoNotConvert[moduleName] {
if moduleNameAllowed {
ctx.ModuleErrorf("a module %q cannot be in moduleDoNotConvert and also be in moduleAlwaysConvert", moduleName)
}
return false
}
// This is a tristate value: true, false, or unset.
if ok, directoryPath := bp2buildDefaultTrueRecursively(packagePath, allowlist.defaultConfig); ok {
if moduleNameAllowed {
ctx.ModuleErrorf("A module cannot be in a directory marked Bp2BuildDefaultTrue"+
" or Bp2BuildDefaultTrueRecursively and also be in moduleAlwaysConvert. Directory: '%s'"+
" Module: '%s'", directoryPath, moduleName)
return false
}
// Allow modules to explicitly opt-out.
return proptools.BoolDefault(propValue, true)
}
// Allow modules to explicitly opt-in.
return proptools.BoolDefault(propValue, allowlistConvert)
}
// bp2buildDefaultTrueRecursively checks that the package contains a prefix from the
// set of package prefixes where all modules must be converted. That is, if the
// package is x/y/z, and the list contains either x, x/y, or x/y/z, this function will
// return true.
//
// However, if the package is x/y, and it matches a Bp2BuildDefaultFalse "x/y" entry
// exactly, this module will return false early.
//
// This function will also return false if the package doesn't match anything in
// the config.
//
// This function will also return the allowlist entry which caused a particular
// package to be enabled. Since packages can be enabled via a recursive declaration,
// the path returned will not always be the same as the one provided.
func bp2buildDefaultTrueRecursively(packagePath string, config allowlists.Bp2BuildConfig) (bool, string) {
// Check if the package path has an exact match in the config.
if config[packagePath] == allowlists.Bp2BuildDefaultTrue || config[packagePath] == allowlists.Bp2BuildDefaultTrueRecursively {
return true, packagePath
} else if config[packagePath] == allowlists.Bp2BuildDefaultFalse || config[packagePath] == allowlists.Bp2BuildDefaultFalseRecursively {
return false, packagePath
}
// If not, check for the config recursively.
packagePrefix := packagePath
// e.g. for x/y/z, iterate over x/y, then x, taking the most-specific value from the allowlist.
for strings.Contains(packagePrefix, "/") {
dirIndex := strings.LastIndex(packagePrefix, "/")
packagePrefix = packagePrefix[:dirIndex]
switch value := config[packagePrefix]; value {
case allowlists.Bp2BuildDefaultTrueRecursively:
// package contains this prefix and this prefix should convert all modules
return true, packagePrefix
case allowlists.Bp2BuildDefaultFalseRecursively:
//package contains this prefix and this prefix should NOT convert any modules
return false, packagePrefix
}
// Continue to the next part of the package dir.
}
return false, packagePath
}
func registerBp2buildConversionMutator(ctx RegisterMutatorsContext) {
ctx.BottomUp("bp2build_conversion", bp2buildConversionMutator).Parallel()
ctx.BottomUp("bp2build_deps", bp2buildDepsMutator).Parallel()
}
func bp2buildConversionMutator(ctx BottomUpMutatorContext) {
// If an existing BUILD file in the module directory has a target defined
// with this same name as this module, assume that this is an existing
// definition for this target.
if ctx.Config().HasBazelBuildTargetInSource(ctx.ModuleDir(), ctx.ModuleName()) {
ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE, ctx.ModuleName())
return
}
bModule, ok := ctx.Module().(Bazelable)
if !ok {
ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_TYPE_UNSUPPORTED, "")
return
}
// There may be cases where the target is created by a macro rather than in a BUILD file, those
// should be captured as well.
if bModule.HasHandcraftedLabel() {
// Defer to the BUILD target. Generating an additional target would
// cause a BUILD file conflict.
ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE, "")
return
}
// TODO: b/285631638 - Differentiate between denylisted modules and missing bp2build capabilities.
if !bModule.shouldConvertWithBp2build(ctx, shouldConvertParams{
module: ctx.Module(),
moduleDir: ctx.ModuleDir(),
moduleName: ctx.ModuleName(),
moduleType: ctx.ModuleType(),
}) {
ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_UNSUPPORTED, "")
return
}
if ctx.Module().base().GetUnconvertedReason() != nil {
return
}
bModule.ConvertWithBp2build(ctx)
installCtx := &baseModuleContextToModuleInstallPathContext{ctx}
ctx.Module().base().setPartitionForBp2build(modulePartition(installCtx, true))
if len(ctx.Module().base().Bp2buildTargets()) == 0 && ctx.Module().base().GetUnconvertedReason() == nil {
panic(fmt.Errorf("illegal bp2build invariant: module '%s' was neither converted nor marked unconvertible", ctx.ModuleName()))
}
// If an existing BUILD file in the module directory has a target defined
// with the same name as any target generated by this module, assume that this
// is an existing definition for this target. (These generated target names
// may be different than the module name, as checked at the beginning of this function!)
for _, targetInfo := range ctx.Module().base().Bp2buildTargets() {
if ctx.Config().HasBazelBuildTargetInSource(targetInfo.TargetPackage(), targetInfo.TargetName()) {
// Defer to the BUILD target. Generating an additional target would
// cause a BUILD file conflict.
ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE, targetInfo.TargetName())
return
}
}
}
// TODO: b/285631638 - Add this as a new mutator to the bp2build conversion mutators.
// Currently, this only exists to prepare test coverage for the launch of this feature.
func bp2buildDepsMutator(ctx BottomUpMutatorContext) {
if ctx.Module().base().GetUnconvertedReason() != nil {
return
}
if len(ctx.Module().GetMissingBp2buildDeps()) > 0 {
exampleDep := ctx.Module().GetMissingBp2buildDeps()[0]
ctx.MarkBp2buildUnconvertible(
bp2build_metrics_proto.UnconvertedReasonType_UNCONVERTED_DEP, exampleDep)
}
// Transitively mark modules unconvertible with the following set of conditions.
ctx.VisitDirectDeps(func(dep Module) {
if dep.base().GetUnconvertedReason() == nil {
return
}
if dep.base().GetUnconvertedReason().ReasonType ==
int(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE) {
return
}
if ctx.OtherModuleDependencyTag(dep) != Bp2buildDepTag {
return
}
ctx.MarkBp2buildUnconvertible(
bp2build_metrics_proto.UnconvertedReasonType_UNCONVERTED_DEP, dep.Name())
})
}
// GetMainClassInManifest scans the manifest file specified in filepath and returns
// the value of attribute Main-Class in the manifest file if it exists, or returns error.
// WARNING: this is for bp2build converters of java_* modules only.
func GetMainClassInManifest(c Config, filepath string) (string, error) {
file, err := c.fs.Open(filepath)
if err != nil {
return "", err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Main-Class:") {
return strings.TrimSpace(line[len("Main-Class:"):]), nil
}
}
return "", errors.New("Main-Class is not found.")
}
func AttachValidationActions(ctx ModuleContext, outputFilePath Path, validations Paths) ModuleOutPath {
validatedOutputFilePath := PathForModuleOut(ctx, "validated", outputFilePath.Base())
ctx.Build(pctx, BuildParams{
Rule: CpNoPreserveSymlink,
Description: "run validations " + outputFilePath.Base(),
Output: validatedOutputFilePath,
Input: outputFilePath,
Validations: validations,
})
return validatedOutputFilePath
}
func RunsOn(hostSupported bool, deviceSupported bool, unitTest bool) []string {
var runsOn []string
if hostSupported && deviceSupported {
runsOn = []string{"host_without_device", "device"}
} else if hostSupported {
if unitTest {
runsOn = []string{"host_without_device"}
} else {
runsOn = []string{"host_with_device"}
}
} else if deviceSupported {
runsOn = []string{"device"}
}
return runsOn
}

1593
android/bazel_handler.go
View File

File diff suppressed because it is too large Load Diff

426
android/bazel_handler_test.go
View File

@@ -1,426 +0,0 @@
package android
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"android/soong/bazel"
"android/soong/bazel/cquery"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
"github.com/google/blueprint/metrics"
"google.golang.org/protobuf/proto"
)
var testConfig = TestConfig("out", nil, "", nil)
type testInvokeBazelContext struct{}
type mockBazelRunner struct {
testHelper *testing.T
// Stores mock behavior. If an issueBazelCommand request is made for command
// k, and {k:v} is present in this map, then the mock will return v.
bazelCommandResults map[bazelCommand]string
// Requests actually made of the mockBazelRunner with issueBazelCommand,
// keyed by the command they represent.
bazelCommandRequests map[bazelCommand]bazel.CmdRequest
}
func (r *mockBazelRunner) bazelCommandForRequest(cmdRequest bazel.CmdRequest) bazelCommand {
for _, arg := range cmdRequest.Argv {
for _, cmdType := range allBazelCommands {
if arg == cmdType.command {
return cmdType
}
}
}
r.testHelper.Fatalf("Unrecognized bazel request: %s", cmdRequest)
return cqueryCmd
}
func (r *mockBazelRunner) issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (string, string, error) {
command := r.bazelCommandForRequest(cmdRequest)
r.bazelCommandRequests[command] = cmdRequest
return r.bazelCommandResults[command], "", nil
}
func (t *testInvokeBazelContext) GetEventHandler() *metrics.EventHandler {
return &metrics.EventHandler{}
}
func TestRequestResultsAfterInvokeBazel(t *testing.T) {
label_foo := "@//foo:foo"
label_bar := "@//foo:bar"
apexKey := ApexConfigKey{
WithinApex: true,
ApexSdkVersion: "29",
ApiDomain: "myapex",
}
cfg_foo := configKey{"arm64_armv8-a", Android, apexKey}
cfg_bar := configKey{arch: "arm64_armv8-a", osType: Android}
cmd_results := []string{
`@//foo:foo|arm64_armv8-a|android|within_apex|29|myapex>>out/foo/foo.txt`,
`@//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`,
}
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{cqueryCmd: strings.Join(cmd_results, "\n")})
bazelContext.QueueBazelRequest(label_foo, cquery.GetOutputFiles, cfg_foo)
bazelContext.QueueBazelRequest(label_bar, cquery.GetOutputFiles, cfg_bar)
err := bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}
verifyCqueryResult(t, bazelContext, label_foo, cfg_foo, "out/foo/foo.txt")
verifyCqueryResult(t, bazelContext, label_bar, cfg_bar, "out/foo/bar.txt")
}
func verifyCqueryResult(t *testing.T, ctx *mixedBuildBazelContext, label string, cfg configKey, result string) {
g, err := ctx.GetOutputFiles(label, cfg)
if err != nil {
t.Errorf("Expected cquery results after running InvokeBazel(), but got err %v", err)
} else if w := []string{result}; !reflect.DeepEqual(w, g) {
t.Errorf("Expected output %s, got %s", w, g)
}
}
func TestInvokeBazelWritesBazelFiles(t *testing.T) {
bazelContext, baseDir := testBazelContext(t, map[bazelCommand]string{})
err := bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}
if _, err := os.Stat(filepath.Join(baseDir, "soong_injection", "mixed_builds", "main.bzl")); os.IsNotExist(err) {
t.Errorf("Expected main.bzl to exist, but it does not")
} else if err != nil {
t.Errorf("Unexpected error stating main.bzl %s", err)
}
if _, err := os.Stat(filepath.Join(baseDir, "soong_injection", "mixed_builds", "BUILD.bazel")); os.IsNotExist(err) {
t.Errorf("Expected BUILD.bazel to exist, but it does not")
} else if err != nil {
t.Errorf("Unexpected error stating BUILD.bazel %s", err)
}
if _, err := os.Stat(filepath.Join(baseDir, "soong_injection", "WORKSPACE.bazel")); os.IsNotExist(err) {
t.Errorf("Expected WORKSPACE.bazel to exist, but it does not")
} else if err != nil {
t.Errorf("Unexpected error stating WORKSPACE.bazel %s", err)
}
}
func TestInvokeBazelPopulatesBuildStatements(t *testing.T) {
type testCase struct {
input string
command string
}
var testCases = []testCase{
{`
{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 },
{ "id": 2, "path_fragment_id": 2 }],
"actions": [{
"target_Id": 1,
"action_Key": "x",
"mnemonic": "x",
"arguments": ["touch", "foo"],
"input_dep_set_ids": [1],
"output_Ids": [1],
"primary_output_id": 1
}],
"dep_set_of_files": [
{ "id": 1, "direct_artifact_ids": [1, 2] }],
"path_fragments": [
{ "id": 1, "label": "one" },
{ "id": 2, "label": "two" }]
}`,
"cd 'test/exec_root' && rm -rf 'one' && touch foo",
}, {`
{
"artifacts": [
{ "id": 1, "path_fragment_id": 10 },
{ "id": 2, "path_fragment_id": 20 }],
"actions": [{
"target_Id": 100,
"action_Key": "x",
"mnemonic": "x",
"arguments": ["bogus", "command"],
"output_Ids": [1, 2],
"primary_output_id": 1
}],
"path_fragments": [
{ "id": 10, "label": "one", "parent_id": 30 },
{ "id": 20, "label": "one.d", "parent_id": 30 },
{ "id": 30, "label": "parent" }]
}`,
`cd 'test/exec_root' && rm -rf 'parent/one' && bogus command && sed -i'' -E 's@(^|\s|")bazel-out/@\1test/bazel_out/@g' 'parent/one.d'`,
},
}
for i, testCase := range testCases {
data, err := JsonToActionGraphContainer(testCase.input)
if err != nil {
t.Error(err)
}
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{aqueryCmd: string(data)})
err = bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("testCase #%d: did not expect error invoking Bazel, but got %s", i+1, err)
}
got := bazelContext.BuildStatementsToRegister()
if want := 1; len(got) != want {
t.Fatalf("expected %d registered build statements, but got %#v", want, got)
}
cmd := RuleBuilderCommand{}
ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "")
if actual, expected := cmd.buf.String(), testCase.command; expected != actual {
t.Errorf("expected: [%s], actual: [%s]", expected, actual)
}
}
}
func TestMixedBuildSandboxedAction(t *testing.T) {
input := `{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 },
{ "id": 2, "path_fragment_id": 2 }],
"actions": [{
"target_Id": 1,
"action_Key": "x",
"mnemonic": "x",
"arguments": ["touch", "foo"],
"input_dep_set_ids": [1],
"output_Ids": [1],
"primary_output_id": 1
}],
"dep_set_of_files": [
{ "id": 1, "direct_artifact_ids": [1, 2] }],
"path_fragments": [
{ "id": 1, "label": "one" },
{ "id": 2, "label": "two" }]
}`
data, err := JsonToActionGraphContainer(input)
if err != nil {
t.Error(err)
}
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{aqueryCmd: string(data)})
err = bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("TestMixedBuildSandboxedAction did not expect error invoking Bazel, but got %s", err)
}
statement := bazelContext.BuildStatementsToRegister()[0]
statement.ShouldRunInSbox = true
cmd := RuleBuilderCommand{}
ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "")
// Assert that the output is generated in an intermediate directory
// fe05bcdcdc4928012781a5f1a2a77cbb5398e106 is the sha1 checksum of "one"
if actual, expected := cmd.outputs[0].String(), "out/soong/mixed_build_sbox_intermediates/fe05bcdcdc4928012781a5f1a2a77cbb5398e106/test/exec_root/one"; expected != actual {
t.Errorf("expected: [%s], actual: [%s]", expected, actual)
}
// Assert the actual command remains unchanged inside the sandbox
if actual, expected := cmd.buf.String(), "mkdir -p 'test/exec_root' && cd 'test/exec_root' && rm -rf 'one' && touch foo"; expected != actual {
t.Errorf("expected: [%s], actual: [%s]", expected, actual)
}
}
func TestCoverageFlagsAfterInvokeBazel(t *testing.T) {
testConfig.productVariables.ClangCoverage = boolPtr(true)
testConfig.productVariables.NativeCoveragePaths = []string{"foo1", "foo2"}
testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1", "bar2"}
verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1,+foo2,-bar1,-bar2")
testConfig.productVariables.NativeCoveragePaths = []string{"foo1"}
testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"}
verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1,-bar1")
testConfig.productVariables.NativeCoveragePaths = []string{"foo1"}
testConfig.productVariables.NativeCoverageExcludePaths = nil
verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1")
testConfig.productVariables.NativeCoveragePaths = nil
testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"}
verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=-bar1")
testConfig.productVariables.NativeCoveragePaths = []string{"*"}
testConfig.productVariables.NativeCoverageExcludePaths = nil
verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+.*")
testConfig.productVariables.ClangCoverage = boolPtr(false)
verifyAqueryDoesNotContainSubstrings(t, testConfig, "collect_code_coverage", "instrumentation_filter")
}
func TestBazelRequestsSorted(t *testing.T) {
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{})
cfgKeyArm64Android := configKey{arch: "arm64_armv8-a", osType: Android}
cfgKeyArm64Linux := configKey{arch: "arm64_armv8-a", osType: Linux}
cfgKeyOtherAndroid := configKey{arch: "otherarch", osType: Android}
bazelContext.QueueBazelRequest("zzz", cquery.GetOutputFiles, cfgKeyArm64Android)
bazelContext.QueueBazelRequest("ccc", cquery.GetApexInfo, cfgKeyArm64Android)
bazelContext.QueueBazelRequest("duplicate", cquery.GetOutputFiles, cfgKeyArm64Android)
bazelContext.QueueBazelRequest("duplicate", cquery.GetOutputFiles, cfgKeyArm64Android)
bazelContext.QueueBazelRequest("xxx", cquery.GetOutputFiles, cfgKeyArm64Linux)
bazelContext.QueueBazelRequest("aaa", cquery.GetOutputFiles, cfgKeyArm64Android)
bazelContext.QueueBazelRequest("aaa", cquery.GetOutputFiles, cfgKeyOtherAndroid)
bazelContext.QueueBazelRequest("bbb", cquery.GetOutputFiles, cfgKeyOtherAndroid)
if len(bazelContext.requests) != 7 {
t.Error("Expected 7 request elements, but got", len(bazelContext.requests))
}
lastString := ""
for _, val := range bazelContext.requests {
thisString := val.String()
if thisString <= lastString {
t.Errorf("Requests are not ordered correctly. '%s' came before '%s'", lastString, thisString)
}
lastString = thisString
}
}
func TestIsModuleNameAllowed(t *testing.T) {
libDisabled := "lib_disabled"
libEnabled := "lib_enabled"
libDclaWithinApex := "lib_dcla_within_apex"
libDclaNonApex := "lib_dcla_non_apex"
libNotConverted := "lib_not_converted"
disabledModules := map[string]bool{
libDisabled: true,
}
enabledModules := map[string]bool{
libEnabled: true,
}
dclaEnabledModules := map[string]bool{
libDclaWithinApex: true,
libDclaNonApex: true,
}
bazelContext := &mixedBuildBazelContext{
bazelEnabledModules: enabledModules,
bazelDisabledModules: disabledModules,
bazelDclaEnabledModules: dclaEnabledModules,
}
if bazelContext.IsModuleNameAllowed(libDisabled, true) {
t.Fatalf("%s shouldn't be allowed for mixed build", libDisabled)
}
if !bazelContext.IsModuleNameAllowed(libEnabled, true) {
t.Fatalf("%s should be allowed for mixed build", libEnabled)
}
if !bazelContext.IsModuleNameAllowed(libDclaWithinApex, true) {
t.Fatalf("%s should be allowed for mixed build", libDclaWithinApex)
}
if bazelContext.IsModuleNameAllowed(libDclaNonApex, false) {
t.Fatalf("%s shouldn't be allowed for mixed build", libDclaNonApex)
}
if bazelContext.IsModuleNameAllowed(libNotConverted, true) {
t.Fatalf("%s shouldn't be allowed for mixed build", libNotConverted)
}
}
func verifyAqueryContainsFlags(t *testing.T, config Config, expected ...string) {
t.Helper()
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{})
err := bazelContext.InvokeBazel(config, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}
sliceContains := func(slice []string, x string) bool {
for _, s := range slice {
if s == x {
return true
}
}
return false
}
aqueryArgv := bazelContext.bazelRunner.(*mockBazelRunner).bazelCommandRequests[aqueryCmd].Argv
for _, expectedFlag := range expected {
if !sliceContains(aqueryArgv, expectedFlag) {
t.Errorf("aquery does not contain expected flag %#v. Argv was: %#v", expectedFlag, aqueryArgv)
}
}
}
func verifyAqueryDoesNotContainSubstrings(t *testing.T, config Config, substrings ...string) {
t.Helper()
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{})
err := bazelContext.InvokeBazel(config, &testInvokeBazelContext{})
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}
sliceContainsSubstring := func(slice []string, substring string) bool {
for _, s := range slice {
if strings.Contains(s, substring) {
return true
}
}
return false
}
aqueryArgv := bazelContext.bazelRunner.(*mockBazelRunner).bazelCommandRequests[aqueryCmd].Argv
for _, substring := range substrings {
if sliceContainsSubstring(aqueryArgv, substring) {
t.Errorf("aquery contains unexpected substring %#v. Argv was: %#v", substring, aqueryArgv)
}
}
}
func testBazelContext(t *testing.T, bazelCommandResults map[bazelCommand]string) (*mixedBuildBazelContext, string) {
t.Helper()
p := bazelPaths{
soongOutDir: t.TempDir(),
outputBase: "outputbase",
workspaceDir: "workspace_dir",
}
if _, exists := bazelCommandResults[aqueryCmd]; !exists {
bazelCommandResults[aqueryCmd] = ""
}
runner := &mockBazelRunner{
testHelper: t,
bazelCommandResults: bazelCommandResults,
bazelCommandRequests: map[bazelCommand]bazel.CmdRequest{},
}
return &mixedBuildBazelContext{
bazelRunner: runner,
paths: &p,
}, p.soongOutDir
}
// Transform the json format to ActionGraphContainer
func JsonToActionGraphContainer(inputString string) ([]byte, error) {
var aqueryProtoResult analysis_v2_proto.ActionGraphContainer
err := json.Unmarshal([]byte(inputString), &aqueryProtoResult)
if err != nil {
return []byte(""), err
}
data, _ := proto.Marshal(&aqueryProtoResult)
return data, err
}

675
android/bazel_paths.go
View File

@@ -1,675 +0,0 @@
// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"fmt"
"os"
"path/filepath"
"strings"
"android/soong/bazel"
"github.com/google/blueprint"
"github.com/google/blueprint/pathtools"
)
// bazel_paths contains methods to:
// * resolve Soong path and module references into bazel.LabelList
// * resolve Bazel path references into Soong-compatible paths
//
// There is often a similar method for Bazel as there is for Soong path handling and should be used
// in similar circumstances
//
// Bazel Soong
// ==============================================================
// BazelLabelForModuleSrc PathForModuleSrc
// BazelLabelForModuleSrcExcludes PathForModuleSrcExcludes
// BazelLabelForModuleDeps n/a
// tbd PathForSource
// tbd ExistentPathsForSources
// PathForBazelOut PathForModuleOut
//
// Use cases:
// * Module contains a property (often tagged `android:"path"`) that expects paths *relative to the
// module directory*:
// * BazelLabelForModuleSrcExcludes, if the module also contains an excludes_<propname> property
// * BazelLabelForModuleSrc, otherwise
// * Converting references to other modules to Bazel Labels:
// BazelLabelForModuleDeps
// * Converting a path obtained from bazel_handler cquery results:
// PathForBazelOut
//
// NOTE: all Soong globs are expanded within Soong rather than being converted to a Bazel glob
// syntax. This occurs because Soong does not have a concept of crossing package boundaries,
// so the glob as computed by Soong may contain paths that cross package-boundaries. These
// would be unknowingly omitted if the glob were handled by Bazel. By expanding globs within
// Soong, we support identification and detection (within Bazel) use of paths that cross
// package boundaries.
//
// Path resolution:
// * filepath/globs: resolves as itself or is converted to an absolute Bazel label (e.g.
// //path/to/dir:<filepath>) if path exists in a separate package or subpackage.
// * references to other modules (using the ":name{.tag}" syntax). These resolve as a Bazel label
// for a target. If the Bazel target is in the local module directory, it will be returned
// relative to the current package (e.g. ":<target>"). Otherwise, it will be returned as an
// absolute Bazel label (e.g. "//path/to/dir:<target>"). If the reference to another module
// cannot be resolved,the function will panic. This is often due to the dependency not being added
// via an AddDependency* method.
// BazelConversionContext is a minimal context interface to check if a module should be converted by bp2build,
// with functions containing information to match against allowlists and denylists.
// If a module is deemed to be convertible by bp2build, then it should rely on a
// BazelConversionPathContext for more functions for dep/path features.
type BazelConversionContext interface {
Config() Config
Module() Module
OtherModuleType(m blueprint.Module) string
OtherModuleName(m blueprint.Module) string
OtherModuleDir(m blueprint.Module) string
ModuleErrorf(format string, args ...interface{})
}
// A subset of the ModuleContext methods which are sufficient to resolve references to paths/deps in
// order to form a Bazel-compatible label for conversion.
type BazelConversionPathContext interface {
EarlyModulePathContext
BazelConversionContext
ModuleName() string
ModuleType() string
ModuleErrorf(fmt string, args ...interface{})
PropertyErrorf(property, fmt string, args ...interface{})
GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag)
ModuleFromName(name string) (blueprint.Module, bool)
AddUnconvertedBp2buildDep(string)
AddMissingBp2buildDep(dep string)
}
// BazelLabelForModuleDeps expects a list of reference to other modules, ("<module>"
// or ":<module>") and returns a Bazel-compatible label which corresponds to dependencies on the
// module within the given ctx.
func BazelLabelForModuleDeps(ctx Bp2buildMutatorContext, modules []string) bazel.LabelList {
return BazelLabelForModuleDepsWithFn(ctx, modules, BazelModuleLabel, true)
}
// BazelLabelForModuleWholeDepsExcludes expects two lists: modules (containing modules to include in
// the list), and excludes (modules to exclude from the list). Both of these should contain
// references to other modules, ("<module>" or ":<module>"). It returns a Bazel-compatible label
// list which corresponds to dependencies on the module within the given ctx, and the excluded
// dependencies. Prebuilt dependencies will be appended with _alwayslink so they can be handled as
// whole static libraries.
func BazelLabelForModuleDepsExcludes(ctx Bp2buildMutatorContext, modules, excludes []string) bazel.LabelList {
return BazelLabelForModuleDepsExcludesWithFn(ctx, modules, excludes, BazelModuleLabel)
}
// BazelLabelForModuleDepsWithFn expects a list of reference to other modules, ("<module>"
// or ":<module>") and applies moduleToLabelFn to determine and return a Bazel-compatible label
// which corresponds to dependencies on the module within the given ctx.
func BazelLabelForModuleDepsWithFn(ctx Bp2buildMutatorContext, modules []string,
moduleToLabelFn func(BazelConversionPathContext, blueprint.Module) string,
markAsDeps bool) bazel.LabelList {
var labels bazel.LabelList
// In some cases, a nil string list is different than an explicitly empty list.
if len(modules) == 0 && modules != nil {
labels.Includes = []bazel.Label{}
return labels
}
modules = FirstUniqueStrings(modules)
for _, module := range modules {
bpText := module
if m := SrcIsModule(module); m == "" {
module = ":" + module
}
if m, t := SrcIsModuleWithTag(module); m != "" {
l := getOtherModuleLabel(ctx, m, t, moduleToLabelFn, markAsDeps)
if l != nil {
l.OriginalModuleName = bpText
labels.Includes = append(labels.Includes, *l)
}
} else {
ctx.ModuleErrorf("%q, is not a module reference", module)
}
}
return labels
}
// BazelLabelForModuleDepsExcludesWithFn expects two lists: modules (containing modules to include in the
// list), and excludes (modules to exclude from the list). Both of these should contain references
// to other modules, ("<module>" or ":<module>"). It applies moduleToLabelFn to determine and return a
// Bazel-compatible label list which corresponds to dependencies on the module within the given ctx, and
// the excluded dependencies.
func BazelLabelForModuleDepsExcludesWithFn(ctx Bp2buildMutatorContext, modules, excludes []string,
moduleToLabelFn func(BazelConversionPathContext, blueprint.Module) string) bazel.LabelList {
moduleLabels := BazelLabelForModuleDepsWithFn(ctx, RemoveListFromList(modules, excludes), moduleToLabelFn, true)
if len(excludes) == 0 {
return moduleLabels
}
excludeLabels := BazelLabelForModuleDepsWithFn(ctx, excludes, moduleToLabelFn, false)
return bazel.LabelList{
Includes: moduleLabels.Includes,
Excludes: excludeLabels.Includes,
}
}
func BazelLabelForModuleSrcSingle(ctx Bp2buildMutatorContext, path string) bazel.Label {
if srcs := BazelLabelForModuleSrcExcludes(ctx, []string{path}, []string(nil)).Includes; len(srcs) > 0 {
return srcs[0]
}
return bazel.Label{}
}
func BazelLabelForModuleDepSingle(ctx Bp2buildMutatorContext, path string) bazel.Label {
if srcs := BazelLabelForModuleDepsExcludes(ctx, []string{path}, []string(nil)).Includes; len(srcs) > 0 {
return srcs[0]
}
return bazel.Label{}
}
// BazelLabelForModuleSrc expects a list of path (relative to local module directory) and module
// references (":<module>") and returns a bazel.LabelList{} containing the resolved references in
// paths, relative to the local module, or Bazel-labels (absolute if in a different package or
// relative if within the same package).
// Properties must have been annotated with struct tag `android:"path"` so that dependencies modules
// will have already been handled by the pathdeps mutator.
func BazelLabelForModuleSrc(ctx Bp2buildMutatorContext, paths []string) bazel.LabelList {
return BazelLabelForModuleSrcExcludes(ctx, paths, []string(nil))
}
// BazelLabelForModuleSrc expects lists of path and excludes (relative to local module directory)
// and module references (":<module>") and returns a bazel.LabelList{} containing the resolved
// references in paths, minus those in excludes, relative to the local module, or Bazel-labels
// (absolute if in a different package or relative if within the same package).
// Properties must have been annotated with struct tag `android:"path"` so that dependencies modules
// will have already been handled by the pathdeps mutator.
func BazelLabelForModuleSrcExcludes(ctx Bp2buildMutatorContext, paths, excludes []string) bazel.LabelList {
excludeLabels := expandSrcsForBazel(ctx, excludes, []string(nil), false)
excluded := make([]string, 0, len(excludeLabels.Includes))
for _, e := range excludeLabels.Includes {
excluded = append(excluded, e.Label)
}
labels := expandSrcsForBazel(ctx, paths, excluded, true)
labels.Excludes = excludeLabels.Includes
labels = TransformSubpackagePaths(ctx.Config(), ctx.ModuleDir(), labels)
return labels
}
func BazelLabelForSrcPatternExcludes(ctx BazelConversionPathContext, dir, pattern string, excludes []string) bazel.LabelList {
topRelPaths, err := ctx.GlobWithDeps(filepath.Join(dir, pattern), excludes)
if err != nil {
ctx.ModuleErrorf("Could not search dir: %s for pattern %s due to %v\n", dir, pattern, err)
}
// An intermediate list of labels relative to `dir` that assumes that there no subpacakges beneath `dir`
dirRelLabels := []bazel.Label{}
for _, topRelPath := range topRelPaths {
dirRelPath := Rel(ctx, dir, topRelPath)
dirRelLabels = append(dirRelLabels, bazel.Label{Label: "./" + dirRelPath})
}
// Return the package boudary resolved labels
return TransformSubpackagePaths(ctx.Config(), dir, bazel.MakeLabelList(dirRelLabels))
}
// Returns true if a prefix + components[:i] is a package boundary.
//
// A package boundary is determined by a BUILD file in the directory. This can happen in 2 cases:
//
// 1. An Android.bp exists, which bp2build will always convert to a sibling BUILD file.
// 2. An Android.bp doesn't exist, but a checked-in BUILD/BUILD.bazel file exists, and that file
// is allowlisted by the bp2build configuration to be merged into the symlink forest workspace.
func isPackageBoundary(config Config, prefix string, components []string, componentIndex int) bool {
isSymlink := func(c Config, path string) bool {
f, err := c.fs.Lstat(path)
if err != nil {
// The file does not exist
return false
}
return f.Mode()&os.ModeSymlink == os.ModeSymlink
}
prefix = filepath.Join(prefix, filepath.Join(components[:componentIndex+1]...))
if exists, _, _ := config.fs.Exists(filepath.Join(prefix, "Android.bp")); exists {
return true
} else if config.Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(prefix) || isSymlink(config, prefix) {
if exists, _, _ := config.fs.Exists(filepath.Join(prefix, "BUILD")); exists {
return true
} else if exists, _, _ := config.fs.Exists(filepath.Join(prefix, "BUILD.bazel")); exists {
return true
}
}
return false
}
// Transform a path (if necessary) to acknowledge package boundaries
//
// e.g. something like
//
// async_safe/include/async_safe/CHECK.h
//
// might become
//
// //bionic/libc/async_safe:include/async_safe/CHECK.h
//
// if the "async_safe" directory is actually a package and not just a directory.
//
// In particular, paths that extend into packages are transformed into absolute labels beginning with //.
func transformSubpackagePath(cfg Config, dir string, path bazel.Label) bazel.Label {
var newPath bazel.Label
// Don't transform OriginalModuleName
newPath.OriginalModuleName = path.OriginalModuleName
// if it wasn't a module, store the original path. We may need the original path to replace
// references if it is actually in another package
if path.OriginalModuleName == "" {
newPath.OriginalModuleName = path.Label
}
if strings.HasPrefix(path.Label, "//") {
// Assume absolute labels are already correct (e.g. //path/to/some/package:foo.h)
newPath.Label = path.Label
return newPath
}
if strings.HasPrefix(path.Label, "./") {
// Drop "./" for consistent handling of paths.
// Specifically, to not let "." be considered a package boundary.
// Say `inputPath` is `x/Android.bp` and that file has some module
// with `srcs=["y/a.c", "z/b.c"]`.
// And say the directory tree is:
// x
// ├── Android.bp
// ├── y
// │ ├── a.c
// │ └── Android.bp
// └── z
// └── b.c
// Then bazel equivalent labels in srcs should be:
// //x/y:a.c, x/z/b.c
// The above should still be the case if `x/Android.bp` had
// srcs=["./y/a.c", "./z/b.c"]
// However, if we didn't strip "./", we'd get
// //x/./y:a.c, //x/.:z/b.c
path.Label = strings.TrimPrefix(path.Label, "./")
}
pathComponents := strings.Split(path.Label, "/")
newLabel := ""
foundPackageBoundary := false
// Check the deepest subdirectory first and work upwards
for i := len(pathComponents) - 1; i >= 0; i-- {
pathComponent := pathComponents[i]
var sep string
if !foundPackageBoundary && isPackageBoundary(cfg, dir, pathComponents, i) {
sep = ":"
foundPackageBoundary = true
} else {
sep = "/"
}
if newLabel == "" {
newLabel = pathComponent
} else {
newLabel = pathComponent + sep + newLabel
}
}
if foundPackageBoundary {
// Ensure paths end up looking like //bionic/... instead of //./bionic/...
moduleDir := dir
if strings.HasPrefix(moduleDir, ".") {
moduleDir = moduleDir[1:]
}
// Make the path into an absolute label (e.g. //bionic/libc/foo:bar.h instead of just foo:bar.h)
if moduleDir == "" {
newLabel = "//" + newLabel
} else {
newLabel = "//" + moduleDir + "/" + newLabel
}
}
newPath.Label = newLabel
return newPath
}
// Transform paths to acknowledge package boundaries
// See transformSubpackagePath() for more information
func TransformSubpackagePaths(cfg Config, dir string, paths bazel.LabelList) bazel.LabelList {
var newPaths bazel.LabelList
for _, include := range paths.Includes {
newPaths.Includes = append(newPaths.Includes, transformSubpackagePath(cfg, dir, include))
}
for _, exclude := range paths.Excludes {
newPaths.Excludes = append(newPaths.Excludes, transformSubpackagePath(cfg, dir, exclude))
}
return newPaths
}
// Converts root-relative Paths to a list of bazel.Label relative to the module in ctx.
func RootToModuleRelativePaths(ctx BazelConversionPathContext, paths Paths) []bazel.Label {
var newPaths []bazel.Label
for _, path := range PathsWithModuleSrcSubDir(ctx, paths, "") {
s := path.Rel()
newPaths = append(newPaths, bazel.Label{Label: s})
}
return newPaths
}
var Bp2buildDepTag bp2buildDepTag
type bp2buildDepTag struct {
blueprint.BaseDependencyTag
}
// expandSrcsForBazel returns bazel.LabelList with paths rooted from the module's local source
// directory and Bazel target labels, excluding those included in the excludes argument (which
// should already be expanded to resolve references to Soong-modules). Valid elements of paths
// include:
// - filepath, relative to local module directory, resolves as a filepath relative to the local
// source directory
// - glob, relative to the local module directory, resolves as filepath(s), relative to the local
// module directory. Because Soong does not have a concept of crossing package boundaries, the
// glob as computed by Soong may contain paths that cross package-boundaries that would be
// unknowingly omitted if the glob were handled by Bazel. To allow identification and detect
// (within Bazel) use of paths that cross package boundaries, we expand globs within Soong rather
// than converting Soong glob syntax to Bazel glob syntax. **Invalid for excludes.**
// - other modules using the ":name{.tag}" syntax. These modules must implement SourceFileProducer
// or OutputFileProducer. These resolve as a Bazel label for a target. If the Bazel target is in
// the local module directory, it will be returned relative to the current package (e.g.
// ":<target>"). Otherwise, it will be returned as an absolute Bazel label (e.g.
// "//path/to/dir:<target>"). If the reference to another module cannot be resolved,the function
// will panic.
//
// Properties passed as the paths or excludes argument must have been annotated with struct tag
// `android:"path"` so that dependencies on other modules will have already been handled by the
// pathdeps mutator.
func expandSrcsForBazel(ctx Bp2buildMutatorContext, paths, expandedExcludes []string, markAsDeps bool) bazel.LabelList {
if paths == nil {
return bazel.LabelList{}
}
labels := bazel.LabelList{
Includes: []bazel.Label{},
}
// expandedExcludes contain module-dir relative paths, but root-relative paths
// are needed for GlobFiles later.
var rootRelativeExpandedExcludes []string
for _, e := range expandedExcludes {
rootRelativeExpandedExcludes = append(rootRelativeExpandedExcludes, filepath.Join(ctx.ModuleDir(), e))
}
for _, p := range paths {
if m, tag := SrcIsModuleWithTag(p); m != "" {
l := getOtherModuleLabel(ctx, m, tag, BazelModuleLabel, markAsDeps)
if l != nil && !InList(l.Label, expandedExcludes) {
if strings.HasPrefix(m, "//") {
// this is a module in a soong namespace
// It appears as //<namespace>:<module_name> in srcs, and not ://<namespace>:<module_name>
l.OriginalModuleName = m
} else {
l.OriginalModuleName = fmt.Sprintf(":%s", m)
}
labels.Includes = append(labels.Includes, *l)
}
} else {
var expandedPaths []bazel.Label
if pathtools.IsGlob(p) {
// e.g. turn "math/*.c" in
// external/arm-optimized-routines to external/arm-optimized-routines/math/*.c
rootRelativeGlobPath := pathForModuleSrc(ctx, p).String()
expandedPaths = RootToModuleRelativePaths(ctx, GlobFiles(ctx, rootRelativeGlobPath, rootRelativeExpandedExcludes))
} else {
if !InList(p, expandedExcludes) {
expandedPaths = append(expandedPaths, bazel.Label{Label: p})
}
}
labels.Includes = append(labels.Includes, expandedPaths...)
}
}
return labels
}
// getOtherModuleLabel returns a bazel.Label for the given dependency/tag combination for the
// module. The label will be relative to the current directory if appropriate. The dependency must
// already be resolved by either deps mutator or path deps mutator.
func getOtherModuleLabel(ctx Bp2buildMutatorContext, dep, tag string,
labelFromModule func(BazelConversionPathContext, blueprint.Module) string,
markAsDep bool) *bazel.Label {
m, _ := ctx.ModuleFromName(dep)
// The module was not found in an Android.bp file, this is often due to:
// * a limited manifest
// * a required module not being converted from Android.mk
if m == nil {
ctx.AddMissingBp2buildDep(dep)
return &bazel.Label{
Label: ":" + dep + "__BP2BUILD__MISSING__DEP",
}
}
// Returns true if a dependency from the current module to the target module
// should be skipped; doing so is a hack to circumvent certain problematic
// scenarios that will be addressed in the future.
shouldSkipDep := func(dep string) bool {
// Don't count dependencies of "libc". This is a hack to circumvent the
// fact that, in a variantless build graph, "libc" has a dependency on itself.
if ctx.ModuleName() == "libc" {
return true
}
// TODO: b/303307672: Dependencies on this module happen to "work" because
// there is a source file with the same name as this module in the
// same directory. We should remove this hack and enforce the underlying
// module of this name is the actual one used.
if dep == "mke2fs.conf" {
return true
}
// TODO: b/303310285: Remove this special-casing once all dependencies of
// crtbegin_dynamic are convertible
if ctx.ModuleName() == "crtbegin_dynamic" {
return true
}
return false
}
if markAsDep && !shouldSkipDep(dep) {
ctx.AddDependency(ctx.Module(), Bp2buildDepTag, dep)
}
if !convertedToBazel(ctx, m) {
ctx.AddUnconvertedBp2buildDep(dep)
}
label := BazelModuleLabel(ctx, ctx.Module())
otherLabel := labelFromModule(ctx, m)
// TODO(b/165114590): Convert tag (":name{.tag}") to corresponding Bazel implicit output targets.
if (tag != "" && m.Name() == "framework-res") ||
(tag == ".generated_srcjars" && ctx.OtherModuleType(m) == "java_aconfig_library") {
otherLabel += tag
}
if samePackage(label, otherLabel) {
otherLabel = bazelShortLabel(otherLabel)
}
return &bazel.Label{
Label: otherLabel,
}
}
func BazelModuleLabel(ctx BazelConversionPathContext, module blueprint.Module) string {
// TODO(b/165114590): Convert tag (":name{.tag}") to corresponding Bazel implicit output targets.
if !convertedToBazel(ctx, module) || isGoModule(module) {
return bp2buildModuleLabel(ctx, module)
}
b, _ := module.(Bazelable)
return b.GetBazelLabel(ctx, module)
}
func bazelShortLabel(label string) string {
i := strings.Index(label, ":")
if i == -1 {
panic(fmt.Errorf("Could not find the ':' character in '%s', expected a fully qualified label.", label))
}
return label[i:]
}
func bazelPackage(label string) string {
i := strings.Index(label, ":")
if i == -1 {
panic(fmt.Errorf("Could not find the ':' character in '%s', expected a fully qualified label.", label))
}
return label[0:i]
}
func samePackage(label1, label2 string) bool {
return bazelPackage(label1) == bazelPackage(label2)
}
func bp2buildModuleLabel(ctx BazelConversionContext, module blueprint.Module) string {
moduleName := moduleNameWithPossibleOverride(ctx, module, ctx.OtherModuleName(module))
moduleDir := moduleDirWithPossibleOverride(ctx, module, ctx.OtherModuleDir(module))
if moduleDir == Bp2BuildTopLevel {
moduleDir = ""
}
if a, ok := module.(Module); ok && IsModulePrebuilt(a) {
moduleName = RemoveOptionalPrebuiltPrefix(moduleName)
}
return fmt.Sprintf("//%s:%s", moduleDir, moduleName)
}
// BazelOutPath is a Bazel output path compatible to be used for mixed builds within Soong/Ninja.
type BazelOutPath struct {
OutputPath
}
// ensure BazelOutPath implements Path
var _ Path = BazelOutPath{}
// ensure BazelOutPath implements genPathProvider
var _ genPathProvider = BazelOutPath{}
// ensure BazelOutPath implements objPathProvider
var _ objPathProvider = BazelOutPath{}
func (p BazelOutPath) genPathWithExt(ctx ModuleOutPathContext, subdir, ext string) ModuleGenPath {
return PathForModuleGen(ctx, subdir, pathtools.ReplaceExtension(p.path, ext))
}
func (p BazelOutPath) objPathWithExt(ctx ModuleOutPathContext, subdir, ext string) ModuleObjPath {
return PathForModuleObj(ctx, subdir, pathtools.ReplaceExtension(p.path, ext))
}
// PathForBazelOutRelative returns a BazelOutPath representing the path under an output directory dedicated to
// bazel-owned outputs. Calling .Rel() on the result will give the input path as relative to the given
// relativeRoot.
func PathForBazelOutRelative(ctx PathContext, relativeRoot string, path string) BazelOutPath {
validatedPath, err := validatePath(filepath.Join("execroot", "__main__", path))
if err != nil {
reportPathError(ctx, err)
}
var relativeRootPath string
if pathComponents := strings.SplitN(path, "/", 4); len(pathComponents) >= 3 &&
pathComponents[0] == "bazel-out" && pathComponents[2] == "bin" {
// If the path starts with something like: bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/
// make it relative to that folder. bazel-out/volatile-status.txt is an example
// of something that starts with bazel-out but is not relative to the bin folder
relativeRootPath = filepath.Join("execroot", "__main__", pathComponents[0], pathComponents[1], pathComponents[2], relativeRoot)
} else {
relativeRootPath = filepath.Join("execroot", "__main__", relativeRoot)
}
var relPath string
if relPath, err = filepath.Rel(relativeRootPath, validatedPath); err != nil || strings.HasPrefix(relPath, "../") {
// We failed to make this path relative to execroot/__main__, fall back to a non-relative path
// One case where this happens is when path is ../bazel_tools/something
relativeRootPath = ""
relPath = validatedPath
}
outputPath := OutputPath{
basePath{"", ""},
ctx.Config().soongOutDir,
ctx.Config().BazelContext.OutputBase(),
}
return BazelOutPath{
// .withRel() appends its argument onto the current path, and only the most
// recently appended part is returned by outputPath.rel().
// So outputPath.rel() will return relPath.
OutputPath: outputPath.withRel(relativeRootPath).withRel(relPath),
}
}
// PathForBazelOut returns a BazelOutPath representing the path under an output directory dedicated to
// bazel-owned outputs.
func PathForBazelOut(ctx PathContext, path string) BazelOutPath {
return PathForBazelOutRelative(ctx, "", path)
}
// PathsForBazelOut returns a list of paths representing the paths under an output directory
// dedicated to Bazel-owned outputs.
func PathsForBazelOut(ctx PathContext, paths []string) Paths {
outs := make(Paths, 0, len(paths))
for _, p := range paths {
outs = append(outs, PathForBazelOut(ctx, p))
}
return outs
}
// BazelStringOrLabelFromProp splits a Soong module property that can be
// either a string literal, path (with android:path tag) or a module reference
// into separate bazel string or label attributes. Bazel treats string and label
// attributes as distinct types, so this function categorizes a string property
// into either one of them.
//
// e.g. apex.private_key = "foo.pem" can either refer to:
//
// 1. "foo.pem" in the current directory -> file target
// 2. "foo.pem" module -> rule target
// 3. "foo.pem" file in a different directory, prefixed by a product variable handled
// in a bazel macro. -> string literal
//
// For the first two cases, they are defined using the label attribute. For the third case,
// it's defined with the string attribute.
func BazelStringOrLabelFromProp(
ctx Bp2buildMutatorContext,
propToDistinguish *string) (bazel.LabelAttribute, bazel.StringAttribute) {
var labelAttr bazel.LabelAttribute
var strAttr bazel.StringAttribute
if propToDistinguish == nil {
// nil pointer
return labelAttr, strAttr
}
prop := String(propToDistinguish)
if SrcIsModule(prop) != "" {
// If it's a module (SrcIsModule will return the module name), set the
// resolved label to the label attribute.
labelAttr.SetValue(BazelLabelForModuleDepSingle(ctx, prop))
} else {
// Not a module name. This could be a string literal or a file target in
// the current dir. Check if the path exists:
path := ExistentPathForSource(ctx, ctx.ModuleDir(), prop)
if path.Valid() && parentDir(path.String()) == ctx.ModuleDir() {
// If it exists and the path is relative to the current dir, resolve the bazel label
// for the _file target_ and set it to the label attribute.
//
// Resolution is necessary because this could be a file in a subpackage.
labelAttr.SetValue(BazelLabelForModuleSrcSingle(ctx, prop))
} else {
// Otherwise, treat it as a string literal and assign to the string attribute.
strAttr.Value = propToDistinguish
}
}
return labelAttr, strAttr
}

240
android/bazel_paths_test.go
View File

@@ -1,240 +0,0 @@
// Copyright 2022 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"fmt"
"path/filepath"
"testing"
"android/soong/bazel"
"github.com/google/blueprint"
"github.com/google/blueprint/pathtools"
)
type TestBazelPathContext struct{}
func (*TestBazelPathContext) Config() Config {
cfg := NullConfig("out", "out/soong")
cfg.BazelContext = MockBazelContext{
OutputBaseDir: "out/bazel",
}
return cfg
}
func (*TestBazelPathContext) AddNinjaFileDeps(...string) {
panic("Unimplemented")
}
func TestPathForBazelOut(t *testing.T) {
ctx := &TestBazelPathContext{}
out := PathForBazelOut(ctx, "foo/bar/baz/boq.txt")
expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/foo/bar/baz/boq.txt")
if out.String() != expectedPath {
t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String())
}
expectedRelPath := "foo/bar/baz/boq.txt"
if out.Rel() != expectedRelPath {
t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel())
}
}
func TestPathForBazelOutRelative(t *testing.T) {
ctx := &TestBazelPathContext{}
out := PathForBazelOutRelative(ctx, "foo/bar", "foo/bar/baz/boq.txt")
expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/foo/bar/baz/boq.txt")
if out.String() != expectedPath {
t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String())
}
expectedRelPath := "baz/boq.txt"
if out.Rel() != expectedRelPath {
t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel())
}
}
func TestPathForBazelOutRelativeUnderBinFolder(t *testing.T) {
ctx := &TestBazelPathContext{}
out := PathForBazelOutRelative(ctx, "foo/bar", "bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/foo/bar/baz/boq.txt")
expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/foo/bar/baz/boq.txt")
if out.String() != expectedPath {
t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String())
}
expectedRelPath := "baz/boq.txt"
if out.Rel() != expectedRelPath {
t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel())
}
}
func TestPathForBazelOutOutsideOfExecroot(t *testing.T) {
ctx := &TestBazelPathContext{}
out := PathForBazelOut(ctx, "../bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar")
expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar")
if out.String() != expectedPath {
t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String())
}
expectedRelPath := "execroot/bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar"
if out.Rel() != expectedRelPath {
t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel())
}
}
func TestPathForBazelOutRelativeWithParentDirectoryRoot(t *testing.T) {
ctx := &TestBazelPathContext{}
out := PathForBazelOutRelative(ctx, "../bazel_tools", "../bazel_tools/foo/bar/baz.sh")
expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/bazel_tools/foo/bar/baz.sh")
if out.String() != expectedPath {
t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String())
}
expectedRelPath := "foo/bar/baz.sh"
if out.Rel() != expectedRelPath {
t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel())
}
}
type TestBazelConversionPathContext struct {
TestBazelConversionContext
moduleDir string
cfg Config
mockGlobResults *[]string
}
func (ctx *TestBazelConversionPathContext) AddNinjaFileDeps(...string) {
panic("Unimplemented")
}
func (ctx *TestBazelConversionPathContext) GlobWithDeps(string, []string) ([]string, error) {
if ctx.mockGlobResults == nil {
return []string{}, fmt.Errorf("Set mock glob results first")
}
return *ctx.mockGlobResults, nil
}
func (ctx *TestBazelConversionPathContext) PropertyErrorf(string, string, ...interface{}) {
panic("Unimplemented")
}
func (ctx *TestBazelConversionPathContext) GetDirectDep(string) (blueprint.Module, blueprint.DependencyTag) {
panic("Unimplemented")
}
func (ctx *TestBazelConversionPathContext) ModuleFromName(string) (blueprint.Module, bool) {
panic("Unimplemented")
}
func (ctx *TestBazelConversionPathContext) AddUnconvertedBp2buildDep(string) {
panic("Unimplemented")
}
func (ctx *TestBazelConversionPathContext) AddMissingBp2buildDep(string) {
panic("Unimplemented")
}
func (ctx *TestBazelConversionPathContext) Module() Module {
panic("Unimplemented")
}
func (ctx *TestBazelConversionPathContext) Config() Config {
return ctx.cfg
}
func (ctx *TestBazelConversionPathContext) ModuleDir() string {
return ctx.moduleDir
}
func (ctx *TestBazelConversionPathContext) ModuleName() string {
panic("Unimplemented")
}
func (ctx *TestBazelConversionPathContext) ModuleType() string {
panic("Unimplemented")
}
func TestTransformSubpackagePath(t *testing.T) {
cfg := NullConfig("out", "out/soong")
cfg.fs = pathtools.MockFs(map[string][]byte{
"x/Android.bp": nil,
"x/y/Android.bp": nil,
})
var ctx BazelConversionPathContext = &TestBazelConversionPathContext{
moduleDir: "x",
cfg: cfg,
}
pairs := map[string]string{
"y/a.c": "//x/y:a.c",
"./y/a.c": "//x/y:a.c",
"z/b.c": "z/b.c",
"./z/b.c": "z/b.c",
}
for in, out := range pairs {
actual := transformSubpackagePath(ctx.Config(), ctx.ModuleDir(), bazel.Label{Label: in}).Label
if actual != out {
t.Errorf("expected:\n%v\nactual:\n%v", out, actual)
}
}
}
// Check that the files in a specific directory are returned with labels that respect package boundaries
// Since the test uses a mock for GlobWithDeps, the params passed to BazelLabelForSrcPatternExcludes are no-ops
func TestBazelLabelForSrcPatternExcludes(t *testing.T) {
cfg := NullConfig("out", "out/soong")
cfg.fs = pathtools.MockFs(map[string][]byte{
"x/Android.bp": nil,
"x/y/Android.bp": nil,
// .proto files
"foo.proto": nil,
"x/bar.proto": nil,
"x/baz.proto": nil,
"x/y/qux.proto": nil,
})
var ctx BazelConversionPathContext = &TestBazelConversionPathContext{
cfg: cfg,
}
// Root dir
ctx.(*TestBazelConversionPathContext).mockGlobResults = &[]string{"foo.proto", "x/bar.proto", "x/baz.proto", "x/y/qux.proto"}
actualLabelsFromRoot := BazelLabelForSrcPatternExcludes(ctx, ".", "**/*.proto", []string{})
expectedLabelsAsString := []string{"foo.proto", "//x:bar.proto", "//x:baz.proto", "//x/y:qux.proto"}
for i, actual := range actualLabelsFromRoot.Includes {
AssertStringEquals(t, "Error in finding src labels relative to root directory", expectedLabelsAsString[i], actual.Label)
}
// x dir
ctx.(*TestBazelConversionPathContext).mockGlobResults = &[]string{"x/bar.proto", "x/baz.proto", "x/y/qux.proto"}
actualLabelsFromRoot = BazelLabelForSrcPatternExcludes(ctx, "x", "**/*.proto", []string{})
expectedLabelsAsString = []string{"bar.proto", "baz.proto", "//x/y:qux.proto"}
for i, actual := range actualLabelsFromRoot.Includes {
AssertStringEquals(t, "Error in finding src labels relative to x directory", expectedLabelsAsString[i], actual.Label)
}
// y dir
ctx.(*TestBazelConversionPathContext).mockGlobResults = &[]string{"x/y/qux.proto"}
actualLabelsFromRoot = BazelLabelForSrcPatternExcludes(ctx, "x/y", "**/*.proto", []string{})
expectedLabelsAsString = []string{"qux.proto"}
for i, actual := range actualLabelsFromRoot.Includes {
AssertStringEquals(t, "Error in finding src labels relative to x/y directory", expectedLabelsAsString[i], actual.Label)
}
}

592
android/bazel_test.go
View File

@@ -1,592 +0,0 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"fmt"
"testing"
"android/soong/android/allowlists"
"android/soong/bazel"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
)
func TestConvertAllModulesInPackage(t *testing.T) {
testCases := []struct {
prefixes allowlists.Bp2BuildConfig
packageDir string
}{
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a/b": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a/b",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a/b": allowlists.Bp2BuildDefaultTrueRecursively,
"a/b/c": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a/b",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultTrueRecursively,
"d/e/f": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a/b",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultFalse,
"a/b": allowlists.Bp2BuildDefaultTrueRecursively,
"a/b/c": allowlists.Bp2BuildDefaultFalse,
},
packageDir: "a/b",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultTrueRecursively,
"a/b": allowlists.Bp2BuildDefaultFalse,
"a/b/c": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultFalseRecursively,
"a/b": allowlists.Bp2BuildDefaultTrue,
},
packageDir: "a/b",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultFalseRecursively,
"a/b": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a/b/c",
},
}
for _, test := range testCases {
if ok, _ := bp2buildDefaultTrueRecursively(test.packageDir, test.prefixes); !ok {
t.Errorf("Expected to convert all modules in %s based on %v, but failed.", test.packageDir, test.prefixes)
}
}
}
func TestModuleOptIn(t *testing.T) {
testCases := []struct {
prefixes allowlists.Bp2BuildConfig
packageDir string
}{
{
prefixes: allowlists.Bp2BuildConfig{
"a/b": allowlists.Bp2BuildDefaultFalse,
},
packageDir: "a/b",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultFalse,
"a/b": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a/b": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a", // opt-in by default
},
{
prefixes: allowlists.Bp2BuildConfig{
"a/b/c": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a/b",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultTrueRecursively,
"d/e/f": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "foo/bar",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultTrueRecursively,
"a/b": allowlists.Bp2BuildDefaultFalse,
"a/b/c": allowlists.Bp2BuildDefaultTrueRecursively,
},
packageDir: "a/b",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultFalse,
"a/b": allowlists.Bp2BuildDefaultTrueRecursively,
"a/b/c": allowlists.Bp2BuildDefaultFalse,
},
packageDir: "a",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultFalseRecursively,
"a/b": allowlists.Bp2BuildDefaultTrue,
},
packageDir: "a/b/c",
},
{
prefixes: allowlists.Bp2BuildConfig{
"a": allowlists.Bp2BuildDefaultTrueRecursively,
"a/b": allowlists.Bp2BuildDefaultFalseRecursively,
},
packageDir: "a/b/c",
},
}
for _, test := range testCases {
if ok, _ := bp2buildDefaultTrueRecursively(test.packageDir, test.prefixes); ok {
t.Errorf("Expected to allow module opt-in in %s based on %v, but failed.", test.packageDir, test.prefixes)
}
}
}
type TestBazelModule struct {
bazel.TestModuleInfo
BazelModuleBase
}
var _ blueprint.Module = TestBazelModule{}
func (m TestBazelModule) Name() string {
return m.TestModuleInfo.ModuleName
}
func (m TestBazelModule) GenerateBuildActions(blueprint.ModuleContext) {
}
type TestBazelConversionContext struct {
omc bazel.OtherModuleTestContext
allowlist Bp2BuildConversionAllowlist
errors []string
}
var _ bazelOtherModuleContext = &TestBazelConversionContext{}
func (bcc *TestBazelConversionContext) OtherModuleType(m blueprint.Module) string {
return bcc.omc.OtherModuleType(m)
}
func (bcc *TestBazelConversionContext) OtherModuleName(m blueprint.Module) string {
return bcc.omc.OtherModuleName(m)
}
func (bcc *TestBazelConversionContext) OtherModuleDir(m blueprint.Module) string {
return bcc.omc.OtherModuleDir(m)
}
func (bcc *TestBazelConversionContext) ModuleErrorf(format string, args ...interface{}) {
bcc.errors = append(bcc.errors, fmt.Sprintf(format, args...))
}
func (bcc *TestBazelConversionContext) Config() Config {
return Config{
&config{
Bp2buildPackageConfig: bcc.allowlist,
},
}
}
var bazelableBazelModuleBase = BazelModuleBase{
bazelProperties: properties{
Bazel_module: BazelModuleProperties{
CanConvertToBazel: true,
},
},
}
func TestBp2BuildAllowlist(t *testing.T) {
testCases := []struct {
description string
shouldConvert bool
expectedErrors []string
module TestBazelModule
allowlist Bp2BuildConversionAllowlist
}{
{
description: "allowlist enables module",
shouldConvert: true,
module: TestBazelModule{
TestModuleInfo: bazel.TestModuleInfo{
ModuleName: "foo",
Typ: "rule1",
Dir: "dir1",
},
BazelModuleBase: bazelableBazelModuleBase,
},
allowlist: Bp2BuildConversionAllowlist{
moduleAlwaysConvert: map[string]bool{
"foo": true,
},
},
},
{
description: "module in name allowlist and type allowlist fails",
shouldConvert: false,
expectedErrors: []string{"A module \"foo\" of type \"rule1\" cannot be in moduleAlwaysConvert and also be in moduleTypeAlwaysConvert"},
module: TestBazelModule{
TestModuleInfo: bazel.TestModuleInfo{
ModuleName: "foo",
Typ: "rule1",
Dir: "dir1",
},
BazelModuleBase: bazelableBazelModuleBase,
},
allowlist: Bp2BuildConversionAllowlist{
moduleAlwaysConvert: map[string]bool{
"foo": true,
},
moduleTypeAlwaysConvert: map[string]bool{
"rule1": true,
},
},
},
{
description: "module in allowlist and denylist fails",
shouldConvert: false,
expectedErrors: []string{"a module \"foo\" cannot be in moduleDoNotConvert and also be in moduleAlwaysConvert"},
module: TestBazelModule{
TestModuleInfo: bazel.TestModuleInfo{
ModuleName: "foo",
Typ: "rule1",
Dir: "dir1",
},
BazelModuleBase: bazelableBazelModuleBase,
},
allowlist: Bp2BuildConversionAllowlist{
moduleAlwaysConvert: map[string]bool{
"foo": true,
},
moduleDoNotConvert: map[string]bool{
"foo": true,
},
},
},
{
description: "module allowlist and enabled directory",
shouldConvert: false,
expectedErrors: []string{"A module cannot be in a directory marked Bp2BuildDefaultTrue or Bp2BuildDefaultTrueRecursively and also be in moduleAlwaysConvert. Directory: 'existing/build/dir' Module: 'foo'"},
module: TestBazelModule{
TestModuleInfo: bazel.TestModuleInfo{
ModuleName: "foo",
Typ: "rule1",
Dir: "existing/build/dir",
},
BazelModuleBase: bazelableBazelModuleBase,
},
allowlist: Bp2BuildConversionAllowlist{
moduleAlwaysConvert: map[string]bool{
"foo": true,
},
defaultConfig: allowlists.Bp2BuildConfig{
"existing/build/dir": allowlists.Bp2BuildDefaultTrue,
},
},
},
{
description: "module allowlist and enabled subdirectory",
shouldConvert: false,
expectedErrors: []string{"A module cannot be in a directory marked Bp2BuildDefaultTrue or Bp2BuildDefaultTrueRecursively and also be in moduleAlwaysConvert. Directory: 'existing/build/dir' Module: 'foo'"},
module: TestBazelModule{
TestModuleInfo: bazel.TestModuleInfo{
ModuleName: "foo",
Typ: "rule1",
Dir: "existing/build/dir/subdir",
},
BazelModuleBase: bazelableBazelModuleBase,
},
allowlist: Bp2BuildConversionAllowlist{
moduleAlwaysConvert: map[string]bool{
"foo": true,
},
defaultConfig: allowlists.Bp2BuildConfig{
"existing/build/dir": allowlists.Bp2BuildDefaultTrueRecursively,
},
},
},
{
description: "module enabled in unit test short-circuits other allowlists",
shouldConvert: true,
module: TestBazelModule{
TestModuleInfo: bazel.TestModuleInfo{
ModuleName: "foo",
Typ: "rule1",
Dir: ".",
},
BazelModuleBase: BazelModuleBase{
bazelProperties: properties{
Bazel_module: BazelModuleProperties{
CanConvertToBazel: true,
Bp2build_available: proptools.BoolPtr(true),
},
},
},
},
allowlist: Bp2BuildConversionAllowlist{
moduleAlwaysConvert: map[string]bool{
"foo": true,
},
moduleDoNotConvert: map[string]bool{
"foo": true,
},
},
},
}
for _, test := range testCases {
t.Run(test.description, func(t *testing.T) {
bcc := &TestBazelConversionContext{
omc: bazel.OtherModuleTestContext{
Modules: []bazel.TestModuleInfo{
test.module.TestModuleInfo,
},
},
allowlist: test.allowlist,
}
shouldConvert := test.module.shouldConvertWithBp2build(bcc,
shouldConvertParams{
module: test.module.TestModuleInfo,
moduleDir: test.module.TestModuleInfo.Dir,
moduleType: test.module.TestModuleInfo.Typ,
moduleName: test.module.TestModuleInfo.ModuleName,
},
)
if test.shouldConvert != shouldConvert {
t.Errorf("Module shouldConvert expected to be: %v, but was: %v", test.shouldConvert, shouldConvert)
}
errorsMatch := true
if len(test.expectedErrors) != len(bcc.errors) {
errorsMatch = false
} else {
for i, err := range test.expectedErrors {
if err != bcc.errors[i] {
errorsMatch = false
}
}
}
if !errorsMatch {
t.Errorf("Expected errors to be: %v, but were: %v", test.expectedErrors, bcc.errors)
}
})
}
}
func TestBp2buildAllowList(t *testing.T) {
allowlist := GetBp2BuildAllowList()
for k, v := range allowlists.Bp2buildDefaultConfig {
if allowlist.defaultConfig[k] != v {
t.Errorf("bp2build default config of %s: expected: %v, got: %v", k, v, allowlist.defaultConfig[k])
}
}
for k, v := range allowlists.Bp2buildKeepExistingBuildFile {
if allowlist.keepExistingBuildFile[k] != v {
t.Errorf("bp2build keep existing build file of %s: expected: %v, got: %v", k, v, allowlist.keepExistingBuildFile[k])
}
}
for _, k := range allowlists.Bp2buildModuleTypeAlwaysConvertList {
if !allowlist.moduleTypeAlwaysConvert[k] {
t.Errorf("bp2build module type always convert of %s: expected: true, got: %v", k, allowlist.moduleTypeAlwaysConvert[k])
}
}
for _, k := range allowlists.Bp2buildModuleDoNotConvertList {
if !allowlist.moduleDoNotConvert[k] {
t.Errorf("bp2build module do not convert of %s: expected: true, got: %v", k, allowlist.moduleDoNotConvert[k])
}
}
}
func TestShouldKeepExistingBuildFileForDir(t *testing.T) {
allowlist := NewBp2BuildAllowlist()
// entry "a/b2/c2" is moot because of its parent "a/b2"
allowlist.SetKeepExistingBuildFile(map[string]bool{"a": false, "a/b1": false, "a/b2": true, "a/b1/c1": true, "a/b2/c2": false})
truths := []string{"a", "a/b1", "a/b2", "a/b1/c1", "a/b2/c", "a/b2/c2", "a/b2/c2/d"}
falsities := []string{"a1", "a/b", "a/b1/c"}
for _, dir := range truths {
if !allowlist.ShouldKeepExistingBuildFileForDir(dir) {
t.Errorf("%s expected TRUE but was FALSE", dir)
}
}
for _, dir := range falsities {
if allowlist.ShouldKeepExistingBuildFileForDir(dir) {
t.Errorf("%s expected FALSE but was TRUE", dir)
}
}
}
type mixedBuildModule struct {
ModuleBase
BazelModuleBase
props struct {
Deps []string
Mixed_build_incompatible *bool
QueuedBazelCall bool `blueprint:"mutated"`
}
}
type mixedBuildModuleInfo struct {
QueuedBazelCall bool
}
var mixedBuildModuleProvider = blueprint.NewProvider(mixedBuildModuleInfo{})
func mixedBuildModuleFactory() Module {
m := &mixedBuildModule{}
m.AddProperties(&m.props)
InitAndroidArchModule(m, HostAndDeviceDefault, MultilibBoth)
InitBazelModule(m)
return m
}
func (m *mixedBuildModule) ConvertWithBp2build(ctx Bp2buildMutatorContext) {
}
func (m *mixedBuildModule) DepsMutator(ctx BottomUpMutatorContext) {
ctx.AddDependency(ctx.Module(), installDepTag{}, m.props.Deps...)
}
func (m *mixedBuildModule) GenerateAndroidBuildActions(ctx ModuleContext) {
}
func (m *mixedBuildModule) IsMixedBuildSupported(ctx BaseModuleContext) bool {
return !proptools.Bool(m.props.Mixed_build_incompatible)
}
func (m *mixedBuildModule) QueueBazelCall(ctx BaseModuleContext) {
m.props.QueuedBazelCall = true
}
func (m *mixedBuildModule) ProcessBazelQueryResponse(ctx ModuleContext) {
ctx.SetProvider(mixedBuildModuleProvider, mixedBuildModuleInfo{
QueuedBazelCall: m.props.QueuedBazelCall,
})
}
var prepareForMixedBuildTests = FixtureRegisterWithContext(func(ctx RegistrationContext) {
ctx.RegisterModuleType("deps", mixedBuildModuleFactory)
RegisterMixedBuildsMutator(ctx)
})
func TestMixedBuildsEnabledForType(t *testing.T) {
baseBp := `
deps {
name: "foo",
deps: ["bar"],
target: { windows: { enabled: true } },
%s
}
`
depBp := `
deps {
name: "bar",
target: {
windows: {
enabled: true,
},
},
}
`
testCases := []struct {
desc string
variant *string
missingDeps bool
extraBpInfo string
mixedBuildsEnabled bool
}{
{
desc: "mixed builds works",
mixedBuildsEnabled: true,
extraBpInfo: `bazel_module: { bp2build_available: true },`,
},
{
desc: "missing deps",
missingDeps: true,
mixedBuildsEnabled: false,
extraBpInfo: `bazel_module: { bp2build_available: true },`,
},
{
desc: "windows no mixed builds",
mixedBuildsEnabled: false,
variant: proptools.StringPtr("windows_x86"),
extraBpInfo: `bazel_module: { bp2build_available: true },`,
},
{
desc: "mixed builds disabled by type",
mixedBuildsEnabled: false,
extraBpInfo: `mixed_build_incompatible: true,
bazel_module: { bp2build_available: true },`,
},
{
desc: "mixed builds not bp2build available",
mixedBuildsEnabled: false,
extraBpInfo: `bazel_module: { bp2build_available: false },`,
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
handlers := GroupFixturePreparers(
prepareForMixedBuildTests,
PrepareForTestWithArchMutator,
FixtureModifyConfig(func(config Config) {
config.BazelContext = MockBazelContext{
OutputBaseDir: "base",
}
config.Targets[Windows] = []Target{
{Windows, Arch{ArchType: X86_64}, NativeBridgeDisabled, "", "", true},
{Windows, Arch{ArchType: X86}, NativeBridgeDisabled, "", "", true},
}
}),
)
bp := fmt.Sprintf(baseBp, tc.extraBpInfo)
if tc.missingDeps {
handlers = GroupFixturePreparers(
handlers,
PrepareForTestWithAllowMissingDependencies,
)
} else {
bp += depBp
}
result := handlers.RunTestWithBp(t, bp)
variant := proptools.StringDefault(tc.variant, "android_arm64_armv8-a")
m := result.ModuleForTests("foo", variant)
mixedBuildModuleInfo := result.TestContext.ModuleProvider(m.Module(), mixedBuildModuleProvider).(mixedBuildModuleInfo)
if w, g := tc.mixedBuildsEnabled, mixedBuildModuleInfo.QueuedBazelCall; w != g {
t.Errorf("Expected mixed builds enabled %t, got mixed builds enabled %t", w, g)
}
})
}
}

180
android/config.go
View File

@@ -84,21 +84,13 @@ type CmdArgs struct {
SoongOutDir string SoongOutDir string
SoongVariables string SoongVariables string
SymlinkForestMarker string BazelQueryViewDir string
Bp2buildMarker string ModuleGraphFile string
BazelQueryViewDir string ModuleActionsFile string
ModuleGraphFile string DocFile string
ModuleActionsFile string
DocFile string
MultitreeBuild bool MultitreeBuild bool
BazelMode bool
BazelModeStaging bool
BazelForceEnabledModules string
UseBazelProxy bool
BuildFromSourceStub bool BuildFromSourceStub bool
EnsureAllowlistIntegrity bool EnsureAllowlistIntegrity bool
@@ -109,12 +101,6 @@ const (
// Don't use bazel at all during module analysis. // Don't use bazel at all during module analysis.
AnalysisNoBazel SoongBuildMode = iota AnalysisNoBazel SoongBuildMode = iota
// Symlink fores mode: merge two directory trees into a symlink forest
SymlinkForest
// Bp2build mode: Generate BUILD files from blueprint files and exit.
Bp2build
// Generate BUILD files which faithfully represent the dependency graph of // Generate BUILD files which faithfully represent the dependency graph of
// blueprint modules. Individual BUILD targets will not, however, faitfhully // blueprint modules. Individual BUILD targets will not, however, faitfhully
// express build semantics. // express build semantics.
@@ -125,15 +111,6 @@ const (
// Generate a documentation file for module type definitions and exit. // Generate a documentation file for module type definitions and exit.
GenerateDocFile GenerateDocFile
// Use bazel during analysis of a few allowlisted build modules. The allowlist
// is considered "staging, as these are modules being prepared to be released
// into prod mode shortly after.
BazelStagingMode
// Use bazel during analysis of build modules from an allowlist carefully
// curated by the build team to be proven stable.
BazelProdMode
) )
// SoongOutDir returns the build output directory for the configuration. // SoongOutDir returns the build output directory for the configuration.
@@ -250,10 +227,6 @@ type config struct {
// Only available on configs created by TestConfig // Only available on configs created by TestConfig
TestProductVariables *ProductVariables TestProductVariables *ProductVariables
// A specialized context object for Bazel/Soong mixed builds and migration
// purposes.
BazelContext BazelContext
ProductVariablesFileName string ProductVariablesFileName string
// BuildOS stores the OsType for the OS that the build is running on. // BuildOS stores the OsType for the OS that the build is running on.
@@ -295,9 +268,7 @@ type config struct {
fs pathtools.FileSystem fs pathtools.FileSystem
mockBpList string mockBpList string
BuildMode SoongBuildMode BuildMode SoongBuildMode
Bp2buildPackageConfig Bp2BuildConversionAllowlist
Bp2buildSoongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions
// If MultitreeBuild is true then this is one inner tree of a multitree // If MultitreeBuild is true then this is one inner tree of a multitree
// build directed by the multitree orchestrator. // build directed by the multitree orchestrator.
@@ -313,29 +284,6 @@ type config struct {
OncePer OncePer
// These fields are only used for metrics collection. A module should be added
// to these maps only if its implementation supports Bazel handling in mixed
// builds. A module being in the "enabled" list indicates that there is a
// variant of that module for which bazel-handling actually took place.
// A module being in the "disabled" list indicates that there is a variant of
// that module for which bazel-handling was denied.
mixedBuildsLock sync.Mutex
mixedBuildEnabledModules map[string]struct{}
mixedBuildDisabledModules map[string]struct{}
// These are modules to be built with Bazel beyond the allowlisted/build-mode
// specified modules. They are passed via the command-line flag
// "--bazel-force-enabled-modules"
bazelForceEnabledModules map[string]struct{}
// Names of Bazel targets as defined by BUILD files in the source tree,
// keyed by the directory in which they are defined.
bazelTargetsByDir map[string][]string
// If true, for any requests to Bazel, communicate with a Bazel proxy using
// unix sockets, instead of spawning Bazel as a subprocess.
UseBazelProxy bool
// If buildFromSourceStub is true then the Java API stubs are // If buildFromSourceStub is true then the Java API stubs are
// built from the source Java files, not the signature text files. // built from the source Java files, not the signature text files.
buildFromSourceStub bool buildFromSourceStub bool
@@ -546,14 +494,10 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error)
runGoTests: cmdArgs.RunGoTests, runGoTests: cmdArgs.RunGoTests,
multilibConflicts: make(map[ArchType]bool), multilibConflicts: make(map[ArchType]bool),
moduleListFile: cmdArgs.ModuleListFile, moduleListFile: cmdArgs.ModuleListFile,
fs: pathtools.NewOsFs(absSrcDir), fs: pathtools.NewOsFs(absSrcDir),
mixedBuildDisabledModules: make(map[string]struct{}),
mixedBuildEnabledModules: make(map[string]struct{}),
bazelForceEnabledModules: make(map[string]struct{}),
MultitreeBuild: cmdArgs.MultitreeBuild, MultitreeBuild: cmdArgs.MultitreeBuild,
UseBazelProxy: cmdArgs.UseBazelProxy,
buildFromSourceStub: cmdArgs.BuildFromSourceStub, buildFromSourceStub: cmdArgs.BuildFromSourceStub,
} }
@@ -646,28 +590,9 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error)
config.BuildMode = mode config.BuildMode = mode
} }
} }
setBazelMode := func(arg bool, argName string, mode SoongBuildMode) {
if arg {
if config.BuildMode != AnalysisNoBazel {
fmt.Fprintf(os.Stderr, "buildMode is already set, illegal argument: %s", argName)
os.Exit(1)
}
config.BuildMode = mode
}
}
setBuildMode(cmdArgs.SymlinkForestMarker, SymlinkForest)
setBuildMode(cmdArgs.Bp2buildMarker, Bp2build)
setBuildMode(cmdArgs.BazelQueryViewDir, GenerateQueryView) setBuildMode(cmdArgs.BazelQueryViewDir, GenerateQueryView)
setBuildMode(cmdArgs.ModuleGraphFile, GenerateModuleGraph) setBuildMode(cmdArgs.ModuleGraphFile, GenerateModuleGraph)
setBuildMode(cmdArgs.DocFile, GenerateDocFile) setBuildMode(cmdArgs.DocFile, GenerateDocFile)
setBazelMode(cmdArgs.BazelMode, "--bazel-mode", BazelProdMode)
setBazelMode(cmdArgs.BazelModeStaging, "--bazel-mode-staging", BazelStagingMode)
for _, module := range getForceEnabledModulesFromFlag(cmdArgs.BazelForceEnabledModules) {
config.bazelForceEnabledModules[module] = struct{}{}
}
config.BazelContext, err = NewBazelContext(config)
config.Bp2buildPackageConfig = GetBp2BuildAllowList()
// TODO(b/276958307): Replace the hardcoded list to a sdk_library local prop. // TODO(b/276958307): Replace the hardcoded list to a sdk_library local prop.
config.apiLibraries = map[string]struct{}{ config.apiLibraries = map[string]struct{}{
@@ -704,13 +629,6 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error)
return Config{config}, err return Config{config}, err
} }
func getForceEnabledModulesFromFlag(forceEnabledFlag string) []string {
if forceEnabledFlag == "" {
return []string{}
}
return strings.Split(forceEnabledFlag, ",")
}
// mockFileSystem replaces all reads with accesses to the provided map of // mockFileSystem replaces all reads with accesses to the provided map of
// filenames to contents stored as a byte slice. // filenames to contents stored as a byte slice.
func (c *config) mockFileSystem(bp string, fs map[string][]byte) { func (c *config) mockFileSystem(bp string, fs map[string][]byte) {
@@ -741,41 +659,6 @@ func (c *config) mockFileSystem(bp string, fs map[string][]byte) {
c.mockBpList = blueprint.MockModuleListFile c.mockBpList = blueprint.MockModuleListFile
} }
// TODO(b/265062549): Add a field to our collected (and uploaded) metrics which
// describes a reason that we fell back to non-mixed builds.
// Returns true if "Bazel builds" is enabled. In this mode, part of build
// analysis is handled by Bazel.
func (c *config) IsMixedBuildsEnabled() bool {
globalMixedBuildsSupport := c.Once(OnceKey{"globalMixedBuildsSupport"}, func() interface{} {
if c.productVariables.DeviceArch != nil && *c.productVariables.DeviceArch == "riscv64" {
return false
}
// Disable Bazel when Kythe is running
if c.EmitXrefRules() {
return false
}
if c.IsEnvTrue("GLOBAL_THINLTO") {
return false
}
if len(c.productVariables.SanitizeHost) > 0 {
return false
}
if len(c.productVariables.SanitizeDevice) > 0 {
return false
}
if len(c.productVariables.SanitizeDeviceDiag) > 0 {
return false
}
if len(c.productVariables.SanitizeDeviceArch) > 0 {
return false
}
return true
}).(bool)
bazelModeEnabled := c.BuildMode == BazelProdMode || c.BuildMode == BazelStagingMode
return globalMixedBuildsSupport && bazelModeEnabled
}
func (c *config) SetAllowMissingDependencies() { func (c *config) SetAllowMissingDependencies() {
c.productVariables.Allow_missing_dependencies = proptools.BoolPtr(true) c.productVariables.Allow_missing_dependencies = proptools.BoolPtr(true)
} }
@@ -1051,8 +934,6 @@ func (c *config) AllSupportedApiLevels() []ApiLevel {
// DefaultAppTargetSdk returns the API level that platform apps are targeting. // DefaultAppTargetSdk returns the API level that platform apps are targeting.
// This converts a codename to the exact ApiLevel it represents. // This converts a codename to the exact ApiLevel it represents.
func (c *config) DefaultAppTargetSdk(ctx EarlyModuleContext) ApiLevel { func (c *config) DefaultAppTargetSdk(ctx EarlyModuleContext) ApiLevel {
// This logic is replicated in starlark, if changing logic here update starlark code too
// https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=72;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061
if Bool(c.productVariables.Platform_sdk_final) { if Bool(c.productVariables.Platform_sdk_final) {
return c.PlatformSdkVersion() return c.PlatformSdkVersion()
} }
@@ -1410,10 +1291,6 @@ func (c *config) PrebuiltHiddenApiDir(_ PathContext) string {
return String(c.productVariables.PrebuiltHiddenApiDir) return String(c.productVariables.PrebuiltHiddenApiDir)
} }
func (c *config) BazelModulesForceEnabledByFlag() map[string]struct{} {
return c.bazelForceEnabledModules
}
func (c *config) IsVndkDeprecated() bool { func (c *config) IsVndkDeprecated() bool {
return !Bool(c.productVariables.KeepVndk) return !Bool(c.productVariables.KeepVndk)
} }
@@ -2026,38 +1903,6 @@ func (c *config) UseHostMusl() bool {
return Bool(c.productVariables.HostMusl) return Bool(c.productVariables.HostMusl)
} }
func (c *config) GetMixedBuildsEnabledModules() map[string]struct{} {
return c.mixedBuildEnabledModules
}
func (c *config) GetMixedBuildsDisabledModules() map[string]struct{} {
return c.mixedBuildDisabledModules
}
func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) {
moduleName := ctx.Module().Name()
c.mixedBuildsLock.Lock()
defer c.mixedBuildsLock.Unlock()
if useBazel {
c.mixedBuildEnabledModules[moduleName] = struct{}{}
} else {
c.mixedBuildDisabledModules[moduleName] = struct{}{}
}
}
func (c *config) HasBazelBuildTargetInSource(dir string, target string) bool {
for _, existingTarget := range c.bazelTargetsByDir[dir] {
if target == existingTarget {
return true
}
}
return false
}
func (c *config) SetBazelBuildFileTargets(bazelTargetsByDir map[string][]string) {
c.bazelTargetsByDir = bazelTargetsByDir
}
// ApiSurfaces directory returns the source path inside the api_surfaces repo // ApiSurfaces directory returns the source path inside the api_surfaces repo
// (relative to workspace root). // (relative to workspace root).
func (c *config) ApiSurfacesDir(s ApiSurface, version string) string { func (c *config) ApiSurfacesDir(s ApiSurface, version string) string {
@@ -2091,12 +1936,6 @@ func (c *config) SetBuildFromTextStub(b bool) {
c.productVariables.Build_from_text_stub = boolPtr(b) c.productVariables.Build_from_text_stub = boolPtr(b)
} }
func (c *config) AddForceEnabledModules(forceEnabled []string) {
for _, forceEnabledModule := range forceEnabled {
c.bazelForceEnabledModules[forceEnabledModule] = struct{}{}
}
}
func (c *config) SetApiLibraries(libs []string) { func (c *config) SetApiLibraries(libs []string) {
c.apiLibraries = make(map[string]struct{}) c.apiLibraries = make(map[string]struct{})
for _, lib := range libs { for _, lib := range libs {
@@ -2108,11 +1947,6 @@ func (c *config) GetApiLibraries() map[string]struct{} {
return c.apiLibraries return c.apiLibraries
} }
// Bp2buildMode indicates whether the config is for bp2build mode of Soong
func (c *config) Bp2buildMode() bool {
return c.BuildMode == Bp2build
}
func (c *deviceConfig) CheckVendorSeappViolations() bool { func (c *deviceConfig) CheckVendorSeappViolations() bool {
return Bool(c.config.productVariables.CheckVendorSeappViolations) return Bool(c.config.productVariables.CheckVendorSeappViolations)
} }

18
android/metrics.go
View File

@@ -19,7 +19,6 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"runtime" "runtime"
"sort"
"strconv" "strconv"
"time" "time"
@@ -93,23 +92,6 @@ func collectMetrics(config Config, eventHandler *metrics.EventHandler) *soong_me
} }
metrics.Events = append(metrics.Events, &perfInfo) metrics.Events = append(metrics.Events, &perfInfo)
} }
mixedBuildsInfo := soong_metrics_proto.MixedBuildsInfo{}
mixedBuildEnabledModules := make([]string, 0, len(config.mixedBuildEnabledModules))
for module, _ := range config.mixedBuildEnabledModules {
mixedBuildEnabledModules = append(mixedBuildEnabledModules, module)
}
mixedBuildDisabledModules := make([]string, 0, len(config.mixedBuildDisabledModules))
for module, _ := range config.mixedBuildDisabledModules {
mixedBuildDisabledModules = append(mixedBuildDisabledModules, module)
}
// Sorted for deterministic output.
sort.Strings(mixedBuildEnabledModules)
sort.Strings(mixedBuildDisabledModules)
mixedBuildsInfo.MixedBuildEnabledModules = mixedBuildEnabledModules
mixedBuildsInfo.MixedBuildDisabledModules = mixedBuildDisabledModules
metrics.MixedBuildsInfo = &mixedBuildsInfo
return metrics return metrics
} }

412
android/module.go
View File

@@ -16,7 +16,6 @@ package android
import ( import (
"android/soong/bazel" "android/soong/bazel"
"android/soong/ui/metrics/bp2build_metrics_proto"
"crypto/md5" "crypto/md5"
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
@@ -96,16 +95,6 @@ type Module interface {
AddProperties(props ...interface{}) AddProperties(props ...interface{})
GetProperties() []interface{} GetProperties() []interface{}
// If this module should not have bazel BUILD definitions generated by bp2build,
// GetUnconvertedReason returns a reason this is the case.
GetUnconvertedReason() *UnconvertedReason
// Bp2buildTargets returns the target(s) generated for Bazel via bp2build for this module
Bp2buildTargets() []bp2buildInfo
GetUnconvertedBp2buildDeps() []string
GetMissingBp2buildDeps() []string
GetPartitionForBp2build() string
BuildParamsForTests() []BuildParams BuildParamsForTests() []BuildParams
RuleParamsForTests() map[blueprint.Rule]blueprint.RuleParams RuleParamsForTests() map[blueprint.Rule]blueprint.RuleParams
VariablesForTests() map[string]string VariablesForTests() map[string]string
@@ -520,9 +509,6 @@ type commonProperties struct {
// constants in image.go, but can also be set to a custom value by individual module types. // constants in image.go, but can also be set to a custom value by individual module types.
ImageVariation string `blueprint:"mutated"` ImageVariation string `blueprint:"mutated"`
// Bazel conversion status
BazelConversionStatus BazelConversionStatus `blueprint:"mutated"`
// SoongConfigTrace records accesses to VendorVars (soong_config). The trace will be hashed // SoongConfigTrace records accesses to VendorVars (soong_config). The trace will be hashed
// and used as a subdir of PathForModuleOut. Note that we mainly focus on incremental // and used as a subdir of PathForModuleOut. Note that we mainly focus on incremental
// builds among similar products (e.g. aosp_cf_x86_64_phone and aosp_cf_x86_64_foldable), // builds among similar products (e.g. aosp_cf_x86_64_phone and aosp_cf_x86_64_foldable),
@@ -532,41 +518,6 @@ type commonProperties struct {
SoongConfigTraceHash string `blueprint:"mutated"` SoongConfigTraceHash string `blueprint:"mutated"`
} }
// CommonAttributes represents the common Bazel attributes from which properties
// in `commonProperties` are translated/mapped; such properties are annotated in
// a list their corresponding attribute. It is embedded within `bp2buildInfo`.
type CommonAttributes struct {
// Soong nameProperties -> Bazel name
Name string
// Data mapped from: Required
Data bazel.LabelListAttribute
// SkipData is neither a Soong nor Bazel target attribute
// If true, this will not fill the data attribute automatically
// This is useful for Soong modules that have 1:many Bazel targets
// Some of the generated Bazel targets might not have a data attribute
SkipData *bool
Tags bazel.StringListAttribute
Applicable_licenses bazel.LabelListAttribute
Testonly *bool
// Dir is neither a Soong nor Bazel target attribute
// If set, the bazel target will be created in this directory
// If unset, the bazel target will default to be created in the directory of the visited soong module
Dir *string
}
// constraintAttributes represents Bazel attributes pertaining to build constraints,
// which make restrict building a Bazel target for some set of platforms.
type constraintAttributes struct {
// Constraint values this target can be built for.
Target_compatible_with bazel.LabelListAttribute
}
type distProperties struct { type distProperties struct {
// configuration to distribute output files from this module to the distribution // configuration to distribute output files from this module to the distribution
// directory (default: $OUT/dist, configurable with $DIST_DIR) // directory (default: $OUT/dist, configurable with $DIST_DIR)
@@ -804,234 +755,6 @@ func InitCommonOSAndroidMultiTargetsArchModule(m Module, hod HostOrDeviceSupport
m.base().commonProperties.CreateCommonOSVariant = true m.base().commonProperties.CreateCommonOSVariant = true
} }
func (attrs *CommonAttributes) getRequiredWithoutCycles(ctx *bottomUpMutatorContext, props *commonProperties) []string {
// Treat `required` as if it's empty if data should be skipped for this target,
// as `required` is only used for the `data` attribute at this time, and we want
// to avoid lookups of labels that won't actually be dependencies of this target.
// TODO: b/202299295 - Refactor this to use `required` dependencies, once they
// are handled other than passing to `data`.
if proptools.Bool(attrs.SkipData) {
return []string{}
}
// The required property can contain the module itself. This causes a cycle
// when generated as the 'data' label list attribute in Bazel. Remove it if
// it exists. See b/247985196.
_, requiredWithoutCycles := RemoveFromList(ctx.ModuleName(), props.Required)
return FirstUniqueStrings(requiredWithoutCycles)
}
func (attrs *CommonAttributes) fillCommonBp2BuildModuleAttrs(ctx *bottomUpMutatorContext,
enabledPropertyOverrides bazel.BoolAttribute) constraintAttributes {
mod := ctx.Module().base()
// Assert passed-in attributes include Name
if len(attrs.Name) == 0 {
if ctx.ModuleType() != "package" {
ctx.ModuleErrorf("CommonAttributes in fillCommonBp2BuildModuleAttrs expects a `.Name`!")
}
}
depsToLabelList := func(deps []string) bazel.LabelListAttribute {
return bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, deps))
}
var enabledProperty bazel.BoolAttribute
onlyAndroid := false
neitherHostNorDevice := false
osSupport := map[string]bool{}
// if the target is enabled and supports arch variance, determine the defaults based on the module
// type's host or device property and host_supported/device_supported properties
if mod.commonProperties.ArchSpecific {
moduleSupportsDevice := mod.DeviceSupported()
moduleSupportsHost := mod.HostSupported()
if moduleSupportsHost && !moduleSupportsDevice {
// for host only, we specify as unsupported on android rather than listing all host osSupport
// TODO(b/220874839): consider replacing this with a constraint that covers all host osSupport
// instead
enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(false))
} else if moduleSupportsDevice && !moduleSupportsHost {
enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(true))
// specify as a positive to ensure any target-specific enabled can be resolved
// also save that a target is only android, as if there is only the positive restriction on
// android, it'll be dropped, so we may need to add it back later
onlyAndroid = true
} else if !moduleSupportsHost && !moduleSupportsDevice {
neitherHostNorDevice = true
}
for _, osType := range OsTypeList() {
if osType.Class == Host {
osSupport[osType.Name] = moduleSupportsHost
} else if osType.Class == Device {
osSupport[osType.Name] = moduleSupportsDevice
}
}
}
if neitherHostNorDevice {
// we can't build this, disable
enabledProperty.Value = proptools.BoolPtr(false)
} else if mod.commonProperties.Enabled != nil {
enabledProperty.SetValue(mod.commonProperties.Enabled)
if !*mod.commonProperties.Enabled {
for oss, enabled := range osSupport {
if val := enabledProperty.SelectValue(bazel.OsConfigurationAxis, oss); enabled && val != nil && *val {
// if this should be disabled by default, clear out any enabling we've done
enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, oss, nil)
}
}
}
}
attrs.Applicable_licenses = bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, mod.commonProperties.Licenses))
requiredWithoutCycles := attrs.getRequiredWithoutCycles(ctx, &mod.commonProperties)
required := depsToLabelList(requiredWithoutCycles)
archVariantProps := mod.GetArchVariantProperties(ctx, &commonProperties{})
for axis, configToProps := range archVariantProps {
for config, _props := range configToProps {
if archProps, ok := _props.(*commonProperties); ok {
requiredWithoutCycles := attrs.getRequiredWithoutCycles(ctx, archProps)
required.SetSelectValue(axis, config, depsToLabelList(requiredWithoutCycles).Value)
if !neitherHostNorDevice {
if archProps.Enabled != nil {
if axis != bazel.OsConfigurationAxis || osSupport[config] {
enabledProperty.SetSelectValue(axis, config, archProps.Enabled)
}
}
}
}
}
}
if !neitherHostNorDevice {
if enabledPropertyOverrides.Value != nil {
enabledProperty.Value = enabledPropertyOverrides.Value
}
for _, axis := range enabledPropertyOverrides.SortedConfigurationAxes() {
configToBools := enabledPropertyOverrides.ConfigurableValues[axis]
for cfg, val := range configToBools {
if axis != bazel.OsConfigurationAxis || osSupport[cfg] || val /*If enabled is explicitly requested via overrides */ {
enabledProperty.SetSelectValue(axis, cfg, &val)
}
}
}
}
productConfigEnabledAttribute := bazel.LabelListAttribute{}
// TODO(b/234497586): Soong config variables and product variables have different overriding behavior, we
// should handle it correctly
if !proptools.BoolDefault(enabledProperty.Value, true) && !neitherHostNorDevice {
// If the module is not enabled by default, then we can check if a
// product variable enables it
productConfigEnabledAttribute = productVariableConfigEnableAttribute(ctx)
if len(productConfigEnabledAttribute.ConfigurableValues) > 0 {
// In this case, an existing product variable configuration overrides any
// module-level `enable: false` definition
newValue := true
enabledProperty.Value = &newValue
}
}
platformEnabledAttribute, err := enabledProperty.ToLabelListAttribute(
bazel.LabelList{[]bazel.Label{{Label: "@platforms//:incompatible"}}, nil},
bazel.LabelList{[]bazel.Label{}, nil})
if err != nil {
ctx.ModuleErrorf("Error processing platform enabled attribute: %s", err)
}
// if android is the only arch/os enabled, then add a restriction to only be compatible with android
if platformEnabledAttribute.IsNil() && onlyAndroid {
l := bazel.LabelAttribute{}
l.SetValue(bazel.Label{Label: bazel.OsConfigurationAxis.SelectKey(Android.Name)})
platformEnabledAttribute.Add(&l)
}
attrs.Data.Append(required)
// SkipData is not an attribute of any Bazel target
// Set this to nil so that it does not appear in the generated build file
attrs.SkipData = nil
moduleEnableConstraints := bazel.LabelListAttribute{}
moduleEnableConstraints.Append(platformEnabledAttribute)
moduleEnableConstraints.Append(productConfigEnabledAttribute)
addCompatibilityConstraintForCompileMultilib(ctx, &moduleEnableConstraints)
return constraintAttributes{Target_compatible_with: moduleEnableConstraints}
}
var (
incompatible = bazel.LabelList{[]bazel.Label{{Label: "@platforms//:incompatible"}}, nil}
)
// If compile_mulitilib is set to
// 1. 32: Add an incompatibility constraint for non-32 arches
// 1. 64: Add an incompatibility constraint for non-64 arches
func addCompatibilityConstraintForCompileMultilib(ctx *bottomUpMutatorContext, enabled *bazel.LabelListAttribute) {
mod := ctx.Module().base()
multilib, _ := decodeMultilib(mod, mod.commonProperties.CompileOS, ctx.Config().IgnorePrefer32OnDevice())
switch multilib {
case "32":
// Add an incompatibility constraint for all known 64-bit arches
enabled.SetSelectValue(bazel.ArchConfigurationAxis, "arm64", incompatible)
enabled.SetSelectValue(bazel.ArchConfigurationAxis, "x86_64", incompatible)
enabled.SetSelectValue(bazel.ArchConfigurationAxis, "riscv64", incompatible)
case "64":
// Add an incompatibility constraint for all known 32-bit arches
enabled.SetSelectValue(bazel.ArchConfigurationAxis, "arm", incompatible)
enabled.SetSelectValue(bazel.ArchConfigurationAxis, "x86", incompatible)
case "both":
// Do nothing: "both" is trivially compatible with 32-bit and 64-bit
// The top level rule (e.g. apex/partition) will be responsible for building this module in both variants via an
// outgoing_transition.
default: // e.g. first, common
// TODO - b/299135307: Add bp2build support for these properties.
}
}
// Check product variables for `enabled: true` flag override.
// Returns a list of the constraint_value targets who enable this override.
func productVariableConfigEnableAttribute(ctx *bottomUpMutatorContext) bazel.LabelListAttribute {
result := bazel.LabelListAttribute{}
productVariableProps, errs := ProductVariableProperties(ctx, ctx.Module())
for _, err := range errs {
ctx.ModuleErrorf("ProductVariableProperties error: %s", err)
}
if productConfigProps, exists := productVariableProps["Enabled"]; exists {
for productConfigProp, prop := range productConfigProps {
flag, ok := prop.(*bool)
if !ok {
ctx.ModuleErrorf("Could not convert product variable enabled property")
}
if flag == nil {
// soong config var is not used to set `enabled`. nothing to do.
continue
} else if *flag {
axis := productConfigProp.ConfigurationAxis()
result.SetSelectValue(axis, bazel.ConditionsDefaultConfigKey, bazel.MakeLabelList([]bazel.Label{{Label: "@platforms//:incompatible"}}))
result.SetSelectValue(axis, productConfigProp.SelectKey(), bazel.LabelList{Includes: []bazel.Label{}})
} else if scp, isSoongConfigProperty := productConfigProp.(SoongConfigProperty); isSoongConfigProperty && scp.value == bazel.ConditionsDefaultConfigKey {
// productVariableConfigEnableAttribute runs only if `enabled: false` is set at the top-level outside soong_config_variables
// conditions_default { enabled: false} is a no-op in this case
continue
} else {
// TODO(b/210546943): handle negative case where `enabled: false`
ctx.ModuleErrorf("`enabled: false` is not currently supported for configuration variables. See b/210546943")
}
}
}
return result
}
// A ModuleBase object contains the properties that are common to all Android // A ModuleBase object contains the properties that are common to all Android
// modules. It should be included as an anonymous field in every module // modules. It should be included as an anonymous field in every module
// struct definition. InitAndroidModule should then be called from the module's // struct definition. InitAndroidModule should then be called from the module's
@@ -1146,81 +869,6 @@ type ModuleBase struct {
licenseMetadataFile WritablePath licenseMetadataFile WritablePath
} }
// A struct containing all relevant information about a Bazel target converted via bp2build.
type bp2buildInfo struct {
Dir string
BazelProps bazel.BazelTargetModuleProperties
CommonAttrs CommonAttributes
ConstraintAttrs constraintAttributes
Attrs interface{}
}
// TargetName returns the Bazel target name of a bp2build converted target.
func (b bp2buildInfo) TargetName() string {
return b.CommonAttrs.Name
}
// TargetPackage returns the Bazel package of a bp2build converted target.
func (b bp2buildInfo) TargetPackage() string {
return b.Dir
}
// BazelRuleClass returns the Bazel rule class of a bp2build converted target.
func (b bp2buildInfo) BazelRuleClass() string {
return b.BazelProps.Rule_class
}
// BazelRuleLoadLocation returns the location of the Bazel rule of a bp2build converted target.
// This may be empty as native Bazel rules do not need to be loaded.
func (b bp2buildInfo) BazelRuleLoadLocation() string {
return b.BazelProps.Bzl_load_location
}
// BazelAttributes returns the Bazel attributes of a bp2build converted target.
func (b bp2buildInfo) BazelAttributes() []interface{} {
return []interface{}{&b.CommonAttrs, &b.ConstraintAttrs, b.Attrs}
}
func (m *ModuleBase) addBp2buildInfo(info bp2buildInfo) {
m.commonProperties.BazelConversionStatus.Bp2buildInfo = append(m.commonProperties.BazelConversionStatus.Bp2buildInfo, info)
}
func (m *ModuleBase) setPartitionForBp2build(partition string) {
m.commonProperties.BazelConversionStatus.Partition = partition
}
func (m *ModuleBase) setBp2buildUnconvertible(reasonType bp2build_metrics_proto.UnconvertedReasonType, detail string) {
m.commonProperties.BazelConversionStatus.UnconvertedReason = &UnconvertedReason{
ReasonType: int(reasonType),
Detail: detail,
}
}
func (m *ModuleBase) GetUnconvertedReason() *UnconvertedReason {
return m.commonProperties.BazelConversionStatus.UnconvertedReason
}
// Bp2buildTargets returns the Bazel targets bp2build generated for this module.
func (m *ModuleBase) Bp2buildTargets() []bp2buildInfo {
return m.commonProperties.BazelConversionStatus.Bp2buildInfo
}
// Bp2buildTargets returns the Bazel targets bp2build generated for this module.
func (m *ModuleBase) GetPartitionForBp2build() string {
return m.commonProperties.BazelConversionStatus.Partition
}
// GetUnconvertedBp2buildDeps returns the list of module names of this module's direct dependencies that
// were not converted to Bazel.
func (m *ModuleBase) GetUnconvertedBp2buildDeps() []string {
return FirstUniqueStrings(m.commonProperties.BazelConversionStatus.UnconvertedDeps)
}
// GetMissingBp2buildDeps returns the list of module names that were not found in Android.bp files.
func (m *ModuleBase) GetMissingBp2buildDeps() []string {
return FirstUniqueStrings(m.commonProperties.BazelConversionStatus.MissingDeps)
}
func (m *ModuleBase) AddJSONData(d *map[string]interface{}) { func (m *ModuleBase) AddJSONData(d *map[string]interface{}) {
(*d)["Android"] = map[string]interface{}{ (*d)["Android"] = map[string]interface{}{
// Properties set in Blueprint or in blueprint of a defaults modules // Properties set in Blueprint or in blueprint of a defaults modules
@@ -2031,11 +1679,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
return return
} }
if mixedBuildMod, handled := m.isHandledByBazel(ctx); handled { m.module.GenerateAndroidBuildActions(ctx)
mixedBuildMod.ProcessBazelQueryResponse(ctx)
} else {
m.module.GenerateAndroidBuildActions(ctx)
}
if ctx.Failed() { if ctx.Failed() {
return return
} }
@@ -2092,15 +1736,6 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
m.variables = ctx.variables m.variables = ctx.variables
} }
func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) {
if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok {
if mixedBuildMod.IsMixedBuildSupported(ctx) && (MixedBuildsEnabled(ctx) == MixedBuildEnabled) {
return mixedBuildMod, true
}
}
return nil, false
}
// Check the supplied dist structure to make sure that it is valid. // Check the supplied dist structure to make sure that it is valid.
// //
// property - the base property, e.g. dist or dists[1], which is combined with the // property - the base property, e.g. dist or dists[1], which is combined with the
@@ -2193,6 +1828,18 @@ func (m *ModuleBase) IsNativeBridgeSupported() bool {
return proptools.Bool(m.commonProperties.Native_bridge_supported) return proptools.Bool(m.commonProperties.Native_bridge_supported)
} }
// ModuleNameWithPossibleOverride returns the name of the OverrideModule that overrides the current
// variant of this OverridableModule, or ctx.ModuleName() if this module is not an OverridableModule
// or if this variant is not overridden.
func ModuleNameWithPossibleOverride(ctx BaseModuleContext) string {
if overridable, ok := ctx.Module().(OverridableModule); ok {
if o := overridable.GetOverriddenBy(); o != "" {
return o
}
}
return ctx.ModuleName()
}
// SrcIsModule decodes module references in the format ":unqualified-name" or "//namespace:name" // SrcIsModule decodes module references in the format ":unqualified-name" or "//namespace:name"
// into the module name, or empty string if the input was not a module reference. // into the module name, or empty string if the input was not a module reference.
func SrcIsModule(s string) (module string) { func SrcIsModule(s string) (module string) {
@@ -2615,36 +2262,3 @@ func (s *soongConfigTraceSingleton) GenerateBuildActions(ctx SingletonContext) {
WriteFileRule(ctx, outFile, string(j)) WriteFileRule(ctx, outFile, string(j))
ctx.Phony("soong_config_trace", outFile) ctx.Phony("soong_config_trace", outFile)
} }
// Interface implemented by xsd_config which has 1:many mappings in bp2build workspace
// This interface exists because we want to
// 1. Determine the name of the additional targets generated by the primary soong module
// 2. Enable distinguishing an xsd_config module from other Soong modules using type assertion
type XsdConfigBp2buildTargets interface {
CppBp2buildTargetName() string
JavaBp2buildTargetName() string
}
// XsdModuleToTargetName is a function that takes an XsdConfigBp2buildTarget
type XsdModuleToTargetName func(xsd XsdConfigBp2buildTargets) string
// XsdLabelMapper returns a bazel.LabelMapper for partitioning XSD sources/headers given an
// XsdModuleToTargetName function.
func XsdLabelMapper(targetName XsdModuleToTargetName) bazel.LabelMapper {
return func(ctx bazel.OtherModuleContext, label bazel.Label) (string, bool) {
mod, exists := ctx.ModuleFromName(label.OriginalModuleName)
if !exists {
return label.Label, false
}
xsdMod, isXsd := mod.(XsdConfigBp2buildTargets)
if !isXsd {
return label.Label, false
}
// Remove the base module name
ret := strings.TrimSuffix(label.Label, mod.Name())
// Append the language specific target name
ret += targetName(xsdMod)
return ret, true
}
}

299
android/mutator.go
View File

@@ -15,11 +15,6 @@
package android package android
import ( import (
"path/filepath"
"android/soong/bazel"
"android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint" "github.com/google/blueprint"
) )
@@ -32,40 +27,9 @@ import (
// run FinalDeps mutators (CreateVariations disallowed in this phase) // run FinalDeps mutators (CreateVariations disallowed in this phase)
// continue on to GenerateAndroidBuildActions // continue on to GenerateAndroidBuildActions
// RegisterMutatorsForBazelConversion is a alternate registration pipeline for bp2build. Exported for testing.
func RegisterMutatorsForBazelConversion(ctx *Context, preArchMutators []RegisterMutatorFunc) {
bp2buildMutators := append(preArchMutators, registerBp2buildConversionMutator)
registerMutatorsForBazelConversion(ctx, bp2buildMutators)
}
func registerMutatorsForBazelConversion(ctx *Context, bp2buildMutators []RegisterMutatorFunc) {
mctx := &registerMutatorsContext{
bazelConversionMode: true,
}
allMutators := append([]RegisterMutatorFunc{
RegisterNamespaceMutator,
RegisterDefaultsPreArchMutators,
// TODO(b/165114590): this is required to resolve deps that are only prebuilts, but we should
// evaluate the impact on conversion.
RegisterPrebuiltsPreArchMutators,
RegisterPrebuiltsPostDepsMutators,
},
bp2buildMutators...)
// Register bp2build mutators
for _, f := range allMutators {
f(mctx)
}
mctx.mutators.registerAll(ctx)
}
// collateGloballyRegisteredMutators constructs the list of mutators that have been registered // collateGloballyRegisteredMutators constructs the list of mutators that have been registered
// with the InitRegistrationContext and will be used at runtime. // with the InitRegistrationContext and will be used at runtime.
func collateGloballyRegisteredMutators() sortableComponents { func collateGloballyRegisteredMutators() sortableComponents {
// ensure mixed builds mutator is the last mutator
finalDeps = append(finalDeps, registerMixedBuildsMutator)
return collateRegisteredMutators(preArch, preDeps, postDeps, finalDeps) return collateRegisteredMutators(preArch, preDeps, postDeps, finalDeps)
} }
@@ -94,9 +58,8 @@ func collateRegisteredMutators(preArch, preDeps, postDeps, finalDeps []RegisterM
} }
type registerMutatorsContext struct { type registerMutatorsContext struct {
mutators sortableComponents mutators sortableComponents
finalPhase bool finalPhase bool
bazelConversionMode bool
} }
type RegisterMutatorsContext interface { type RegisterMutatorsContext interface {
@@ -219,58 +182,6 @@ func FinalDepsMutators(f RegisterMutatorFunc) {
finalDeps = append(finalDeps, f) finalDeps = append(finalDeps, f)
} }
var bp2buildPreArchMutators = []RegisterMutatorFunc{}
// A minimal context for Bp2build conversion
type Bp2buildMutatorContext interface {
BazelConversionPathContext
BaseMutatorContext
// AddDependency adds a dependency to the given module. It returns a slice of modules for each
// dependency (some entries may be nil).
//
// If the mutator is parallel (see MutatorHandle.Parallel), this method will pause until the
// new dependencies have had the current mutator called on them. If the mutator is not
// parallel this method does not affect the ordering of the current mutator pass, but will
// be ordered correctly for all future mutator passes.
AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) []blueprint.Module
// CreateBazelTargetModule creates a BazelTargetModule by calling the
// factory method, just like in CreateModule, but also requires
// BazelTargetModuleProperties containing additional metadata for the
// bp2build codegenerator.
CreateBazelTargetModule(bazel.BazelTargetModuleProperties, CommonAttributes, interface{})
// CreateBazelTargetModuleWithRestrictions creates a BazelTargetModule by calling the
// factory method, just like in CreateModule, but also requires
// BazelTargetModuleProperties containing additional metadata for the
// bp2build codegenerator. The generated target is restricted to only be buildable for certain
// platforms, as dictated by a given bool attribute: the target will not be buildable in
// any platform for which this bool attribute is false.
CreateBazelTargetModuleWithRestrictions(bazel.BazelTargetModuleProperties, CommonAttributes, interface{}, bazel.BoolAttribute)
// MarkBp2buildUnconvertible registers the current module as "unconvertible to bp2build" for the
// given reason.
MarkBp2buildUnconvertible(reasonType bp2build_metrics_proto.UnconvertedReasonType, detail string)
// CreateBazelTargetAliasInDir creates an alias definition in `dir` directory.
// This function can be used to create alias definitions in a directory that is different
// from the directory of the visited Soong module.
CreateBazelTargetAliasInDir(dir string, name string, actual bazel.Label)
// CreateBazelConfigSetting creates a config_setting in <dir>/BUILD.bazel
// build/bazel has several static config_setting(s) that are used in Bazel builds.
// This function can be used to createa additional config_setting(s) based on the build graph
// (e.g. a config_setting specific to an apex variant)
CreateBazelConfigSetting(csa bazel.ConfigSettingAttributes, ca CommonAttributes, dir string)
}
// PreArchBp2BuildMutators adds mutators to be register for converting Android Blueprint modules
// into Bazel BUILD targets that should run prior to deps and conversion.
func PreArchBp2BuildMutators(f RegisterMutatorFunc) {
bp2buildPreArchMutators = append(bp2buildPreArchMutators, f)
}
type BaseMutatorContext interface { type BaseMutatorContext interface {
BaseModuleContext BaseModuleContext
@@ -301,7 +212,15 @@ type BottomUpMutator func(BottomUpMutatorContext)
type BottomUpMutatorContext interface { type BottomUpMutatorContext interface {
BaseMutatorContext BaseMutatorContext
Bp2buildMutatorContext
// AddDependency adds a dependency to the given module. It returns a slice of modules for each
// dependency (some entries may be nil).
//
// If the mutator is parallel (see MutatorHandle.Parallel), this method will pause until the
// new dependencies have had the current mutator called on them. If the mutator is not
// parallel this method does not affect the ordering of the current mutator pass, but will
// be ordered correctly for all future mutator passes.
AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) []blueprint.Module
// AddReverseDependency adds a dependency from the destination to the given module. // AddReverseDependency adds a dependency from the destination to the given module.
// Does not affect the ordering of the current mutator pass, but will be ordered // Does not affect the ordering of the current mutator pass, but will be ordered
@@ -416,10 +335,9 @@ type bottomUpMutatorContext struct {
} }
func bottomUpMutatorContextFactory(ctx blueprint.BottomUpMutatorContext, a Module, func bottomUpMutatorContextFactory(ctx blueprint.BottomUpMutatorContext, a Module,
finalPhase, bazelConversionMode bool) BottomUpMutatorContext { finalPhase bool) BottomUpMutatorContext {
moduleContext := a.base().baseModuleContextFactory(ctx) moduleContext := a.base().baseModuleContextFactory(ctx)
moduleContext.bazelConversionMode = bazelConversionMode
return &bottomUpMutatorContext{ return &bottomUpMutatorContext{
bp: ctx, bp: ctx,
@@ -430,10 +348,9 @@ func bottomUpMutatorContextFactory(ctx blueprint.BottomUpMutatorContext, a Modul
func (x *registerMutatorsContext) BottomUp(name string, m BottomUpMutator) MutatorHandle { func (x *registerMutatorsContext) BottomUp(name string, m BottomUpMutator) MutatorHandle {
finalPhase := x.finalPhase finalPhase := x.finalPhase
bazelConversionMode := x.bazelConversionMode
f := func(ctx blueprint.BottomUpMutatorContext) { f := func(ctx blueprint.BottomUpMutatorContext) {
if a, ok := ctx.Module().(Module); ok { if a, ok := ctx.Module().(Module); ok {
m(bottomUpMutatorContextFactory(ctx, a, finalPhase, bazelConversionMode)) m(bottomUpMutatorContextFactory(ctx, a, finalPhase))
} }
} }
mutator := &mutator{name: x.mutatorName(name), bottomUpMutator: f} mutator := &mutator{name: x.mutatorName(name), bottomUpMutator: f}
@@ -550,15 +467,13 @@ type TransitionMutator interface {
} }
type androidTransitionMutator struct { type androidTransitionMutator struct {
finalPhase bool finalPhase bool
bazelConversionMode bool mutator TransitionMutator
mutator TransitionMutator
} }
func (a *androidTransitionMutator) Split(ctx blueprint.BaseModuleContext) []string { func (a *androidTransitionMutator) Split(ctx blueprint.BaseModuleContext) []string {
if m, ok := ctx.Module().(Module); ok { if m, ok := ctx.Module().(Module); ok {
moduleContext := m.base().baseModuleContextFactory(ctx) moduleContext := m.base().baseModuleContextFactory(ctx)
moduleContext.bazelConversionMode = a.bazelConversionMode
return a.mutator.Split(&moduleContext) return a.mutator.Split(&moduleContext)
} else { } else {
return []string{""} return []string{""}
@@ -607,15 +522,14 @@ func (a *androidTransitionMutator) IncomingTransition(ctx blueprint.IncomingTran
func (a *androidTransitionMutator) Mutate(ctx blueprint.BottomUpMutatorContext, variation string) { func (a *androidTransitionMutator) Mutate(ctx blueprint.BottomUpMutatorContext, variation string) {
if am, ok := ctx.Module().(Module); ok { if am, ok := ctx.Module().(Module); ok {
a.mutator.Mutate(bottomUpMutatorContextFactory(ctx, am, a.finalPhase, a.bazelConversionMode), variation) a.mutator.Mutate(bottomUpMutatorContextFactory(ctx, am, a.finalPhase), variation)
} }
} }
func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) {
atm := &androidTransitionMutator{ atm := &androidTransitionMutator{
finalPhase: x.finalPhase, finalPhase: x.finalPhase,
bazelConversionMode: x.bazelConversionMode, mutator: m,
mutator: m,
} }
mutator := &mutator{ mutator := &mutator{
name: name, name: name,
@@ -624,9 +538,6 @@ func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) {
} }
func (x *registerMutatorsContext) mutatorName(name string) string { func (x *registerMutatorsContext) mutatorName(name string) string {
if x.bazelConversionMode {
return name + "_bp2build"
}
return name return name
} }
@@ -634,7 +545,6 @@ func (x *registerMutatorsContext) TopDown(name string, m TopDownMutator) Mutator
f := func(ctx blueprint.TopDownMutatorContext) { f := func(ctx blueprint.TopDownMutatorContext) {
if a, ok := ctx.Module().(Module); ok { if a, ok := ctx.Module().(Module); ok {
moduleContext := a.base().baseModuleContextFactory(ctx) moduleContext := a.base().baseModuleContextFactory(ctx)
moduleContext.bazelConversionMode = x.bazelConversionMode
actx := &topDownMutatorContext{ actx := &topDownMutatorContext{
bp: ctx, bp: ctx,
baseModuleContext: moduleContext, baseModuleContext: moduleContext,
@@ -698,179 +608,6 @@ func registerDepsMutator(ctx RegisterMutatorsContext) {
ctx.BottomUp("deps", depsMutator).Parallel() ctx.BottomUp("deps", depsMutator).Parallel()
} }
func (t *bottomUpMutatorContext) CreateBazelTargetModule(
bazelProps bazel.BazelTargetModuleProperties,
commonAttrs CommonAttributes,
attrs interface{}) {
t.createBazelTargetModule(bazelProps, commonAttrs, attrs, bazel.BoolAttribute{})
}
func (t *bottomUpMutatorContext) CreateBazelTargetModuleWithRestrictions(
bazelProps bazel.BazelTargetModuleProperties,
commonAttrs CommonAttributes,
attrs interface{},
enabledProperty bazel.BoolAttribute) {
t.createBazelTargetModule(bazelProps, commonAttrs, attrs, enabledProperty)
}
func (t *bottomUpMutatorContext) MarkBp2buildUnconvertible(
reasonType bp2build_metrics_proto.UnconvertedReasonType, detail string) {
mod := t.Module()
mod.base().setBp2buildUnconvertible(reasonType, detail)
}
var (
bazelAliasModuleProperties = bazel.BazelTargetModuleProperties{
Rule_class: "alias",
}
)
type bazelAliasAttributes struct {
Actual *bazel.LabelAttribute
}
func (t *bottomUpMutatorContext) CreateBazelTargetAliasInDir(
dir string,
name string,
actual bazel.Label) {
mod := t.Module()
attrs := &bazelAliasAttributes{
Actual: bazel.MakeLabelAttribute(actual.Label),
}
info := bp2buildInfo{
Dir: dir,
BazelProps: bazelAliasModuleProperties,
CommonAttrs: CommonAttributes{Name: name},
ConstraintAttrs: constraintAttributes{},
Attrs: attrs,
}
mod.base().addBp2buildInfo(info)
}
// Returns the directory in which the bazel target will be generated
// If ca.Dir is not nil, use that
// Otherwise default to the directory of the soong module
func dirForBazelTargetGeneration(t *bottomUpMutatorContext, ca *CommonAttributes) string {
dir := t.OtherModuleDir(t.Module())
if ca.Dir != nil {
dir = *ca.Dir
// Restrict its use to dirs that contain an Android.bp file.
// There are several places in bp2build where we use the existence of Android.bp/BUILD on the filesystem
// to curate a compatible label for src files (e.g. headers for cc).
// If we arbritrarily create BUILD files, then it might render those curated labels incompatible.
if exists, _, _ := t.Config().fs.Exists(filepath.Join(dir, "Android.bp")); !exists {
t.ModuleErrorf("Cannot use ca.Dir to create a BazelTarget in dir: %v since it does not contain an Android.bp file", dir)
}
// Set ca.Dir to nil so that it does not get emitted to the BUILD files
ca.Dir = nil
}
return dir
}
func (t *bottomUpMutatorContext) CreateBazelConfigSetting(
csa bazel.ConfigSettingAttributes,
ca CommonAttributes,
dir string) {
mod := t.Module()
info := bp2buildInfo{
Dir: dir,
BazelProps: bazel.BazelTargetModuleProperties{
Rule_class: "config_setting",
},
CommonAttrs: ca,
ConstraintAttrs: constraintAttributes{},
Attrs: &csa,
}
mod.base().addBp2buildInfo(info)
}
// ApexAvailableTags converts the apex_available property value of an ApexModule
// module and returns it as a list of keyed tags.
func ApexAvailableTags(mod Module) bazel.StringListAttribute {
attr := bazel.StringListAttribute{}
// Transform specific attributes into tags.
if am, ok := mod.(ApexModule); ok {
// TODO(b/218841706): hidl_interface has the apex_available prop, but it's
// defined directly as a prop and not via ApexModule, so this doesn't
// pick those props up.
apexAvailable := am.apexModuleBase().ApexAvailable()
// If a user does not specify apex_available in Android.bp, then soong provides a default.
// To avoid verbosity of BUILD files, remove this default from user-facing BUILD files.
if len(am.apexModuleBase().ApexProperties.Apex_available) == 0 {
apexAvailable = []string{}
}
attr.Value = ConvertApexAvailableToTags(apexAvailable)
}
return attr
}
func ApexAvailableTagsWithoutTestApexes(ctx BaseModuleContext, mod Module) bazel.StringListAttribute {
attr := bazel.StringListAttribute{}
if am, ok := mod.(ApexModule); ok {
apexAvailableWithoutTestApexes := removeTestApexes(ctx, am.apexModuleBase().ApexAvailable())
// If a user does not specify apex_available in Android.bp, then soong provides a default.
// To avoid verbosity of BUILD files, remove this default from user-facing BUILD files.
if len(am.apexModuleBase().ApexProperties.Apex_available) == 0 {
apexAvailableWithoutTestApexes = []string{}
}
attr.Value = ConvertApexAvailableToTags(apexAvailableWithoutTestApexes)
}
return attr
}
func removeTestApexes(ctx BaseModuleContext, apex_available []string) []string {
testApexes := []string{}
for _, aa := range apex_available {
// ignore the wildcards
if InList(aa, AvailableToRecognziedWildcards) {
continue
}
mod, _ := ctx.ModuleFromName(aa)
if apex, ok := mod.(ApexTestInterface); ok && apex.IsTestApex() {
testApexes = append(testApexes, aa)
}
}
return RemoveListFromList(CopyOf(apex_available), testApexes)
}
func ConvertApexAvailableToTags(apexAvailable []string) []string {
if len(apexAvailable) == 0 {
// We need nil specifically to make bp2build not add the tags property at all,
// instead of adding it with an empty list
return nil
}
result := make([]string, 0, len(apexAvailable))
for _, a := range apexAvailable {
result = append(result, "apex_available="+a)
}
return result
}
// ConvertApexAvailableToTagsWithoutTestApexes converts a list of apex names to a list of bazel tags
// This function drops any test apexes from the input.
func ConvertApexAvailableToTagsWithoutTestApexes(ctx BaseModuleContext, apexAvailable []string) []string {
noTestApexes := removeTestApexes(ctx, apexAvailable)
return ConvertApexAvailableToTags(noTestApexes)
}
func (t *bottomUpMutatorContext) createBazelTargetModule(
bazelProps bazel.BazelTargetModuleProperties,
commonAttrs CommonAttributes,
attrs interface{},
enabledProperty bazel.BoolAttribute) {
constraintAttributes := commonAttrs.fillCommonBp2BuildModuleAttrs(t, enabledProperty)
mod := t.Module()
info := bp2buildInfo{
Dir: dirForBazelTargetGeneration(t, &commonAttrs),
BazelProps: bazelProps,
CommonAttrs: commonAttrs,
ConstraintAttrs: constraintAttributes,
Attrs: attrs,
}
mod.base().addBp2buildInfo(info)
}
// android.topDownMutatorContext either has to embed blueprint.TopDownMutatorContext, in which case every method that // android.topDownMutatorContext either has to embed blueprint.TopDownMutatorContext, in which case every method that
// has an overridden version in android.BaseModuleContext has to be manually forwarded to BaseModuleContext to avoid // has an overridden version in android.BaseModuleContext has to be manually forwarded to BaseModuleContext to avoid
// ambiguous method errors, or it has to store a blueprint.TopDownMutatorContext non-embedded, in which case every // ambiguous method errors, or it has to store a blueprint.TopDownMutatorContext non-embedded, in which case every

20
android/mutator_test.go
View File

@@ -16,7 +16,6 @@ package android
import ( import (
"fmt" "fmt"
"reflect"
"strings" "strings"
"testing" "testing"
@@ -268,22 +267,3 @@ func TestNoCreateVariationsInFinalDeps(t *testing.T) {
FixtureWithRootAndroidBp(`test {name: "foo"}`), FixtureWithRootAndroidBp(`test {name: "foo"}`),
).RunTest(t) ).RunTest(t)
} }
func TestConvertApexAvailableToTags(t *testing.T) {
input := []string{
"com.android.adbd",
"//apex_available:platform",
}
actual := ConvertApexAvailableToTags(input)
expected := []string{
"apex_available=com.android.adbd",
"apex_available=//apex_available:platform",
}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("Expected: %v, actual: %v", expected, actual)
}
if ConvertApexAvailableToTags(nil) != nil {
t.Errorf("Expected providing nil to return nil")
}
}

39
android/override_module.go
View File

@@ -214,17 +214,6 @@ func (b *OverridableModuleBase) override(ctx BaseModuleContext, m Module, o Over
} }
b.overridableModuleProperties.OverriddenBy = o.Name() b.overridableModuleProperties.OverriddenBy = o.Name()
b.overridableModuleProperties.OverriddenByModuleDir = o.ModuleDir() b.overridableModuleProperties.OverriddenByModuleDir = o.ModuleDir()
if oBazelable, ok := o.base().module.(Bazelable); ok {
if bBazelable, ok := m.(Bazelable); ok {
oProps := oBazelable.bazelProps()
bProps := bBazelable.bazelProps()
bProps.Bazel_module.Bp2build_available = oProps.Bazel_module.Bp2build_available
bProps.Bazel_module.Label = oProps.Bazel_module.Label
} else {
ctx.ModuleErrorf("Override type cannot be Bazelable if original module type is not Bazelable %v %v.", o.Name(), m.Name())
}
}
} }
// GetOverriddenBy returns the name of the override module that has overridden this module. // GetOverriddenBy returns the name of the override module that has overridden this module.
@@ -348,31 +337,3 @@ func replaceDepsOnOverridingModuleMutator(ctx BottomUpMutatorContext) {
} }
} }
} }
// ModuleNameWithPossibleOverride returns the name of the OverrideModule that overrides the current
// variant of this OverridableModule, or ctx.ModuleName() if this module is not an OverridableModule
// or if this variant is not overridden.
func ModuleNameWithPossibleOverride(ctx BazelConversionContext) string {
return moduleNameWithPossibleOverride(ctx, ctx.Module(), ctx.OtherModuleName(ctx.Module()))
}
func moduleNameWithPossibleOverride(ctx shouldConvertModuleContext, module blueprint.Module, name string) string {
if overridable, ok := module.(OverridableModule); ok {
if o := overridable.GetOverriddenBy(); o != "" {
return o
}
}
return name
}
// moduleDirWithPossibleOverride returns the dir of the OverrideModule that overrides the current
// variant of the given OverridableModule, or ctx.OtherModuleName() if the module is not an
// OverridableModule or if the variant is not overridden.
func moduleDirWithPossibleOverride(ctx shouldConvertModuleContext, module blueprint.Module, dir string) string {
if overridable, ok := module.(OverridableModule); ok {
if o := overridable.GetOverriddenByModuleDir(); o != "" {
return o
}
}
return dir
}

57
android/prebuilt.go
View File

@@ -461,10 +461,6 @@ func PrebuiltSelectModuleMutator(ctx BottomUpMutatorContext) {
// Propagate the provider received from `all_apex_contributions` // Propagate the provider received from `all_apex_contributions`
// to the source module // to the source module
ctx.VisitDirectDepsWithTag(acDepTag, func(am Module) { ctx.VisitDirectDepsWithTag(acDepTag, func(am Module) {
if ctx.Config().Bp2buildMode() {
// This provider key is not applicable in bp2build
return
}
psi := ctx.OtherModuleProvider(am, PrebuiltSelectionInfoProvider).(PrebuiltSelectionInfoMap) psi := ctx.OtherModuleProvider(am, PrebuiltSelectionInfoProvider).(PrebuiltSelectionInfoMap)
ctx.SetProvider(PrebuiltSelectionInfoProvider, psi) ctx.SetProvider(PrebuiltSelectionInfoProvider, psi)
}) })
@@ -570,55 +566,20 @@ func (p *Prebuilt) usePrebuilt(ctx BaseMutatorContext, source Module, prebuilt M
// fall back to the existing source vs prebuilt selection. // fall back to the existing source vs prebuilt selection.
// TODO: Drop the fallback mechanisms // TODO: Drop the fallback mechanisms
if !ctx.Config().Bp2buildMode() { if p.srcsSupplier != nil && len(p.srcsSupplier(ctx, prebuilt)) == 0 {
if p.srcsSupplier != nil && len(p.srcsSupplier(ctx, prebuilt)) == 0 { return false
return false }
}
// Skip prebuilt modules under unexported namespaces so that we won't // Skip prebuilt modules under unexported namespaces so that we won't
// end up shadowing non-prebuilt module when prebuilt module under same // end up shadowing non-prebuilt module when prebuilt module under same
// name happens to have a `Prefer` property set to true. // name happens to have a `Prefer` property set to true.
if ctx.Config().KatiEnabled() && !prebuilt.ExportedToMake() { if ctx.Config().KatiEnabled() && !prebuilt.ExportedToMake() {
return false return false
}
} }
// If source is not available or is disabled then always use the prebuilt. // If source is not available or is disabled then always use the prebuilt.
if source == nil || !source.Enabled() { if source == nil || !source.Enabled() {
// If in bp2build mode, we need to check product variables & Soong config variables, which may return true
// have overridden the "enabled" property but have not been merged into the property value as
// they would in a non-bp2build mode invocation
if ctx.Config().Bp2buildMode() && source != nil {
productVariableProps, errs := ProductVariableProperties(ctx, source)
if productConfigProps, exists := productVariableProps["Enabled"]; len(errs) == 0 && exists && len(productConfigProps) == 1 {
var prop ProductConfigOrSoongConfigProperty
var value bool
for p, v := range productConfigProps {
prop = p
actual, ok := v.(*bool)
if ok {
value = proptools.Bool(actual)
}
}
if scv, ok := prop.(SoongConfigProperty); ok {
// If the product config var is enabled but the value of enabled is false still, the
// prebuilt is preferred. Otherwise, check if the prebulit is explicitly preferred
if ctx.Config().VendorConfig(scv.namespace).Bool(strings.ToLower(scv.name)) && !value {
return true
}
} else {
// TODO: b/300998219 - handle product vars
// We don't handle product variables yet, so return based on the non-product specific
// value of enabled
return true
}
} else {
// No "enabled" property override, return true since this module isn't enabled
return true
}
} else {
return true
}
} }
// If the use_source_config_var property is set then it overrides the prefer property setting. // If the use_source_config_var property is set then it overrides the prefer property setting.

79
android/register.go
View File

@@ -15,15 +15,9 @@
package android package android
import ( import (
"bufio"
"fmt" "fmt"
"path/filepath"
"reflect"
"regexp"
"android/soong/shared"
"github.com/google/blueprint" "github.com/google/blueprint"
"reflect"
) )
// A sortable component is one whose registration order affects the order in which it is executed // A sortable component is one whose registration order affects the order in which it is executed
@@ -166,75 +160,6 @@ func NewContext(config Config) *Context {
return ctx return ctx
} }
// Helper function to register the module types used in bp2build.
func registerModuleTypes(ctx *Context) {
for _, t := range moduleTypes {
t.register(ctx)
}
// Required for SingletonModule types, even though we are not using them.
for _, t := range singletons {
t.register(ctx)
}
}
// RegisterForBazelConversion registers an alternate shadow pipeline of
// singletons, module types and mutators to register for converting Blueprint
// files to semantically equivalent BUILD files.
func (ctx *Context) RegisterForBazelConversion() {
registerModuleTypes(ctx)
RegisterMutatorsForBazelConversion(ctx, bp2buildPreArchMutators)
}
// RegisterExistingBazelTargets reads Bazel BUILD.bazel and BUILD files under
// the workspace, and returns a map containing names of Bazel targets defined in
// these BUILD files.
// For example, maps "//foo/bar" to ["baz", "qux"] if `//foo/bar:{baz,qux}` exist.
func (c *Context) RegisterExistingBazelTargets(topDir string, existingBazelFiles []string) error {
result := map[string][]string{}
// Search for instances of `name = "$NAME"` (with arbitrary spacing).
targetNameRegex := regexp.MustCompile(`(?m)^\s*name\s*=\s*\"([^\"]+)\"`)
parseBuildFile := func(path string) error {
fullPath := shared.JoinPath(topDir, path)
sourceDir := filepath.Dir(path)
fileInfo, err := c.Config().fs.Stat(fullPath)
if err != nil {
return fmt.Errorf("Error accessing Bazel file '%s': %s", path, err)
}
if !fileInfo.IsDir() &&
(fileInfo.Name() == "BUILD" || fileInfo.Name() == "BUILD.bazel") {
f, err := c.Config().fs.Open(fullPath)
if err != nil {
return fmt.Errorf("Error reading Bazel file '%s': %s", path, err)
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
matches := targetNameRegex.FindAllStringSubmatch(line, -1)
for _, match := range matches {
result[sourceDir] = append(result[sourceDir], match[1])
}
}
}
return nil
}
for _, path := range existingBazelFiles {
if !c.Config().Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(filepath.Dir(path)) {
continue
}
err := parseBuildFile(path)
if err != nil {
return err
}
}
c.Config().SetBazelBuildFileTargets(result)
return nil
}
// Register the pipeline of singletons, module types, and mutators for // Register the pipeline of singletons, module types, and mutators for
// generating build.ninja and other files for Kati, from Android.bp files. // generating build.ninja and other files for Kati, from Android.bp files.
func (ctx *Context) Register() { func (ctx *Context) Register() {
@@ -260,8 +185,6 @@ func (ctx *Context) registerSingletonMakeVarsProvider(makevars SingletonMakeVars
func collateGloballyRegisteredSingletons() sortableComponents { func collateGloballyRegisteredSingletons() sortableComponents {
allSingletons := append(sortableComponents(nil), singletons...) allSingletons := append(sortableComponents(nil), singletons...)
allSingletons = append(allSingletons, allSingletons = append(allSingletons,
singleton{parallel: true, name: "bazeldeps", factory: BazelSingleton},
// Register phony just before makevars so it can write out its phony rules as Make rules // Register phony just before makevars so it can write out its phony rules as Make rules
singleton{parallel: false, name: "phony", factory: phonySingletonFactory}, singleton{parallel: false, name: "phony", factory: phonySingletonFactory},

6
android/test_config.go
View File

@@ -61,11 +61,7 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string
// passed to PathForSource or PathForModuleSrc. // passed to PathForSource or PathForModuleSrc.
TestAllowNonExistentPaths: true, TestAllowNonExistentPaths: true,
BazelContext: noopBazelContext{}, BuildMode: AnalysisNoBazel,
BuildMode: BazelProdMode,
mixedBuildDisabledModules: make(map[string]struct{}),
mixedBuildEnabledModules: make(map[string]struct{}),
bazelForceEnabledModules: make(map[string]struct{}),
} }
config.deviceConfig = &deviceConfig{ config.deviceConfig = &deviceConfig{
config: config, config: config,

10
android/testing.go
View File

@@ -219,10 +219,6 @@ func (ctx *TestContext) FinalDepsMutators(f RegisterMutatorFunc) {
ctx.finalDeps = append(ctx.finalDeps, f) ctx.finalDeps = append(ctx.finalDeps, f)
} }
func (ctx *TestContext) RegisterBp2BuildConfig(config Bp2BuildConversionAllowlist) {
ctx.config.Bp2buildPackageConfig = config
}
// PreArchBp2BuildMutators adds mutators to be register for converting Android Blueprint modules // PreArchBp2BuildMutators adds mutators to be register for converting Android Blueprint modules
// into Bazel BUILD targets that should run prior to deps and conversion. // into Bazel BUILD targets that should run prior to deps and conversion.
func (ctx *TestContext) PreArchBp2BuildMutators(f RegisterMutatorFunc) { func (ctx *TestContext) PreArchBp2BuildMutators(f RegisterMutatorFunc) {
@@ -449,12 +445,6 @@ func (ctx *TestContext) Register() {
ctx.singletonOrder = componentsToNames(singletons) ctx.singletonOrder = componentsToNames(singletons)
} }
// RegisterForBazelConversion prepares a test context for bp2build conversion.
func (ctx *TestContext) RegisterForBazelConversion() {
ctx.config.BuildMode = Bp2build
RegisterMutatorsForBazelConversion(ctx.Context, ctx.bp2buildPreArch)
}
func (ctx *TestContext) ParseFileList(rootDir string, filePaths []string) (deps []string, errs []error) { func (ctx *TestContext) ParseFileList(rootDir string, filePaths []string) (deps []string, errs []error) {
// This function adapts the old style ParseFileList calls that are spread throughout the tests // This function adapts the old style ParseFileList calls that are spread throughout the tests
// to the new style that takes a config. // to the new style that takes a config.

6
android/util.go
View File

@@ -591,3 +591,9 @@ func CheckDuplicate(values []string) (duplicate string, found bool) {
} }
return "", false return "", false
} }
func AddToStringSet(set map[string]bool, items []string) {
for _, item := range items {
set[item] = true
}
}

43
android/variable.go
View File

@@ -20,7 +20,6 @@ import (
"runtime" "runtime"
"strings" "strings"
"android/soong/android/soongconfig"
"android/soong/bazel" "android/soong/bazel"
"github.com/google/blueprint/proptools" "github.com/google/blueprint/proptools"
@@ -746,44 +745,6 @@ func (p SoongConfigProperty) SelectKey() string {
// property, like ["-DDEFINES"] for cflags. // property, like ["-DDEFINES"] for cflags.
type ProductConfigProperties map[string]map[ProductConfigOrSoongConfigProperty]interface{} type ProductConfigProperties map[string]map[ProductConfigOrSoongConfigProperty]interface{}
// ProductVariableProperties returns a ProductConfigProperties containing only the properties which
// have been set for the given module.
func ProductVariableProperties(ctx ArchVariantContext, module Module) (ProductConfigProperties, []error) {
var errs []error
moduleBase := module.base()
productConfigProperties := ProductConfigProperties{}
if moduleBase.variableProperties != nil {
productVariablesProperty := proptools.FieldNameForProperty("product_variables")
if moduleBase.ArchSpecific() {
for /* axis */ _, configToProps := range moduleBase.GetArchVariantProperties(ctx, moduleBase.variableProperties) {
for config, props := range configToProps {
variableValues := reflect.ValueOf(props).Elem().FieldByName(productVariablesProperty)
productConfigProperties.AddProductConfigProperties(variableValues, config)
}
}
} else {
variableValues := reflect.ValueOf(moduleBase.variableProperties).Elem().FieldByName(productVariablesProperty)
productConfigProperties.AddProductConfigProperties(variableValues, "")
}
}
if m, ok := module.(Bazelable); ok && m.namespacedVariableProps() != nil {
for namespace, namespacedVariableProps := range m.namespacedVariableProps() {
for _, namespacedVariableProp := range namespacedVariableProps {
variableValues := reflect.ValueOf(namespacedVariableProp).Elem().FieldByName(soongconfig.SoongConfigProperty)
err := productConfigProperties.AddSoongConfigProperties(namespace, variableValues)
if err != nil {
errs = append(errs, err)
}
}
}
}
return productConfigProperties, errs
}
func (p *ProductConfigProperties) AddProductConfigProperty( func (p *ProductConfigProperties) AddProductConfigProperty(
propertyName, productVariableName, arch string, propertyValue interface{}) { propertyName, productVariableName, arch string, propertyValue interface{}) {
@@ -833,10 +794,6 @@ func (p *ProductConfigProperties) AddEitherProperty(
} }
} }
var (
conditionsDefaultField string = proptools.FieldNameForProperty(bazel.ConditionsDefaultConfigKey)
)
// maybeExtractConfigVarProp attempts to read this value as a config var struct // maybeExtractConfigVarProp attempts to read this value as a config var struct
// wrapped by interfaces and ptrs. If it's not the right type, the second return // wrapped by interfaces and ptrs. If it's not the right type, the second return
// value is false. // value is false.

6
bp2build/Android.bp
View File

@@ -7,16 +7,11 @@ bootstrap_go_package {
pkgPath: "android/soong/bp2build", pkgPath: "android/soong/bp2build",
srcs: [ srcs: [
"androidbp_to_build_templates.go", "androidbp_to_build_templates.go",
"bp2build.go",
"bp2build_product_config.go",
"build_conversion.go", "build_conversion.go",
"bzl_conversion.go", "bzl_conversion.go",
"configurability.go", "configurability.go",
"constants.go", "constants.go",
"conversion.go", "conversion.go",
"metrics.go",
"symlink_forest.go",
"testing.go",
], ],
deps: [ deps: [
"blueprint-bootstrap", "blueprint-bootstrap",
@@ -41,7 +36,6 @@ bootstrap_go_package {
], ],
testSrcs: [ testSrcs: [
"conversion_test.go", "conversion_test.go",
"performance_test.go",
], ],
pluginFor: [ pluginFor: [
"soong_build", "soong_build",

145
bp2build/bp2build.go
View File

@@ -1,145 +0,0 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"fmt"
"os"
"path/filepath"
"strings"
"android/soong/android"
"android/soong/bazel"
"android/soong/shared"
"android/soong/starlark_import"
)
func deleteFilesExcept(ctx *CodegenContext, rootOutputPath android.OutputPath, except []BazelFile) {
// Delete files that should no longer be present.
bp2buildDirAbs := shared.JoinPath(ctx.topDir, rootOutputPath.String())
filesToDelete := make(map[string]struct{})
err := filepath.Walk(bp2buildDirAbs,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
relPath, err := filepath.Rel(bp2buildDirAbs, path)
if err != nil {
return err
}
filesToDelete[relPath] = struct{}{}
}
return nil
})
if err != nil {
fmt.Printf("ERROR reading %s: %s", bp2buildDirAbs, err)
os.Exit(1)
}
for _, bazelFile := range except {
filePath := filepath.Join(bazelFile.Dir, bazelFile.Basename)
delete(filesToDelete, filePath)
}
for f, _ := range filesToDelete {
absPath := shared.JoinPath(bp2buildDirAbs, f)
if err := os.RemoveAll(absPath); err != nil {
fmt.Printf("ERROR deleting %s: %s", absPath, err)
os.Exit(1)
}
}
}
// Codegen is the backend of bp2build. The code generator is responsible for
// writing .bzl files that are equivalent to Android.bp files that are capable
// of being built with Bazel.
func Codegen(ctx *CodegenContext) *CodegenMetrics {
ctx.Context().BeginEvent("Codegen")
defer ctx.Context().EndEvent("Codegen")
// This directory stores BUILD files that could be eventually checked-in.
bp2buildDir := android.PathForOutput(ctx, "bp2build")
res, errs := GenerateBazelTargets(ctx, true)
if len(errs) > 0 {
errMsgs := make([]string, len(errs))
for i, err := range errs {
errMsgs[i] = fmt.Sprintf("%q", err)
}
fmt.Printf("ERROR: Encountered %d error(s): \nERROR: %s", len(errs), strings.Join(errMsgs, "\n"))
os.Exit(1)
}
var bp2buildFiles []BazelFile
productConfig, err := createProductConfigFiles(ctx, res.moduleNameToPartition, res.metrics.convertedModulePathMap)
ctx.Context().EventHandler.Do("CreateBazelFile", func() {
allTargets := make(map[string]BazelTargets)
for k, v := range res.buildFileToTargets {
allTargets[k] = append(allTargets[k], v...)
}
for k, v := range productConfig.bp2buildTargets {
allTargets[k] = append(allTargets[k], v...)
}
bp2buildFiles = CreateBazelFiles(nil, allTargets, ctx.mode)
})
bp2buildFiles = append(bp2buildFiles, productConfig.bp2buildFiles...)
injectionFiles, err := createSoongInjectionDirFiles(ctx, res.metrics)
if err != nil {
fmt.Printf("%s\n", err.Error())
os.Exit(1)
}
injectionFiles = append(injectionFiles, productConfig.injectionFiles...)
writeFiles(ctx, bp2buildDir, bp2buildFiles)
// Delete files under the bp2build root which weren't just written. An
// alternative would have been to delete the whole directory and write these
// files. However, this would regenerate files which were otherwise unchanged
// since the last bp2build run, which would have negative incremental
// performance implications.
deleteFilesExcept(ctx, bp2buildDir, bp2buildFiles)
writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles)
starlarkDeps, err := starlark_import.GetNinjaDeps()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(1)
}
ctx.AddNinjaFileDeps(starlarkDeps...)
return &res.metrics
}
// Get the output directory and create it if it doesn't exist.
func getOrCreateOutputDir(outputDir android.OutputPath, ctx android.PathContext, dir string) android.OutputPath {
dirPath := outputDir.Join(ctx, dir)
if err := android.CreateOutputDirIfNonexistent(dirPath, os.ModePerm); err != nil {
fmt.Printf("ERROR: path %s: %s", dirPath, err.Error())
}
return dirPath
}
// writeFiles materializes a list of BazelFile rooted at outputDir.
func writeFiles(ctx android.PathContext, outputDir android.OutputPath, files []BazelFile) {
for _, f := range files {
p := getOrCreateOutputDir(outputDir, ctx, f.Dir).Join(ctx, f.Basename)
if err := writeFile(p, f.Contents); err != nil {
panic(fmt.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err))
}
}
}
func writeFile(pathToFile android.OutputPath, content string) error {
// These files are made editable to allow users to modify and iterate on them
// in the source tree.
return android.WriteFileToOutputDir(pathToFile, []byte(content), 0644)
}

885
bp2build/bp2build_product_config.go
View File

@@ -1,885 +0,0 @@
package bp2build
import (
"encoding/json"
"fmt"
"path/filepath"
"reflect"
"sort"
"strings"
"android/soong/android"
"android/soong/android/soongconfig"
"android/soong/starlark_import"
"github.com/google/blueprint/proptools"
"go.starlark.net/starlark"
)
type createProductConfigFilesResult struct {
injectionFiles []BazelFile
bp2buildFiles []BazelFile
bp2buildTargets map[string]BazelTargets
}
type bazelLabel struct {
repo string
pkg string
target string
}
const releaseAconfigValueSetsName = "release_aconfig_value_sets"
func (l *bazelLabel) Less(other *bazelLabel) bool {
if l.repo < other.repo {
return true
}
if l.repo > other.repo {
return false
}
if l.pkg < other.pkg {
return true
}
if l.pkg > other.pkg {
return false
}
return l.target < other.target
}
func (l *bazelLabel) String() string {
return fmt.Sprintf("@%s//%s:%s", l.repo, l.pkg, l.target)
}
func createProductConfigFiles(
ctx *CodegenContext,
moduleNameToPartition map[string]string,
convertedModulePathMap map[string]string) (createProductConfigFilesResult, error) {
cfg := &ctx.config
targetProduct := "unknown"
if cfg.HasDeviceProduct() {
targetProduct = cfg.DeviceProduct()
}
targetBuildVariant := "user"
if cfg.Eng() {
targetBuildVariant = "eng"
} else if cfg.Debuggable() {
targetBuildVariant = "userdebug"
}
var res createProductConfigFilesResult
productVariables := ctx.Config().ProductVariables()
// TODO(b/306243251): For some reason, using the real value of native_coverage makes some select
// statements ambiguous
productVariables.Native_coverage = nil
productVariablesBytes, err := json.Marshal(productVariables)
if err != nil {
return res, err
}
currentProductFolder := fmt.Sprintf("build/bazel/products/%s", targetProduct)
if len(productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory) > 0 {
currentProductFolder = fmt.Sprintf("%s%s", productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory, targetProduct)
}
productReplacer := strings.NewReplacer(
"{PRODUCT}", targetProduct,
"{VARIANT}", targetBuildVariant,
"{PRODUCT_FOLDER}", currentProductFolder)
productsForTestingMap, err := starlark_import.GetStarlarkValue[map[string]map[string]starlark.Value]("products_for_testing")
if err != nil {
return res, err
}
productsForTesting := android.SortedKeys(productsForTestingMap)
for i := range productsForTesting {
productsForTesting[i] = fmt.Sprintf(" \"@//build/bazel/tests/products:%s\",", productsForTesting[i])
}
productLabelsToVariables := make(map[bazelLabel]*android.ProductVariables)
productLabelsToVariables[bazelLabel{
repo: "",
pkg: currentProductFolder,
target: targetProduct,
}] = &productVariables
for product, productVariablesStarlark := range productsForTestingMap {
productVariables, err := starlarkMapToProductVariables(productVariablesStarlark)
if err != nil {
return res, err
}
productLabelsToVariables[bazelLabel{
repo: "",
pkg: "build/bazel/tests/products",
target: product,
}] = &productVariables
}
res.bp2buildTargets = make(map[string]BazelTargets)
res.bp2buildTargets[currentProductFolder] = append(res.bp2buildTargets[currentProductFolder], BazelTarget{
name: productReplacer.Replace("{PRODUCT}"),
packageName: currentProductFolder,
content: productReplacer.Replace(`android_product(
name = "{PRODUCT}",
soong_variables = _soong_variables,
)`),
ruleClass: "android_product",
loads: []BazelLoad{
{
file: ":soong.variables.bzl",
symbols: []BazelLoadSymbol{{
symbol: "variables",
alias: "_soong_variables",
}},
},
{
file: "//build/bazel/product_config:android_product.bzl",
symbols: []BazelLoadSymbol{{symbol: "android_product"}},
},
},
})
createTargets(ctx, productLabelsToVariables, moduleNameToPartition, convertedModulePathMap, res.bp2buildTargets)
platformMappingContent, err := platformMappingContent(
productLabelsToVariables,
ctx.Config().Bp2buildSoongConfigDefinitions,
convertedModulePathMap)
if err != nil {
return res, err
}
res.injectionFiles = []BazelFile{
newFile(
"product_config_platforms",
"BUILD.bazel",
productReplacer.Replace(`
package(default_visibility = [
"@//build/bazel/product_config:__subpackages__",
"@soong_injection//product_config_platforms:__subpackages__",
])
load("@//{PRODUCT_FOLDER}:soong.variables.bzl", _soong_variables = "variables")
load("@//build/bazel/product_config:android_product.bzl", "android_product")
# Bazel will qualify its outputs by the platform name. When switching between products, this
# means that soong-built files that depend on bazel-built files will suddenly get different
# dependency files, because the path changes, and they will be rebuilt. In order to avoid this
# extra rebuilding, make mixed builds always use a single platform so that the bazel artifacts
# are always under the same path.
android_product(
name = "mixed_builds_product",
soong_variables = _soong_variables,
extra_constraints = ["@//build/bazel/platforms:mixed_builds"],
)
`)),
newFile(
"product_config_platforms",
"product_labels.bzl",
productReplacer.Replace(`
# This file keeps a list of all the products in the android source tree, because they're
# discovered as part of a preprocessing step before bazel runs.
# TODO: When we start generating the platforms for more than just the
# currently lunched product, they should all be listed here
product_labels = [
"@soong_injection//product_config_platforms:mixed_builds_product",
"@//{PRODUCT_FOLDER}:{PRODUCT}",
`)+strings.Join(productsForTesting, "\n")+"\n]\n"),
newFile(
"product_config_platforms",
"common.bazelrc",
productReplacer.Replace(`
build --platform_mappings=platform_mappings
build --platforms @//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64
build --//build/bazel/product_config:target_build_variant={VARIANT}
build:android --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}
build:linux_x86 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86
build:linux_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64
build:linux_bionic_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_bionic_x86_64
build:linux_musl_x86 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_musl_x86
build:linux_musl_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_musl_x86_64
`)),
newFile(
"product_config_platforms",
"linux.bazelrc",
productReplacer.Replace(`
build --host_platform @//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64
`)),
newFile(
"product_config_platforms",
"darwin.bazelrc",
productReplacer.Replace(`
build --host_platform @//{PRODUCT_FOLDER}:{PRODUCT}_darwin_x86_64
`)),
}
res.bp2buildFiles = []BazelFile{
newFile(
"",
"platform_mappings",
platformMappingContent),
newFile(
currentProductFolder,
"soong.variables.bzl",
`variables = json.decode("""`+strings.ReplaceAll(string(productVariablesBytes), "\\", "\\\\")+`""")`),
}
return res, nil
}
func platformMappingContent(
productLabelToVariables map[bazelLabel]*android.ProductVariables,
soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions,
convertedModulePathMap map[string]string) (string, error) {
var result strings.Builder
mergedConvertedModulePathMap := make(map[string]string)
for k, v := range convertedModulePathMap {
mergedConvertedModulePathMap[k] = v
}
additionalModuleNamesToPackages, err := starlark_import.GetStarlarkValue[map[string]string]("additional_module_names_to_packages")
if err != nil {
return "", err
}
for k, v := range additionalModuleNamesToPackages {
mergedConvertedModulePathMap[k] = v
}
productLabels := make([]bazelLabel, 0, len(productLabelToVariables))
for k := range productLabelToVariables {
productLabels = append(productLabels, k)
}
sort.Slice(productLabels, func(i, j int) bool {
return productLabels[i].Less(&productLabels[j])
})
result.WriteString("platforms:\n")
for _, productLabel := range productLabels {
platformMappingSingleProduct(productLabel, productLabelToVariables[productLabel], soongConfigDefinitions, mergedConvertedModulePathMap, &result)
}
return result.String(), nil
}
var bazelPlatformSuffixes = []string{
"",
"_darwin_arm64",
"_darwin_x86_64",
"_linux_bionic_arm64",
"_linux_bionic_x86_64",
"_linux_musl_x86",
"_linux_musl_x86_64",
"_linux_x86",
"_linux_x86_64",
"_windows_x86",
"_windows_x86_64",
}
func platformMappingSingleProduct(
label bazelLabel,
productVariables *android.ProductVariables,
soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions,
convertedModulePathMap map[string]string,
result *strings.Builder) {
platform_sdk_version := -1
if productVariables.Platform_sdk_version != nil {
platform_sdk_version = *productVariables.Platform_sdk_version
}
defaultAppCertificateFilegroup := "//build/bazel/utils:empty_filegroup"
if proptools.String(productVariables.DefaultAppCertificate) != "" {
defaultAppCertificateFilegroup = "@//" + filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) + ":generated_android_certificate_directory"
}
// TODO: b/301598690 - commas can't be escaped in a string-list passed in a platform mapping,
// so commas are switched for ":" here, and must be back-substituted into commas
// wherever the AAPTCharacteristics product config variable is used.
AAPTConfig := []string{}
for _, conf := range productVariables.AAPTConfig {
AAPTConfig = append(AAPTConfig, strings.Replace(conf, ",", ":", -1))
}
for _, suffix := range bazelPlatformSuffixes {
result.WriteString(" ")
result.WriteString(label.String())
result.WriteString(suffix)
result.WriteString("\n")
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_characteristics=%s\n", proptools.String(productVariables.AAPTCharacteristics)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_config=%s\n", strings.Join(AAPTConfig, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_preferred_config=%s\n", proptools.String(productVariables.AAPTPreferredConfig)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:always_use_prebuilt_sdks=%t\n", proptools.Bool(productVariables.Always_use_prebuilt_sdks)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:arc=%t\n", proptools.Bool(productVariables.Arc)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:apex_global_min_sdk_version_override=%s\n", proptools.String(productVariables.ApexGlobalMinSdkVersionOverride)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:binder32bit=%t\n", proptools.Bool(productVariables.Binder32bit)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_from_text_stub=%t\n", proptools.Bool(productVariables.Build_from_text_stub)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_broken_incorrect_partition_images=%t\n", productVariables.BuildBrokenIncorrectPartitionImages))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_id=%s\n", proptools.String(productVariables.BuildId)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_version_tags=%s\n", strings.Join(productVariables.BuildVersionTags, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:cfi_exclude_paths=%s\n", strings.Join(productVariables.CFIExcludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:cfi_include_paths=%s\n", strings.Join(productVariables.CFIIncludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:compressed_apex=%t\n", proptools.Bool(productVariables.CompressedApex)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:default_app_certificate=%s\n", proptools.String(productVariables.DefaultAppCertificate)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:default_app_certificate_filegroup=%s\n", defaultAppCertificateFilegroup))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_abi=%s\n", strings.Join(productVariables.DeviceAbi, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_max_page_size_supported=%s\n", proptools.String(productVariables.DeviceMaxPageSizeSupported)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_name=%s\n", proptools.String(productVariables.DeviceName)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_no_bionic_page_size_macro=%t\n", proptools.Bool(productVariables.DeviceNoBionicPageSizeMacro)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_product=%s\n", proptools.String(productVariables.DeviceProduct)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_platform=%s\n", label.String()))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:enable_cfi=%t\n", proptools.BoolDefault(productVariables.EnableCFI, true)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:enforce_vintf_manifest=%t\n", proptools.Bool(productVariables.Enforce_vintf_manifest)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_not_svelte=%t\n", proptools.Bool(productVariables.Malloc_not_svelte)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_pattern_fill_contents=%t\n", proptools.Bool(productVariables.Malloc_pattern_fill_contents)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_zero_contents=%t\n", proptools.Bool(productVariables.Malloc_zero_contents)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_exclude_paths=%s\n", strings.Join(productVariables.MemtagHeapExcludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_async_include_paths=%s\n", strings.Join(productVariables.MemtagHeapAsyncIncludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_sync_include_paths=%s\n", strings.Join(productVariables.MemtagHeapSyncIncludePaths, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:manifest_package_name_overrides=%s\n", strings.Join(productVariables.ManifestPackageNameOverrides, ",")))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:native_coverage=%t\n", proptools.Bool(productVariables.Native_coverage)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_sdk_final=%t\n", proptools.Bool(productVariables.Platform_sdk_final)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_security_patch=%s\n", proptools.String(productVariables.Platform_security_patch)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_version_last_stable=%s\n", proptools.String(productVariables.Platform_version_last_stable)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_version_name=%s\n", proptools.String(productVariables.Platform_version_name)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:product_brand=%s\n", productVariables.ProductBrand))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:product_manufacturer=%s\n", productVariables.ProductManufacturer))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_aconfig_flag_default_permission=%s\n", productVariables.ReleaseAconfigFlagDefaultPermission))
releaseAconfigValueSets := "//build/bazel/product_config:empty_aconfig_value_sets"
if len(productVariables.ReleaseAconfigValueSets) > 0 {
releaseAconfigValueSets = "@//" + label.pkg + ":" + releaseAconfigValueSetsName + "_" + label.target
}
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_aconfig_value_sets=%s\n", releaseAconfigValueSets))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_version=%s\n", productVariables.ReleaseVersion))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_sdk_version=%d\n", platform_sdk_version))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:safestack=%t\n", proptools.Bool(productVariables.Safestack)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:treble_linker_namespaces=%t\n", proptools.Bool(productVariables.Treble_linker_namespaces)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:tidy_checks=%s\n", proptools.String(productVariables.TidyChecks)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:uml=%t\n", proptools.Bool(productVariables.Uml)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:unbundled_build=%t\n", proptools.Bool(productVariables.Unbundled_build)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:unbundled_build_apps=%s\n", strings.Join(productVariables.Unbundled_build_apps, ",")))
for _, override := range productVariables.CertificateOverrides {
parts := strings.SplitN(override, ":", 2)
if apexPath, ok := convertedModulePathMap[parts[0]]; ok {
if overrideCertPath, ok := convertedModulePathMap[parts[1]]; ok {
result.WriteString(fmt.Sprintf(" --%s:%s_certificate_override=%s:%s\n", apexPath, parts[0], overrideCertPath, parts[1]))
}
}
}
for _, namespace := range android.SortedKeys(productVariables.VendorVars) {
for _, variable := range android.SortedKeys(productVariables.VendorVars[namespace]) {
value := productVariables.VendorVars[namespace][variable]
key := namespace + "__" + variable
_, hasBool := soongConfigDefinitions.BoolVars[key]
_, hasString := soongConfigDefinitions.StringVars[key]
_, hasValue := soongConfigDefinitions.ValueVars[key]
if !hasBool && !hasString && !hasValue {
// Not all soong config variables are defined in Android.bp files. For example,
// prebuilt_bootclasspath_fragment uses soong config variables in a nonstandard
// way, that causes them to be present in the soong.variables file but not
// defined in an Android.bp file. There's also nothing stopping you from setting
// a variable in make that doesn't exist in soong. We only generate build
// settings for the ones that exist in soong, so skip all others.
continue
}
if hasBool && hasString || hasBool && hasValue || hasString && hasValue {
panic(fmt.Sprintf("Soong config variable %s:%s appears to be of multiple types. bool? %t, string? %t, value? %t", namespace, variable, hasBool, hasString, hasValue))
}
if hasBool {
// Logic copied from soongConfig.Bool()
value = strings.ToLower(value)
if value == "1" || value == "y" || value == "yes" || value == "on" || value == "true" {
value = "true"
} else {
value = "false"
}
}
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config/soong_config_variables:%s=%s\n", strings.ToLower(key), value))
}
}
}
}
func starlarkMapToProductVariables(in map[string]starlark.Value) (android.ProductVariables, error) {
result := android.ProductVariables{}
productVarsReflect := reflect.ValueOf(&result).Elem()
for i := 0; i < productVarsReflect.NumField(); i++ {
field := productVarsReflect.Field(i)
fieldType := productVarsReflect.Type().Field(i)
name := fieldType.Name
if name == "BootJars" || name == "ApexBootJars" || name == "VendorSnapshotModules" ||
name == "RecoverySnapshotModules" {
// These variables have more complicated types, and we don't need them right now
continue
}
if _, ok := in[name]; ok {
if name == "VendorVars" {
vendorVars, err := starlark_import.Unmarshal[map[string]map[string]string](in[name])
if err != nil {
return result, err
}
field.Set(reflect.ValueOf(vendorVars))
continue
}
switch field.Type().Kind() {
case reflect.Bool:
val, err := starlark_import.Unmarshal[bool](in[name])
if err != nil {
return result, err
}
field.SetBool(val)
case reflect.String:
val, err := starlark_import.Unmarshal[string](in[name])
if err != nil {
return result, err
}
field.SetString(val)
case reflect.Slice:
if field.Type().Elem().Kind() != reflect.String {
return result, fmt.Errorf("slices of types other than strings are unimplemented")
}
val, err := starlark_import.UnmarshalReflect(in[name], field.Type())
if err != nil {
return result, err
}
field.Set(val)
case reflect.Pointer:
switch field.Type().Elem().Kind() {
case reflect.Bool:
val, err := starlark_import.UnmarshalNoneable[bool](in[name])
if err != nil {
return result, err
}
field.Set(reflect.ValueOf(val))
case reflect.String:
val, err := starlark_import.UnmarshalNoneable[string](in[name])
if err != nil {
return result, err
}
field.Set(reflect.ValueOf(val))
case reflect.Int:
val, err := starlark_import.UnmarshalNoneable[int](in[name])
if err != nil {
return result, err
}
field.Set(reflect.ValueOf(val))
default:
return result, fmt.Errorf("pointers of types other than strings/bools are unimplemented: %s", field.Type().Elem().Kind().String())
}
default:
return result, fmt.Errorf("unimplemented type: %s", field.Type().String())
}
}
}
result.Native_coverage = proptools.BoolPtr(
proptools.Bool(result.GcovCoverage) ||
proptools.Bool(result.ClangCoverage))
return result, nil
}
func createTargets(
ctx *CodegenContext,
productLabelsToVariables map[bazelLabel]*android.ProductVariables,
moduleNameToPartition map[string]string,
convertedModulePathMap map[string]string,
res map[string]BazelTargets) {
createGeneratedAndroidCertificateDirectories(productLabelsToVariables, res)
createAvbKeyFilegroups(productLabelsToVariables, res)
createReleaseAconfigValueSetsFilegroup(productLabelsToVariables, res)
for label, variables := range productLabelsToVariables {
createSystemPartition(ctx, label, &variables.PartitionVarsForBazelMigrationOnlyDoNotUse, moduleNameToPartition, convertedModulePathMap, res)
}
}
func createGeneratedAndroidCertificateDirectories(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) {
var allDefaultAppCertificateDirs []string
for _, productVariables := range productLabelsToVariables {
if proptools.String(productVariables.DefaultAppCertificate) != "" {
d := filepath.Dir(proptools.String(productVariables.DefaultAppCertificate))
if !android.InList(d, allDefaultAppCertificateDirs) {
allDefaultAppCertificateDirs = append(allDefaultAppCertificateDirs, d)
}
}
}
for _, dir := range allDefaultAppCertificateDirs {
content := `filegroup(
name = "generated_android_certificate_directory",
srcs = glob([
"*.pk8",
"*.pem",
"*.avbpubkey",
]),
visibility = ["//visibility:public"],
)`
targets[dir] = append(targets[dir], BazelTarget{
name: "generated_android_certificate_directory",
packageName: dir,
content: content,
ruleClass: "filegroup",
})
}
}
func createReleaseAconfigValueSetsFilegroup(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) {
for label, productVariables := range productLabelsToVariables {
if len(productVariables.ReleaseAconfigValueSets) > 0 {
key := label.target
dir := label.pkg
var value_sets strings.Builder
for _, value_set := range productVariables.ReleaseAconfigValueSets {
value_sets.WriteString(" \"" + value_set + "\",\n")
}
name := releaseAconfigValueSetsName + "_" + key
content := "aconfig_value_sets(\n" +
" name = \"" + name + "\",\n" +
" value_sets = [\n" +
value_sets.String() +
" ],\n" +
" visibility = [\"//visibility:public\"],\n" +
")"
targets[dir] = append(targets[dir], BazelTarget{
name: name,
packageName: dir,
content: content,
ruleClass: "aconfig_value_sets",
loads: []BazelLoad{{
file: "//build/bazel/rules/aconfig:aconfig_value_sets.bzl",
symbols: []BazelLoadSymbol{{
symbol: "aconfig_value_sets",
}},
}},
})
}
}
}
func createAvbKeyFilegroups(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) {
var allAvbKeys []string
for _, productVariables := range productLabelsToVariables {
for _, partitionVariables := range productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.PartitionQualifiedVariables {
if partitionVariables.BoardAvbKeyPath != "" {
if !android.InList(partitionVariables.BoardAvbKeyPath, allAvbKeys) {
allAvbKeys = append(allAvbKeys, partitionVariables.BoardAvbKeyPath)
}
}
}
}
for _, key := range allAvbKeys {
dir := filepath.Dir(key)
name := filepath.Base(key)
content := fmt.Sprintf(`filegroup(
name = "%s_filegroup",
srcs = ["%s"],
visibility = ["//visibility:public"],
)`, name, name)
targets[dir] = append(targets[dir], BazelTarget{
name: name + "_filegroup",
packageName: dir,
content: content,
ruleClass: "filegroup",
})
}
}
func createSystemPartition(
ctx *CodegenContext,
platformLabel bazelLabel,
variables *android.PartitionVariables,
moduleNameToPartition map[string]string,
convertedModulePathMap map[string]string,
targets map[string]BazelTargets) {
if !variables.PartitionQualifiedVariables["system"].BuildingImage {
return
}
qualifiedVariables := variables.PartitionQualifiedVariables["system"]
imageProps := generateImagePropDictionary(variables, "system")
imageProps["skip_fsck"] = "true"
var properties strings.Builder
for _, prop := range android.SortedKeys(imageProps) {
properties.WriteString(prop)
properties.WriteRune('=')
properties.WriteString(imageProps[prop])
properties.WriteRune('\n')
}
var extraProperties strings.Builder
if variables.BoardAvbEnable {
extraProperties.WriteString(" avb_enable = True,\n")
extraProperties.WriteString(fmt.Sprintf(" avb_add_hashtree_footer_args = %q,\n", qualifiedVariables.BoardAvbAddHashtreeFooterArgs))
keypath := qualifiedVariables.BoardAvbKeyPath
if keypath != "" {
extraProperties.WriteString(fmt.Sprintf(" avb_key = \"//%s:%s\",\n", filepath.Dir(keypath), filepath.Base(keypath)+"_filegroup"))
extraProperties.WriteString(fmt.Sprintf(" avb_algorithm = %q,\n", qualifiedVariables.BoardAvbAlgorithm))
extraProperties.WriteString(fmt.Sprintf(" avb_rollback_index = %s,\n", qualifiedVariables.BoardAvbRollbackIndex))
extraProperties.WriteString(fmt.Sprintf(" avb_rollback_index_location = %s,\n", qualifiedVariables.BoardAvbRollbackIndexLocation))
}
}
var deps []string
for _, mod := range variables.ProductPackages {
if path, ok := convertedModulePathMap[mod]; ok && ctx.Config().BazelContext.IsModuleNameAllowed(mod, false) {
if partition, ok := moduleNameToPartition[mod]; ok && partition == "system" {
if path == "//." {
path = "//"
}
deps = append(deps, fmt.Sprintf(" \"%s:%s\",\n", path, mod))
}
}
}
if len(deps) > 0 {
sort.Strings(deps)
extraProperties.WriteString(" deps = [\n")
for _, dep := range deps {
extraProperties.WriteString(dep)
}
extraProperties.WriteString(" ],\n")
}
targets[platformLabel.pkg] = append(targets[platformLabel.pkg], BazelTarget{
name: "system_image",
packageName: platformLabel.pkg,
content: fmt.Sprintf(`partition(
name = "system_image",
base_staging_dir = "//build/bazel/bazel_sandwich:system_staging_dir",
base_staging_dir_file_list = "//build/bazel/bazel_sandwich:system_staging_dir_file_list",
root_dir = "//build/bazel/bazel_sandwich:root_staging_dir",
selinux_file_contexts = "//build/bazel/bazel_sandwich:selinux_file_contexts",
image_properties = """
%s
""",
%s
type = "system",
)`, properties.String(), extraProperties.String()),
ruleClass: "partition",
loads: []BazelLoad{{
file: "//build/bazel/rules/partitions:partition.bzl",
symbols: []BazelLoadSymbol{{
symbol: "partition",
}},
}},
}, BazelTarget{
name: "system_image_test",
packageName: platformLabel.pkg,
content: `partition_diff_test(
name = "system_image_test",
partition1 = "//build/bazel/bazel_sandwich:make_system_image",
partition2 = ":system_image",
)`,
ruleClass: "partition_diff_test",
loads: []BazelLoad{{
file: "//build/bazel/rules/partitions/diff:partition_diff.bzl",
symbols: []BazelLoadSymbol{{
symbol: "partition_diff_test",
}},
}},
}, BazelTarget{
name: "run_system_image_test",
packageName: platformLabel.pkg,
content: `run_test_in_build(
name = "run_system_image_test",
test = ":system_image_test",
)`,
ruleClass: "run_test_in_build",
loads: []BazelLoad{{
file: "//build/bazel/bazel_sandwich:run_test_in_build.bzl",
symbols: []BazelLoadSymbol{{
symbol: "run_test_in_build",
}},
}},
})
}
var allPartitionTypes = []string{
"system",
"vendor",
"cache",
"userdata",
"product",
"system_ext",
"oem",
"odm",
"vendor_dlkm",
"odm_dlkm",
"system_dlkm",
}
// An equivalent of make's generate-image-prop-dictionary function
func generateImagePropDictionary(variables *android.PartitionVariables, partitionType string) map[string]string {
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
ret := map[string]string{}
if partitionType == "system" {
if len(variables.PartitionQualifiedVariables["system_other"].BoardPartitionSize) > 0 {
ret["system_other_size"] = variables.PartitionQualifiedVariables["system_other"].BoardPartitionSize
}
if len(partitionQualifiedVariables.ProductHeadroom) > 0 {
ret["system_headroom"] = partitionQualifiedVariables.ProductHeadroom
}
addCommonRoFlagsToImageProps(variables, partitionType, ret)
}
// TODO: other partition-specific logic
if variables.TargetUserimagesUseExt2 {
ret["fs_type"] = "ext2"
} else if variables.TargetUserimagesUseExt3 {
ret["fs_type"] = "ext3"
} else if variables.TargetUserimagesUseExt4 {
ret["fs_type"] = "ext4"
}
if !variables.TargetUserimagesSparseExtDisabled {
ret["extfs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseErofsDisabled {
ret["erofs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseSquashfsDisabled {
ret["squashfs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseF2fsDisabled {
ret["f2fs_sparse_flag"] = "-S"
}
erofsCompressor := variables.BoardErofsCompressor
if len(erofsCompressor) == 0 && hasErofsPartition(variables) {
if len(variables.BoardErofsUseLegacyCompression) > 0 {
erofsCompressor = "lz4"
} else {
erofsCompressor = "lz4hc,9"
}
}
if len(erofsCompressor) > 0 {
ret["erofs_default_compressor"] = erofsCompressor
}
if len(variables.BoardErofsCompressorHints) > 0 {
ret["erofs_default_compress_hints"] = variables.BoardErofsCompressorHints
}
if len(variables.BoardErofsCompressorHints) > 0 {
ret["erofs_default_compress_hints"] = variables.BoardErofsCompressorHints
}
if len(variables.BoardErofsPclusterSize) > 0 {
ret["erofs_pcluster_size"] = variables.BoardErofsPclusterSize
}
if len(variables.BoardErofsShareDupBlocks) > 0 {
ret["erofs_share_dup_blocks"] = variables.BoardErofsShareDupBlocks
}
if len(variables.BoardErofsUseLegacyCompression) > 0 {
ret["erofs_use_legacy_compression"] = variables.BoardErofsUseLegacyCompression
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
if len(variables.BoardFlashLogicalBlockSize) > 0 {
ret["flash_logical_block_size"] = variables.BoardFlashLogicalBlockSize
}
if len(variables.BoardFlashEraseBlockSize) > 0 {
ret["flash_erase_block_size"] = variables.BoardFlashEraseBlockSize
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
for _, partitionType := range allPartitionTypes {
if qualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]; ok && len(qualifiedVariables.ProductVerityPartition) > 0 {
ret[partitionType+"_verity_block_device"] = qualifiedVariables.ProductVerityPartition
}
}
// TODO: Vboot
// TODO: AVB
if variables.BoardUsesRecoveryAsBoot {
ret["recovery_as_boot"] = "true"
}
if variables.ProductUseDynamicPartitionSize {
ret["use_dynamic_partition_size"] = "true"
}
if variables.CopyImagesForTargetFilesZip {
ret["use_fixed_timestamp"] = "true"
}
return ret
}
// Soong equivalent of make's add-common-ro-flags-to-image-props
func addCommonRoFlagsToImageProps(variables *android.PartitionVariables, partitionType string, ret map[string]string) {
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
if len(partitionQualifiedVariables.BoardErofsCompressor) > 0 {
ret[partitionType+"_erofs_compressor"] = partitionQualifiedVariables.BoardErofsCompressor
}
if len(partitionQualifiedVariables.BoardErofsCompressHints) > 0 {
ret[partitionType+"_erofs_compress_hints"] = partitionQualifiedVariables.BoardErofsCompressHints
}
if len(partitionQualifiedVariables.BoardErofsPclusterSize) > 0 {
ret[partitionType+"_erofs_pcluster_size"] = partitionQualifiedVariables.BoardErofsPclusterSize
}
if len(partitionQualifiedVariables.BoardExtfsRsvPct) > 0 {
ret[partitionType+"_extfs_rsv_pct"] = partitionQualifiedVariables.BoardExtfsRsvPct
}
if len(partitionQualifiedVariables.BoardF2fsSloadCompressFlags) > 0 {
ret[partitionType+"_f2fs_sldc_flags"] = partitionQualifiedVariables.BoardF2fsSloadCompressFlags
}
if len(partitionQualifiedVariables.BoardFileSystemCompress) > 0 {
ret[partitionType+"_f2fs_compress"] = partitionQualifiedVariables.BoardFileSystemCompress
}
if len(partitionQualifiedVariables.BoardFileSystemType) > 0 {
ret[partitionType+"_fs_type"] = partitionQualifiedVariables.BoardFileSystemType
}
if len(partitionQualifiedVariables.BoardJournalSize) > 0 {
ret[partitionType+"_journal_size"] = partitionQualifiedVariables.BoardJournalSize
}
if len(partitionQualifiedVariables.BoardPartitionReservedSize) > 0 {
ret[partitionType+"_reserved_size"] = partitionQualifiedVariables.BoardPartitionReservedSize
}
if len(partitionQualifiedVariables.BoardPartitionSize) > 0 {
ret[partitionType+"_size"] = partitionQualifiedVariables.BoardPartitionSize
}
if len(partitionQualifiedVariables.BoardSquashfsBlockSize) > 0 {
ret[partitionType+"_squashfs_block_size"] = partitionQualifiedVariables.BoardSquashfsBlockSize
}
if len(partitionQualifiedVariables.BoardSquashfsCompressor) > 0 {
ret[partitionType+"_squashfs_compressor"] = partitionQualifiedVariables.BoardSquashfsCompressor
}
if len(partitionQualifiedVariables.BoardSquashfsCompressorOpt) > 0 {
ret[partitionType+"_squashfs_compressor_opt"] = partitionQualifiedVariables.BoardSquashfsCompressorOpt
}
if len(partitionQualifiedVariables.BoardSquashfsDisable4kAlign) > 0 {
ret[partitionType+"_squashfs_disable_4k_align"] = partitionQualifiedVariables.BoardSquashfsDisable4kAlign
}
if len(partitionQualifiedVariables.BoardPartitionSize) == 0 && len(partitionQualifiedVariables.BoardPartitionReservedSize) == 0 && len(partitionQualifiedVariables.ProductHeadroom) == 0 {
ret[partitionType+"_disable_sparse"] = "true"
}
addCommonFlagsToImageProps(variables, partitionType, ret)
}
func hasErofsPartition(variables *android.PartitionVariables) bool {
return variables.PartitionQualifiedVariables["product"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system_ext"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["odm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["vendor"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["vendor_dlkm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["odm_dlkm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system_dlkm"].BoardFileSystemType == "erofs"
}
// Soong equivalent of make's add-common-flags-to-image-props
func addCommonFlagsToImageProps(variables *android.PartitionVariables, partitionType string, ret map[string]string) {
// The selinux_fc will be handled separately
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
ret["building_"+partitionType+"_image"] = boolToMakeString(partitionQualifiedVariables.BuildingImage)
}
func boolToMakeString(b bool) string {
if b {
return "true"
}
return ""
}

605
bp2build/build_conversion.go
View File

@@ -28,10 +28,7 @@ import (
"android/soong/android" "android/soong/android"
"android/soong/bazel" "android/soong/bazel"
"android/soong/starlark_fmt" "android/soong/starlark_fmt"
"android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint" "github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/proptools" "github.com/google/blueprint/proptools"
) )
@@ -201,18 +198,13 @@ func (ctx *CodegenContext) Mode() CodegenMode {
type CodegenMode int type CodegenMode int
const ( const (
// Bp2Build - generate BUILD files with targets buildable by Bazel directly.
//
// This mode is used for the Soong->Bazel build definition conversion.
Bp2Build CodegenMode = iota
// QueryView - generate BUILD files with targets representing fully mutated // QueryView - generate BUILD files with targets representing fully mutated
// Soong modules, representing the fully configured Soong module graph with // Soong modules, representing the fully configured Soong module graph with
// variants and dependency edges. // variants and dependency edges.
// //
// This mode is used for discovering and introspecting the existing Soong // This mode is used for discovering and introspecting the existing Soong
// module graph. // module graph.
QueryView QueryView CodegenMode = iota
) )
type unconvertedDepsMode int type unconvertedDepsMode int
@@ -227,8 +219,6 @@ const (
func (mode CodegenMode) String() string { func (mode CodegenMode) String() string {
switch mode { switch mode {
case Bp2Build:
return "Bp2Build"
case QueryView: case QueryView:
return "QueryView" return "QueryView"
default: default:
@@ -256,9 +246,6 @@ func (ctx *CodegenContext) Context() *android.Context { return ctx.context }
// writing BUILD files in the output directory. // writing BUILD files in the output directory.
func NewCodegenContext(config android.Config, context *android.Context, mode CodegenMode, topDir string) *CodegenContext { func NewCodegenContext(config android.Config, context *android.Context, mode CodegenMode, topDir string) *CodegenContext {
var unconvertedDeps unconvertedDepsMode var unconvertedDeps unconvertedDepsMode
if config.IsEnvTrue("BP2BUILD_ERROR_UNCONVERTED") {
unconvertedDeps = errorModulesUnconvertedDeps
}
return &CodegenContext{ return &CodegenContext{
context: context, context: context,
config: config, config: config,
@@ -281,526 +268,30 @@ func propsToAttributes(props map[string]string) string {
type conversionResults struct { type conversionResults struct {
buildFileToTargets map[string]BazelTargets buildFileToTargets map[string]BazelTargets
moduleNameToPartition map[string]string moduleNameToPartition map[string]string
metrics CodegenMetrics
} }
func (r conversionResults) BuildDirToTargets() map[string]BazelTargets { func (r conversionResults) BuildDirToTargets() map[string]BazelTargets {
return r.buildFileToTargets return r.buildFileToTargets
} }
// struct to store state of b bazel targets (e.g. go targets which do not implement android.Module)
// this implements bp2buildModule interface and is passed to generateBazelTargets
type bTarget struct {
targetName string
targetPackage string
bazelRuleClass string
bazelRuleLoadLocation string
bazelAttributes []interface{}
}
var _ bp2buildModule = (*bTarget)(nil)
func (b bTarget) TargetName() string {
return b.targetName
}
func (b bTarget) TargetPackage() string {
return b.targetPackage
}
func (b bTarget) BazelRuleClass() string {
return b.bazelRuleClass
}
func (b bTarget) BazelRuleLoadLocation() string {
return b.bazelRuleLoadLocation
}
func (b bTarget) BazelAttributes() []interface{} {
return b.bazelAttributes
}
// Creates a target_compatible_with entry that is *not* compatible with android
func targetNotCompatibleWithAndroid() bazel.LabelListAttribute {
ret := bazel.LabelListAttribute{}
ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsAndroid,
bazel.MakeLabelList(
[]bazel.Label{
bazel.Label{
Label: "@platforms//:incompatible",
},
},
),
)
return ret
}
// helper function to return labels for srcs used in bootstrap_go_package and bootstrap_go_binary
// this function has the following limitations which make it unsuitable for widespread use
// - wildcard patterns in srcs
// This is ok for go since build/blueprint does not support it.
//
// Prefer to use `BazelLabelForModuleSrc` instead
func goSrcLabels(cfg android.Config, moduleDir string, srcs []string, linuxSrcs, darwinSrcs []string) bazel.LabelListAttribute {
labels := func(srcs []string) bazel.LabelList {
ret := []bazel.Label{}
for _, src := range srcs {
srcLabel := bazel.Label{
Label: src,
}
ret = append(ret, srcLabel)
}
// Respect package boundaries
return android.TransformSubpackagePaths(
cfg,
moduleDir,
bazel.MakeLabelList(ret),
)
}
ret := bazel.LabelListAttribute{}
// common
ret.SetSelectValue(bazel.NoConfigAxis, "", labels(srcs))
// linux
ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsLinux, labels(linuxSrcs))
// darwin
ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsDarwin, labels(darwinSrcs))
return ret
}
func goDepLabels(deps []string, goModulesMap nameToGoLibraryModule) bazel.LabelListAttribute {
labels := []bazel.Label{}
for _, dep := range deps {
moduleDir := goModulesMap[dep].Dir
if moduleDir == "." {
moduleDir = ""
}
label := bazel.Label{
Label: fmt.Sprintf("//%s:%s", moduleDir, dep),
}
labels = append(labels, label)
}
return bazel.MakeLabelListAttribute(bazel.MakeLabelList(labels))
}
// attributes common to blueprint_go_binary and bootstap_go_package
type goAttributes struct {
Importpath bazel.StringAttribute
Srcs bazel.LabelListAttribute
Deps bazel.LabelListAttribute
Data bazel.LabelListAttribute
Target_compatible_with bazel.LabelListAttribute
// attributes for the dynamically generated go_test target
Embed bazel.LabelListAttribute
}
type goTestProperties struct {
name string
dir string
testSrcs []string
linuxTestSrcs []string
darwinTestSrcs []string
testData []string
// Name of the target that should be compiled together with the test
embedName string
}
// Creates a go_test target for bootstrap_go_package / blueprint_go_binary
func generateBazelTargetsGoTest(ctx *android.Context, goModulesMap nameToGoLibraryModule, gp goTestProperties) (BazelTarget, error) {
ca := android.CommonAttributes{
Name: gp.name,
}
ga := goAttributes{
Srcs: goSrcLabels(ctx.Config(), gp.dir, gp.testSrcs, gp.linuxTestSrcs, gp.darwinTestSrcs),
Data: goSrcLabels(ctx.Config(), gp.dir, gp.testData, []string{}, []string{}),
Embed: bazel.MakeLabelListAttribute(
bazel.MakeLabelList(
[]bazel.Label{bazel.Label{Label: ":" + gp.embedName}},
),
),
Target_compatible_with: targetNotCompatibleWithAndroid(),
}
libTest := bTarget{
targetName: gp.name,
targetPackage: gp.dir,
bazelRuleClass: "go_test",
bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
bazelAttributes: []interface{}{&ca, &ga},
}
return generateBazelTarget(ctx, libTest)
}
// TODO - b/288491147: testSrcs of certain bootstrap_go_package/blueprint_go_binary are not hermetic and depend on
// testdata checked into the filesystem.
// Denylist the generation of go_test targets for these Soong modules.
// The go_library/go_binary will still be generated, since those are hermitic.
var (
goTestsDenylist = []string{
"android-archive-zip",
"bazel_notice_gen",
"blueprint-bootstrap-bpdoc",
"blueprint-microfactory",
"blueprint-pathtools",
"bssl_ar",
"compliance_checkmetadata",
"compliance_checkshare",
"compliance_dumpgraph",
"compliance_dumpresolutions",
"compliance_listshare",
"compliance-module",
"compliancenotice_bom",
"compliancenotice_shippedlibs",
"compliance_rtrace",
"compliance_sbom",
"golang-protobuf-internal-fuzz-jsonfuzz",
"golang-protobuf-internal-fuzz-textfuzz",
"golang-protobuf-internal-fuzz-wirefuzz",
"htmlnotice",
"protoc-gen-go",
"rbcrun-module",
"spdx-tools-builder",
"spdx-tools-builder2v1",
"spdx-tools-builder2v2",
"spdx-tools-builder2v3",
"spdx-tools-idsearcher",
"spdx-tools-spdx-json",
"spdx-tools-utils",
"soong-ui-build",
"textnotice",
"xmlnotice",
}
)
func testOfGoPackageIsIncompatible(g *bootstrap.GoPackage) bool {
return android.InList(g.Name(), goTestsDenylist) ||
// Denylist tests of soong_build
// Theses tests have a guard that prevent usage outside a test environment
// The guard (`ensureTestOnly`) looks for a `-test` in os.Args, which is present in soong's gotestrunner, but missing in `b test`
g.IsPluginFor("soong_build") ||
// soong-android is a dep of soong_build
// This dependency is created by soong_build by listing it in its deps explicitly in Android.bp, and not via `plugin_for` in `soong-android`
g.Name() == "soong-android"
}
func testOfGoBinaryIsIncompatible(g *bootstrap.GoBinary) bool {
return android.InList(g.Name(), goTestsDenylist)
}
func generateBazelTargetsGoPackage(ctx *android.Context, g *bootstrap.GoPackage, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) {
ca := android.CommonAttributes{
Name: g.Name(),
}
// For this bootstrap_go_package dep chain,
// A --> B --> C ( ---> depends on)
// Soong provides the convenience of only listing B as deps of A even if a src file of A imports C
// Bazel OTOH
// 1. requires C to be listed in `deps` expllicity.
// 2. does not require C to be listed if src of A does not import C
//
// bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps
transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap)
ga := goAttributes{
Importpath: bazel.StringAttribute{
Value: proptools.StringPtr(g.GoPkgPath()),
},
Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()),
Deps: goDepLabels(
android.FirstUniqueStrings(transitiveDeps),
goModulesMap,
),
Target_compatible_with: targetNotCompatibleWithAndroid(),
}
lib := bTarget{
targetName: g.Name(),
targetPackage: ctx.ModuleDir(g),
bazelRuleClass: "go_library",
bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
bazelAttributes: []interface{}{&ca, &ga},
}
retTargets := []BazelTarget{}
var retErrs []error
if libTarget, err := generateBazelTarget(ctx, lib); err == nil {
retTargets = append(retTargets, libTarget)
} else {
retErrs = []error{err}
}
// If the library contains test srcs, create an additional go_test target
if !testOfGoPackageIsIncompatible(g) && (len(g.TestSrcs()) > 0 || len(g.LinuxTestSrcs()) > 0 || len(g.DarwinTestSrcs()) > 0) {
gp := goTestProperties{
name: g.Name() + "-test",
dir: ctx.ModuleDir(g),
testSrcs: g.TestSrcs(),
linuxTestSrcs: g.LinuxTestSrcs(),
darwinTestSrcs: g.DarwinTestSrcs(),
testData: g.TestData(),
embedName: g.Name(), // embed the source go_library in the test so that its .go files are included in the compilation unit
}
if libTestTarget, err := generateBazelTargetsGoTest(ctx, goModulesMap, gp); err == nil {
retTargets = append(retTargets, libTestTarget)
} else {
retErrs = append(retErrs, err)
}
}
return retTargets, retErrs
}
type goLibraryModule struct {
Dir string
Deps []string
}
type buildConversionMetadata struct {
nameToGoLibraryModule nameToGoLibraryModule
ndkHeaders []blueprint.Module
}
type nameToGoLibraryModule map[string]goLibraryModule
// Visit each module in the graph, and collect metadata about the build graph
// If a module is of type `bootstrap_go_package`, return a map containing metadata like its dir and deps
// If a module is of type `ndk_headers`, add it to a list and return the list
func createBuildConversionMetadata(ctx *android.Context) buildConversionMetadata {
goMap := nameToGoLibraryModule{}
ndkHeaders := []blueprint.Module{}
ctx.VisitAllModules(func(m blueprint.Module) {
moduleType := ctx.ModuleType(m)
// We do not need to store information about blueprint_go_binary since it does not have any rdeps
if moduleType == "bootstrap_go_package" {
goMap[m.Name()] = goLibraryModule{
Dir: ctx.ModuleDir(m),
Deps: m.(*bootstrap.GoPackage).Deps(),
}
} else if moduleType == "ndk_headers" || moduleType == "versioned_ndk_headers" {
ndkHeaders = append(ndkHeaders, m)
}
})
return buildConversionMetadata{
nameToGoLibraryModule: goMap,
ndkHeaders: ndkHeaders,
}
}
// Returns the deps in the transitive closure of a go target
func transitiveGoDeps(directDeps []string, goModulesMap nameToGoLibraryModule) []string {
allDeps := directDeps
i := 0
for i < len(allDeps) {
curr := allDeps[i]
allDeps = append(allDeps, goModulesMap[curr].Deps...)
i += 1
}
allDeps = android.SortedUniqueStrings(allDeps)
return allDeps
}
func generateBazelTargetsGoBinary(ctx *android.Context, g *bootstrap.GoBinary, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) {
ca := android.CommonAttributes{
Name: g.Name(),
}
retTargets := []BazelTarget{}
var retErrs []error
// For this bootstrap_go_package dep chain,
// A --> B --> C ( ---> depends on)
// Soong provides the convenience of only listing B as deps of A even if a src file of A imports C
// Bazel OTOH
// 1. requires C to be listed in `deps` expllicity.
// 2. does not require C to be listed if src of A does not import C
//
// bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps
transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap)
goSource := ""
// If the library contains test srcs, create an additional go_test target
// The go_test target will embed a go_source containining the source .go files it tests
if !testOfGoBinaryIsIncompatible(g) && (len(g.TestSrcs()) > 0 || len(g.LinuxTestSrcs()) > 0 || len(g.DarwinTestSrcs()) > 0) {
// Create a go_source containing the source .go files of go_library
// This target will be an `embed` of the go_binary and go_test
goSource = g.Name() + "-source"
ca := android.CommonAttributes{
Name: goSource,
}
ga := goAttributes{
Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()),
Deps: goDepLabels(transitiveDeps, goModulesMap),
Target_compatible_with: targetNotCompatibleWithAndroid(),
}
libTestSource := bTarget{
targetName: goSource,
targetPackage: ctx.ModuleDir(g),
bazelRuleClass: "go_source",
bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
bazelAttributes: []interface{}{&ca, &ga},
}
if libSourceTarget, err := generateBazelTarget(ctx, libTestSource); err == nil {
retTargets = append(retTargets, libSourceTarget)
} else {
retErrs = append(retErrs, err)
}
// Create a go_test target
gp := goTestProperties{
name: g.Name() + "-test",
dir: ctx.ModuleDir(g),
testSrcs: g.TestSrcs(),
linuxTestSrcs: g.LinuxTestSrcs(),
darwinTestSrcs: g.DarwinTestSrcs(),
testData: g.TestData(),
// embed the go_source in the test
embedName: g.Name() + "-source",
}
if libTestTarget, err := generateBazelTargetsGoTest(ctx, goModulesMap, gp); err == nil {
retTargets = append(retTargets, libTestTarget)
} else {
retErrs = append(retErrs, err)
}
}
// Create a go_binary target
ga := goAttributes{
Deps: goDepLabels(transitiveDeps, goModulesMap),
Target_compatible_with: targetNotCompatibleWithAndroid(),
}
// If the binary has testSrcs, embed the common `go_source`
if goSource != "" {
ga.Embed = bazel.MakeLabelListAttribute(
bazel.MakeLabelList(
[]bazel.Label{bazel.Label{Label: ":" + goSource}},
),
)
} else {
ga.Srcs = goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs())
}
bin := bTarget{
targetName: g.Name(),
targetPackage: ctx.ModuleDir(g),
bazelRuleClass: "go_binary",
bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
bazelAttributes: []interface{}{&ca, &ga},
}
if binTarget, err := generateBazelTarget(ctx, bin); err == nil {
retTargets = append(retTargets, binTarget)
} else {
retErrs = []error{err}
}
return retTargets, retErrs
}
func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (conversionResults, []error) { func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (conversionResults, []error) {
ctx.Context().BeginEvent("GenerateBazelTargets") ctx.Context().BeginEvent("GenerateBazelTargets")
defer ctx.Context().EndEvent("GenerateBazelTargets") defer ctx.Context().EndEvent("GenerateBazelTargets")
buildFileToTargets := make(map[string]BazelTargets) buildFileToTargets := make(map[string]BazelTargets)
// Simple metrics tracking for bp2build
metrics := CreateCodegenMetrics()
dirs := make(map[string]bool) dirs := make(map[string]bool)
moduleNameToPartition := make(map[string]string) moduleNameToPartition := make(map[string]string)
var errs []error var errs []error
// Visit go libraries in a pre-run and store its state in a map
// The time complexity remains O(N), and this does not add significant wall time.
meta := createBuildConversionMetadata(ctx.Context())
nameToGoLibMap := meta.nameToGoLibraryModule
ndkHeaders := meta.ndkHeaders
bpCtx := ctx.Context() bpCtx := ctx.Context()
bpCtx.VisitAllModules(func(m blueprint.Module) { bpCtx.VisitAllModules(func(m blueprint.Module) {
dir := bpCtx.ModuleDir(m) dir := bpCtx.ModuleDir(m)
moduleType := bpCtx.ModuleType(m)
dirs[dir] = true dirs[dir] = true
var targets []BazelTarget var targets []BazelTarget
var targetErrs []error
switch ctx.Mode() { switch ctx.Mode() {
case Bp2Build:
if aModule, ok := m.(android.Module); ok {
reason := aModule.GetUnconvertedReason()
if reason != nil {
// If this module was force-enabled, cause an error.
if _, ok := ctx.Config().BazelModulesForceEnabledByFlag()[m.Name()]; ok && m.Name() != "" {
err := fmt.Errorf("Force Enabled Module %s not converted", m.Name())
errs = append(errs, err)
}
// Log the module isn't to be converted by bp2build.
// TODO: b/291598248 - Log handcrafted modules differently than other unconverted modules.
metrics.AddUnconvertedModule(m, moduleType, dir, *reason)
return
}
if len(aModule.Bp2buildTargets()) == 0 {
panic(fmt.Errorf("illegal bp2build invariant: module '%s' was neither converted nor marked unconvertible", aModule.Name()))
}
// Handle modules converted to generated targets.
targets, targetErrs = generateBazelTargets(bpCtx, aModule)
errs = append(errs, targetErrs...)
for _, t := range targets {
// A module can potentially generate more than 1 Bazel
// target, each of a different rule class.
metrics.IncrementRuleClassCount(t.ruleClass)
}
// record the partition
moduleNameToPartition[android.RemoveOptionalPrebuiltPrefix(aModule.Name())] = aModule.GetPartitionForBp2build()
// Log the module.
metrics.AddConvertedModule(aModule, moduleType, dir)
// Handle modules with unconverted deps. By default, emit a warning.
if unconvertedDeps := aModule.GetUnconvertedBp2buildDeps(); len(unconvertedDeps) > 0 {
msg := fmt.Sprintf("%s %s:%s depends on unconverted modules: %s",
moduleType, bpCtx.ModuleDir(m), m.Name(), strings.Join(unconvertedDeps, ", "))
switch ctx.unconvertedDepMode {
case warnUnconvertedDeps:
metrics.moduleWithUnconvertedDepsMsgs = append(metrics.moduleWithUnconvertedDepsMsgs, msg)
case errorModulesUnconvertedDeps:
errs = append(errs, fmt.Errorf(msg))
return
}
}
if unconvertedDeps := aModule.GetMissingBp2buildDeps(); len(unconvertedDeps) > 0 {
msg := fmt.Sprintf("%s %s:%s depends on missing modules: %s",
moduleType, bpCtx.ModuleDir(m), m.Name(), strings.Join(unconvertedDeps, ", "))
switch ctx.unconvertedDepMode {
case warnUnconvertedDeps:
metrics.moduleWithMissingDepsMsgs = append(metrics.moduleWithMissingDepsMsgs, msg)
case errorModulesUnconvertedDeps:
errs = append(errs, fmt.Errorf(msg))
return
}
}
} else if glib, ok := m.(*bootstrap.GoPackage); ok {
targets, targetErrs = generateBazelTargetsGoPackage(bpCtx, glib, nameToGoLibMap)
errs = append(errs, targetErrs...)
metrics.IncrementRuleClassCount("bootstrap_go_package")
metrics.AddConvertedModule(glib, "bootstrap_go_package", dir)
} else if gbin, ok := m.(*bootstrap.GoBinary); ok {
targets, targetErrs = generateBazelTargetsGoBinary(bpCtx, gbin, nameToGoLibMap)
errs = append(errs, targetErrs...)
metrics.IncrementRuleClassCount("blueprint_go_binary")
metrics.AddConvertedModule(gbin, "blueprint_go_binary", dir)
} else {
metrics.AddUnconvertedModule(m, moduleType, dir, android.UnconvertedReason{
ReasonType: int(bp2build_metrics_proto.UnconvertedReasonType_TYPE_UNSUPPORTED),
})
return
}
case QueryView: case QueryView:
// Blocklist certain module types from being generated. // Blocklist certain module types from being generated.
if canonicalizeModuleType(bpCtx.ModuleType(m)) == "package" { if canonicalizeModuleType(bpCtx.ModuleType(m)) == "package" {
@@ -824,37 +315,6 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
} }
}) })
// Create an ndk_sysroot target that has a dependency edge on every target corresponding to Soong's ndk_headers
// This root target will provide headers to sdk variants of jni libraries
if ctx.Mode() == Bp2Build {
var depLabels bazel.LabelList
for _, ndkHeader := range ndkHeaders {
depLabel := bazel.Label{
Label: "//" + bpCtx.ModuleDir(ndkHeader) + ":" + ndkHeader.Name(),
}
depLabels.Add(&depLabel)
}
a := struct {
Deps bazel.LabelListAttribute
}{
Deps: bazel.MakeLabelListAttribute(bazel.UniqueSortedBazelLabelList(depLabels)),
}
ndkSysroot := bTarget{
targetName: "ndk_sysroot",
targetPackage: "build/bazel/rules/cc", // The location is subject to change, use build/bazel for now
bazelRuleClass: "cc_library_headers",
bazelRuleLoadLocation: "//build/bazel/rules/cc:cc_library_headers.bzl",
bazelAttributes: []interface{}{&a},
}
if t, err := generateBazelTarget(bpCtx, ndkSysroot); err == nil {
dir := ndkSysroot.targetPackage
buildFileToTargets[dir] = append(buildFileToTargets[dir], t)
} else {
errs = append(errs, err)
}
}
if len(errs) > 0 { if len(errs) > 0 {
return conversionResults{}, errs return conversionResults{}, errs
} }
@@ -883,72 +343,9 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
return conversionResults{ return conversionResults{
buildFileToTargets: buildFileToTargets, buildFileToTargets: buildFileToTargets,
moduleNameToPartition: moduleNameToPartition, moduleNameToPartition: moduleNameToPartition,
metrics: metrics,
}, errs }, errs
} }
func generateBazelTargets(ctx bpToBuildContext, m android.Module) ([]BazelTarget, []error) {
var targets []BazelTarget
var errs []error
for _, m := range m.Bp2buildTargets() {
target, err := generateBazelTarget(ctx, m)
if err != nil {
errs = append(errs, err)
return targets, errs
}
targets = append(targets, target)
}
return targets, errs
}
type bp2buildModule interface {
TargetName() string
TargetPackage() string
BazelRuleClass() string
BazelRuleLoadLocation() string
BazelAttributes() []interface{}
}
func generateBazelTarget(ctx bpToBuildContext, m bp2buildModule) (BazelTarget, error) {
ruleClass := m.BazelRuleClass()
bzlLoadLocation := m.BazelRuleLoadLocation()
// extract the bazel attributes from the module.
attrs := m.BazelAttributes()
props, err := extractModuleProperties(attrs, true)
if err != nil {
return BazelTarget{}, err
}
// name is handled in a special manner
delete(props.Attrs, "name")
// Return the Bazel target with rule class and attributes, ready to be
// code-generated.
attributes := propsToAttributes(props.Attrs)
var content string
targetName := m.TargetName()
if targetName != "" {
content = fmt.Sprintf(ruleTargetTemplate, ruleClass, targetName, attributes)
} else {
content = fmt.Sprintf(unnamedRuleTargetTemplate, ruleClass, attributes)
}
var loads []BazelLoad
if bzlLoadLocation != "" {
loads = append(loads, BazelLoad{
file: bzlLoadLocation,
symbols: []BazelLoadSymbol{{symbol: ruleClass}},
})
}
return BazelTarget{
name: targetName,
packageName: m.TargetPackage(),
ruleClass: ruleClass,
loads: loads,
content: content,
}, nil
}
// Convert a module and its deps and props into a Bazel macro/rule // Convert a module and its deps and props into a Bazel macro/rule
// representation in the BUILD file. // representation in the BUILD file.
func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) (BazelTarget, error) { func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) (BazelTarget, error) {

101
bp2build/conversion.go
View File

@@ -1,15 +1,10 @@
package bp2build package bp2build
import ( import (
"encoding/json"
"fmt"
"reflect" "reflect"
"strconv"
"strings" "strings"
"android/soong/android" "android/soong/android"
"android/soong/starlark_fmt"
"github.com/google/blueprint/proptools" "github.com/google/blueprint/proptools"
) )
@@ -19,79 +14,6 @@ type BazelFile struct {
Contents string Contents string
} }
// createSoongInjectionDirFiles returns most of the files to write to the soong_injection directory.
// Some other files also come from CreateProductConfigFiles
func createSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) ([]BazelFile, error) {
cfg := ctx.Config()
var files []BazelFile
files = append(files, newFile("android", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package.
files = append(files, newFile("android", "constants.bzl", android.BazelCcToolchainVars(cfg)))
if buf, err := json.MarshalIndent(metrics.convertedModuleWithType, "", " "); err != nil {
return []BazelFile{}, err
} else {
files = append(files, newFile("metrics", "converted_modules.json", string(buf)))
}
convertedModulePathMap, err := json.MarshalIndent(metrics.convertedModulePathMap, "", "\t")
if err != nil {
panic(err)
}
files = append(files, newFile("metrics", GeneratedBuildFileName, "")) // Creates a //metrics package.
files = append(files, newFile("metrics", "converted_modules_path_map.json", string(convertedModulePathMap)))
files = append(files, newFile("metrics", "converted_modules_path_map.bzl", "modules = "+strings.ReplaceAll(string(convertedModulePathMap), "\\", "\\\\")))
files = append(files, newFile("product_config", "soong_config_variables.bzl", cfg.Bp2buildSoongConfigDefinitions.String()))
files = append(files, newFile("product_config", "arch_configuration.bzl", android.StarlarkArchConfigurations()))
apiLevelsMap, err := android.GetApiLevelsMap(cfg)
if err != nil {
return nil, err
}
apiLevelsContent, err := json.Marshal(apiLevelsMap)
if err != nil {
return nil, err
}
files = append(files, newFile("api_levels", GeneratedBuildFileName, `exports_files(["api_levels.json"])`))
// TODO(b/269691302) value of apiLevelsContent is product variable dependent and should be avoided for soong injection
files = append(files, newFile("api_levels", "api_levels.json", string(apiLevelsContent)))
files = append(files, newFile("api_levels", "platform_versions.bzl", platformVersionContents(cfg)))
files = append(files, newFile("allowlists", GeneratedBuildFileName, ""))
// TODO(b/262781701): Create an alternate soong_build entrypoint for writing out these files only when requested
files = append(files, newFile("allowlists", "mixed_build_prod_allowlist.txt", strings.Join(android.GetBazelEnabledModules(android.BazelProdMode), "\n")+"\n"))
files = append(files, newFile("allowlists", "mixed_build_staging_allowlist.txt", strings.Join(android.GetBazelEnabledModules(android.BazelStagingMode), "\n")+"\n"))
return files, nil
}
func platformVersionContents(cfg android.Config) string {
// Despite these coming from cfg.productVariables, they are actually hardcoded in global
// makefiles, not set in individual product config makesfiles, so they're safe to just export
// and load() directly.
platformVersionActiveCodenames := make([]string, 0, len(cfg.PlatformVersionActiveCodenames()))
for _, codename := range cfg.PlatformVersionActiveCodenames() {
platformVersionActiveCodenames = append(platformVersionActiveCodenames, fmt.Sprintf("%q", codename))
}
platformSdkVersion := "None"
if cfg.RawPlatformSdkVersion() != nil {
platformSdkVersion = strconv.Itoa(*cfg.RawPlatformSdkVersion())
}
return fmt.Sprintf(`
platform_versions = struct(
platform_sdk_final = %s,
platform_sdk_version = %s,
platform_sdk_codename = %q,
platform_version_active_codenames = [%s],
)
`, starlark_fmt.PrintBool(cfg.PlatformSdkFinal()), platformSdkVersion, cfg.PlatformSdkCodename(), strings.Join(platformVersionActiveCodenames, ", "))
}
func CreateBazelFiles(ruleShims map[string]RuleShim, buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile { func CreateBazelFiles(ruleShims map[string]RuleShim, buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile {
var files []BazelFile var files []BazelFile
@@ -125,20 +47,7 @@ func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode)
targets.sort() targets.sort()
var content string var content string
if mode == Bp2Build { if mode == QueryView {
content = `# READ THIS FIRST:
# This file was automatically generated by bp2build for the Bazel migration project.
# Feel free to edit or test it, but do *not* check it into your version control system.
`
content += targets.LoadStatements()
content += "\n\n"
// Get package rule from the handcrafted BUILD file, otherwise emit the default one.
prText := "package(default_visibility = [\"//visibility:public\"])\n"
if pr := targets.packageRule(); pr != nil {
prText = pr.content
}
content += prText
} else if mode == QueryView {
content = soongModuleLoad content = soongModuleLoad
} }
if content != "" { if content != "" {
@@ -161,14 +70,6 @@ func newFile(dir, basename, content string) BazelFile {
const ( const (
bazelRulesSubDir = "build/bazel/queryview_rules" bazelRulesSubDir = "build/bazel/queryview_rules"
// additional files:
// * workspace file
// * base BUILD file
// * rules BUILD file
// * rules providers.bzl file
// * rules soong_module.bzl file
numAdditionalFiles = 5
) )
var ( var (

231
bp2build/metrics.go
View File

@@ -1,231 +0,0 @@
package bp2build
import (
"fmt"
"os"
"path/filepath"
"strings"
"android/soong/android"
"android/soong/shared"
"android/soong/ui/metrics/bp2build_metrics_proto"
"google.golang.org/protobuf/proto"
"github.com/google/blueprint"
)
type moduleInfo struct {
Name string `json:"name"`
Type string `json:"type"`
}
// CodegenMetrics represents information about the Blueprint-to-BUILD
// conversion process.
// Use CreateCodegenMetrics() to get a properly initialized instance
type CodegenMetrics struct {
serialized *bp2build_metrics_proto.Bp2BuildMetrics
// List of modules with unconverted deps
// NOTE: NOT in the .proto
moduleWithUnconvertedDepsMsgs []string
// List of modules with missing deps
// NOTE: NOT in the .proto
moduleWithMissingDepsMsgs []string
// Map of converted modules and paths to call
// NOTE: NOT in the .proto
convertedModulePathMap map[string]string
// Name and type of converted modules
convertedModuleWithType []moduleInfo
}
func CreateCodegenMetrics() CodegenMetrics {
return CodegenMetrics{
serialized: &bp2build_metrics_proto.Bp2BuildMetrics{
RuleClassCount: make(map[string]uint64),
ConvertedModuleTypeCount: make(map[string]uint64),
TotalModuleTypeCount: make(map[string]uint64),
UnconvertedModules: make(map[string]*bp2build_metrics_proto.UnconvertedReason),
},
convertedModulePathMap: make(map[string]string),
}
}
// Serialize returns the protoized version of CodegenMetrics: bp2build_metrics_proto.Bp2BuildMetrics
func (metrics *CodegenMetrics) Serialize() *bp2build_metrics_proto.Bp2BuildMetrics {
return metrics.serialized
}
// Print the codegen metrics to stdout.
func (metrics *CodegenMetrics) Print() {
generatedTargetCount := uint64(0)
for _, ruleClass := range android.SortedKeys(metrics.serialized.RuleClassCount) {
count := metrics.serialized.RuleClassCount[ruleClass]
fmt.Printf("[bp2build] %s: %d targets\n", ruleClass, count)
generatedTargetCount += count
}
fmt.Printf(
`[bp2build] Converted %d Android.bp modules to %d total generated BUILD targets. Included %d handcrafted BUILD targets. There are %d total Android.bp modules.
%d converted modules have unconverted deps:
%s
%d converted modules have missing deps:
%s
`,
metrics.serialized.GeneratedModuleCount,
generatedTargetCount,
metrics.serialized.HandCraftedModuleCount,
metrics.TotalModuleCount(),
len(metrics.moduleWithUnconvertedDepsMsgs),
strings.Join(metrics.moduleWithUnconvertedDepsMsgs, "\n\t"),
len(metrics.moduleWithMissingDepsMsgs),
strings.Join(metrics.moduleWithMissingDepsMsgs, "\n\t"),
)
}
const bp2buildMetricsFilename = "bp2build_metrics.pb"
// fail prints $PWD to stderr, followed by the given printf string and args (vals),
// then the given alert, and then exits with 1 for failure
func fail(err error, alertFmt string, vals ...interface{}) {
cwd, wderr := os.Getwd()
if wderr != nil {
cwd = "FAILED TO GET $PWD: " + wderr.Error()
}
fmt.Fprintf(os.Stderr, "\nIn "+cwd+":\n"+alertFmt+"\n"+err.Error()+"\n", vals...)
os.Exit(1)
}
// Write the bp2build-protoized codegen metrics into the given directory
func (metrics *CodegenMetrics) Write(dir string) {
if _, err := os.Stat(dir); os.IsNotExist(err) {
// The metrics dir doesn't already exist, so create it (and parents)
if err := os.MkdirAll(dir, 0755); err != nil { // rx for all; w for user
fail(err, "Failed to `mkdir -p` %s", dir)
}
} else if err != nil {
fail(err, "Failed to `stat` %s", dir)
}
metricsFile := filepath.Join(dir, bp2buildMetricsFilename)
if err := metrics.dump(metricsFile); err != nil {
fail(err, "Error outputting %s", metricsFile)
}
if _, err := os.Stat(metricsFile); err != nil {
if os.IsNotExist(err) {
fail(err, "MISSING BP2BUILD METRICS OUTPUT: %s", metricsFile)
} else {
fail(err, "FAILED TO `stat` BP2BUILD METRICS OUTPUT: %s", metricsFile)
}
}
}
// ReadCodegenMetrics loads CodegenMetrics from `dir`
// returns a nil pointer if the file doesn't exist
func ReadCodegenMetrics(dir string) *CodegenMetrics {
metricsFile := filepath.Join(dir, bp2buildMetricsFilename)
if _, err := os.Stat(metricsFile); err != nil {
if os.IsNotExist(err) {
return nil
} else {
fail(err, "FAILED TO `stat` BP2BUILD METRICS OUTPUT: %s", metricsFile)
panic("unreachable after fail")
}
}
if buf, err := os.ReadFile(metricsFile); err != nil {
fail(err, "FAILED TO READ BP2BUILD METRICS OUTPUT: %s", metricsFile)
panic("unreachable after fail")
} else {
bp2BuildMetrics := bp2build_metrics_proto.Bp2BuildMetrics{
RuleClassCount: make(map[string]uint64),
ConvertedModuleTypeCount: make(map[string]uint64),
TotalModuleTypeCount: make(map[string]uint64),
}
if err := proto.Unmarshal(buf, &bp2BuildMetrics); err != nil {
fail(err, "FAILED TO PARSE BP2BUILD METRICS OUTPUT: %s", metricsFile)
}
return &CodegenMetrics{
serialized: &bp2BuildMetrics,
convertedModulePathMap: make(map[string]string),
}
}
}
func (metrics *CodegenMetrics) IncrementRuleClassCount(ruleClass string) {
metrics.serialized.RuleClassCount[ruleClass] += 1
}
func (metrics *CodegenMetrics) AddEvent(event *bp2build_metrics_proto.Event) {
metrics.serialized.Events = append(metrics.serialized.Events, event)
}
func (metrics *CodegenMetrics) SetSymlinkCount(n uint64) {
if m := metrics.serialized.WorkspaceSymlinkCount; m != 0 {
fmt.Fprintf(os.Stderr, "unexpected non-zero workspaceSymlinkCount of %d", m)
}
metrics.serialized.WorkspaceSymlinkCount = n
}
func (metrics *CodegenMetrics) SetMkDirCount(n uint64) {
if m := metrics.serialized.WorkspaceMkDirCount; m != 0 {
fmt.Fprintf(os.Stderr, "unexpected non-zero workspaceDirCount of %d", m)
}
metrics.serialized.WorkspaceMkDirCount = n
}
func (metrics *CodegenMetrics) TotalModuleCount() uint64 {
return metrics.serialized.HandCraftedModuleCount +
metrics.serialized.GeneratedModuleCount +
metrics.serialized.UnconvertedModuleCount
}
// Dump serializes the metrics to the given filename
func (metrics *CodegenMetrics) dump(filename string) (err error) {
ser := metrics.Serialize()
return shared.Save(ser, filename)
}
type ConversionType int
const (
Generated ConversionType = iota
Handcrafted
)
func (metrics *CodegenMetrics) AddConvertedModule(m blueprint.Module, moduleType string, dir string) {
//a package module has empty name
if moduleType == "package" {
return
}
// Undo prebuilt_ module name prefix modifications
moduleName := android.RemoveOptionalPrebuiltPrefix(m.Name())
metrics.serialized.ConvertedModules = append(metrics.serialized.ConvertedModules, moduleName)
metrics.convertedModuleWithType = append(metrics.convertedModuleWithType, moduleInfo{
moduleName,
moduleType,
})
metrics.convertedModulePathMap[moduleName] = "//" + dir
metrics.serialized.ConvertedModuleTypeCount[moduleType] += 1
metrics.serialized.TotalModuleTypeCount[moduleType] += 1
metrics.serialized.GeneratedModuleCount += 1
}
func (metrics *CodegenMetrics) AddUnconvertedModule(m blueprint.Module, moduleType string, dir string,
reason android.UnconvertedReason) {
//a package module has empty name
if moduleType == "package" {
return
}
// Undo prebuilt_ module name prefix modifications
moduleName := android.RemoveOptionalPrebuiltPrefix(m.Name())
metrics.serialized.UnconvertedModules[moduleName] = &bp2build_metrics_proto.UnconvertedReason{
Type: bp2build_metrics_proto.UnconvertedReasonType(reason.ReasonType),
Detail: reason.Detail,
}
metrics.serialized.UnconvertedModuleCount += 1
metrics.serialized.TotalModuleTypeCount[moduleType] += 1
if reason.ReasonType == int(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE) {
metrics.serialized.HandCraftedModuleCount += 1
}
}

219
bp2build/performance_test.go
View File

@@ -1,219 +0,0 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
// to run the benchmarks in this file, you must run go test with the -bench.
// The benchmarked portion will run for the specified time (can be set via -benchtime)
// This can mean if you are benchmarking a faster portion of a larger operation, it will take
// longer.
// If you are seeing a small number of iterations for a specific run, the data is less reliable, to
// run for longer, set -benchtime to a larger value.
import (
"fmt"
"math"
"strings"
"testing"
"android/soong/android"
)
const (
performance_test_dir = "."
)
func genCustomModule(i int, convert bool) string {
var conversionString string
if convert {
conversionString = `bazel_module: { bp2build_available: true },`
}
return fmt.Sprintf(`
custom {
name: "arch_paths_%[1]d",
string_list_prop: ["\t", "\n"],
string_prop: "a\t\n\r",
arch_paths: ["outer", ":outer_dep_%[1]d"],
arch: {
x86: {
arch_paths: ["abc", ":x86_dep_%[1]d"],
},
x86_64: {
arch_paths: ["64bit"],
arch_paths_exclude: ["outer"],
},
},
%[2]s
}
custom {
name: "outer_dep_%[1]d",
%[2]s
}
custom {
name: "x86_dep_%[1]d",
%[2]s
}
`, i, conversionString)
}
func genCustomModuleBp(pctConverted float64) string {
modules := 100
bp := make([]string, 0, modules)
toConvert := int(math.Round(float64(modules) * pctConverted))
for i := 0; i < modules; i++ {
bp = append(bp, genCustomModule(i, i < toConvert))
}
return strings.Join(bp, "\n\n")
}
type testConfig struct {
config android.Config
ctx *android.TestContext
codegenCtx *CodegenContext
}
func (tc testConfig) parse() []error {
_, errs := tc.ctx.ParseFileList(performance_test_dir, []string{"Android.bp"})
return errs
}
func (tc testConfig) resolveDependencies() []error {
_, errs := tc.ctx.ResolveDependencies(tc.config)
return errs
}
func (tc testConfig) convert() {
generateBazelTargetsForDir(tc.codegenCtx, performance_test_dir)
}
func setup(builddir string, tcSize float64) testConfig {
config := android.TestConfig(buildDir, nil, genCustomModuleBp(tcSize), nil)
ctx := android.NewTestContext(config)
registerCustomModuleForBp2buildConversion(ctx)
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
return testConfig{
config,
ctx,
codegenCtx,
}
}
var pctToConvert = []float64{0.0, 0.01, 0.05, 0.10, 0.25, 0.5, 0.75, 1.0}
// This is not intended to test performance, but to verify performance infra continues to work
func TestConvertManyModulesFull(t *testing.T) {
for _, tcSize := range pctToConvert {
t.Run(fmt.Sprintf("pctConverted %f", tcSize), func(t *testing.T) {
testConfig := setup(buildDir, tcSize)
errs := testConfig.parse()
if len(errs) > 0 {
t.Fatalf("Unexpected errors: %s", errs)
}
errs = testConfig.resolveDependencies()
if len(errs) > 0 {
t.Fatalf("Unexpected errors: %s", errs)
}
testConfig.convert()
})
}
}
func BenchmarkManyModulesFull(b *testing.B) {
for _, tcSize := range pctToConvert {
b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) {
for n := 0; n < b.N; n++ {
b.StopTimer()
testConfig := setup(buildDir, tcSize)
b.StartTimer()
errs := testConfig.parse()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
errs = testConfig.resolveDependencies()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
testConfig.convert()
b.StopTimer()
}
})
}
}
func BenchmarkManyModulesResolveDependencies(b *testing.B) {
for _, tcSize := range pctToConvert {
b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) {
for n := 0; n < b.N; n++ {
b.StopTimer()
// setup we don't want to measure
testConfig := setup(buildDir, tcSize)
errs := testConfig.parse()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
b.StartTimer()
errs = testConfig.resolveDependencies()
b.StopTimer()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
testConfig.convert()
}
})
}
}
func BenchmarkManyModulesGenerateBazelTargetsForDir(b *testing.B) {
for _, tcSize := range pctToConvert {
b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) {
for n := 0; n < b.N; n++ {
b.StopTimer()
// setup we don't want to measure
testConfig := setup(buildDir, tcSize)
errs := testConfig.parse()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
errs = testConfig.resolveDependencies()
if len(errs) > 0 {
b.Fatalf("Unexpected errors: %s", errs)
}
b.StartTimer()
testConfig.convert()
b.StopTimer()
}
})
}
}

511
bp2build/symlink_forest.go
View File

@@ -1,511 +0,0 @@
// Copyright 2022 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"sync"
"sync/atomic"
"android/soong/shared"
"github.com/google/blueprint/pathtools"
)
// A tree structure that describes what to do at each directory in the created
// symlink tree. Currently, it is used to enumerate which files/directories
// should be excluded from symlinking. Each instance of "node" represents a file
// or a directory. If excluded is true, then that file/directory should be
// excluded from symlinking. Otherwise, the node is not excluded, but one of its
// descendants is (otherwise the node in question would not exist)
type instructionsNode struct {
name string
excluded bool // If false, this is just an intermediate node
children map[string]*instructionsNode
}
type symlinkForestContext struct {
verbose bool
topdir string // $TOPDIR
// State
wg sync.WaitGroup
depCh chan string
mkdirCount atomic.Uint64
symlinkCount atomic.Uint64
}
// Ensures that the node for the given path exists in the tree and returns it.
func ensureNodeExists(root *instructionsNode, path string) *instructionsNode {
if path == "" {
return root
}
if path[len(path)-1] == '/' {
path = path[:len(path)-1] // filepath.Split() leaves a trailing slash
}
dir, base := filepath.Split(path)
// First compute the parent node...
dn := ensureNodeExists(root, dir)
// then create the requested node as its direct child, if needed.
if child, ok := dn.children[base]; ok {
return child
} else {
dn.children[base] = &instructionsNode{base, false, make(map[string]*instructionsNode)}
return dn.children[base]
}
}
// Turns a list of paths to be excluded into a tree
func instructionsFromExcludePathList(paths []string) *instructionsNode {
result := &instructionsNode{"", false, make(map[string]*instructionsNode)}
for _, p := range paths {
ensureNodeExists(result, p).excluded = true
}
return result
}
func mergeBuildFiles(output string, srcBuildFile string, generatedBuildFile string, verbose bool) error {
srcBuildFileContent, err := os.ReadFile(srcBuildFile)
if err != nil {
return err
}
generatedBuildFileContent, err := os.ReadFile(generatedBuildFile)
if err != nil {
return err
}
// There can't be a package() call in both the source and generated BUILD files.
// bp2build will generate a package() call for licensing information, but if
// there's no licensing information, it will still generate a package() call
// that just sets default_visibility=public. If the handcrafted build file
// also has a package() call, we'll allow it to override the bp2build
// generated one if it doesn't have any licensing information. If the bp2build
// one has licensing information and the handcrafted one exists, we'll leave
// them both in for bazel to throw an error.
packageRegex := regexp.MustCompile(`(?m)^package\s*\(`)
packageDefaultVisibilityRegex := regexp.MustCompile(`(?m)^package\s*\(\s*default_visibility\s*=\s*\[\s*"//visibility:public",?\s*]\s*\)`)
if packageRegex.Find(srcBuildFileContent) != nil {
if verbose && packageDefaultVisibilityRegex.Find(generatedBuildFileContent) != nil {
fmt.Fprintf(os.Stderr, "Both '%s' and '%s' have a package() target, removing the first one\n",
generatedBuildFile, srcBuildFile)
}
generatedBuildFileContent = packageDefaultVisibilityRegex.ReplaceAll(generatedBuildFileContent, []byte{})
}
newContents := generatedBuildFileContent
if newContents[len(newContents)-1] != '\n' {
newContents = append(newContents, '\n')
}
newContents = append(newContents, srcBuildFileContent...)
// Say you run bp2build 4 times:
// - The first time there's only an Android.bp file. bp2build will convert it to a build file
// under out/soong/bp2build, then symlink from the forest to that generated file
// - Then you add a handcrafted BUILD file in the same directory. bp2build will merge this with
// the generated one, and write the result to the output file in the forest. But the output
// file was a symlink to out/soong/bp2build from the previous step! So we erroneously update
// the file in out/soong/bp2build instead. So far this doesn't cause any problems...
// - You run a 3rd bp2build with no relevant changes. Everything continues to work.
// - You then add a comment to the handcrafted BUILD file. This causes a merge with the
// generated file again. But since we wrote to the generated file in step 2, the generated
// file has an old copy of the handcrafted file in it! This probably causes duplicate bazel
// targets.
// To solve this, if we see that the output file is a symlink from a previous build, remove it.
stat, err := os.Lstat(output)
if err != nil && !os.IsNotExist(err) {
return err
} else if err == nil {
if stat.Mode()&os.ModeSymlink == os.ModeSymlink {
if verbose {
fmt.Fprintf(os.Stderr, "Removing symlink so that we can replace it with a merged file: %s\n", output)
}
err = os.Remove(output)
if err != nil {
return err
}
}
}
return pathtools.WriteFileIfChanged(output, newContents, 0666)
}
// Calls readdir() and returns it as a map from the basename of the files in dir
// to os.FileInfo.
func readdirToMap(dir string) map[string]os.FileInfo {
entryList, err := ioutil.ReadDir(dir)
result := make(map[string]os.FileInfo)
if err != nil {
if os.IsNotExist(err) {
// It's okay if a directory doesn't exist; it just means that one of the
// trees to be merged contains parts the other doesn't
return result
} else {
fmt.Fprintf(os.Stderr, "Cannot readdir '%s': %s\n", dir, err)
os.Exit(1)
}
}
for _, fi := range entryList {
result[fi.Name()] = fi
}
return result
}
// Creates a symbolic link at dst pointing to src
func symlinkIntoForest(topdir, dst, src string) uint64 {
srcPath := shared.JoinPath(topdir, src)
dstPath := shared.JoinPath(topdir, dst)
// Check whether a symlink already exists.
if dstInfo, err := os.Lstat(dstPath); err != nil {
if !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Failed to lstat '%s': %s", dst, err)
os.Exit(1)
}
} else {
if dstInfo.Mode()&os.ModeSymlink != 0 {
// Assume that the link's target is correct, i.e. no manual tampering.
// E.g. OUT_DIR could have been previously used with a different source tree check-out!
return 0
} else {
if err := os.RemoveAll(dstPath); err != nil {
fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", dst, err)
os.Exit(1)
}
}
}
// Create symlink.
if err := os.Symlink(srcPath, dstPath); err != nil {
fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err)
os.Exit(1)
}
return 1
}
func isDir(path string, fi os.FileInfo) bool {
if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
return fi.IsDir()
}
fi2, statErr := os.Stat(path)
if statErr == nil {
return fi2.IsDir()
}
// Check if this is a dangling symlink. If so, treat it like a file, not a dir.
_, lstatErr := os.Lstat(path)
if lstatErr != nil {
fmt.Fprintf(os.Stderr, "Cannot stat or lstat '%s': %s\n%s\n", path, statErr, lstatErr)
os.Exit(1)
}
return false
}
// Returns the mtime of the soong_build binary to determine whether we should
// force symlink_forest to re-execute
func getSoongBuildMTime() (int64, error) {
binaryPath, err := os.Executable()
if err != nil {
return 0, err
}
info, err := os.Stat(binaryPath)
if err != nil {
return 0, err
}
return info.ModTime().UnixMilli(), nil
}
// cleanSymlinkForest will remove the whole symlink forest directory
func cleanSymlinkForest(topdir, forest string) error {
return os.RemoveAll(shared.JoinPath(topdir, forest))
}
// This returns whether symlink forest should clean and replant symlinks.
// It compares the mtime of this executable with the mtime of the last-run
// soong_build binary. If they differ, then we should clean and replant.
func shouldCleanSymlinkForest(topdir string, forest string, soongBuildMTime int64) (bool, error) {
mtimeFilePath := shared.JoinPath(topdir, forest, "soong_build_mtime")
mtimeFileContents, err := os.ReadFile(mtimeFilePath)
if err != nil {
if os.IsNotExist(err) {
// This is likely the first time this has run with this functionality - clean away!
return true, nil
} else {
return false, err
}
}
return strconv.FormatInt(soongBuildMTime, 10) != string(mtimeFileContents), nil
}
func writeSoongBuildMTimeFile(topdir, forest string, mtime int64) error {
mtimeFilePath := shared.JoinPath(topdir, forest, "soong_build_mtime")
contents := []byte(strconv.FormatInt(mtime, 10))
return os.WriteFile(mtimeFilePath, contents, 0666)
}
// Recursively plants a symlink forest at forestDir. The symlink tree will
// contain every file in buildFilesDir and srcDir excluding the files in
// instructions. Collects every directory encountered during the traversal of
// srcDir .
func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *instructionsNode, forestDir string, buildFilesDir string, srcDir string) {
defer context.wg.Done()
if instructions != nil && instructions.excluded {
// Excluded paths are skipped at the level of the non-excluded parent.
fmt.Fprintf(os.Stderr, "may not specify a root-level exclude directory '%s'", srcDir)
os.Exit(1)
}
// We don't add buildFilesDir here because the bp2build files marker files is
// already a dependency which covers it. If we ever wanted to turn this into
// a generic symlink forest creation tool, we'd need to add it, too.
context.depCh <- srcDir
srcDirMap := readdirToMap(shared.JoinPath(context.topdir, srcDir))
buildFilesMap := readdirToMap(shared.JoinPath(context.topdir, buildFilesDir))
renamingBuildFile := false
if _, ok := srcDirMap["BUILD"]; ok {
if _, ok := srcDirMap["BUILD.bazel"]; !ok {
if _, ok := buildFilesMap["BUILD.bazel"]; ok {
renamingBuildFile = true
srcDirMap["BUILD.bazel"] = srcDirMap["BUILD"]
delete(srcDirMap, "BUILD")
if instructions != nil {
if _, ok := instructions.children["BUILD"]; ok {
instructions.children["BUILD.bazel"] = instructions.children["BUILD"]
delete(instructions.children, "BUILD")
}
}
}
}
}
allEntries := make([]string, 0, len(srcDirMap)+len(buildFilesMap))
for n := range srcDirMap {
allEntries = append(allEntries, n)
}
for n := range buildFilesMap {
if _, ok := srcDirMap[n]; !ok {
allEntries = append(allEntries, n)
}
}
// Tests read the error messages generated, so ensure their order is deterministic
sort.Strings(allEntries)
fullForestPath := shared.JoinPath(context.topdir, forestDir)
createForestDir := false
if fi, err := os.Lstat(fullForestPath); err != nil {
if os.IsNotExist(err) {
createForestDir = true
} else {
fmt.Fprintf(os.Stderr, "Could not read info for '%s': %s\n", forestDir, err)
}
} else if fi.Mode()&os.ModeDir == 0 {
if err := os.RemoveAll(fullForestPath); err != nil {
fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", forestDir, err)
os.Exit(1)
}
createForestDir = true
}
if createForestDir {
if err := os.MkdirAll(fullForestPath, 0777); err != nil {
fmt.Fprintf(os.Stderr, "Could not mkdir '%s': %s\n", forestDir, err)
os.Exit(1)
}
context.mkdirCount.Add(1)
}
// Start with a list of items that already exist in the forest, and remove
// each element as it is processed in allEntries. Any remaining items in
// forestMapForDeletion must be removed. (This handles files which were
// removed since the previous forest generation).
forestMapForDeletion := readdirToMap(shared.JoinPath(context.topdir, forestDir))
for _, f := range allEntries {
if f[0] == '.' {
continue // Ignore dotfiles
}
delete(forestMapForDeletion, f)
// todo add deletionCount metric
// The full paths of children in the input trees and in the output tree
forestChild := shared.JoinPath(forestDir, f)
srcChild := shared.JoinPath(srcDir, f)
if f == "BUILD.bazel" && renamingBuildFile {
srcChild = shared.JoinPath(srcDir, "BUILD")
}
buildFilesChild := shared.JoinPath(buildFilesDir, f)
// Descend in the instruction tree if it exists
var instructionsChild *instructionsNode
if instructions != nil {
instructionsChild = instructions.children[f]
}
srcChildEntry, sExists := srcDirMap[f]
buildFilesChildEntry, bExists := buildFilesMap[f]
if instructionsChild != nil && instructionsChild.excluded {
if bExists {
context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild))
}
continue
}
sDir := sExists && isDir(shared.JoinPath(context.topdir, srcChild), srcChildEntry)
bDir := bExists && isDir(shared.JoinPath(context.topdir, buildFilesChild), buildFilesChildEntry)
if !sExists {
if bDir && instructionsChild != nil {
// Not in the source tree, but we have to exclude something from under
// this subtree, so descend
context.wg.Add(1)
go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
} else {
// Not in the source tree, symlink BUILD file
context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild))
}
} else if !bExists {
if sDir && instructionsChild != nil {
// Not in the build file tree, but we have to exclude something from
// under this subtree, so descend
context.wg.Add(1)
go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
} else {
// Not in the build file tree, symlink source tree, carry on
context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, srcChild))
}
} else if sDir && bDir {
// Both are directories. Descend.
context.wg.Add(1)
go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
} else if !sDir && !bDir {
// Neither is a directory. Merge them.
srcBuildFile := shared.JoinPath(context.topdir, srcChild)
generatedBuildFile := shared.JoinPath(context.topdir, buildFilesChild)
// The Android.bp file that codegen used to produce `buildFilesChild` is
// already a dependency, we can ignore `buildFilesChild`.
context.depCh <- srcChild
if err := mergeBuildFiles(shared.JoinPath(context.topdir, forestChild), srcBuildFile, generatedBuildFile, context.verbose); err != nil {
fmt.Fprintf(os.Stderr, "Error merging %s and %s: %s",
srcBuildFile, generatedBuildFile, err)
os.Exit(1)
}
} else {
// Both exist and one is a file. This is an error.
fmt.Fprintf(os.Stderr,
"Conflict in workspace symlink tree creation: both '%s' and '%s' exist and exactly one is a directory\n",
srcChild, buildFilesChild)
os.Exit(1)
}
}
// Remove all files in the forest that exist in neither the source
// tree nor the build files tree. (This handles files which were removed
// since the previous forest generation).
for f := range forestMapForDeletion {
var instructionsChild *instructionsNode
if instructions != nil {
instructionsChild = instructions.children[f]
}
if instructionsChild != nil && instructionsChild.excluded {
// This directory may be excluded because bazel writes to it under the
// forest root. Thus this path is intentionally left alone.
continue
}
forestChild := shared.JoinPath(context.topdir, forestDir, f)
if err := os.RemoveAll(forestChild); err != nil {
fmt.Fprintf(os.Stderr, "Failed to remove '%s/%s': %s", forestDir, f, err)
os.Exit(1)
}
}
}
// PlantSymlinkForest Creates a symlink forest by merging the directory tree at "buildFiles" and
// "srcDir" while excluding paths listed in "exclude". Returns the set of paths
// under srcDir on which readdir() had to be called to produce the symlink
// forest.
func PlantSymlinkForest(verbose bool, topdir string, forest string, buildFiles string, exclude []string) (deps []string, mkdirCount, symlinkCount uint64) {
context := &symlinkForestContext{
verbose: verbose,
topdir: topdir,
depCh: make(chan string),
mkdirCount: atomic.Uint64{},
symlinkCount: atomic.Uint64{},
}
// Check whether soong_build has been modified since the last run
soongBuildMTime, err := getSoongBuildMTime()
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
shouldClean, err := shouldCleanSymlinkForest(topdir, forest, soongBuildMTime)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
} else if shouldClean {
err = cleanSymlinkForest(topdir, forest)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
instructions := instructionsFromExcludePathList(exclude)
go func() {
context.wg.Add(1)
plantSymlinkForestRecursive(context, instructions, forest, buildFiles, ".")
context.wg.Wait()
close(context.depCh)
}()
for dep := range context.depCh {
deps = append(deps, dep)
}
err = writeSoongBuildMTimeFile(topdir, forest, soongBuildMTime)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
return deps, context.mkdirCount.Load(), context.symlinkCount.Load()
}

785
bp2build/testing.go
View File

@@ -1,785 +0,0 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
/*
For shareable/common bp2build testing functionality and dumping ground for
specific-but-shared functionality among tests in package
*/
import (
"fmt"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
"android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint/proptools"
"android/soong/android"
"android/soong/android/allowlists"
"android/soong/bazel"
)
var (
buildDir string
)
var labelRegex = regexp.MustCompile(`^//([^: ]+):([^ ]+)$`)
var simpleModuleNameRegex = regexp.MustCompile(`^[^: /]+$`)
func checkError(t *testing.T, errs []error, expectedErr error) bool {
t.Helper()
if len(errs) != 1 {
return false
}
if strings.Contains(errs[0].Error(), expectedErr.Error()) {
return true
}
return false
}
func errored(t *testing.T, tc Bp2buildTestCase, errs []error) bool {
t.Helper()
if tc.ExpectedErr != nil {
// Rely on checkErrors, as this test case is expected to have an error.
return false
}
if len(errs) > 0 {
for _, err := range errs {
t.Errorf("%s: %s", tc.Description, err)
}
return true
}
// All good, continue execution.
return false
}
func RunBp2BuildTestCaseSimple(t *testing.T, tc Bp2buildTestCase) {
t.Helper()
RunBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, tc)
}
type Bp2buildTestCase struct {
Description string
ModuleTypeUnderTest string
ModuleTypeUnderTestFactory android.ModuleFactory
// Text to add to the toplevel, root Android.bp file. If Dir is not set, all
// ExpectedBazelTargets are assumed to be generated by this file.
Blueprint string
// ExpectedBazelTargets compares the BazelTargets generated in `Dir` (if not empty).
// Otherwise, it checks the BazelTargets generated by `Blueprint` in the root directory.
ExpectedBazelTargets []string
// ExpectedConvertedModules asserts that modules in this list are labeled as "converted
// by bp2build" in the metrics reported by bp2build.
ExpectedConvertedModules []string
// ExpectedHandcraftedModules asserts that modules in this list are labeled as "handcrafted
// in build files" in the metrics reported by bp2build. Such modules are either explicitly
// defined in a BUILD file (by name), or registered as "otherwise implicitly handled"
// by bp2build (for example, by macros owned by other modules).
ExpectedHandcraftedModules []string
// AlreadyExistingBuildContents, if non-empty, simulates an already-present source BUILD file
// in the directory under test. The BUILD file has the given contents. This BUILD file
// will also be treated as "BUILD file to keep" by the simulated bp2build environment.
AlreadyExistingBuildContents string
// StubbedBuildDefinitions, if non-empty, adds stub definitions to already-present source
// BUILD files for each bazel label given. The BUILD files with these stub definitions
// are added to the BUILD file given in AlreadyExistingBuildContents.
// Labels may be of the form //pkg/to:target_name (which would be defined in pkg/to/BUILD.bazel)
// or `target_name` (which would be defined in ./BUILD.bazel).
StubbedBuildDefinitions []string
Filesystem map[string]string
// Dir sets the directory which will be compared against the targets in ExpectedBazelTargets.
// This should used in conjunction with the Filesystem property to check for targets
// generated from a directory that is not the root.
// If not set, all ExpectedBazelTargets are assumed to be generated by the text in the
// Blueprint property.
Dir string
// An error with a string contained within the string of the expected error
ExpectedErr error
UnconvertedDepsMode unconvertedDepsMode
// For every directory listed here, the BUILD file for that directory will
// be merged with the generated BUILD file. This allows custom BUILD targets
// to be used in tests, or use BUILD files to draw package boundaries.
KeepBuildFileForDirs []string
// An extra FixturePreparer to use when running the test. If you need multiple extra
// FixturePreparers, use android.GroupFixturePreparers()
ExtraFixturePreparer android.FixturePreparer
// If bp2build_product_config.go should run as part of the test.
RunBp2buildProductConfig bool
}
func RunBp2BuildTestCase(t *testing.T, registerModuleTypes func(ctx android.RegistrationContext), tc Bp2buildTestCase) {
t.Helper()
preparers := []android.FixturePreparer{
android.FixtureRegisterWithContext(registerModuleTypes),
}
if tc.ExtraFixturePreparer != nil {
preparers = append(preparers, tc.ExtraFixturePreparer)
}
preparers = append(preparers, android.FixtureSetTestRunner(&bazelTestRunner{generateProductConfigTargets: tc.RunBp2buildProductConfig}))
bp2buildSetup := android.GroupFixturePreparers(
preparers...,
)
runBp2BuildTestCaseWithSetup(t, bp2buildSetup, tc)
}
func runBp2BuildTestCaseWithSetup(t *testing.T, extraPreparer android.FixturePreparer, tc Bp2buildTestCase) {
t.Helper()
if tc.Filesystem == nil {
tc.Filesystem = map[string]string{}
}
checkDir := "."
if tc.Dir != "" {
checkDir = tc.Dir
}
keepExistingBuildDirs := tc.KeepBuildFileForDirs
buildFilesToParse := []string{}
if len(tc.StubbedBuildDefinitions) > 0 {
for _, buildDef := range tc.StubbedBuildDefinitions {
globalLabelMatch := labelRegex.FindStringSubmatch(buildDef)
var dir, targetName string
if len(globalLabelMatch) > 0 {
dir = globalLabelMatch[1]
targetName = globalLabelMatch[2]
} else {
if !simpleModuleNameRegex.MatchString(buildDef) {
t.Errorf("Stubbed build definition '%s' must be either a simple module name or of global target syntax (//foo/bar:baz).", buildDef)
return
}
dir = "."
targetName = buildDef
}
buildFilePath := filepath.Join(dir, "BUILD")
tc.Filesystem[buildFilePath] +=
MakeBazelTarget(
"bp2build_test_stub",
targetName,
AttrNameToString{})
keepExistingBuildDirs = append(keepExistingBuildDirs, dir)
buildFilesToParse = append(buildFilesToParse, buildFilePath)
}
}
if len(tc.AlreadyExistingBuildContents) > 0 {
buildFilePath := filepath.Join(checkDir, "BUILD")
tc.Filesystem[buildFilePath] += tc.AlreadyExistingBuildContents
keepExistingBuildDirs = append(keepExistingBuildDirs, checkDir)
buildFilesToParse = append(buildFilesToParse, buildFilePath)
}
filesystem := make(map[string][]byte)
for f, content := range tc.Filesystem {
filesystem[f] = []byte(content)
}
preparers := []android.FixturePreparer{
extraPreparer,
android.FixtureMergeMockFs(filesystem),
android.FixtureWithRootAndroidBp(tc.Blueprint),
android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) {
ctx.RegisterModuleType(tc.ModuleTypeUnderTest, tc.ModuleTypeUnderTestFactory)
}),
android.FixtureModifyContextWithMockFs(func(ctx *android.TestContext) {
// A default configuration for tests to not have to specify bp2build_available on top level
// targets.
bp2buildConfig := android.NewBp2BuildAllowlist().SetDefaultConfig(
allowlists.Bp2BuildConfig{
android.Bp2BuildTopLevel: allowlists.Bp2BuildDefaultTrueRecursively,
},
)
for _, f := range keepExistingBuildDirs {
bp2buildConfig.SetKeepExistingBuildFile(map[string]bool{
f: /*recursive=*/ false,
})
}
ctx.RegisterBp2BuildConfig(bp2buildConfig)
// This setting is added to bp2build invocations. It prevents bp2build
// from cloning modules to their original state after mutators run. This
// would lose some data intentionally set by these mutators.
ctx.SkipCloneModulesAfterMutators = true
err := ctx.RegisterExistingBazelTargets(".", buildFilesToParse)
if err != nil {
t.Errorf("error parsing build files in test setup: %s", err)
}
}),
android.FixtureModifyEnv(func(env map[string]string) {
if tc.UnconvertedDepsMode == errorModulesUnconvertedDeps {
env["BP2BUILD_ERROR_UNCONVERTED"] = "true"
}
}),
}
preparer := android.GroupFixturePreparers(preparers...)
if tc.ExpectedErr != nil {
pattern := "\\Q" + tc.ExpectedErr.Error() + "\\E"
preparer = preparer.ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern(pattern))
}
result := preparer.RunTestWithCustomResult(t).(*BazelTestResult)
if len(result.Errs) > 0 {
return
}
expectedTargets := map[string][]string{
checkDir: tc.ExpectedBazelTargets,
}
result.CompareAllBazelTargets(t, tc, expectedTargets, true)
}
// bazelTestRunner customizes the test fixture mechanism to run tests of the bp2build build mode.
type bazelTestRunner struct {
generateProductConfigTargets bool
}
func (b *bazelTestRunner) FinalPreparer(result *android.TestResult) android.CustomTestResult {
ctx := result.TestContext
ctx.RegisterForBazelConversion()
return &BazelTestResult{TestResult: result}
}
func (b *bazelTestRunner) PostParseProcessor(result android.CustomTestResult) {
bazelResult := result.(*BazelTestResult)
ctx := bazelResult.TestContext
config := bazelResult.Config
_, errs := ctx.ResolveDependencies(config)
if bazelResult.CollateErrs(errs) {
return
}
codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "")
res, errs := GenerateBazelTargets(codegenCtx, false)
if bazelResult.CollateErrs(errs) {
return
}
if b.generateProductConfigTargets {
productConfig, err := createProductConfigFiles(codegenCtx, res.moduleNameToPartition, res.metrics.convertedModulePathMap)
if err != nil {
bazelResult.CollateErrs([]error{err})
return
}
for k, v := range productConfig.bp2buildTargets {
res.buildFileToTargets[k] = append(res.buildFileToTargets[k], v...)
}
}
// Store additional data for access by tests.
bazelResult.conversionResults = res
}
// BazelTestResult is a wrapper around android.TestResult to provide type safe access to the bazel
// specific data stored by the bazelTestRunner.
type BazelTestResult struct {
*android.TestResult
// The result returned by the GenerateBazelTargets function.
conversionResults
}
// CompareAllBazelTargets compares the BazelTargets produced by the test for all the directories
// with the supplied set of expected targets.
//
// If ignoreUnexpected=false then this enforces an exact match where every BazelTarget produced must
// have a corresponding expected BazelTarget.
//
// If ignoreUnexpected=true then it will ignore directories for which there are no expected targets.
func (b BazelTestResult) CompareAllBazelTargets(t *testing.T, tc Bp2buildTestCase, expectedTargets map[string][]string, ignoreUnexpected bool) {
t.Helper()
actualTargets := b.buildFileToTargets
// Generate the sorted set of directories to check.
dirsToCheck := android.SortedKeys(expectedTargets)
if !ignoreUnexpected {
// This needs to perform an exact match so add the directories in which targets were
// produced to the list of directories to check.
dirsToCheck = append(dirsToCheck, android.SortedKeys(actualTargets)...)
dirsToCheck = android.SortedUniqueStrings(dirsToCheck)
}
for _, dir := range dirsToCheck {
expected := expectedTargets[dir]
actual := actualTargets[dir]
if expected == nil {
if actual != nil {
t.Errorf("did not expect any bazel modules in %q but found %d", dir, len(actual))
}
} else if actual == nil {
expectedCount := len(expected)
if expectedCount > 0 {
t.Errorf("expected %d bazel modules in %q but did not find any", expectedCount, dir)
}
} else {
b.CompareBazelTargets(t, tc.Description, expected, actual)
}
}
for _, module := range tc.ExpectedConvertedModules {
if _, found := b.metrics.convertedModulePathMap[module]; !found {
t.Errorf("expected %s to be generated by bp2build, but was not. Map of converted modules: %s", module, b.metrics.convertedModulePathMap)
}
}
for _, module := range tc.ExpectedHandcraftedModules {
if reason, found := b.metrics.serialized.UnconvertedModules[module]; !found {
t.Errorf("expected %s to be marked 'unconverted' by bp2build, but was not found. Full list: %s",
module, b.metrics.serialized.UnconvertedModules)
} else {
if reason.Type != bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE {
t.Errorf("expected %s to be marked 'handcrafted' by bp2build, but was disabled for another reason: %s", module, reason)
}
}
}
}
func (b BazelTestResult) CompareBazelTargets(t *testing.T, description string, expectedContents []string, actualTargets BazelTargets) {
t.Helper()
if actualCount, expectedCount := len(actualTargets), len(expectedContents); actualCount != expectedCount {
t.Errorf("%s: Expected %d bazel target (%s), got %d (%s)",
description, expectedCount, expectedContents, actualCount, actualTargets)
} else {
sort.SliceStable(actualTargets, func(i, j int) bool {
return actualTargets[i].name < actualTargets[j].name
})
sort.SliceStable(expectedContents, func(i, j int) bool {
return getTargetName(expectedContents[i]) < getTargetName(expectedContents[j])
})
for i, actualTarget := range actualTargets {
if w, g := expectedContents[i], actualTarget.content; w != g {
t.Errorf(
"%s[%d]: Expected generated Bazel target to be `%s`, got `%s`",
description, i, w, g)
}
}
}
}
type nestedProps struct {
Nested_prop *string
}
type EmbeddedProps struct {
Embedded_prop *string
}
type OtherEmbeddedProps struct {
Other_embedded_prop *string
}
type customProps struct {
EmbeddedProps
*OtherEmbeddedProps
Bool_prop bool
Bool_ptr_prop *bool
// Ensure that properties tagged `blueprint:mutated` are omitted
Int_prop int `blueprint:"mutated"`
Int64_ptr_prop *int64
String_prop string
String_literal_prop *string `android:"arch_variant"`
String_ptr_prop *string
String_list_prop []string
Nested_props nestedProps
Nested_props_ptr *nestedProps
Arch_paths []string `android:"path,arch_variant"`
Arch_paths_exclude []string `android:"path,arch_variant"`
// Prop used to indicate this conversion should be 1 module -> multiple targets
One_to_many_prop *bool
// Prop used to simulate an unsupported property in bp2build conversion. If this
// is true, this module should be treated as "unconvertible" via bp2build.
Does_not_convert_to_bazel *bool
Api *string // File describing the APIs of this module
Test_config_setting *bool // Used to test generation of config_setting targets
Dir *string // Dir in which the Bazel Target will be created
}
type customModule struct {
android.ModuleBase
android.BazelModuleBase
props customProps
}
// OutputFiles is needed because some instances of this module use dist with a
// tag property which requires the module implements OutputFileProducer.
func (m *customModule) OutputFiles(tag string) (android.Paths, error) {
return android.PathsForTesting("path" + tag), nil
}
func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// nothing for now.
}
func customModuleFactoryBase() android.Module {
module := &customModule{}
module.AddProperties(&module.props)
android.InitBazelModule(module)
return module
}
func customModuleFactoryHostAndDevice() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibBoth)
return m
}
func customModuleFactoryDeviceSupported() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.DeviceSupported, android.MultilibBoth)
return m
}
func customModuleFactoryHostSupported() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.HostSupported, android.MultilibBoth)
return m
}
func customModuleFactoryHostAndDeviceDefault() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.HostAndDeviceDefault, android.MultilibBoth)
return m
}
func customModuleFactoryNeitherHostNorDeviceSupported() android.Module {
m := customModuleFactoryBase()
android.InitAndroidArchModule(m, android.NeitherHostNorDeviceSupported, android.MultilibBoth)
return m
}
type testProps struct {
Test_prop struct {
Test_string_prop string
}
}
type customTestModule struct {
android.ModuleBase
props customProps
test_props testProps
}
func (m *customTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// nothing for now.
}
func customTestModuleFactoryBase() android.Module {
m := &customTestModule{}
m.AddProperties(&m.props)
m.AddProperties(&m.test_props)
return m
}
func customTestModuleFactory() android.Module {
m := customTestModuleFactoryBase()
android.InitAndroidModule(m)
return m
}
type customDefaultsModule struct {
android.ModuleBase
android.DefaultsModuleBase
}
func customDefaultsModuleFactoryBase() android.DefaultsModule {
module := &customDefaultsModule{}
module.AddProperties(&customProps{})
return module
}
func customDefaultsModuleFactoryBasic() android.Module {
return customDefaultsModuleFactoryBase()
}
func customDefaultsModuleFactory() android.Module {
m := customDefaultsModuleFactoryBase()
android.InitDefaultsModule(m)
return m
}
type EmbeddedAttr struct {
Embedded_attr *string
}
type OtherEmbeddedAttr struct {
Other_embedded_attr *string
}
type customBazelModuleAttributes struct {
EmbeddedAttr
*OtherEmbeddedAttr
String_literal_prop bazel.StringAttribute
String_ptr_prop *string
String_list_prop []string
Arch_paths bazel.LabelListAttribute
Api bazel.LabelAttribute
}
func (m *customModule) dir() *string {
return m.props.Dir
}
func (m *customModule) ConvertWithBp2build(ctx android.Bp2buildMutatorContext) {
if p := m.props.Does_not_convert_to_bazel; p != nil && *p {
ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_PROPERTY_UNSUPPORTED, "")
return
}
if p := m.props.One_to_many_prop; p != nil && *p {
customBp2buildOneToMany(ctx, m)
return
}
paths := bazel.LabelListAttribute{}
strAttr := bazel.StringAttribute{}
for axis, configToProps := range m.GetArchVariantProperties(ctx, &customProps{}) {
for config, props := range configToProps {
if custProps, ok := props.(*customProps); ok {
if custProps.Arch_paths != nil {
paths.SetSelectValue(axis, config, android.BazelLabelForModuleSrcExcludes(ctx, custProps.Arch_paths, custProps.Arch_paths_exclude))
}
if custProps.String_literal_prop != nil {
strAttr.SetSelectValue(axis, config, custProps.String_literal_prop)
}
}
}
}
productVariableProps, errs := android.ProductVariableProperties(ctx, ctx.Module())
for _, err := range errs {
ctx.ModuleErrorf("ProductVariableProperties error: %s", err)
}
if props, ok := productVariableProps["String_literal_prop"]; ok {
for c, p := range props {
if val, ok := p.(*string); ok {
strAttr.SetSelectValue(c.ConfigurationAxis(), c.SelectKey(), val)
}
}
}
paths.ResolveExcludes()
attrs := &customBazelModuleAttributes{
String_literal_prop: strAttr,
String_ptr_prop: m.props.String_ptr_prop,
String_list_prop: m.props.String_list_prop,
Arch_paths: paths,
}
attrs.Embedded_attr = m.props.Embedded_prop
if m.props.OtherEmbeddedProps != nil {
attrs.OtherEmbeddedAttr = &OtherEmbeddedAttr{Other_embedded_attr: m.props.OtherEmbeddedProps.Other_embedded_prop}
}
props := bazel.BazelTargetModuleProperties{
Rule_class: "custom",
}
ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name(), Dir: m.dir()}, attrs)
if proptools.Bool(m.props.Test_config_setting) {
m.createConfigSetting(ctx)
}
}
func (m *customModule) createConfigSetting(ctx android.Bp2buildMutatorContext) {
csa := bazel.ConfigSettingAttributes{
Flag_values: bazel.StringMapAttribute{
"//build/bazel/rules/my_string_setting": m.Name(),
},
}
ca := android.CommonAttributes{
Name: m.Name() + "_config_setting",
}
ctx.CreateBazelConfigSetting(
csa,
ca,
ctx.ModuleDir(),
)
}
// A bp2build mutator that uses load statements and creates a 1:M mapping from
// module to target.
func customBp2buildOneToMany(ctx android.Bp2buildMutatorContext, m *customModule) {
baseName := m.Name()
attrs := &customBazelModuleAttributes{}
myLibraryProps := bazel.BazelTargetModuleProperties{
Rule_class: "my_library",
Bzl_load_location: "//build/bazel/rules:rules.bzl",
}
ctx.CreateBazelTargetModule(myLibraryProps, android.CommonAttributes{Name: baseName}, attrs)
protoLibraryProps := bazel.BazelTargetModuleProperties{
Rule_class: "proto_library",
Bzl_load_location: "//build/bazel/rules:proto.bzl",
}
ctx.CreateBazelTargetModule(protoLibraryProps, android.CommonAttributes{Name: baseName + "_proto_library_deps"}, attrs)
myProtoLibraryProps := bazel.BazelTargetModuleProperties{
Rule_class: "my_proto_library",
Bzl_load_location: "//build/bazel/rules:proto.bzl",
}
ctx.CreateBazelTargetModule(myProtoLibraryProps, android.CommonAttributes{Name: baseName + "_my_proto_library_deps"}, attrs)
}
// Helper method for tests to easily access the targets in a dir.
func generateBazelTargetsForDir(codegenCtx *CodegenContext, dir string) (BazelTargets, []error) {
// TODO: Set generateFilegroups to true and/or remove the generateFilegroups argument completely
res, err := GenerateBazelTargets(codegenCtx, false)
if err != nil {
return BazelTargets{}, err
}
return res.buildFileToTargets[dir], err
}
func registerCustomModuleForBp2buildConversion(ctx *android.TestContext) {
ctx.RegisterModuleType("custom", customModuleFactoryHostAndDevice)
ctx.RegisterForBazelConversion()
}
func simpleModule(typ, name string) string {
return fmt.Sprintf(`
%s {
name: "%s",
}`, typ, name)
}
type AttrNameToString map[string]string
func (a AttrNameToString) clone() AttrNameToString {
newAttrs := make(AttrNameToString, len(a))
for k, v := range a {
newAttrs[k] = v
}
return newAttrs
}
// makeBazelTargetNoRestrictions returns bazel target build file definition that can be host or
// device specific, or independent of host/device.
func makeBazelTargetHostOrDevice(typ, name string, attrs AttrNameToString, hod android.HostOrDeviceSupported) string {
if _, ok := attrs["target_compatible_with"]; !ok {
switch hod {
case android.HostSupported:
attrs["target_compatible_with"] = `select({
"//build/bazel_common_rules/platforms/os:android": ["@platforms//:incompatible"],
"//conditions:default": [],
})`
case android.DeviceSupported:
attrs["target_compatible_with"] = `["//build/bazel_common_rules/platforms/os:android"]`
}
}
attrStrings := make([]string, 0, len(attrs)+1)
if name != "" {
attrStrings = append(attrStrings, fmt.Sprintf(` name = "%s",`, name))
}
for _, k := range android.SortedKeys(attrs) {
attrStrings = append(attrStrings, fmt.Sprintf(" %s = %s,", k, attrs[k]))
}
return fmt.Sprintf(`%s(
%s
)`, typ, strings.Join(attrStrings, "\n"))
}
// MakeBazelTargetNoRestrictions returns bazel target build file definition that does not add a
// target_compatible_with. This is useful for module types like filegroup and genrule that arch not
// arch variant
func MakeBazelTargetNoRestrictions(typ, name string, attrs AttrNameToString) string {
return makeBazelTargetHostOrDevice(typ, name, attrs, android.HostAndDeviceDefault)
}
// makeBazelTargetNoRestrictions returns bazel target build file definition that is device specific
// as this is the most common default in Soong.
func MakeBazelTarget(typ, name string, attrs AttrNameToString) string {
return makeBazelTargetHostOrDevice(typ, name, attrs, android.DeviceSupported)
}
type ExpectedRuleTarget struct {
Rule string
Name string
Attrs AttrNameToString
Hod android.HostOrDeviceSupported
}
func (ebr ExpectedRuleTarget) String() string {
return makeBazelTargetHostOrDevice(ebr.Rule, ebr.Name, ebr.Attrs, ebr.Hod)
}
func makeCcStubSuiteTargets(name string, attrs AttrNameToString) string {
if _, hasStubs := attrs["stubs_symbol_file"]; !hasStubs {
return ""
}
STUB_SUITE_ATTRS := map[string]string{
"api_surface": "api_surface",
"stubs_symbol_file": "symbol_file",
"stubs_versions": "versions",
"soname": "soname",
"source_library_label": "source_library_label",
}
stubSuiteAttrs := AttrNameToString{}
for key, _ := range attrs {
if _, stubSuiteAttr := STUB_SUITE_ATTRS[key]; stubSuiteAttr {
stubSuiteAttrs[STUB_SUITE_ATTRS[key]] = attrs[key]
} else {
panic(fmt.Sprintf("unused cc_stub_suite attr %q\n", key))
}
}
return MakeBazelTarget("cc_stub_suite", name+"_stub_libs", stubSuiteAttrs)
}
func MakeNeverlinkDuplicateTarget(moduleType string, name string) string {
return MakeNeverlinkDuplicateTargetWithAttrs(moduleType, name, AttrNameToString{
"sdk_version": `"current"`, // use as default
})
}
func MakeNeverlinkDuplicateTargetWithAttrs(moduleType string, name string, extraAttrs AttrNameToString) string {
attrs := extraAttrs
attrs["neverlink"] = `True`
attrs["exports"] = `[":` + name + `"]`
return MakeBazelTarget(moduleType, name+"-neverlink", attrs)
}
func getTargetName(targetContent string) string {
data := strings.Split(targetContent, "name = \"")
if len(data) < 2 {
return ""
} else {
endIndex := strings.Index(data[1], "\"")
return data[1][:endIndex]
}
}

344
cmd/soong_build/main.go
View File

@@ -28,8 +28,6 @@ import (
"android/soong/android/allowlists" "android/soong/android/allowlists"
"android/soong/bp2build" "android/soong/bp2build"
"android/soong/shared" "android/soong/shared"
"android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint" "github.com/google/blueprint"
"github.com/google/blueprint/bootstrap" "github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/deptools" "github.com/google/blueprint/deptools"
@@ -74,16 +72,10 @@ func init() {
flag.StringVar(&cmdlineArgs.ModuleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules") flag.StringVar(&cmdlineArgs.ModuleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules")
flag.StringVar(&cmdlineArgs.DocFile, "soong_docs", "", "build documentation file to output") flag.StringVar(&cmdlineArgs.DocFile, "soong_docs", "", "build documentation file to output")
flag.StringVar(&cmdlineArgs.BazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top") flag.StringVar(&cmdlineArgs.BazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
flag.StringVar(&cmdlineArgs.Bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
flag.StringVar(&cmdlineArgs.SymlinkForestMarker, "symlink_forest_marker", "", "If set, create the bp2build symlink forest, touch the specified marker file, then exit")
flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output") flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output")
flag.StringVar(&cmdlineArgs.SoongVariables, "soong_variables", "soong.variables", "the file contains all build variables") flag.StringVar(&cmdlineArgs.SoongVariables, "soong_variables", "soong.variables", "the file contains all build variables")
flag.StringVar(&cmdlineArgs.BazelForceEnabledModules, "bazel-force-enabled-modules", "", "additional modules to build with Bazel. Comma-delimited")
flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file") flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file")
flag.BoolVar(&cmdlineArgs.MultitreeBuild, "multitree-build", false, "this is a multitree build") flag.BoolVar(&cmdlineArgs.MultitreeBuild, "multitree-build", false, "this is a multitree build")
flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules")
flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses")
flag.BoolVar(&cmdlineArgs.BuildFromSourceStub, "build-from-source-stub", false, "build Java stubs from source files instead of API text files") flag.BoolVar(&cmdlineArgs.BuildFromSourceStub, "build-from-source-stub", false, "build Java stubs from source files instead of API text files")
flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built") flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built")
// Flags that probably shouldn't be flags of soong_build, but we haven't found // Flags that probably shouldn't be flags of soong_build, but we haven't found
@@ -110,40 +102,6 @@ func newContext(configuration android.Config) *android.Context {
return ctx return ctx
} }
// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a
// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata
// for modules that should be handled by Bazel.
func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string {
ctx.EventHandler.Begin("mixed_build")
defer ctx.EventHandler.End("mixed_build")
bazelHook := func() error {
err := ctx.Config().BazelContext.QueueBazelSandwichCqueryRequests(ctx.Config())
if err != nil {
return err
}
return ctx.Config().BazelContext.InvokeBazel(ctx.Config(), ctx)
}
ctx.SetBeforePrepareBuildActionsHook(bazelHook)
ninjaDeps, err := bootstrap.RunBlueprint(cmdlineArgs.Args, bootstrap.DoEverything, ctx.Context, ctx.Config())
maybeQuit(err, "")
ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
bazelPaths, err := readFileLines(ctx.Config().Getenv("BAZEL_DEPS_FILE"))
if err != nil {
panic("Bazel deps file not found: " + err.Error())
}
ninjaDeps = append(ninjaDeps, bazelPaths...)
ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
if needToWriteNinjaHint(ctx) {
writeNinjaHint(ctx)
}
return cmdlineArgs.OutFile
}
func needToWriteNinjaHint(ctx *android.Context) bool { func needToWriteNinjaHint(ctx *android.Context) bool {
switch ctx.Config().GetenvWithDefault("SOONG_GENERATES_NINJA_HINT", "") { switch ctx.Config().GetenvWithDefault("SOONG_GENERATES_NINJA_HINT", "") {
case "always": case "always":
@@ -228,67 +186,6 @@ func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandl
maybeQuit(err, "error writing soong_build metrics %s", metricsFile) maybeQuit(err, "error writing soong_build metrics %s", metricsFile)
} }
// Errors out if any modules expected to be mixed_built were not, unless
// the modules did not exist.
func checkForAllowlistIntegrityError(configuration android.Config, isStagingMode bool) error {
modules := findMisconfiguredModules(configuration, isStagingMode)
if len(modules) == 0 {
return nil
}
return fmt.Errorf("Error: expected the following modules to be mixed_built: %s", modules)
}
// Returns true if the given module has all of the following true:
// 1. Is allowlisted to be built with Bazel.
// 2. Has a variant which is *not* built with Bazel.
// 3. Has no variant which is built with Bazel.
//
// This indicates the allowlisting of this variant had no effect.
// TODO(b/280457637): Return true for nonexistent modules.
func isAllowlistMisconfiguredForModule(module string, mixedBuildsEnabled map[string]struct{}, mixedBuildsDisabled map[string]struct{}) bool {
_, enabled := mixedBuildsEnabled[module]
if enabled {
return false
}
_, disabled := mixedBuildsDisabled[module]
return disabled
}
// Returns the list of modules that should have been mixed_built (per the
// allowlists and cmdline flags) but were not.
// Note: nonexistent modules are excluded from the list. See b/280457637
func findMisconfiguredModules(configuration android.Config, isStagingMode bool) []string {
retval := []string{}
forceEnabledModules := configuration.BazelModulesForceEnabledByFlag()
mixedBuildsEnabled := configuration.GetMixedBuildsEnabledModules()
mixedBuildsDisabled := configuration.GetMixedBuildsDisabledModules()
for _, module := range allowlists.ProdMixedBuildsEnabledList {
if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) {
retval = append(retval, module)
}
}
if isStagingMode {
for _, module := range allowlists.StagingMixedBuildsEnabledList {
if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) {
retval = append(retval, module)
}
}
}
for module, _ := range forceEnabledModules {
if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) {
retval = append(retval, module)
}
}
return retval
}
func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) { func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) {
graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile)) graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile))
maybeQuit(graphErr, "graph err") maybeQuit(graphErr, "graph err")
@@ -424,37 +321,9 @@ func main() {
ctx := newContext(configuration) ctx := newContext(configuration)
android.StartBackgroundMetrics(configuration) android.StartBackgroundMetrics(configuration)
var finalOutputFile string ctx.Register()
finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps)
// Run Soong for a specific activity, like bp2build, queryview writeMetrics(configuration, ctx.EventHandler, metricsDir)
// or the actual Soong build for the build.ninja file.
switch configuration.BuildMode {
case android.SymlinkForest:
finalOutputFile = runSymlinkForestCreation(ctx, extraNinjaDeps, metricsDir)
case android.Bp2build:
// Run the alternate pipeline of bp2build mutators and singleton to convert
// Blueprint to BUILD files before everything else.
finalOutputFile = runBp2Build(ctx, extraNinjaDeps, metricsDir)
default:
ctx.Register()
isMixedBuildsEnabled := configuration.IsMixedBuildsEnabled()
if isMixedBuildsEnabled {
finalOutputFile = runMixedModeBuild(ctx, extraNinjaDeps)
if cmdlineArgs.EnsureAllowlistIntegrity {
if err := checkForAllowlistIntegrityError(configuration, cmdlineArgs.BazelModeStaging); err != nil {
maybeQuit(err, "")
}
}
} else {
finalOutputFile = runSoongOnlyBuild(ctx, extraNinjaDeps)
}
writeMetrics(configuration, ctx.EventHandler, metricsDir)
}
// Register this environment variablesas being an implicit dependencies of
// soong_build. Changes to this environment variable will result in
// retriggering soong_build.
configuration.Getenv("USE_BAZEL_VERSION")
writeUsedEnvironmentFile(configuration) writeUsedEnvironmentFile(configuration)
@@ -497,213 +366,6 @@ func touch(path string) {
maybeQuit(err, "error touching '%s'", path) maybeQuit(err, "error touching '%s'", path)
} }
// Read the bazel.list file that the Soong Finder already dumped earlier (hopefully)
// It contains the locations of BUILD files, BUILD.bazel files, etc. in the source dir
func getExistingBazelRelatedFiles(topDir string) ([]string, error) {
bazelFinderFile := filepath.Join(filepath.Dir(cmdlineArgs.ModuleListFile), "bazel.list")
if !filepath.IsAbs(bazelFinderFile) {
// Assume this was a relative path under topDir
bazelFinderFile = filepath.Join(topDir, bazelFinderFile)
}
return readFileLines(bazelFinderFile)
}
func bazelArtifacts() []string {
return []string{
"bazel-bin",
"bazel-genfiles",
"bazel-out",
"bazel-testlogs",
"bazel-workspace",
"bazel-" + filepath.Base(topDir),
}
}
// This could in theory easily be separated into a binary that generically
// merges two directories into a symlink tree. The main obstacle is that this
// function currently depends on both Bazel-specific knowledge (the existence
// of bazel-* symlinks) and configuration (the set of BUILD.bazel files that
// should and should not be kept)
//
// Ideally, bp2build would write a file that contains instructions to the
// symlink tree creation binary. Then the latter would not need to depend on
// the very heavy-weight machinery of soong_build .
func runSymlinkForestCreation(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string {
var ninjaDeps []string
var mkdirCount, symlinkCount uint64
ctx.EventHandler.Do("symlink_forest", func() {
ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
verbose := ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE")
// PlantSymlinkForest() returns all the directories that were readdir()'ed.
// Such a directory SHOULD be added to `ninjaDeps` so that a child directory
// or file created/deleted under it would trigger an update of the symlink forest.
generatedRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "bp2build")
workspaceRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "workspace")
var symlinkForestDeps []string
ctx.EventHandler.Do("plant", func() {
symlinkForestDeps, mkdirCount, symlinkCount = bp2build.PlantSymlinkForest(
verbose, topDir, workspaceRoot, generatedRoot, excludedFromSymlinkForest(ctx, verbose))
})
ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
})
writeDepFile(cmdlineArgs.SymlinkForestMarker, ctx.EventHandler, ninjaDeps)
touch(shared.JoinPath(topDir, cmdlineArgs.SymlinkForestMarker))
codegenMetrics := bp2build.ReadCodegenMetrics(metricsDir)
if codegenMetrics == nil {
m := bp2build.CreateCodegenMetrics()
codegenMetrics = &m
} else {
//TODO (usta) we cannot determine if we loaded a stale file, i.e. from an unrelated prior
//invocation of codegen. We should simply use a separate .pb file
}
codegenMetrics.SetSymlinkCount(symlinkCount)
codegenMetrics.SetMkDirCount(mkdirCount)
writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
return cmdlineArgs.SymlinkForestMarker
}
func excludedFromSymlinkForest(ctx *android.Context, verbose bool) []string {
excluded := bazelArtifacts()
if cmdlineArgs.OutDir[0] != '/' {
excluded = append(excluded, cmdlineArgs.OutDir)
}
// Find BUILD files in the srcDir which are not in the allowlist
// (android.Bp2BuildConversionAllowlist#ShouldKeepExistingBuildFileForDir)
// and return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
existingBazelFiles, err := getExistingBazelRelatedFiles(topDir)
maybeQuit(err, "Error determining existing Bazel-related files")
for _, path := range existingBazelFiles {
fullPath := shared.JoinPath(topDir, path)
fileInfo, err2 := os.Stat(fullPath)
if err2 != nil {
// Warn about error, but continue trying to check files
fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fullPath, err2)
continue
}
// Exclude only files named 'BUILD' or 'BUILD.bazel' and unless forcibly kept
if fileInfo.IsDir() ||
(fileInfo.Name() != "BUILD" && fileInfo.Name() != "BUILD.bazel") ||
ctx.Config().Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(filepath.Dir(path)) {
// Don't ignore this existing build file
continue
}
if verbose {
fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", path)
}
excluded = append(excluded, path)
}
// Temporarily exclude stuff to make `bazel build //external/...` (and `bazel build //frameworks/...`) work
excluded = append(excluded,
// FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite
// symlink expansion error for Bazel
"external/autotest/venv/autotest_lib",
"external/autotest/autotest_lib",
"external/autotest/client/autotest_lib/client",
// FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison
// It contains several symlinks back to real source dirs, and those source dirs contain
// BUILD files we want to ignore
"external/google-fruit/extras/bazel_root/third_party/fruit",
// FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue
"frameworks/compile/slang",
)
return excluded
}
// Run Soong in the bp2build mode. This creates a standalone context that registers
// an alternate pipeline of mutators and singletons specifically for generating
// Bazel BUILD files instead of Ninja files.
func runBp2Build(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string {
var codegenMetrics *bp2build.CodegenMetrics
ctx.EventHandler.Do("bp2build", func() {
ctx.EventHandler.Do("read_build", func() {
existingBazelFiles, err := getExistingBazelRelatedFiles(topDir)
maybeQuit(err, "Error determining existing Bazel-related files")
err = ctx.RegisterExistingBazelTargets(topDir, existingBazelFiles)
maybeQuit(err, "Error parsing existing Bazel-related files")
})
// Propagate "allow misssing dependencies" bit. This is normally set in
// newContext(), but we create ctx without calling that method.
ctx.SetAllowMissingDependencies(ctx.Config().AllowMissingDependencies())
ctx.SetNameInterface(newNameResolver(ctx.Config()))
ctx.RegisterForBazelConversion()
ctx.SetModuleListFile(cmdlineArgs.ModuleListFile)
// Skip cloning modules during bp2build's blueprint run. Some mutators set
// bp2build-related module values which should be preserved during codegen.
ctx.SkipCloneModulesAfterMutators = true
var ninjaDeps []string
ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
// Run the loading and analysis pipeline to prepare the graph of regular
// Modules parsed from Android.bp files, and the BazelTargetModules mapped
// from the regular Modules.
ctx.EventHandler.Do("bootstrap", func() {
blueprintArgs := cmdlineArgs
bootstrapDeps, err := bootstrap.RunBlueprint(blueprintArgs.Args,
bootstrap.StopBeforePrepareBuildActions, ctx.Context, ctx.Config())
maybeQuit(err, "")
ninjaDeps = append(ninjaDeps, bootstrapDeps...)
})
globListFiles := writeBuildGlobsNinjaFile(ctx)
ninjaDeps = append(ninjaDeps, globListFiles...)
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion codegenMetrics to the user.
codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build, topDir)
codegenMetrics = bp2build.Codegen(codegenContext)
ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
writeDepFile(cmdlineArgs.Bp2buildMarker, ctx.EventHandler, ninjaDeps)
touch(shared.JoinPath(topDir, cmdlineArgs.Bp2buildMarker))
})
// Only report metrics when in bp2build mode. The metrics aren't relevant
// for queryview, since that's a total repo-wide conversion and there's a
// 1:1 mapping for each module.
if ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE") {
codegenMetrics.Print()
}
writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
return cmdlineArgs.Bp2buildMarker
}
// Write Bp2Build metrics into $LOG_DIR
func writeBp2BuildMetrics(codegenMetrics *bp2build.CodegenMetrics, eventHandler *metrics.EventHandler, metricsDir string) {
for _, event := range eventHandler.CompletedEvents() {
codegenMetrics.AddEvent(&bp2build_metrics_proto.Event{
Name: event.Id,
StartTime: uint64(event.Start.UnixNano()),
RealTime: event.RuntimeNanoseconds(),
})
}
if len(metricsDir) < 1 {
fmt.Fprintf(os.Stderr, "\nMissing required env var for generating bp2build metrics: LOG_DIR\n")
os.Exit(1)
}
codegenMetrics.Write(metricsDir)
}
func readFileLines(path string) ([]string, error) {
data, err := os.ReadFile(path)
if err == nil {
return strings.Split(strings.TrimSpace(string(data)), "\n"), nil
}
return nil, err
}
func maybeQuit(err error, format string, args ...interface{}) { func maybeQuit(err error, format string, args ...interface{}) {
if err == nil { if err == nil {
return return