From b63d7b3af71b13d94f30f1d973d375a5fadfdd48 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 7 Dec 2023 16:54:51 -0800 Subject: [PATCH] Remove infrastructure to run bp2build Bug: 315353489 Test: m blueprint_tests Change-Id: Idcf6377d389b94c39e4e6ff4b8efa8a9f9e78b17 --- android/Android.bp | 9 - android/apex_contributions.go | 3 - android/arch.go | 4 +- android/base_module_context.go | 45 - android/bazel.go | 784 ------------- android/bazel_handler.go | 1593 --------------------------- android/bazel_handler_test.go | 426 ------- android/bazel_paths.go | 675 ------------ android/bazel_paths_test.go | 240 ---- android/bazel_test.go | 592 ---------- android/config.go | 180 +-- android/metrics.go | 18 - android/module.go | 412 +------ android/mutator.go | 299 +---- android/mutator_test.go | 20 - android/override_module.go | 39 - android/prebuilt.go | 57 +- android/register.go | 79 +- android/test_config.go | 6 +- android/testing.go | 10 - android/util.go | 6 + android/variable.go | 43 - bp2build/Android.bp | 6 - bp2build/bp2build.go | 145 --- bp2build/bp2build_product_config.go | 885 --------------- bp2build/build_conversion.go | 605 +--------- bp2build/conversion.go | 101 +- bp2build/metrics.go | 231 ---- bp2build/performance_test.go | 219 ---- bp2build/symlink_forest.go | 511 --------- bp2build/testing.go | 785 ------------- cmd/soong_build/main.go | 344 +----- 32 files changed, 62 insertions(+), 9310 deletions(-) delete mode 100644 android/bazel.go delete mode 100644 android/bazel_handler.go delete mode 100644 android/bazel_handler_test.go delete mode 100644 android/bazel_paths.go delete mode 100644 android/bazel_paths_test.go delete mode 100644 android/bazel_test.go delete mode 100644 bp2build/bp2build.go delete mode 100644 bp2build/bp2build_product_config.go delete mode 100644 bp2build/metrics.go delete mode 100644 bp2build/performance_test.go delete mode 100644 bp2build/symlink_forest.go delete mode 100644 bp2build/testing.go diff --git a/android/Android.bp b/android/Android.bp index 62f534c53..2ac1d5fbe 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -12,14 +12,11 @@ bootstrap_go_package { "sbox_proto", "soong", "soong-android-soongconfig", - "soong-bazel", - "soong-cquery", "soong-remoteexec", "soong-response", "soong-shared", "soong-starlark", "soong-starlark-format", - "soong-ui-bp2build_metrics_proto", "soong-ui-metrics_proto", "soong-android-allowlists", @@ -38,9 +35,6 @@ bootstrap_go_package { "arch.go", "arch_list.go", "base_module_context.go", - "bazel.go", - "bazel_handler.go", - "bazel_paths.go", "buildinfo_prop.go", "config.go", "test_config.go", @@ -106,9 +100,6 @@ bootstrap_go_package { "androidmk_test.go", "apex_test.go", "arch_test.go", - "bazel_handler_test.go", - "bazel_paths_test.go", - "bazel_test.go", "config_test.go", "config_bp2build_test.go", "configured_jars_test.go", diff --git a/android/apex_contributions.go b/android/apex_contributions.go index 9b188deed..f13659a6d 100644 --- a/android/apex_contributions.go +++ b/android/apex_contributions.go @@ -111,9 +111,6 @@ func (a *allApexContributions) SetPrebuiltSelectionInfoProvider(ctx BaseModuleCo } } - if ctx.Config().Bp2buildMode() { // Skip bp2build - return - } p := PrebuiltSelectionInfoMap{} ctx.VisitDirectDepsWithTag(acDepTag, func(child Module) { if m, ok := child.(*apexContributions); ok { diff --git a/android/arch.go b/android/arch.go index 152016cd1..7436660ac 100644 --- a/android/arch.go +++ b/android/arch.go @@ -425,7 +425,7 @@ func osMutator(bpctx blueprint.BottomUpMutatorContext) { // blueprint.BottomUpMutatorContext because android.BottomUpMutatorContext // filters out non-Soong modules. Now that we've handled them, create a // normal android.BottomUpMutatorContext. - mctx := bottomUpMutatorContextFactory(bpctx, module, false, false) + mctx := bottomUpMutatorContextFactory(bpctx, module, false) base := module.base() @@ -570,7 +570,7 @@ func archMutator(bpctx blueprint.BottomUpMutatorContext) { // blueprint.BottomUpMutatorContext because android.BottomUpMutatorContext // filters out non-Soong modules. Now that we've handled them, create a // normal android.BottomUpMutatorContext. - mctx := bottomUpMutatorContextFactory(bpctx, module, false, false) + mctx := bottomUpMutatorContextFactory(bpctx, module, false) base := module.base() diff --git a/android/base_module_context.go b/android/base_module_context.go index ec9c888be..4312e9bd4 100644 --- a/android/base_module_context.go +++ b/android/base_module_context.go @@ -112,8 +112,6 @@ type BaseModuleContext interface { // the first DependencyTag. GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) - ModuleFromName(name string) (blueprint.Module, bool) - // VisitDirectDepsBlueprint calls visit for each direct dependency. If there are multiple // direct dependencies on the same module visit will be called multiple times on that module // and OtherModuleDependencyTag will return a different tag for each. @@ -209,12 +207,6 @@ type BaseModuleContext interface { // Calling this function prevents adding new dependencies. getMissingDependencies() []string - // AddUnconvertedBp2buildDep stores module name of a direct dependency that was not converted via bp2build - AddUnconvertedBp2buildDep(dep string) - - // AddMissingBp2buildDep stores the module name of a direct dependency that was not found. - AddMissingBp2buildDep(dep string) - Target() Target TargetPrimary() bool @@ -243,12 +235,8 @@ type baseModuleContext struct { strictVisitDeps bool // If true, enforce that all dependencies are enabled - bazelConversionMode bool } -func (b *baseModuleContext) isBazelConversionMode() bool { - return b.bazelConversionMode -} func (b *baseModuleContext) OtherModuleName(m blueprint.Module) string { return b.bp.OtherModuleName(m) } @@ -296,18 +284,6 @@ func (b *baseModuleContext) blueprintBaseModuleContext() blueprint.BaseModuleCon return b.bp } -// AddUnconvertedBp2buildDep stores module name of a dependency that was not converted to Bazel. -func (b *baseModuleContext) AddUnconvertedBp2buildDep(dep string) { - unconvertedDeps := &b.Module().base().commonProperties.BazelConversionStatus.UnconvertedDeps - *unconvertedDeps = append(*unconvertedDeps, dep) -} - -// AddMissingBp2buildDep stores module name of a dependency that was not found in a Android.bp file. -func (b *baseModuleContext) AddMissingBp2buildDep(dep string) { - missingDeps := &b.Module().base().commonProperties.BazelConversionStatus.MissingDeps - *missingDeps = append(*missingDeps, dep) -} - func (b *baseModuleContext) AddMissingDependencies(deps []string) { if deps != nil { missingDeps := &b.Module().base().commonProperties.MissingDeps @@ -435,27 +411,6 @@ func (b *baseModuleContext) GetDirectDep(name string) (blueprint.Module, bluepri return b.getDirectDepFirstTag(name) } -func (b *baseModuleContext) ModuleFromName(name string) (blueprint.Module, bool) { - if !b.isBazelConversionMode() { - panic("cannot call ModuleFromName if not in bazel conversion mode") - } - var m blueprint.Module - var ok bool - if moduleName, _ := SrcIsModuleWithTag(name); moduleName != "" { - m, ok = b.bp.ModuleFromName(moduleName) - } else { - m, ok = b.bp.ModuleFromName(name) - } - if !ok { - return m, ok - } - // If this module is not preferred, tried to get the prebuilt version instead - if a, aOk := m.(Module); aOk && !IsModulePrebuilt(a) && !IsModulePreferred(a) { - return b.ModuleFromName("prebuilt_" + name) - } - return m, ok -} - func (b *baseModuleContext) VisitDirectDepsBlueprint(visit func(blueprint.Module)) { b.bp.VisitDirectDeps(visit) } diff --git a/android/bazel.go b/android/bazel.go deleted file mode 100644 index 1602b9b02..000000000 --- a/android/bazel.go +++ /dev/null @@ -1,784 +0,0 @@ -// Copyright 2021 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "bufio" - "errors" - "fmt" - "strings" - - "android/soong/ui/metrics/bp2build_metrics_proto" - - "github.com/google/blueprint" - "github.com/google/blueprint/bootstrap" - "github.com/google/blueprint/proptools" - - "android/soong/android/allowlists" -) - -const ( - // A sentinel value to be used as a key in Bp2BuildConfig for modules with - // no package path. This is also the module dir for top level Android.bp - // modules. - Bp2BuildTopLevel = "." -) - -type MixedBuildEnabledStatus int - -const ( - // This module can be mixed_built. - MixedBuildEnabled = iota - - // There is a technical incompatibility preventing this module from being - // bazel-analyzed. Note: the module might also be incompatible. - TechnicalIncompatibility - - // This module cannot be mixed_built due to some incompatibility with it - // that is not a platform incompatibility. Example: the module-type is not - // enabled, or is not bp2build-converted. - ModuleIncompatibility - - // Missing dependencies. We can't query Bazel for modules if it has missing dependencies, there - // will be failures. - ModuleMissingDeps -) - -// FileGroupAsLibrary describes a filegroup module that is converted to some library -// such as aidl_library or proto_library. -type FileGroupAsLibrary interface { - ShouldConvertToAidlLibrary(ctx BazelConversionPathContext) bool - ShouldConvertToProtoLibrary(ctx BazelConversionPathContext) bool - GetAidlLibraryLabel(ctx BazelConversionPathContext) string - GetProtoLibraryLabel(ctx BazelConversionPathContext) string -} - -type BazelConversionStatus struct { - // Information about _all_ bp2build targets generated by this module. Multiple targets are - // supported as Soong handles some things within a single target that we may choose to split into - // multiple targets, e.g. renderscript, protos, yacc within a cc module. - Bp2buildInfo []bp2buildInfo `blueprint:"mutated"` - - // UnconvertedBp2buildDep stores the module names of direct dependency that were not converted to - // Bazel - UnconvertedDeps []string `blueprint:"mutated"` - - // MissingBp2buildDep stores the module names of direct dependency that were not found - MissingDeps []string `blueprint:"mutated"` - - // If non-nil, indicates that the module could not be converted successfully - // with bp2build. This will describe the reason the module could not be converted. - UnconvertedReason *UnconvertedReason - - // The Partition this module will be installed on. - // TODO(b/306200980) Investigate how to handle modules that are installed in multiple - // partitions. - Partition string `blueprint:"mutated"` -} - -// The reason a module could not be converted to a BUILD target via bp2build. -// This should match bp2build_metrics_proto.UnconvertedReason, but omits private -// proto-related fields that prevent copying this struct. -type UnconvertedReason struct { - // Should correspond to a valid value in bp2build_metrics_proto.UnconvertedReasonType. - // A raw int is used here instead, because blueprint logic requires that all transitive - // fields of module definitions be primitives. - ReasonType int - Detail string -} - -type BazelModuleProperties struct { - // The label of the Bazel target replacing this Soong module. When run in conversion mode, this - // will import the handcrafted build target into the autogenerated file. Note: this may result in - // a conflict due to duplicate targets if bp2build_available is also set. - Label *string - - // If true, bp2build will generate the converted Bazel target for this module. Note: this may - // cause a conflict due to the duplicate targets if label is also set. - // - // This is a bool pointer to support tristates: true, false, not set. - // - // To opt in a module, set bazel_module: { bp2build_available: true } - // To opt out a module, set bazel_module: { bp2build_available: false } - // To defer the default setting for the directory, do not set the value. - Bp2build_available *bool - - // CanConvertToBazel is set via InitBazelModule to indicate that a module type can be converted to - // Bazel with Bp2build. - CanConvertToBazel bool `blueprint:"mutated"` -} - -// Properties contains common module properties for Bazel migration purposes. -type properties struct { - // In "Bazel mixed build" mode, this represents the Bazel target replacing - // this Soong module. - Bazel_module BazelModuleProperties -} - -// namespacedVariableProperties is a map from a string representing a Soong -// config variable namespace, like "android" or "vendor_name" to a slice of -// pointer to a struct containing a single field called Soong_config_variables -// whose value mirrors the structure in the Blueprint file. -type namespacedVariableProperties map[string][]interface{} - -// BazelModuleBase contains the property structs with metadata for modules which can be converted to -// Bazel. -type BazelModuleBase struct { - bazelProperties properties - - // namespacedVariableProperties is used for soong_config_module_type support - // in bp2build. Soong config modules allow users to set module properties - // based on custom product variables defined in Android.bp files. These - // variables are namespaced to prevent clobbering, especially when set from - // Makefiles. - namespacedVariableProperties namespacedVariableProperties - - // baseModuleType is set when this module was created from a module type - // defined by a soong_config_module_type. Every soong_config_module_type - // "wraps" another module type, e.g. a soong_config_module_type can wrap a - // cc_defaults to a custom_cc_defaults, or cc_binary to a custom_cc_binary. - // This baseModuleType is set to the wrapped module type. - baseModuleType string -} - -// Bazelable is specifies the interface for modules that can be converted to Bazel. -type Bazelable interface { - bazelProps() *properties - HasHandcraftedLabel() bool - HandcraftedLabel() string - GetBazelLabel(ctx BazelConversionPathContext, module blueprint.Module) string - ShouldConvertWithBp2build(ctx ShouldConvertWithBazelContext) bool - shouldConvertWithBp2build(shouldConvertModuleContext, shouldConvertParams) bool - - // ConvertWithBp2build either converts the module to a Bazel build target or - // declares the module as unconvertible (for logging and metrics). - // Modules must implement this function to be bp2build convertible. The function - // must either create at least one Bazel target module (using ctx.CreateBazelTargetModule or - // its related functions), or declare itself unconvertible using ctx.MarkBp2buildUnconvertible. - ConvertWithBp2build(ctx Bp2buildMutatorContext) - - // namespacedVariableProps is a map from a soong config variable namespace - // (e.g. acme, android) to a map of interfaces{}, which are really - // reflect.Struct pointers, representing the value of the - // soong_config_variables property of a module. The struct pointer is the - // one with the single member called Soong_config_variables, which itself is - // a struct containing fields for each supported feature in that namespace. - // - // The reason for using a slice of interface{} is to support defaults - // propagation of the struct pointers. - namespacedVariableProps() namespacedVariableProperties - setNamespacedVariableProps(props namespacedVariableProperties) - BaseModuleType() string - SetBaseModuleType(baseModuleType string) -} - -// ApiProvider is implemented by modules that contribute to an API surface -type ApiProvider interface { - ConvertWithApiBp2build(ctx TopDownMutatorContext) -} - -// MixedBuildBuildable is an interface that module types should implement in order -// to be "handled by Bazel" in a mixed build. -type MixedBuildBuildable interface { - // IsMixedBuildSupported returns true if and only if this module should be - // "handled by Bazel" in a mixed build. - // This "escape hatch" allows modules with corner-case scenarios to opt out - // of being built with Bazel. - IsMixedBuildSupported(ctx BaseModuleContext) bool - - // QueueBazelCall invokes request-queueing functions on the BazelContext - // so that these requests are handled when Bazel's cquery is invoked. - QueueBazelCall(ctx BaseModuleContext) - - // ProcessBazelQueryResponse uses Bazel information (obtained from the BazelContext) - // to set module fields and providers to propagate this module's metadata upstream. - // This effectively "bridges the gap" between Bazel and Soong in a mixed build. - // Soong modules depending on this module should be oblivious to the fact that - // this module was handled by Bazel. - ProcessBazelQueryResponse(ctx ModuleContext) -} - -// BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules. -type BazelModule interface { - Module - Bazelable -} - -// InitBazelModule is a wrapper function that decorates a BazelModule with Bazel-conversion -// properties. -func InitBazelModule(module BazelModule) { - module.AddProperties(module.bazelProps()) - module.bazelProps().Bazel_module.CanConvertToBazel = true -} - -// BazelHandcraftedHook is a load hook to possibly register the current module as -// a "handcrafted" Bazel target of a given name. If the current module should be -// registered in this way, the hook function should return the target name. If -// it should not be registered in this way, this function should return the empty string. -type BazelHandcraftedHook func(ctx LoadHookContext) string - -// AddBazelHandcraftedHook adds a load hook to (maybe) mark the given module so that -// it is treated by bp2build as if it has a handcrafted Bazel target. -func AddBazelHandcraftedHook(module BazelModule, hook BazelHandcraftedHook) { - AddLoadHook(module, func(ctx LoadHookContext) { - var targetName string = hook(ctx) - if len(targetName) > 0 { - moduleDir := ctx.ModuleDir() - if moduleDir == Bp2BuildTopLevel { - moduleDir = "" - } - label := fmt.Sprintf("//%s:%s", moduleDir, targetName) - module.bazelProps().Bazel_module.Label = &label - } - }) -} - -// bazelProps returns the Bazel properties for the given BazelModuleBase. -func (b *BazelModuleBase) bazelProps() *properties { - return &b.bazelProperties -} - -func (b *BazelModuleBase) namespacedVariableProps() namespacedVariableProperties { - return b.namespacedVariableProperties -} - -func (b *BazelModuleBase) setNamespacedVariableProps(props namespacedVariableProperties) { - b.namespacedVariableProperties = props -} - -func (b *BazelModuleBase) BaseModuleType() string { - return b.baseModuleType -} - -func (b *BazelModuleBase) SetBaseModuleType(baseModuleType string) { - b.baseModuleType = baseModuleType -} - -// HasHandcraftedLabel returns whether this module has a handcrafted Bazel label. -func (b *BazelModuleBase) HasHandcraftedLabel() bool { - return b.bazelProperties.Bazel_module.Label != nil -} - -// HandcraftedLabel returns the handcrafted label for this module, or empty string if there is none -func (b *BazelModuleBase) HandcraftedLabel() string { - return proptools.String(b.bazelProperties.Bazel_module.Label) -} - -// GetBazelLabel returns the Bazel label for the given BazelModuleBase. -func (b *BazelModuleBase) GetBazelLabel(ctx BazelConversionPathContext, module blueprint.Module) string { - if b.HasHandcraftedLabel() { - return b.HandcraftedLabel() - } - if b.ShouldConvertWithBp2build(ctx) { - return bp2buildModuleLabel(ctx, module) - } - panic(fmt.Errorf("requested non-existent label for module %s", module.Name())) -} - -type Bp2BuildConversionAllowlist struct { - // Configure modules in these directories to enable bp2build_available: true or false by default. - defaultConfig allowlists.Bp2BuildConfig - - // Keep any existing BUILD files (and do not generate new BUILD files) for these directories - // in the synthetic Bazel workspace. - keepExistingBuildFile map[string]bool - - // Per-module allowlist to always opt modules into both bp2build and Bazel Dev Mode mixed - // builds. These modules are usually in directories with many other modules that are not ready - // for conversion. - // - // A module can either be in this list or its directory allowlisted entirely - // in bp2buildDefaultConfig, but not both at the same time. - moduleAlwaysConvert map[string]bool - - // Per-module-type allowlist to always opt modules in to both bp2build and - // Bazel Dev Mode mixed builds when they have the same type as one listed. - moduleTypeAlwaysConvert map[string]bool - - // Per-module denylist to always opt modules out of bp2build conversion. - moduleDoNotConvert map[string]bool -} - -// NewBp2BuildAllowlist creates a new, empty Bp2BuildConversionAllowlist -// which can be populated using builder pattern Set* methods -func NewBp2BuildAllowlist() Bp2BuildConversionAllowlist { - return Bp2BuildConversionAllowlist{ - allowlists.Bp2BuildConfig{}, - map[string]bool{}, - map[string]bool{}, - map[string]bool{}, - map[string]bool{}, - } -} - -// SetDefaultConfig copies the entries from defaultConfig into the allowlist -func (a Bp2BuildConversionAllowlist) SetDefaultConfig(defaultConfig allowlists.Bp2BuildConfig) Bp2BuildConversionAllowlist { - if a.defaultConfig == nil { - a.defaultConfig = allowlists.Bp2BuildConfig{} - } - for k, v := range defaultConfig { - a.defaultConfig[k] = v - } - - return a -} - -// SetKeepExistingBuildFile copies the entries from keepExistingBuildFile into the allowlist -func (a Bp2BuildConversionAllowlist) SetKeepExistingBuildFile(keepExistingBuildFile map[string]bool) Bp2BuildConversionAllowlist { - if a.keepExistingBuildFile == nil { - a.keepExistingBuildFile = map[string]bool{} - } - for k, v := range keepExistingBuildFile { - a.keepExistingBuildFile[k] = v - } - - return a -} - -// SetModuleAlwaysConvertList copies the entries from moduleAlwaysConvert into the allowlist -func (a Bp2BuildConversionAllowlist) SetModuleAlwaysConvertList(moduleAlwaysConvert []string) Bp2BuildConversionAllowlist { - if a.moduleAlwaysConvert == nil { - a.moduleAlwaysConvert = map[string]bool{} - } - for _, m := range moduleAlwaysConvert { - a.moduleAlwaysConvert[m] = true - } - - return a -} - -// SetModuleTypeAlwaysConvertList copies the entries from moduleTypeAlwaysConvert into the allowlist -func (a Bp2BuildConversionAllowlist) SetModuleTypeAlwaysConvertList(moduleTypeAlwaysConvert []string) Bp2BuildConversionAllowlist { - if a.moduleTypeAlwaysConvert == nil { - a.moduleTypeAlwaysConvert = map[string]bool{} - } - for _, m := range moduleTypeAlwaysConvert { - a.moduleTypeAlwaysConvert[m] = true - } - - return a -} - -// SetModuleDoNotConvertList copies the entries from moduleDoNotConvert into the allowlist -func (a Bp2BuildConversionAllowlist) SetModuleDoNotConvertList(moduleDoNotConvert []string) Bp2BuildConversionAllowlist { - if a.moduleDoNotConvert == nil { - a.moduleDoNotConvert = map[string]bool{} - } - for _, m := range moduleDoNotConvert { - a.moduleDoNotConvert[m] = true - } - - return a -} - -// ShouldKeepExistingBuildFileForDir returns whether an existing BUILD file should be -// added to the build symlink forest based on the current global configuration. -func (a Bp2BuildConversionAllowlist) ShouldKeepExistingBuildFileForDir(dir string) bool { - if _, ok := a.keepExistingBuildFile[dir]; ok { - // Exact dir match - return true - } - var i int - // Check if subtree match - for { - j := strings.Index(dir[i:], "/") - if j == -1 { - return false //default - } - prefix := dir[0 : i+j] - i = i + j + 1 // skip the "/" - if recursive, ok := a.keepExistingBuildFile[prefix]; ok && recursive { - return true - } - } -} - -var bp2BuildAllowListKey = NewOnceKey("Bp2BuildAllowlist") -var bp2buildAllowlist OncePer - -func GetBp2BuildAllowList() Bp2BuildConversionAllowlist { - return bp2buildAllowlist.Once(bp2BuildAllowListKey, func() interface{} { - return NewBp2BuildAllowlist().SetDefaultConfig(allowlists.Bp2buildDefaultConfig). - SetKeepExistingBuildFile(allowlists.Bp2buildKeepExistingBuildFile). - SetModuleAlwaysConvertList(allowlists.Bp2buildModuleAlwaysConvertList). - SetModuleTypeAlwaysConvertList(allowlists.Bp2buildModuleTypeAlwaysConvertList). - SetModuleDoNotConvertList(allowlists.Bp2buildModuleDoNotConvertList) - }).(Bp2BuildConversionAllowlist) -} - -// MixedBuildsEnabled returns a MixedBuildEnabledStatus regarding whether -// a module is ready to be replaced by a converted or handcrafted Bazel target. -// As a side effect, calling this method will also log whether this module is -// mixed build enabled for metrics reporting. -func MixedBuildsEnabled(ctx BaseModuleContext) MixedBuildEnabledStatus { - platformIncompatible := isPlatformIncompatible(ctx.Os(), ctx.Arch().ArchType) - if platformIncompatible { - ctx.Config().LogMixedBuild(ctx, false) - return TechnicalIncompatibility - } - - if ctx.Config().AllowMissingDependencies() { - missingDeps := ctx.getMissingDependencies() - // If there are missing dependencies, querying Bazel will fail. Soong instead fails at execution - // time, not loading/analysis. disable mixed builds and fall back to Soong to maintain that - // behavior. - if len(missingDeps) > 0 { - ctx.Config().LogMixedBuild(ctx, false) - return ModuleMissingDeps - } - } - - module := ctx.Module() - apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) - withinApex := !apexInfo.IsForPlatform() - mixedBuildEnabled := ctx.Config().IsMixedBuildsEnabled() && - module.Enabled() && - convertedToBazel(ctx, module) && - ctx.Config().BazelContext.IsModuleNameAllowed(module.Name(), withinApex) - ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled) - - if mixedBuildEnabled { - return MixedBuildEnabled - } - return ModuleIncompatibility -} - -func isGoModule(module blueprint.Module) bool { - if _, ok := module.(*bootstrap.GoPackage); ok { - return true - } - if _, ok := module.(*bootstrap.GoBinary); ok { - return true - } - return false -} - -// ConvertedToBazel returns whether this module has been converted (with bp2build or manually) to Bazel. -func convertedToBazel(ctx BazelConversionContext, module blueprint.Module) bool { - // Special-case bootstrap_go_package and bootstrap_go_binary - // These do not implement Bazelable, but have been converted - if isGoModule(module) { - return true - } - b, ok := module.(Bazelable) - if !ok { - return false - } - - return b.HasHandcraftedLabel() || b.shouldConvertWithBp2build(ctx, shouldConvertParams{ - module: module, - moduleDir: ctx.OtherModuleDir(module), - moduleName: ctx.OtherModuleName(module), - moduleType: ctx.OtherModuleType(module), - }) -} - -type ShouldConvertWithBazelContext interface { - ModuleErrorf(format string, args ...interface{}) - Module() Module - Config() Config - ModuleType() string - ModuleName() string - ModuleDir() string -} - -// ShouldConvertWithBp2build returns whether the given BazelModuleBase should be converted with bp2build -func (b *BazelModuleBase) ShouldConvertWithBp2build(ctx ShouldConvertWithBazelContext) bool { - return b.shouldConvertWithBp2build(ctx, shouldConvertParams{ - module: ctx.Module(), - moduleDir: ctx.ModuleDir(), - moduleName: ctx.ModuleName(), - moduleType: ctx.ModuleType(), - }) -} - -type bazelOtherModuleContext interface { - ModuleErrorf(format string, args ...interface{}) - Config() Config - OtherModuleType(m blueprint.Module) string - OtherModuleName(m blueprint.Module) string - OtherModuleDir(m blueprint.Module) string -} - -func isPlatformIncompatible(osType OsType, arch ArchType) bool { - return osType == Windows || // Windows toolchains are not currently supported. - osType == LinuxBionic || // Linux Bionic toolchains are not currently supported. - osType == LinuxMusl || // Linux musl toolchains are not currently supported (b/259266326). - arch == Riscv64 // TODO(b/262192655) Riscv64 toolchains are not currently supported. -} - -type shouldConvertModuleContext interface { - ModuleErrorf(format string, args ...interface{}) - Config() Config -} - -type shouldConvertParams struct { - module blueprint.Module - moduleType string - moduleDir string - moduleName string -} - -func (b *BazelModuleBase) shouldConvertWithBp2build(ctx shouldConvertModuleContext, p shouldConvertParams) bool { - if !b.bazelProps().Bazel_module.CanConvertToBazel { - return false - } - - module := p.module - - propValue := b.bazelProperties.Bazel_module.Bp2build_available - packagePath := moduleDirWithPossibleOverride(ctx, module, p.moduleDir) - - // Modules in unit tests which are enabled in the allowlist by type or name - // trigger this conditional because unit tests run under the "." package path - isTestModule := packagePath == Bp2BuildTopLevel && proptools.BoolDefault(propValue, false) - if isTestModule { - return true - } - - moduleName := moduleNameWithPossibleOverride(ctx, module, p.moduleName) - // use "prebuilt_" + original module name as the java_import(_host) module name, - // to avoid the failure that a normal module and a prebuilt module with - // the same name are both allowlisted. This cannot be applied to all the *_import - // module types. For example, android_library_import has to use original module - // name here otherwise the *-nodeps targets cannot be handled correctly. - // TODO(b/304385140): remove this special casing - if p.moduleType == "java_import" || p.moduleType == "java_import_host" { - moduleName = module.Name() - } - - allowlist := ctx.Config().Bp2buildPackageConfig - - moduleNameAllowed := allowlist.moduleAlwaysConvert[moduleName] - moduleTypeAllowed := allowlist.moduleTypeAlwaysConvert[p.moduleType] - allowlistConvert := moduleNameAllowed || moduleTypeAllowed - if moduleNameAllowed && moduleTypeAllowed { - ctx.ModuleErrorf("A module %q of type %q cannot be in moduleAlwaysConvert and also be in moduleTypeAlwaysConvert", moduleName, p.moduleType) - return false - } - - if allowlist.moduleDoNotConvert[moduleName] { - if moduleNameAllowed { - ctx.ModuleErrorf("a module %q cannot be in moduleDoNotConvert and also be in moduleAlwaysConvert", moduleName) - } - return false - } - - // This is a tristate value: true, false, or unset. - if ok, directoryPath := bp2buildDefaultTrueRecursively(packagePath, allowlist.defaultConfig); ok { - if moduleNameAllowed { - ctx.ModuleErrorf("A module cannot be in a directory marked Bp2BuildDefaultTrue"+ - " or Bp2BuildDefaultTrueRecursively and also be in moduleAlwaysConvert. Directory: '%s'"+ - " Module: '%s'", directoryPath, moduleName) - return false - } - - // Allow modules to explicitly opt-out. - return proptools.BoolDefault(propValue, true) - } - - // Allow modules to explicitly opt-in. - return proptools.BoolDefault(propValue, allowlistConvert) -} - -// bp2buildDefaultTrueRecursively checks that the package contains a prefix from the -// set of package prefixes where all modules must be converted. That is, if the -// package is x/y/z, and the list contains either x, x/y, or x/y/z, this function will -// return true. -// -// However, if the package is x/y, and it matches a Bp2BuildDefaultFalse "x/y" entry -// exactly, this module will return false early. -// -// This function will also return false if the package doesn't match anything in -// the config. -// -// This function will also return the allowlist entry which caused a particular -// package to be enabled. Since packages can be enabled via a recursive declaration, -// the path returned will not always be the same as the one provided. -func bp2buildDefaultTrueRecursively(packagePath string, config allowlists.Bp2BuildConfig) (bool, string) { - // Check if the package path has an exact match in the config. - if config[packagePath] == allowlists.Bp2BuildDefaultTrue || config[packagePath] == allowlists.Bp2BuildDefaultTrueRecursively { - return true, packagePath - } else if config[packagePath] == allowlists.Bp2BuildDefaultFalse || config[packagePath] == allowlists.Bp2BuildDefaultFalseRecursively { - return false, packagePath - } - - // If not, check for the config recursively. - packagePrefix := packagePath - - // e.g. for x/y/z, iterate over x/y, then x, taking the most-specific value from the allowlist. - for strings.Contains(packagePrefix, "/") { - dirIndex := strings.LastIndex(packagePrefix, "/") - packagePrefix = packagePrefix[:dirIndex] - switch value := config[packagePrefix]; value { - case allowlists.Bp2BuildDefaultTrueRecursively: - // package contains this prefix and this prefix should convert all modules - return true, packagePrefix - case allowlists.Bp2BuildDefaultFalseRecursively: - //package contains this prefix and this prefix should NOT convert any modules - return false, packagePrefix - } - // Continue to the next part of the package dir. - - } - - return false, packagePath -} - -func registerBp2buildConversionMutator(ctx RegisterMutatorsContext) { - ctx.BottomUp("bp2build_conversion", bp2buildConversionMutator).Parallel() - ctx.BottomUp("bp2build_deps", bp2buildDepsMutator).Parallel() -} - -func bp2buildConversionMutator(ctx BottomUpMutatorContext) { - // If an existing BUILD file in the module directory has a target defined - // with this same name as this module, assume that this is an existing - // definition for this target. - if ctx.Config().HasBazelBuildTargetInSource(ctx.ModuleDir(), ctx.ModuleName()) { - ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE, ctx.ModuleName()) - return - } - bModule, ok := ctx.Module().(Bazelable) - if !ok { - ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_TYPE_UNSUPPORTED, "") - return - } - // There may be cases where the target is created by a macro rather than in a BUILD file, those - // should be captured as well. - if bModule.HasHandcraftedLabel() { - // Defer to the BUILD target. Generating an additional target would - // cause a BUILD file conflict. - ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE, "") - return - } - // TODO: b/285631638 - Differentiate between denylisted modules and missing bp2build capabilities. - if !bModule.shouldConvertWithBp2build(ctx, shouldConvertParams{ - module: ctx.Module(), - moduleDir: ctx.ModuleDir(), - moduleName: ctx.ModuleName(), - moduleType: ctx.ModuleType(), - }) { - ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_UNSUPPORTED, "") - return - } - if ctx.Module().base().GetUnconvertedReason() != nil { - return - } - - bModule.ConvertWithBp2build(ctx) - - installCtx := &baseModuleContextToModuleInstallPathContext{ctx} - ctx.Module().base().setPartitionForBp2build(modulePartition(installCtx, true)) - - if len(ctx.Module().base().Bp2buildTargets()) == 0 && ctx.Module().base().GetUnconvertedReason() == nil { - panic(fmt.Errorf("illegal bp2build invariant: module '%s' was neither converted nor marked unconvertible", ctx.ModuleName())) - } - - // If an existing BUILD file in the module directory has a target defined - // with the same name as any target generated by this module, assume that this - // is an existing definition for this target. (These generated target names - // may be different than the module name, as checked at the beginning of this function!) - for _, targetInfo := range ctx.Module().base().Bp2buildTargets() { - if ctx.Config().HasBazelBuildTargetInSource(targetInfo.TargetPackage(), targetInfo.TargetName()) { - // Defer to the BUILD target. Generating an additional target would - // cause a BUILD file conflict. - ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE, targetInfo.TargetName()) - return - } - } -} - -// TODO: b/285631638 - Add this as a new mutator to the bp2build conversion mutators. -// Currently, this only exists to prepare test coverage for the launch of this feature. -func bp2buildDepsMutator(ctx BottomUpMutatorContext) { - if ctx.Module().base().GetUnconvertedReason() != nil { - return - } - - if len(ctx.Module().GetMissingBp2buildDeps()) > 0 { - exampleDep := ctx.Module().GetMissingBp2buildDeps()[0] - ctx.MarkBp2buildUnconvertible( - bp2build_metrics_proto.UnconvertedReasonType_UNCONVERTED_DEP, exampleDep) - } - - // Transitively mark modules unconvertible with the following set of conditions. - ctx.VisitDirectDeps(func(dep Module) { - if dep.base().GetUnconvertedReason() == nil { - return - } - - if dep.base().GetUnconvertedReason().ReasonType == - int(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE) { - return - } - - if ctx.OtherModuleDependencyTag(dep) != Bp2buildDepTag { - return - } - - ctx.MarkBp2buildUnconvertible( - bp2build_metrics_proto.UnconvertedReasonType_UNCONVERTED_DEP, dep.Name()) - }) -} - -// GetMainClassInManifest scans the manifest file specified in filepath and returns -// the value of attribute Main-Class in the manifest file if it exists, or returns error. -// WARNING: this is for bp2build converters of java_* modules only. -func GetMainClassInManifest(c Config, filepath string) (string, error) { - file, err := c.fs.Open(filepath) - if err != nil { - return "", err - } - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "Main-Class:") { - return strings.TrimSpace(line[len("Main-Class:"):]), nil - } - } - - return "", errors.New("Main-Class is not found.") -} - -func AttachValidationActions(ctx ModuleContext, outputFilePath Path, validations Paths) ModuleOutPath { - validatedOutputFilePath := PathForModuleOut(ctx, "validated", outputFilePath.Base()) - ctx.Build(pctx, BuildParams{ - Rule: CpNoPreserveSymlink, - Description: "run validations " + outputFilePath.Base(), - Output: validatedOutputFilePath, - Input: outputFilePath, - Validations: validations, - }) - return validatedOutputFilePath -} - -func RunsOn(hostSupported bool, deviceSupported bool, unitTest bool) []string { - var runsOn []string - - if hostSupported && deviceSupported { - runsOn = []string{"host_without_device", "device"} - } else if hostSupported { - if unitTest { - runsOn = []string{"host_without_device"} - } else { - runsOn = []string{"host_with_device"} - } - } else if deviceSupported { - runsOn = []string{"device"} - } - - return runsOn -} diff --git a/android/bazel_handler.go b/android/bazel_handler.go deleted file mode 100644 index 0c654154a..000000000 --- a/android/bazel_handler.go +++ /dev/null @@ -1,1593 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "bytes" - "crypto/sha1" - "encoding/hex" - "fmt" - "os" - "path" - "path/filepath" - "regexp" - "runtime" - "sort" - "strings" - "sync" - - "android/soong/android/allowlists" - "android/soong/bazel/cquery" - "android/soong/shared" - "android/soong/starlark_import" - - "android/soong/bazel" - - "github.com/google/blueprint" - "github.com/google/blueprint/metrics" -) - -var ( - _ = pctx.HostBinToolVariable("bazelBuildRunfilesTool", "build-runfiles") - buildRunfilesRule = pctx.AndroidStaticRule("bazelBuildRunfiles", blueprint.RuleParams{ - Command: "${bazelBuildRunfilesTool} ${in} ${outDir}", - Depfile: "", - Description: "", - CommandDeps: []string{"${bazelBuildRunfilesTool}"}, - }, "outDir") -) - -func registerMixedBuildsMutator(ctx RegisterMutatorsContext) { - ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() -} - -func RegisterMixedBuildsMutator(ctx RegistrationContext) { - ctx.FinalDepsMutators(registerMixedBuildsMutator) -} - -func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) { - if m := ctx.Module(); m.Enabled() { - if mixedBuildMod, ok := m.(MixedBuildBuildable); ok { - mixedBuildEnabled := MixedBuildsEnabled(ctx) - queueMixedBuild := mixedBuildMod.IsMixedBuildSupported(ctx) && mixedBuildEnabled == MixedBuildEnabled - if queueMixedBuild { - mixedBuildMod.QueueBazelCall(ctx) - } - } - } -} - -type cqueryRequest interface { - // Name returns a string name for this request type. Such request type names must be unique, - // and must only consist of alphanumeric characters. - Name() string - - // StarlarkFunctionBody returns a starlark function body to process this request type. - // The returned string is the body of a Starlark function which obtains - // all request-relevant information about a target and returns a string containing - // this information. - // The function should have the following properties: - // - The arguments are `target` (a configured target) and `id_string` (the label + configuration). - // - The return value must be a string. - // - The function body should not be indented outside of its own scope. - StarlarkFunctionBody() string -} - -// Portion of cquery map key to describe target configuration. -type configKey struct { - arch string - osType OsType - apexKey ApexConfigKey -} - -type ApexConfigKey struct { - WithinApex bool - ApexSdkVersion string - ApiDomain string -} - -func (c ApexConfigKey) String() string { - return fmt.Sprintf("%s_%s_%s", withinApexToString(c.WithinApex), c.ApexSdkVersion, c.ApiDomain) -} - -func withinApexToString(withinApex bool) string { - if withinApex { - return "within_apex" - } - return "" -} - -func (c configKey) String() string { - return fmt.Sprintf("%s::%s::%s", c.arch, c.osType, c.apexKey) -} - -// Map key to describe bazel cquery requests. -type cqueryKey struct { - label string - requestType cqueryRequest - configKey configKey -} - -func makeCqueryKey(label string, cqueryRequest cqueryRequest, cfgKey configKey) cqueryKey { - if strings.HasPrefix(label, "//") { - // Normalize Bazel labels to specify main repository explicitly. - label = "@" + label - } - return cqueryKey{label, cqueryRequest, cfgKey} -} - -func (c cqueryKey) String() string { - return fmt.Sprintf("cquery(%s,%s,%s)", c.label, c.requestType.Name(), c.configKey) -} - -type invokeBazelContext interface { - GetEventHandler() *metrics.EventHandler -} - -// BazelContext is a context object useful for interacting with Bazel during -// the course of a build. Use of Bazel to evaluate part of the build graph -// is referred to as a "mixed build". (Some modules are managed by Soong, -// some are managed by Bazel). To facilitate interop between these build -// subgraphs, Soong may make requests to Bazel and evaluate their responses -// so that Soong modules may accurately depend on Bazel targets. -type BazelContext interface { - // Add a cquery request to the bazel request queue. All queued requests - // will be sent to Bazel on a subsequent invocation of InvokeBazel. - QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) - - // ** Cquery Results Retrieval Functions - // The below functions pertain to retrieving cquery results from a prior - // InvokeBazel function call and parsing the results. - - // Returns result files built by building the given bazel target label. - GetOutputFiles(label string, cfgKey configKey) ([]string, error) - - // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order). - GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) - - // Returns the results of the GetApexInfo query (including output files) - GetApexInfo(label string, cfgkey configKey) (cquery.ApexInfo, error) - - // Returns the results of the GetCcUnstrippedInfo query - GetCcUnstrippedInfo(label string, cfgkey configKey) (cquery.CcUnstrippedInfo, error) - - // Returns the results of the GetPrebuiltFileInfo query - GetPrebuiltFileInfo(label string, cfgKey configKey) (cquery.PrebuiltFileInfo, error) - - // ** end Cquery Results Retrieval Functions - - // Issues commands to Bazel to receive results for all cquery requests - // queued in the BazelContext. The ctx argument is optional and is only - // used for performance data collection - InvokeBazel(config Config, ctx invokeBazelContext) error - - // Returns true if Bazel handling is enabled for the module with the given name. - // Note that this only implies "bazel mixed build" allowlisting. The caller - // should independently verify the module is eligible for Bazel handling - // (for example, that it is MixedBuildBuildable). - IsModuleNameAllowed(moduleName string, withinApex bool) bool - - // Returns the bazel output base (the root directory for all bazel intermediate outputs). - OutputBase() string - - // Returns build statements which should get registered to reflect Bazel's outputs. - BuildStatementsToRegister() []*bazel.BuildStatement - - // Returns the depsets defined in Bazel's aquery response. - AqueryDepsets() []bazel.AqueryDepset - - QueueBazelSandwichCqueryRequests(config Config) error -} - -type bazelRunner interface { - issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (output string, errorMessage string, error error) -} - -type bazelPaths struct { - homeDir string - bazelPath string - outputBase string - workspaceDir string - soongOutDir string - metricsDir string - bazelDepsFile string -} - -// A context object which tracks queued requests that need to be made to Bazel, -// and their results after the requests have been made. -type mixedBuildBazelContext struct { - bazelRunner - paths *bazelPaths - // cquery requests that have not yet been issued to Bazel. This list is maintained - // in a sorted state, and is guaranteed to have no duplicates. - requests []cqueryKey - requestMutex sync.Mutex // requests can be written in parallel - - results map[cqueryKey]string // Results of cquery requests after Bazel invocations - - // Build statements which should get registered to reflect Bazel's outputs. - buildStatements []*bazel.BuildStatement - - // Depsets which should be used for Bazel's build statements. - depsets []bazel.AqueryDepset - - // Per-module allowlist/denylist functionality to control whether analysis of - // modules are handled by Bazel. For modules which do not have a Bazel definition - // (or do not sufficiently support bazel handling via MixedBuildBuildable), - // this allowlist will have no effect, even if the module is explicitly allowlisted here. - // Per-module denylist to opt modules out of bazel handling. - bazelDisabledModules map[string]bool - // Per-module allowlist to opt modules in to bazel handling. - bazelEnabledModules map[string]bool - // DCLA modules are enabled when used in apex. - bazelDclaEnabledModules map[string]bool - - targetProduct string - targetBuildVariant string -} - -var _ BazelContext = &mixedBuildBazelContext{} - -// A bazel context to use when Bazel is disabled. -type noopBazelContext struct{} - -var _ BazelContext = noopBazelContext{} - -func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) { - panic("unimplemented") -} - -func (n noopBazelContext) QueueBazelSandwichCqueryRequests(config Config) error { - panic("unimplemented") -} - -func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) { - panic("unimplemented") -} - -func (n noopBazelContext) GetCcInfo(_ string, _ configKey) (cquery.CcInfo, error) { - panic("unimplemented") -} - -func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexInfo, error) { - panic("unimplemented") -} - -func (n noopBazelContext) GetCcUnstrippedInfo(_ string, _ configKey) (cquery.CcUnstrippedInfo, error) { - //TODO implement me - panic("implement me") -} - -func (n noopBazelContext) GetPrebuiltFileInfo(_ string, _ configKey) (cquery.PrebuiltFileInfo, error) { - panic("implement me") -} - -func (n noopBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error { - panic("unimplemented") -} - -func (m noopBazelContext) OutputBase() string { - return "" -} - -func (n noopBazelContext) IsModuleNameAllowed(_ string, _ bool) bool { - return false -} - -func (m noopBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement { - return []*bazel.BuildStatement{} -} - -func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset { - return []bazel.AqueryDepset{} -} - -// A bazel context to use for tests. -type MockBazelContext struct { - OutputBaseDir string - - LabelToOutputFiles map[string][]string - LabelToCcInfo map[string]cquery.CcInfo - LabelToPythonBinary map[string]string - LabelToApexInfo map[string]cquery.ApexInfo - LabelToCcBinary map[string]cquery.CcUnstrippedInfo - LabelToPrebuiltFileInfo map[string]cquery.PrebuiltFileInfo - - BazelRequests map[string]bool -} - -func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { - key := BuildMockBazelContextRequestKey(label, requestType, cfgKey.arch, cfgKey.osType, cfgKey.apexKey) - if m.BazelRequests == nil { - m.BazelRequests = make(map[string]bool) - } - m.BazelRequests[key] = true -} - -func (m MockBazelContext) QueueBazelSandwichCqueryRequests(config Config) error { - panic("unimplemented") -} - -func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) { - result, ok := m.LabelToOutputFiles[label] - if !ok { - return []string{}, fmt.Errorf("no target with label %q in LabelToOutputFiles", label) - } - return result, nil -} - -func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { - result, ok := m.LabelToCcInfo[label] - if !ok { - key := BuildMockBazelContextResultKey(label, cfgKey.arch, cfgKey.osType, cfgKey.apexKey) - result, ok = m.LabelToCcInfo[key] - if !ok { - return cquery.CcInfo{}, fmt.Errorf("no target with label %q in LabelToCcInfo", label) - } - } - return result, nil -} - -func (m MockBazelContext) GetApexInfo(label string, _ configKey) (cquery.ApexInfo, error) { - result, ok := m.LabelToApexInfo[label] - if !ok { - return cquery.ApexInfo{}, fmt.Errorf("no target with label %q in LabelToApexInfo", label) - } - return result, nil -} - -func (m MockBazelContext) GetCcUnstrippedInfo(label string, _ configKey) (cquery.CcUnstrippedInfo, error) { - result, ok := m.LabelToCcBinary[label] - if !ok { - return cquery.CcUnstrippedInfo{}, fmt.Errorf("no target with label %q in LabelToCcBinary", label) - } - return result, nil -} - -func (m MockBazelContext) GetPrebuiltFileInfo(label string, _ configKey) (cquery.PrebuiltFileInfo, error) { - result, ok := m.LabelToPrebuiltFileInfo[label] - if !ok { - return cquery.PrebuiltFileInfo{}, fmt.Errorf("no target with label %q in LabelToPrebuiltFileInfo", label) - } - return result, nil -} - -func (m MockBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error { - panic("unimplemented") -} - -func (m MockBazelContext) IsModuleNameAllowed(_ string, _ bool) bool { - return true -} - -func (m MockBazelContext) OutputBase() string { return m.OutputBaseDir } - -func (m MockBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement { - return []*bazel.BuildStatement{} -} - -func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset { - return []bazel.AqueryDepset{} -} - -var _ BazelContext = MockBazelContext{} - -func BuildMockBazelContextRequestKey(label string, request cqueryRequest, arch string, osType OsType, apexKey ApexConfigKey) string { - cfgKey := configKey{ - arch: arch, - osType: osType, - apexKey: apexKey, - } - - return strings.Join([]string{label, request.Name(), cfgKey.String()}, "_") -} - -func BuildMockBazelContextResultKey(label string, arch string, osType OsType, apexKey ApexConfigKey) string { - cfgKey := configKey{ - arch: arch, - osType: osType, - apexKey: apexKey, - } - - return strings.Join([]string{label, cfgKey.String()}, "_") -} - -func (bazelCtx *mixedBuildBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { - key := makeCqueryKey(label, requestType, cfgKey) - bazelCtx.requestMutex.Lock() - defer bazelCtx.requestMutex.Unlock() - - // Insert key into requests, maintaining the sort, and only if it's not duplicate. - keyString := key.String() - foundEqual := false - notLessThanKeyString := func(i int) bool { - s := bazelCtx.requests[i].String() - v := strings.Compare(s, keyString) - if v == 0 { - foundEqual = true - } - return v >= 0 - } - targetIndex := sort.Search(len(bazelCtx.requests), notLessThanKeyString) - if foundEqual { - return - } - - if targetIndex == len(bazelCtx.requests) { - bazelCtx.requests = append(bazelCtx.requests, key) - } else { - bazelCtx.requests = append(bazelCtx.requests[:targetIndex+1], bazelCtx.requests[targetIndex:]...) - bazelCtx.requests[targetIndex] = key - } -} - -func (bazelCtx *mixedBuildBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { - key := makeCqueryKey(label, cquery.GetOutputFiles, cfgKey) - if rawString, ok := bazelCtx.results[key]; ok { - bazelOutput := strings.TrimSpace(rawString) - - return cquery.GetOutputFiles.ParseResult(bazelOutput), nil - } - return nil, fmt.Errorf("no bazel response found for %v", key) -} - -func (bazelCtx *mixedBuildBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { - key := makeCqueryKey(label, cquery.GetCcInfo, cfgKey) - if rawString, ok := bazelCtx.results[key]; ok { - bazelOutput := strings.TrimSpace(rawString) - return cquery.GetCcInfo.ParseResult(bazelOutput) - } - return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key) -} - -func (bazelCtx *mixedBuildBazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexInfo, error) { - key := makeCqueryKey(label, cquery.GetApexInfo, cfgKey) - if rawString, ok := bazelCtx.results[key]; ok { - return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString)) - } - return cquery.ApexInfo{}, fmt.Errorf("no bazel response found for %v", key) -} - -func (bazelCtx *mixedBuildBazelContext) GetCcUnstrippedInfo(label string, cfgKey configKey) (cquery.CcUnstrippedInfo, error) { - key := makeCqueryKey(label, cquery.GetCcUnstrippedInfo, cfgKey) - if rawString, ok := bazelCtx.results[key]; ok { - return cquery.GetCcUnstrippedInfo.ParseResult(strings.TrimSpace(rawString)) - } - return cquery.CcUnstrippedInfo{}, fmt.Errorf("no bazel response for %s", key) -} - -func (bazelCtx *mixedBuildBazelContext) GetPrebuiltFileInfo(label string, cfgKey configKey) (cquery.PrebuiltFileInfo, error) { - key := makeCqueryKey(label, cquery.GetPrebuiltFileInfo, cfgKey) - if rawString, ok := bazelCtx.results[key]; ok { - return cquery.GetPrebuiltFileInfo.ParseResult(strings.TrimSpace(rawString)) - } - return cquery.PrebuiltFileInfo{}, fmt.Errorf("no bazel response for %s", key) -} - -func AddToStringSet(set map[string]bool, items []string) { - for _, item := range items { - set[item] = true - } -} - -func GetBazelEnabledAndDisabledModules(buildMode SoongBuildMode, forceEnabled map[string]struct{}) (map[string]bool, map[string]bool) { - disabledModules := map[string]bool{} - enabledModules := map[string]bool{} - - switch buildMode { - case BazelProdMode: - AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) - for enabledAdHocModule := range forceEnabled { - enabledModules[enabledAdHocModule] = true - } - case BazelStagingMode: - // Staging mode includes all prod modules plus all staging modules. - AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) - AddToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList) - for enabledAdHocModule := range forceEnabled { - enabledModules[enabledAdHocModule] = true - } - default: - panic("Expected BazelProdMode or BazelStagingMode") - } - return enabledModules, disabledModules -} - -func GetBazelEnabledModules(buildMode SoongBuildMode) []string { - enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(buildMode, nil) - enabledList := make([]string, 0, len(enabledModules)) - for module := range enabledModules { - if !disabledModules[module] { - enabledList = append(enabledList, module) - } - } - sort.Strings(enabledList) - return enabledList -} - -func NewBazelContext(c *config) (BazelContext, error) { - if c.BuildMode != BazelProdMode && c.BuildMode != BazelStagingMode { - return noopBazelContext{}, nil - } - - enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(c.BuildMode, c.BazelModulesForceEnabledByFlag()) - - paths := bazelPaths{ - soongOutDir: c.soongOutDir, - } - var missing []string - vars := []struct { - name string - ptr *string - - // True if the environment variable needs to be tracked so that changes to the variable - // cause the ninja file to be regenerated, false otherwise. False should only be set for - // environment variables that have no effect on the generated ninja file. - track bool - }{ - {"BAZEL_HOME", &paths.homeDir, true}, - {"BAZEL_PATH", &paths.bazelPath, true}, - {"BAZEL_OUTPUT_BASE", &paths.outputBase, true}, - {"BAZEL_WORKSPACE", &paths.workspaceDir, true}, - {"BAZEL_METRICS_DIR", &paths.metricsDir, false}, - {"BAZEL_DEPS_FILE", &paths.bazelDepsFile, true}, - } - for _, v := range vars { - if v.track { - if s := c.Getenv(v.name); len(s) > 1 { - *v.ptr = s - continue - } - } else if s, ok := c.env[v.name]; ok { - *v.ptr = s - } else { - missing = append(missing, v.name) - } - } - if len(missing) > 0 { - return nil, fmt.Errorf("missing required env vars to use bazel: %s", missing) - } - - targetBuildVariant := "user" - if c.Eng() { - targetBuildVariant = "eng" - } else if c.Debuggable() { - targetBuildVariant = "userdebug" - } - targetProduct := "unknown" - if c.HasDeviceProduct() { - targetProduct = c.DeviceProduct() - } - dclaMixedBuildsEnabledList := []string{} - if c.BuildMode == BazelProdMode { - dclaMixedBuildsEnabledList = allowlists.ProdDclaMixedBuildsEnabledList - } else if c.BuildMode == BazelStagingMode { - dclaMixedBuildsEnabledList = append(allowlists.ProdDclaMixedBuildsEnabledList, - allowlists.StagingDclaMixedBuildsEnabledList...) - } - dclaEnabledModules := map[string]bool{} - AddToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList) - return &mixedBuildBazelContext{ - bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)}, - paths: &paths, - bazelEnabledModules: enabledModules, - bazelDisabledModules: disabledModules, - bazelDclaEnabledModules: dclaEnabledModules, - targetProduct: targetProduct, - targetBuildVariant: targetBuildVariant, - }, nil -} - -func (p *bazelPaths) BazelMetricsDir() string { - return p.metricsDir -} - -func (context *mixedBuildBazelContext) IsModuleNameAllowed(moduleName string, withinApex bool) bool { - if context.bazelDisabledModules[moduleName] { - return false - } - if context.bazelEnabledModules[moduleName] { - return true - } - if withinApex && context.bazelDclaEnabledModules[moduleName] { - return true - } - - return false -} - -func pwdPrefix() string { - // Darwin doesn't have /proc - if runtime.GOOS != "darwin" { - return "PWD=/proc/self/cwd" - } - return "" -} - -type bazelCommand struct { - command string - // query or label - expression string -} - -type builtinBazelRunner struct { - useBazelProxy bool - outDir string -} - -// Issues the given bazel command with given build label and additional flags. -// Returns (stdout, stderr, error). The first and second return values are strings -// containing the stdout and stderr of the run command, and an error is returned if -// the invocation returned an error code. -func (r *builtinBazelRunner) issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (string, string, error) { - if r.useBazelProxy { - eventHandler.Begin("client_proxy") - defer eventHandler.End("client_proxy") - proxyClient := bazel.NewProxyClient(r.outDir) - resp, err := proxyClient.IssueCommand(cmdRequest) - - if err != nil { - return "", "", err - } - if len(resp.ErrorString) > 0 { - return "", "", fmt.Errorf(resp.ErrorString) - } - return resp.Stdout, resp.Stderr, nil - } else { - eventHandler.Begin("bazel command") - defer eventHandler.End("bazel command") - - stdout, stderr, err := bazel.ExecBazel(paths.bazelPath, absolutePath(paths.syntheticWorkspaceDir()), cmdRequest) - return string(stdout), string(stderr), err - } -} - -func (context *mixedBuildBazelContext) createBazelCommand(config Config, runName bazel.RunName, command bazelCommand, - extraFlags ...string) bazel.CmdRequest { - if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { - panic("Unknown GOOS: " + runtime.GOOS) - } - cmdFlags := []string{ - "--output_base=" + absolutePath(context.paths.outputBase), - command.command, - command.expression, - "--profile=" + shared.BazelMetricsFilename(context.paths, runName), - - "--host_platform=@soong_injection//product_config_platforms:mixed_builds_product_" + runtime.GOOS + "_x86_64", - "--//build/bazel/product_config:target_build_variant=" + context.targetBuildVariant, - // Don't specify --platforms, because on some products/branches (like kernel-build-tools) - // the main platform for mixed_builds_product-variant doesn't exist because an arch isn't - // specified in product config. The derivative platforms that config_node transitions into - // will still work. - - // Suppress noise - "--ui_event_filters=-INFO", - "--noshow_progress", - "--norun_validations", - } - cmdFlags = append(cmdFlags, extraFlags...) - - extraEnv := []string{ - "HOME=" + context.paths.homeDir, - pwdPrefix(), - "BUILD_DIR=" + absolutePath(context.paths.soongOutDir), - // Make OUT_DIR absolute here so build/bazel/bin/bazel uses the correct - // OUT_DIR at /out, instead of /out/soong/workspace/out. - "OUT_DIR=" + absolutePath(context.paths.outDir()), - // Disables local host detection of gcc; toolchain information is defined - // explicitly in BUILD files. - "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1", - } - capturedEnvVars, err := starlark_import.GetStarlarkValue[[]string]("captured_env_vars") - if err != nil { - panic(err) - } - for _, envvar := range capturedEnvVars { - val := config.Getenv(envvar) - if val == "" { - continue - } - extraEnv = append(extraEnv, fmt.Sprintf("%s=%s", envvar, val)) - } - envVars := append(os.Environ(), extraEnv...) - - return bazel.CmdRequest{cmdFlags, envVars} -} - -func (context *mixedBuildBazelContext) printableCqueryCommand(bazelCmd bazel.CmdRequest) string { - args := append([]string{context.paths.bazelPath}, bazelCmd.Argv...) - outputString := strings.Join(bazelCmd.Env, " ") + " \"" + strings.Join(args, "\" \"") + "\"" - return outputString -} - -func (context *mixedBuildBazelContext) mainBzlFileContents() []byte { - // TODO(cparsons): Define configuration transitions programmatically based - // on available archs. - contents := ` -##################################################### -# This file is generated by soong_build. Do not edit. -##################################################### -def _config_node_transition_impl(settings, attr): - if attr.os == "android" and attr.arch == "target": - target = "mixed_builds_product" - else: - target = "mixed_builds_product_%s_%s" % (attr.os, attr.arch) - apex_name = "" - if attr.within_apex: - # //build/bazel/rules/apex:apex_name has to be set to a non_empty value, - # otherwise //build/bazel/rules/apex:non_apex will be true and the - # "-D__ANDROID_APEX__" compiler flag will be missing. Apex_name is used - # in some validation on bazel side which don't really apply in mixed - # build because soong will do the work, so we just set it to a fixed - # value here. - apex_name = "dcla_apex" - outputs = { - "//command_line_option:platforms": "@soong_injection//product_config_platforms:%s" % target, - "@//build/bazel/rules/apex:within_apex": attr.within_apex, - "@//build/bazel/rules/apex:min_sdk_version": attr.apex_sdk_version, - "@//build/bazel/rules/apex:apex_name": apex_name, - "@//build/bazel/rules/apex:api_domain": attr.api_domain, - } - - return outputs - -_config_node_transition = transition( - implementation = _config_node_transition_impl, - inputs = [], - outputs = [ - "//command_line_option:platforms", - "@//build/bazel/rules/apex:within_apex", - "@//build/bazel/rules/apex:min_sdk_version", - "@//build/bazel/rules/apex:apex_name", - "@//build/bazel/rules/apex:api_domain", - ], -) - -def _passthrough_rule_impl(ctx): - return [DefaultInfo(files = depset(ctx.files.deps))] - -config_node = rule( - implementation = _passthrough_rule_impl, - attrs = { - "arch" : attr.string(mandatory = True), - "os" : attr.string(mandatory = True), - "within_apex" : attr.bool(default = False), - "apex_sdk_version" : attr.string(mandatory = True), - "api_domain" : attr.string(mandatory = True), - "deps" : attr.label_list(cfg = _config_node_transition, allow_files = True), - "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), - }, -) - - -# Rule representing the root of the build, to depend on all Bazel targets that -# are required for the build. Building this target will build the entire Bazel -# build tree. -mixed_build_root = rule( - implementation = _passthrough_rule_impl, - attrs = { - "deps" : attr.label_list(), - }, -) - -def _phony_root_impl(ctx): - return [] - -# Rule to depend on other targets but build nothing. -# This is useful as follows: building a target of this rule will generate -# symlink forests for all dependencies of the target, without executing any -# actions of the build. -phony_root = rule( - implementation = _phony_root_impl, - attrs = {"deps" : attr.label_list()}, -) -` - - return []byte(contents) -} - -func (context *mixedBuildBazelContext) mainBuildFileContents() []byte { - // TODO(cparsons): Map label to attribute programmatically; don't use hard-coded - // architecture mapping. - formatString := ` -# This file is generated by soong_build. Do not edit. -load(":main.bzl", "config_node", "mixed_build_root", "phony_root") - -%s - -mixed_build_root(name = "buildroot", - deps = [%s], - testonly = True, # Unblocks testonly deps. -) - -phony_root(name = "phonyroot", - deps = [":buildroot"], - testonly = True, # Unblocks testonly deps. -) -` - configNodeFormatString := ` -config_node(name = "%s", - arch = "%s", - os = "%s", - within_apex = %s, - apex_sdk_version = "%s", - api_domain = "%s", - deps = [%s], - testonly = True, # Unblocks testonly deps. -) -` - - configNodesSection := "" - - labelsByConfig := map[string][]string{} - - for _, val := range context.requests { - labelString := fmt.Sprintf("\"@%s\"", val.label) - configString := getConfigString(val) - labelsByConfig[configString] = append(labelsByConfig[configString], labelString) - } - - // Configs need to be sorted to maintain determinism of the BUILD file. - sortedConfigs := make([]string, 0, len(labelsByConfig)) - for val := range labelsByConfig { - sortedConfigs = append(sortedConfigs, val) - } - sort.Slice(sortedConfigs, func(i, j int) bool { return sortedConfigs[i] < sortedConfigs[j] }) - - allLabels := []string{} - for _, configString := range sortedConfigs { - labels := labelsByConfig[configString] - configTokens := strings.Split(configString, "|") - if len(configTokens) < 2 { - panic(fmt.Errorf("Unexpected config string format: %s", configString)) - } - archString := configTokens[0] - osString := configTokens[1] - withinApex := "False" - apexSdkVerString := "" - apiDomainString := "" - if osString == "android" { - // api domains are meaningful only for device variants - apiDomainString = "system" - } - targetString := fmt.Sprintf("%s_%s", osString, archString) - if len(configTokens) > 2 { - targetString += "_" + configTokens[2] - if configTokens[2] == withinApexToString(true) { - withinApex = "True" - } - } - if len(configTokens) > 3 { - targetString += "_" + configTokens[3] - apexSdkVerString = configTokens[3] - } - if len(configTokens) > 4 { - apiDomainString = configTokens[4] - targetString += "_" + apiDomainString - } - allLabels = append(allLabels, fmt.Sprintf("\":%s\"", targetString)) - labelsString := strings.Join(labels, ",\n ") - configNodesSection += fmt.Sprintf(configNodeFormatString, targetString, archString, osString, withinApex, apexSdkVerString, apiDomainString, - labelsString) - } - - return []byte(fmt.Sprintf(formatString, configNodesSection, strings.Join(allLabels, ",\n "))) -} - -func indent(original string) string { - result := "" - for _, line := range strings.Split(original, "\n") { - result += " " + line + "\n" - } - return result -} - -// Returns the file contents of the buildroot.cquery file that should be used for the cquery -// expression in order to obtain information about buildroot and its dependencies. -// The contents of this file depend on the mixedBuildBazelContext's requests; requests are enumerated -// and grouped by their request type. The data retrieved for each label depends on its -// request type. -func (context *mixedBuildBazelContext) cqueryStarlarkFileContents() []byte { - requestTypeToCqueryIdEntries := map[cqueryRequest][]string{} - requestTypes := []cqueryRequest{} - for _, val := range context.requests { - cqueryId := getCqueryId(val) - mapEntryString := fmt.Sprintf("%q : True", cqueryId) - if _, seenKey := requestTypeToCqueryIdEntries[val.requestType]; !seenKey { - requestTypes = append(requestTypes, val.requestType) - } - requestTypeToCqueryIdEntries[val.requestType] = - append(requestTypeToCqueryIdEntries[val.requestType], mapEntryString) - } - labelRegistrationMapSection := "" - functionDefSection := "" - mainSwitchSection := "" - - mapDeclarationFormatString := ` -%s = { - %s -} -` - functionDefFormatString := ` -def %s(target, id_string): -%s -` - mainSwitchSectionFormatString := ` - if id_string in %s: - return id_string + ">>" + %s(target, id_string) -` - - for _, requestType := range requestTypes { - labelMapName := requestType.Name() + "_Labels" - functionName := requestType.Name() + "_Fn" - labelRegistrationMapSection += fmt.Sprintf(mapDeclarationFormatString, - labelMapName, - strings.Join(requestTypeToCqueryIdEntries[requestType], ",\n ")) - functionDefSection += fmt.Sprintf(functionDefFormatString, - functionName, - indent(requestType.StarlarkFunctionBody())) - mainSwitchSection += fmt.Sprintf(mainSwitchSectionFormatString, - labelMapName, functionName) - } - - formatString := ` -# This file is generated by soong_build. Do not edit. - -{LABEL_REGISTRATION_MAP_SECTION} - -{FUNCTION_DEF_SECTION} - -def get_arch(target): - # TODO(b/199363072): filegroups and file targets aren't associated with any - # specific platform architecture in mixed builds. This is consistent with how - # Soong treats filegroups, but it may not be the case with manually-written - # filegroup BUILD targets. - buildoptions = build_options(target) - - if buildoptions == None: - # File targets do not have buildoptions. File targets aren't associated with - # any specific platform architecture in mixed builds, so use the host. - return "x86_64|linux" - platforms = buildoptions["//command_line_option:platforms"] - if len(platforms) != 1: - # An individual configured target should have only one platform architecture. - # Note that it's fine for there to be multiple architectures for the same label, - # but each is its own configured target. - fail("expected exactly 1 platform for " + str(target.label) + " but got " + str(platforms)) - platform_name = platforms[0].name - if platform_name == "host": - return "HOST" - if not platform_name.startswith("mixed_builds_product"): - fail("expected platform name of the form 'mixed_builds_product_android_' or 'mixed_builds_product_linux_', but was " + str(platforms)) - platform_name = platform_name.removeprefix("mixed_builds_product").removeprefix("_") - config_key = "" - if not platform_name: - config_key = "target|android" - elif platform_name.startswith("android_"): - config_key = platform_name.removeprefix("android_") + "|android" - elif platform_name.startswith("linux_"): - config_key = platform_name.removeprefix("linux_") + "|linux" - else: - fail("expected platform name of the form 'mixed_builds_product_android_' or 'mixed_builds_product_linux_', but was " + str(platforms)) - - within_apex = buildoptions.get("//build/bazel/rules/apex:within_apex") - apex_sdk_version = buildoptions.get("//build/bazel/rules/apex:min_sdk_version") - api_domain = buildoptions.get("//build/bazel/rules/apex:api_domain") - - if within_apex: - config_key += "|within_apex" - if apex_sdk_version != None and len(apex_sdk_version) > 0: - config_key += "|" + apex_sdk_version - if api_domain != None and len(api_domain) > 0: - config_key += "|" + api_domain - - return config_key - -def format(target): - id_string = str(target.label) + "|" + get_arch(target) - - # TODO(b/248106697): Remove once Bazel is updated to always normalize labels. - if id_string.startswith("//"): - id_string = "@" + id_string - - {MAIN_SWITCH_SECTION} - - # This target was not requested via cquery, and thus must be a dependency - # of a requested target. - return id_string + ">>NONE" -` - replacer := strings.NewReplacer( - "{LABEL_REGISTRATION_MAP_SECTION}", labelRegistrationMapSection, - "{FUNCTION_DEF_SECTION}", functionDefSection, - "{MAIN_SWITCH_SECTION}", mainSwitchSection) - - return []byte(replacer.Replace(formatString)) -} - -// Returns a path containing build-related metadata required for interfacing -// with Bazel. Example: out/soong/bazel. -func (p *bazelPaths) intermediatesDir() string { - return filepath.Join(p.soongOutDir, "bazel") -} - -// Returns the path where the contents of the @soong_injection repository live. -// It is used by Soong to tell Bazel things it cannot over the command line. -func (p *bazelPaths) injectedFilesDir() string { - return filepath.Join(p.soongOutDir, bazel.SoongInjectionDirName) -} - -// Returns the path of the synthetic Bazel workspace that contains a symlink -// forest composed the whole source tree and BUILD files generated by bp2build. -func (p *bazelPaths) syntheticWorkspaceDir() string { - return filepath.Join(p.soongOutDir, "workspace") -} - -// Returns the path to the top level out dir ($OUT_DIR). -func (p *bazelPaths) outDir() string { - return filepath.Dir(p.soongOutDir) -} - -const buildrootLabel = "@soong_injection//mixed_builds:buildroot" - -var ( - cqueryCmd = bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)} - aqueryCmd = bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)} - buildCmd = bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"} - allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd} -) - -// This can't be part of bp2build_product_config.go because it would create a circular go package dependency -func getLabelsForBazelSandwichPartitions(variables *ProductVariables) []string { - targetProduct := "unknown" - if variables.DeviceProduct != nil { - targetProduct = *variables.DeviceProduct - } - currentProductFolder := fmt.Sprintf("build/bazel/products/%s", targetProduct) - if len(variables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory) > 0 { - currentProductFolder = fmt.Sprintf("%s%s", variables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory, targetProduct) - } - var ret []string - if variables.PartitionVarsForBazelMigrationOnlyDoNotUse.PartitionQualifiedVariables["system"].BuildingImage { - ret = append(ret, "@//"+currentProductFolder+":system_image") - ret = append(ret, "@//"+currentProductFolder+":run_system_image_test") - } - return ret -} - -func GetBazelSandwichCqueryRequests(config Config) ([]cqueryKey, error) { - partitionLabels := getLabelsForBazelSandwichPartitions(&config.productVariables) - result := make([]cqueryKey, 0, len(partitionLabels)) - labelRegex := regexp.MustCompile("^@?//([a-zA-Z0-9/_-]+):[a-zA-Z0-9_-]+$") - // Note that bazel "targets" are different from soong "targets", the bazel targets are - // synonymous with soong modules, and soong targets are a configuration a module is built in. - for _, target := range partitionLabels { - match := labelRegex.FindStringSubmatch(target) - if match == nil { - return nil, fmt.Errorf("invalid label, must match `^@?//([a-zA-Z0-9/_-]+):[a-zA-Z0-9_-]+$`: %s", target) - } - - // change this to config.BuildOSTarget if we add host targets - soongTarget := config.AndroidCommonTarget - if soongTarget.Os.Class != Device { - // kernel-build-tools seems to set the AndroidCommonTarget to a linux host - // target for some reason, disable device builds in that case. - continue - } - - result = append(result, cqueryKey{ - label: target, - requestType: cquery.GetOutputFiles, - configKey: configKey{ - arch: soongTarget.Arch.String(), - osType: soongTarget.Os, - }, - }) - } - return result, nil -} - -// QueueBazelSandwichCqueryRequests queues cquery requests for all the bazel labels in -// bazel_sandwich_targets. These will later be given phony targets so that they can be built on the -// command line. -func (context *mixedBuildBazelContext) QueueBazelSandwichCqueryRequests(config Config) error { - requests, err := GetBazelSandwichCqueryRequests(config) - if err != nil { - return err - } - for _, request := range requests { - context.QueueBazelRequest(request.label, request.requestType, request.configKey) - } - - return nil -} - -// Issues commands to Bazel to receive results for all cquery requests -// queued in the BazelContext. -func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error { - eventHandler := ctx.GetEventHandler() - eventHandler.Begin("bazel") - defer eventHandler.End("bazel") - - if metricsDir := context.paths.BazelMetricsDir(); metricsDir != "" { - if err := os.MkdirAll(metricsDir, 0777); err != nil { - return err - } - } - context.results = make(map[cqueryKey]string) - if err := context.runCquery(config, ctx); err != nil { - return err - } - if err := context.runAquery(config, ctx); err != nil { - return err - } - if err := context.generateBazelSymlinks(config, ctx); err != nil { - return err - } - - // Clear requests. - context.requests = []cqueryKey{} - return nil -} - -func (context *mixedBuildBazelContext) runCquery(config Config, ctx invokeBazelContext) error { - eventHandler := ctx.GetEventHandler() - eventHandler.Begin("cquery") - defer eventHandler.End("cquery") - soongInjectionPath := absolutePath(context.paths.injectedFilesDir()) - mixedBuildsPath := filepath.Join(soongInjectionPath, "mixed_builds") - if _, err := os.Stat(mixedBuildsPath); os.IsNotExist(err) { - err = os.MkdirAll(mixedBuildsPath, 0777) - if err != nil { - return err - } - } - if err := writeFileBytesIfChanged(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil { - return err - } - if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil { - return err - } - if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil { - return err - } - cqueryFileRelpath := filepath.Join(context.paths.injectedFilesDir(), "buildroot.cquery") - if err := writeFileBytesIfChanged(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil { - return err - } - - extraFlags := []string{"--output=starlark", "--starlark:file=" + absolutePath(cqueryFileRelpath)} - if Bool(config.productVariables.ClangCoverage) { - extraFlags = append(extraFlags, "--collect_code_coverage") - } - - cqueryCmdRequest := context.createBazelCommand(config, bazel.CqueryBuildRootRunName, cqueryCmd, extraFlags...) - cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCmdRequest, context.paths, eventHandler) - if cqueryErr != nil { - return cqueryErr - } - cqueryCommandPrint := fmt.Sprintf("cquery command line:\n %s \n\n\n", context.printableCqueryCommand(cqueryCmdRequest)) - if err := os.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryCommandPrint+cqueryOutput), 0666); err != nil { - return err - } - cqueryResults := map[string]string{} - for _, outputLine := range strings.Split(cqueryOutput, "\n") { - if strings.Contains(outputLine, ">>") { - splitLine := strings.SplitN(outputLine, ">>", 2) - cqueryResults[splitLine[0]] = splitLine[1] - } - } - for _, val := range context.requests { - if cqueryResult, ok := cqueryResults[getCqueryId(val)]; ok { - context.results[val] = cqueryResult - } else { - return fmt.Errorf("missing result for bazel target %s. query output: [%s], cquery err: [%s]", - getCqueryId(val), cqueryOutput, cqueryErrorMessage) - } - } - return nil -} - -func writeFileBytesIfChanged(path string, contents []byte, perm os.FileMode) error { - oldContents, err := os.ReadFile(path) - if err != nil || !bytes.Equal(contents, oldContents) { - err = os.WriteFile(path, contents, perm) - } - return nil -} - -func (context *mixedBuildBazelContext) runAquery(config Config, ctx invokeBazelContext) error { - eventHandler := ctx.GetEventHandler() - eventHandler.Begin("aquery") - defer eventHandler.End("aquery") - // Issue an aquery command to retrieve action information about the bazel build tree. - // - // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's - // proto sources, which would add a number of unnecessary dependencies. - extraFlags := []string{"--output=proto", "--include_file_write_contents"} - if Bool(config.productVariables.ClangCoverage) { - extraFlags = append(extraFlags, "--collect_code_coverage") - paths := make([]string, 0, 2) - if p := config.productVariables.NativeCoveragePaths; len(p) > 0 { - for i := range p { - // TODO(b/259404593) convert path wildcard to regex values - if p[i] == "*" { - p[i] = ".*" - } - } - paths = append(paths, JoinWithPrefixAndSeparator(p, "+", ",")) - } - if p := config.productVariables.NativeCoverageExcludePaths; len(p) > 0 { - paths = append(paths, JoinWithPrefixAndSeparator(p, "-", ",")) - } - if len(paths) > 0 { - extraFlags = append(extraFlags, "--instrumentation_filter="+strings.Join(paths, ",")) - } - } - aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.AqueryBuildRootRunName, aqueryCmd, - extraFlags...), context.paths, eventHandler) - if err != nil { - return err - } - context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput), eventHandler) - return err -} - -func (context *mixedBuildBazelContext) generateBazelSymlinks(config Config, ctx invokeBazelContext) error { - eventHandler := ctx.GetEventHandler() - eventHandler.Begin("symlinks") - defer eventHandler.End("symlinks") - // Issue a build command of the phony root to generate symlink forests for dependencies of the - // Bazel build. This is necessary because aquery invocations do not generate this symlink forest, - // but some of symlinks may be required to resolve source dependencies of the build. - _, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.BazelBuildPhonyRootRunName, buildCmd), context.paths, eventHandler) - return err -} - -func (context *mixedBuildBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement { - return context.buildStatements -} - -func (context *mixedBuildBazelContext) AqueryDepsets() []bazel.AqueryDepset { - return context.depsets -} - -func (context *mixedBuildBazelContext) OutputBase() string { - return context.paths.outputBase -} - -// Singleton used for registering BUILD file ninja dependencies (needed -// for correctness of builds which use Bazel. -func BazelSingleton() Singleton { - return &bazelSingleton{} -} - -type bazelSingleton struct{} - -func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { - // bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled. - if !ctx.Config().IsMixedBuildsEnabled() { - return - } - - // Add ninja file dependencies for files which all bazel invocations require. - bazelBuildList := absolutePath(filepath.Join( - filepath.Dir(ctx.Config().moduleListFile), "bazel.list")) - ctx.AddNinjaFileDeps(bazelBuildList) - - data, err := os.ReadFile(bazelBuildList) - if err != nil { - ctx.Errorf(err.Error()) - } - files := strings.Split(strings.TrimSpace(string(data)), "\n") - for _, file := range files { - ctx.AddNinjaFileDeps(file) - } - - depsetHashToDepset := map[string]bazel.AqueryDepset{} - - for _, depset := range ctx.Config().BazelContext.AqueryDepsets() { - depsetHashToDepset[depset.ContentHash] = depset - - var outputs []Path - var orderOnlies []Path - for _, depsetDepHash := range depset.TransitiveDepSetHashes { - otherDepsetName := bazelDepsetName(depsetDepHash) - outputs = append(outputs, PathForPhony(ctx, otherDepsetName)) - } - for _, artifactPath := range depset.DirectArtifacts { - pathInBazelOut := PathForBazelOut(ctx, artifactPath) - if artifactPath == "bazel-out/volatile-status.txt" { - // See https://bazel.build/docs/user-manual#workspace-status - orderOnlies = append(orderOnlies, pathInBazelOut) - } else { - outputs = append(outputs, pathInBazelOut) - } - } - thisDepsetName := bazelDepsetName(depset.ContentHash) - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Outputs: []WritablePath{PathForPhony(ctx, thisDepsetName)}, - Implicits: outputs, - OrderOnly: orderOnlies, - }) - } - - executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__") - bazelOutDir := path.Join(executionRoot, "bazel-out") - rel, err := filepath.Rel(ctx.Config().OutDir(), executionRoot) - if err != nil { - ctx.Errorf("%s", err.Error()) - } - dotdotsToOutRoot := strings.Repeat("../", strings.Count(rel, "/")+1) - for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() { - // nil build statements are a valid case where we do not create an action because it is - // unnecessary or handled by other processing - if buildStatement == nil { - continue - } - if len(buildStatement.Command) > 0 { - rule := NewRuleBuilder(pctx, ctx) - intermediateDir, intermediateDirHash := intermediatePathForSboxMixedBuildAction(ctx, buildStatement) - if buildStatement.ShouldRunInSbox { - // Create a rule to build the output inside a sandbox - // This will create two changes of working directory - // 1. From ANDROID_BUILD_TOP to sbox top - // 2. From sbox top to a a synthetic mixed build execution root relative to it - // Finally, the outputs will be copied to intermediateDir - rule.Sbox(intermediateDir, - PathForOutput(ctx, "mixed_build_sbox_intermediates", intermediateDirHash+".textproto")). - SandboxInputs(). - // Since we will cd to mixed build execution root, set sbox's out subdir to empty - // Without this, we will try to copy from $SBOX_SANDBOX_DIR/out/out/bazel/output/execroot/__main__/... - SetSboxOutDirDirAsEmpty() - - // Create another set of rules to copy files from the intermediate dir to mixed build execution root - for _, outputPath := range buildStatement.OutputPaths { - ctx.Build(pctx, BuildParams{ - Rule: CpIfChanged, - Input: intermediateDir.Join(ctx, executionRoot, outputPath), - Output: PathForBazelOut(ctx, outputPath), - }) - } - } - createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset, dotdotsToOutRoot) - - desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) - rule.Build(fmt.Sprintf("bazel %d", index), desc) - continue - } - // Certain actions returned by aquery (for instance FileWrite) do not contain a command - // and thus require special treatment. If BuildStatement were an interface implementing - // buildRule(ctx) function, the code here would just call it. - // Unfortunately, the BuildStatement is defined in - // the 'bazel' package, which cannot depend on 'android' package where ctx is defined, - // because this would cause circular dependency. So, until we move aquery processing - // to the 'android' package, we need to handle special cases here. - switch buildStatement.Mnemonic { - case "RepoMappingManifest": - // It appears RepoMappingManifest files currently have - // non-deterministic content. Just emit empty files for - // now because they're unused. - out := PathForBazelOut(ctx, buildStatement.OutputPaths[0]) - WriteFileRuleVerbatim(ctx, out, "") - case "FileWrite", "SourceSymlinkManifest": - out := PathForBazelOut(ctx, buildStatement.OutputPaths[0]) - if buildStatement.IsExecutable { - WriteExecutableFileRuleVerbatim(ctx, out, buildStatement.FileContents) - } else { - WriteFileRuleVerbatim(ctx, out, buildStatement.FileContents) - } - case "SymlinkTree": - // build-runfiles arguments are the manifest file and the target directory - // where it creates the symlink tree according to this manifest (and then - // writes the MANIFEST file to it). - outManifest := PathForBazelOut(ctx, buildStatement.OutputPaths[0]) - outManifestPath := outManifest.String() - if !strings.HasSuffix(outManifestPath, "MANIFEST") { - panic("the base name of the symlink tree action should be MANIFEST, got " + outManifestPath) - } - outDir := filepath.Dir(outManifestPath) - ctx.Build(pctx, BuildParams{ - Rule: buildRunfilesRule, - Output: outManifest, - Inputs: []Path{PathForBazelOut(ctx, buildStatement.InputPaths[0])}, - Description: "symlink tree for " + outDir, - Args: map[string]string{ - "outDir": outDir, - }, - }) - default: - panic(fmt.Sprintf("unhandled build statement: %v", buildStatement)) - } - } - - // Create phony targets for all the bazel sandwich output files - requests, err := GetBazelSandwichCqueryRequests(ctx.Config()) - if err != nil { - ctx.Errorf(err.Error()) - } - for _, request := range requests { - files, err := ctx.Config().BazelContext.GetOutputFiles(request.label, request.configKey) - if err != nil { - ctx.Errorf(err.Error()) - } - filesAsPaths := make([]Path, 0, len(files)) - for _, file := range files { - filesAsPaths = append(filesAsPaths, PathForBazelOut(ctx, file)) - } - ctx.Phony("bazel_sandwich", filesAsPaths...) - } - ctx.Phony("checkbuild", PathForPhony(ctx, "bazel_sandwich")) -} - -// Returns a out dir path for a sandboxed mixed build action -func intermediatePathForSboxMixedBuildAction(ctx PathContext, statement *bazel.BuildStatement) (OutputPath, string) { - // An artifact can be generated by a single buildstatement. - // Use the hash of the first artifact to create a unique path - uniqueDir := sha1.New() - uniqueDir.Write([]byte(statement.OutputPaths[0])) - uniqueDirHashString := hex.EncodeToString(uniqueDir.Sum(nil)) - return PathForOutput(ctx, "mixed_build_sbox_intermediates", uniqueDirHashString), uniqueDirHashString -} - -// Register bazel-owned build statements (obtained from the aquery invocation). -func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset, dotdotsToOutRoot string) { - // executionRoot is the action cwd. - if buildStatement.ShouldRunInSbox { - // mkdir -p ensures that the directory exists when run via sbox - cmd.Text(fmt.Sprintf("mkdir -p '%s' && cd '%s' &&", executionRoot, executionRoot)) - } else { - cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot)) - } - - // Remove old outputs, as some actions might not rerun if the outputs are detected. - if len(buildStatement.OutputPaths) > 0 { - cmd.Text("rm -rf") // -r because outputs can be Bazel dir/tree artifacts. - for _, outputPath := range buildStatement.OutputPaths { - cmd.Text(fmt.Sprintf("'%s'", outputPath)) - } - cmd.Text("&&") - } - - for _, pair := range buildStatement.Env { - // Set per-action env variables, if any. - cmd.Flag(pair.Key + "=" + pair.Value) - } - - command := buildStatement.Command - command = strings.ReplaceAll(command, "{DOTDOTS_TO_OUTPUT_ROOT}", dotdotsToOutRoot) - - // The actual Bazel action. - if len(command) > 16*1024 { - commandFile := PathForBazelOut(ctx, buildStatement.OutputPaths[0]+".sh") - WriteFileRule(ctx, commandFile, command) - - cmd.Text("bash").Text(buildStatement.OutputPaths[0] + ".sh").Implicit(commandFile) - } else { - cmd.Text(command) - } - - for _, outputPath := range buildStatement.OutputPaths { - if buildStatement.ShouldRunInSbox { - // The full path has three components that get joined together - // 1. intermediate output dir that `sbox` will place the artifacts at - // 2. mixed build execution root - // 3. artifact path returned by aquery - intermediateDir, _ := intermediatePathForSboxMixedBuildAction(ctx, buildStatement) - cmd.ImplicitOutput(intermediateDir.Join(ctx, executionRoot, outputPath)) - } else { - cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath)) - } - } - for _, inputPath := range buildStatement.InputPaths { - cmd.Implicit(PathForBazelOut(ctx, inputPath)) - } - for _, inputDepsetHash := range buildStatement.InputDepsetHashes { - if buildStatement.ShouldRunInSbox { - // Bazel depsets are phony targets that are used to group files. - // We need to copy the grouped files into the sandbox - ds, _ := depsetHashToDepset[inputDepsetHash] - cmd.Implicits(PathsForBazelOut(ctx, ds.DirectArtifacts)) - } else { - otherDepsetName := bazelDepsetName(inputDepsetHash) - cmd.Implicit(PathForPhony(ctx, otherDepsetName)) - } - } - for _, implicitPath := range buildStatement.ImplicitDeps { - cmd.Implicit(PathForArbitraryOutput(ctx, implicitPath)) - } - - if depfile := buildStatement.Depfile; depfile != nil { - // The paths in depfile are relative to `executionRoot`. - // Hence, they need to be corrected by replacing "bazel-out" - // with the full `bazelOutDir`. - // Otherwise, implicit outputs and implicit inputs under "bazel-out/" - // would be deemed missing. - // (Note: The regexp uses a capture group because the version of sed - // does not support a look-behind pattern.) - replacement := fmt.Sprintf(`&& sed -i'' -E 's@(^|\s|")bazel-out/@\1%s/@g' '%s'`, - bazelOutDir, *depfile) - cmd.Text(replacement) - cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile)) - } - - for _, symlinkPath := range buildStatement.SymlinkPaths { - cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath)) - } -} - -func getCqueryId(key cqueryKey) string { - return key.label + "|" + getConfigString(key) -} - -func getConfigString(key cqueryKey) string { - arch := key.configKey.arch - if len(arch) == 0 || arch == "common" { - if key.configKey.osType.Class == Device { - // For the generic Android, the expected result is "target|android", which - // corresponds to the product_variable_config named "android_target" in - // build/bazel/platforms/BUILD.bazel. - arch = "target" - } else { - // Use host platform, which is currently hardcoded to be x86_64. - arch = "x86_64" - } - } - osName := key.configKey.osType.Name - if len(osName) == 0 || osName == "common_os" || osName == "linux_glibc" || osName == "linux_musl" { - // Use host OS, which is currently hardcoded to be linux. - osName = "linux" - } - keyString := arch + "|" + osName - if key.configKey.apexKey.WithinApex { - keyString += "|" + withinApexToString(key.configKey.apexKey.WithinApex) - } - - if len(key.configKey.apexKey.ApexSdkVersion) > 0 { - keyString += "|" + key.configKey.apexKey.ApexSdkVersion - } - - if len(key.configKey.apexKey.ApiDomain) > 0 { - keyString += "|" + key.configKey.apexKey.ApiDomain - } - - return keyString -} - -func GetConfigKey(ctx BaseModuleContext) configKey { - return configKey{ - // use string because Arch is not a valid key in go - arch: ctx.Arch().String(), - osType: ctx.Os(), - } -} - -func GetConfigKeyApexVariant(ctx BaseModuleContext, apexKey *ApexConfigKey) configKey { - configKey := GetConfigKey(ctx) - - if apexKey != nil { - configKey.apexKey = ApexConfigKey{ - WithinApex: apexKey.WithinApex, - ApexSdkVersion: apexKey.ApexSdkVersion, - ApiDomain: apexKey.ApiDomain, - } - } - - return configKey -} - -func bazelDepsetName(contentHash string) string { - return fmt.Sprintf("bazel_depset_%s", contentHash) -} diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go deleted file mode 100644 index 9a3c8fc7c..000000000 --- a/android/bazel_handler_test.go +++ /dev/null @@ -1,426 +0,0 @@ -package android - -import ( - "encoding/json" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - - "android/soong/bazel" - "android/soong/bazel/cquery" - analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" - - "github.com/google/blueprint/metrics" - "google.golang.org/protobuf/proto" -) - -var testConfig = TestConfig("out", nil, "", nil) - -type testInvokeBazelContext struct{} - -type mockBazelRunner struct { - testHelper *testing.T - // Stores mock behavior. If an issueBazelCommand request is made for command - // k, and {k:v} is present in this map, then the mock will return v. - bazelCommandResults map[bazelCommand]string - // Requests actually made of the mockBazelRunner with issueBazelCommand, - // keyed by the command they represent. - bazelCommandRequests map[bazelCommand]bazel.CmdRequest -} - -func (r *mockBazelRunner) bazelCommandForRequest(cmdRequest bazel.CmdRequest) bazelCommand { - for _, arg := range cmdRequest.Argv { - for _, cmdType := range allBazelCommands { - if arg == cmdType.command { - return cmdType - } - } - } - r.testHelper.Fatalf("Unrecognized bazel request: %s", cmdRequest) - return cqueryCmd -} - -func (r *mockBazelRunner) issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (string, string, error) { - command := r.bazelCommandForRequest(cmdRequest) - r.bazelCommandRequests[command] = cmdRequest - return r.bazelCommandResults[command], "", nil -} - -func (t *testInvokeBazelContext) GetEventHandler() *metrics.EventHandler { - return &metrics.EventHandler{} -} - -func TestRequestResultsAfterInvokeBazel(t *testing.T) { - label_foo := "@//foo:foo" - label_bar := "@//foo:bar" - apexKey := ApexConfigKey{ - WithinApex: true, - ApexSdkVersion: "29", - ApiDomain: "myapex", - } - cfg_foo := configKey{"arm64_armv8-a", Android, apexKey} - cfg_bar := configKey{arch: "arm64_armv8-a", osType: Android} - cmd_results := []string{ - `@//foo:foo|arm64_armv8-a|android|within_apex|29|myapex>>out/foo/foo.txt`, - `@//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`, - } - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{cqueryCmd: strings.Join(cmd_results, "\n")}) - - bazelContext.QueueBazelRequest(label_foo, cquery.GetOutputFiles, cfg_foo) - bazelContext.QueueBazelRequest(label_bar, cquery.GetOutputFiles, cfg_bar) - err := bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{}) - if err != nil { - t.Fatalf("Did not expect error invoking Bazel, but got %s", err) - } - verifyCqueryResult(t, bazelContext, label_foo, cfg_foo, "out/foo/foo.txt") - verifyCqueryResult(t, bazelContext, label_bar, cfg_bar, "out/foo/bar.txt") -} - -func verifyCqueryResult(t *testing.T, ctx *mixedBuildBazelContext, label string, cfg configKey, result string) { - g, err := ctx.GetOutputFiles(label, cfg) - if err != nil { - t.Errorf("Expected cquery results after running InvokeBazel(), but got err %v", err) - } else if w := []string{result}; !reflect.DeepEqual(w, g) { - t.Errorf("Expected output %s, got %s", w, g) - } -} - -func TestInvokeBazelWritesBazelFiles(t *testing.T) { - bazelContext, baseDir := testBazelContext(t, map[bazelCommand]string{}) - err := bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{}) - if err != nil { - t.Fatalf("Did not expect error invoking Bazel, but got %s", err) - } - if _, err := os.Stat(filepath.Join(baseDir, "soong_injection", "mixed_builds", "main.bzl")); os.IsNotExist(err) { - t.Errorf("Expected main.bzl to exist, but it does not") - } else if err != nil { - t.Errorf("Unexpected error stating main.bzl %s", err) - } - - if _, err := os.Stat(filepath.Join(baseDir, "soong_injection", "mixed_builds", "BUILD.bazel")); os.IsNotExist(err) { - t.Errorf("Expected BUILD.bazel to exist, but it does not") - } else if err != nil { - t.Errorf("Unexpected error stating BUILD.bazel %s", err) - } - - if _, err := os.Stat(filepath.Join(baseDir, "soong_injection", "WORKSPACE.bazel")); os.IsNotExist(err) { - t.Errorf("Expected WORKSPACE.bazel to exist, but it does not") - } else if err != nil { - t.Errorf("Unexpected error stating WORKSPACE.bazel %s", err) - } -} - -func TestInvokeBazelPopulatesBuildStatements(t *testing.T) { - type testCase struct { - input string - command string - } - - var testCases = []testCase{ - {` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_Id": 1, - "action_Key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_Ids": [1], - "primary_output_id": 1 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }] -}`, - "cd 'test/exec_root' && rm -rf 'one' && touch foo", - }, {` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 10 }, - { "id": 2, "path_fragment_id": 20 }], - "actions": [{ - "target_Id": 100, - "action_Key": "x", - "mnemonic": "x", - "arguments": ["bogus", "command"], - "output_Ids": [1, 2], - "primary_output_id": 1 - }], - "path_fragments": [ - { "id": 10, "label": "one", "parent_id": 30 }, - { "id": 20, "label": "one.d", "parent_id": 30 }, - { "id": 30, "label": "parent" }] -}`, - `cd 'test/exec_root' && rm -rf 'parent/one' && bogus command && sed -i'' -E 's@(^|\s|")bazel-out/@\1test/bazel_out/@g' 'parent/one.d'`, - }, - } - - for i, testCase := range testCases { - data, err := JsonToActionGraphContainer(testCase.input) - if err != nil { - t.Error(err) - } - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{aqueryCmd: string(data)}) - - err = bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{}) - if err != nil { - t.Fatalf("testCase #%d: did not expect error invoking Bazel, but got %s", i+1, err) - } - - got := bazelContext.BuildStatementsToRegister() - if want := 1; len(got) != want { - t.Fatalf("expected %d registered build statements, but got %#v", want, got) - } - - cmd := RuleBuilderCommand{} - ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))} - createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "") - if actual, expected := cmd.buf.String(), testCase.command; expected != actual { - t.Errorf("expected: [%s], actual: [%s]", expected, actual) - } - } -} - -func TestMixedBuildSandboxedAction(t *testing.T) { - input := `{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_Id": 1, - "action_Key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_Ids": [1], - "primary_output_id": 1 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }] -}` - data, err := JsonToActionGraphContainer(input) - if err != nil { - t.Error(err) - } - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{aqueryCmd: string(data)}) - - err = bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{}) - if err != nil { - t.Fatalf("TestMixedBuildSandboxedAction did not expect error invoking Bazel, but got %s", err) - } - - statement := bazelContext.BuildStatementsToRegister()[0] - statement.ShouldRunInSbox = true - - cmd := RuleBuilderCommand{} - ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))} - createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "") - // Assert that the output is generated in an intermediate directory - // fe05bcdcdc4928012781a5f1a2a77cbb5398e106 is the sha1 checksum of "one" - if actual, expected := cmd.outputs[0].String(), "out/soong/mixed_build_sbox_intermediates/fe05bcdcdc4928012781a5f1a2a77cbb5398e106/test/exec_root/one"; expected != actual { - t.Errorf("expected: [%s], actual: [%s]", expected, actual) - } - - // Assert the actual command remains unchanged inside the sandbox - if actual, expected := cmd.buf.String(), "mkdir -p 'test/exec_root' && cd 'test/exec_root' && rm -rf 'one' && touch foo"; expected != actual { - t.Errorf("expected: [%s], actual: [%s]", expected, actual) - } -} - -func TestCoverageFlagsAfterInvokeBazel(t *testing.T) { - testConfig.productVariables.ClangCoverage = boolPtr(true) - - testConfig.productVariables.NativeCoveragePaths = []string{"foo1", "foo2"} - testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1", "bar2"} - verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1,+foo2,-bar1,-bar2") - - testConfig.productVariables.NativeCoveragePaths = []string{"foo1"} - testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"} - verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1,-bar1") - - testConfig.productVariables.NativeCoveragePaths = []string{"foo1"} - testConfig.productVariables.NativeCoverageExcludePaths = nil - verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1") - - testConfig.productVariables.NativeCoveragePaths = nil - testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"} - verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=-bar1") - - testConfig.productVariables.NativeCoveragePaths = []string{"*"} - testConfig.productVariables.NativeCoverageExcludePaths = nil - verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+.*") - - testConfig.productVariables.ClangCoverage = boolPtr(false) - verifyAqueryDoesNotContainSubstrings(t, testConfig, "collect_code_coverage", "instrumentation_filter") -} - -func TestBazelRequestsSorted(t *testing.T) { - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) - - cfgKeyArm64Android := configKey{arch: "arm64_armv8-a", osType: Android} - cfgKeyArm64Linux := configKey{arch: "arm64_armv8-a", osType: Linux} - cfgKeyOtherAndroid := configKey{arch: "otherarch", osType: Android} - - bazelContext.QueueBazelRequest("zzz", cquery.GetOutputFiles, cfgKeyArm64Android) - bazelContext.QueueBazelRequest("ccc", cquery.GetApexInfo, cfgKeyArm64Android) - bazelContext.QueueBazelRequest("duplicate", cquery.GetOutputFiles, cfgKeyArm64Android) - bazelContext.QueueBazelRequest("duplicate", cquery.GetOutputFiles, cfgKeyArm64Android) - bazelContext.QueueBazelRequest("xxx", cquery.GetOutputFiles, cfgKeyArm64Linux) - bazelContext.QueueBazelRequest("aaa", cquery.GetOutputFiles, cfgKeyArm64Android) - bazelContext.QueueBazelRequest("aaa", cquery.GetOutputFiles, cfgKeyOtherAndroid) - bazelContext.QueueBazelRequest("bbb", cquery.GetOutputFiles, cfgKeyOtherAndroid) - - if len(bazelContext.requests) != 7 { - t.Error("Expected 7 request elements, but got", len(bazelContext.requests)) - } - - lastString := "" - for _, val := range bazelContext.requests { - thisString := val.String() - if thisString <= lastString { - t.Errorf("Requests are not ordered correctly. '%s' came before '%s'", lastString, thisString) - } - lastString = thisString - } -} - -func TestIsModuleNameAllowed(t *testing.T) { - libDisabled := "lib_disabled" - libEnabled := "lib_enabled" - libDclaWithinApex := "lib_dcla_within_apex" - libDclaNonApex := "lib_dcla_non_apex" - libNotConverted := "lib_not_converted" - - disabledModules := map[string]bool{ - libDisabled: true, - } - enabledModules := map[string]bool{ - libEnabled: true, - } - dclaEnabledModules := map[string]bool{ - libDclaWithinApex: true, - libDclaNonApex: true, - } - - bazelContext := &mixedBuildBazelContext{ - bazelEnabledModules: enabledModules, - bazelDisabledModules: disabledModules, - bazelDclaEnabledModules: dclaEnabledModules, - } - - if bazelContext.IsModuleNameAllowed(libDisabled, true) { - t.Fatalf("%s shouldn't be allowed for mixed build", libDisabled) - } - - if !bazelContext.IsModuleNameAllowed(libEnabled, true) { - t.Fatalf("%s should be allowed for mixed build", libEnabled) - } - - if !bazelContext.IsModuleNameAllowed(libDclaWithinApex, true) { - t.Fatalf("%s should be allowed for mixed build", libDclaWithinApex) - } - - if bazelContext.IsModuleNameAllowed(libDclaNonApex, false) { - t.Fatalf("%s shouldn't be allowed for mixed build", libDclaNonApex) - } - - if bazelContext.IsModuleNameAllowed(libNotConverted, true) { - t.Fatalf("%s shouldn't be allowed for mixed build", libNotConverted) - } -} - -func verifyAqueryContainsFlags(t *testing.T, config Config, expected ...string) { - t.Helper() - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) - - err := bazelContext.InvokeBazel(config, &testInvokeBazelContext{}) - if err != nil { - t.Fatalf("Did not expect error invoking Bazel, but got %s", err) - } - - sliceContains := func(slice []string, x string) bool { - for _, s := range slice { - if s == x { - return true - } - } - return false - } - - aqueryArgv := bazelContext.bazelRunner.(*mockBazelRunner).bazelCommandRequests[aqueryCmd].Argv - - for _, expectedFlag := range expected { - if !sliceContains(aqueryArgv, expectedFlag) { - t.Errorf("aquery does not contain expected flag %#v. Argv was: %#v", expectedFlag, aqueryArgv) - } - } -} - -func verifyAqueryDoesNotContainSubstrings(t *testing.T, config Config, substrings ...string) { - t.Helper() - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) - - err := bazelContext.InvokeBazel(config, &testInvokeBazelContext{}) - if err != nil { - t.Fatalf("Did not expect error invoking Bazel, but got %s", err) - } - - sliceContainsSubstring := func(slice []string, substring string) bool { - for _, s := range slice { - if strings.Contains(s, substring) { - return true - } - } - return false - } - - aqueryArgv := bazelContext.bazelRunner.(*mockBazelRunner).bazelCommandRequests[aqueryCmd].Argv - - for _, substring := range substrings { - if sliceContainsSubstring(aqueryArgv, substring) { - t.Errorf("aquery contains unexpected substring %#v. Argv was: %#v", substring, aqueryArgv) - } - } -} - -func testBazelContext(t *testing.T, bazelCommandResults map[bazelCommand]string) (*mixedBuildBazelContext, string) { - t.Helper() - p := bazelPaths{ - soongOutDir: t.TempDir(), - outputBase: "outputbase", - workspaceDir: "workspace_dir", - } - if _, exists := bazelCommandResults[aqueryCmd]; !exists { - bazelCommandResults[aqueryCmd] = "" - } - runner := &mockBazelRunner{ - testHelper: t, - bazelCommandResults: bazelCommandResults, - bazelCommandRequests: map[bazelCommand]bazel.CmdRequest{}, - } - return &mixedBuildBazelContext{ - bazelRunner: runner, - paths: &p, - }, p.soongOutDir -} - -// Transform the json format to ActionGraphContainer -func JsonToActionGraphContainer(inputString string) ([]byte, error) { - var aqueryProtoResult analysis_v2_proto.ActionGraphContainer - err := json.Unmarshal([]byte(inputString), &aqueryProtoResult) - if err != nil { - return []byte(""), err - } - data, _ := proto.Marshal(&aqueryProtoResult) - return data, err -} diff --git a/android/bazel_paths.go b/android/bazel_paths.go deleted file mode 100644 index f25803ccc..000000000 --- a/android/bazel_paths.go +++ /dev/null @@ -1,675 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "android/soong/bazel" - - "github.com/google/blueprint" - "github.com/google/blueprint/pathtools" -) - -// bazel_paths contains methods to: -// * resolve Soong path and module references into bazel.LabelList -// * resolve Bazel path references into Soong-compatible paths -// -// There is often a similar method for Bazel as there is for Soong path handling and should be used -// in similar circumstances -// -// Bazel Soong -// ============================================================== -// BazelLabelForModuleSrc PathForModuleSrc -// BazelLabelForModuleSrcExcludes PathForModuleSrcExcludes -// BazelLabelForModuleDeps n/a -// tbd PathForSource -// tbd ExistentPathsForSources -// PathForBazelOut PathForModuleOut -// -// Use cases: -// * Module contains a property (often tagged `android:"path"`) that expects paths *relative to the -// module directory*: -// * BazelLabelForModuleSrcExcludes, if the module also contains an excludes_ property -// * BazelLabelForModuleSrc, otherwise -// * Converting references to other modules to Bazel Labels: -// BazelLabelForModuleDeps -// * Converting a path obtained from bazel_handler cquery results: -// PathForBazelOut -// -// NOTE: all Soong globs are expanded within Soong rather than being converted to a Bazel glob -// syntax. This occurs because Soong does not have a concept of crossing package boundaries, -// so the glob as computed by Soong may contain paths that cross package-boundaries. These -// would be unknowingly omitted if the glob were handled by Bazel. By expanding globs within -// Soong, we support identification and detection (within Bazel) use of paths that cross -// package boundaries. -// -// Path resolution: -// * filepath/globs: resolves as itself or is converted to an absolute Bazel label (e.g. -// //path/to/dir:) if path exists in a separate package or subpackage. -// * references to other modules (using the ":name{.tag}" syntax). These resolve as a Bazel label -// for a target. If the Bazel target is in the local module directory, it will be returned -// relative to the current package (e.g. ":"). Otherwise, it will be returned as an -// absolute Bazel label (e.g. "//path/to/dir:"). If the reference to another module -// cannot be resolved,the function will panic. This is often due to the dependency not being added -// via an AddDependency* method. - -// BazelConversionContext is a minimal context interface to check if a module should be converted by bp2build, -// with functions containing information to match against allowlists and denylists. -// If a module is deemed to be convertible by bp2build, then it should rely on a -// BazelConversionPathContext for more functions for dep/path features. -type BazelConversionContext interface { - Config() Config - - Module() Module - OtherModuleType(m blueprint.Module) string - OtherModuleName(m blueprint.Module) string - OtherModuleDir(m blueprint.Module) string - ModuleErrorf(format string, args ...interface{}) -} - -// A subset of the ModuleContext methods which are sufficient to resolve references to paths/deps in -// order to form a Bazel-compatible label for conversion. -type BazelConversionPathContext interface { - EarlyModulePathContext - BazelConversionContext - - ModuleName() string - ModuleType() string - ModuleErrorf(fmt string, args ...interface{}) - PropertyErrorf(property, fmt string, args ...interface{}) - GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) - ModuleFromName(name string) (blueprint.Module, bool) - AddUnconvertedBp2buildDep(string) - AddMissingBp2buildDep(dep string) -} - -// BazelLabelForModuleDeps expects a list of reference to other modules, ("" -// or ":") and returns a Bazel-compatible label which corresponds to dependencies on the -// module within the given ctx. -func BazelLabelForModuleDeps(ctx Bp2buildMutatorContext, modules []string) bazel.LabelList { - return BazelLabelForModuleDepsWithFn(ctx, modules, BazelModuleLabel, true) -} - -// BazelLabelForModuleWholeDepsExcludes expects two lists: modules (containing modules to include in -// the list), and excludes (modules to exclude from the list). Both of these should contain -// references to other modules, ("" or ":"). It returns a Bazel-compatible label -// list which corresponds to dependencies on the module within the given ctx, and the excluded -// dependencies. Prebuilt dependencies will be appended with _alwayslink so they can be handled as -// whole static libraries. -func BazelLabelForModuleDepsExcludes(ctx Bp2buildMutatorContext, modules, excludes []string) bazel.LabelList { - return BazelLabelForModuleDepsExcludesWithFn(ctx, modules, excludes, BazelModuleLabel) -} - -// BazelLabelForModuleDepsWithFn expects a list of reference to other modules, ("" -// or ":") and applies moduleToLabelFn to determine and return a Bazel-compatible label -// which corresponds to dependencies on the module within the given ctx. -func BazelLabelForModuleDepsWithFn(ctx Bp2buildMutatorContext, modules []string, - moduleToLabelFn func(BazelConversionPathContext, blueprint.Module) string, - markAsDeps bool) bazel.LabelList { - var labels bazel.LabelList - // In some cases, a nil string list is different than an explicitly empty list. - if len(modules) == 0 && modules != nil { - labels.Includes = []bazel.Label{} - return labels - } - modules = FirstUniqueStrings(modules) - for _, module := range modules { - bpText := module - if m := SrcIsModule(module); m == "" { - module = ":" + module - } - if m, t := SrcIsModuleWithTag(module); m != "" { - l := getOtherModuleLabel(ctx, m, t, moduleToLabelFn, markAsDeps) - if l != nil { - l.OriginalModuleName = bpText - labels.Includes = append(labels.Includes, *l) - } - } else { - ctx.ModuleErrorf("%q, is not a module reference", module) - } - } - return labels -} - -// BazelLabelForModuleDepsExcludesWithFn expects two lists: modules (containing modules to include in the -// list), and excludes (modules to exclude from the list). Both of these should contain references -// to other modules, ("" or ":"). It applies moduleToLabelFn to determine and return a -// Bazel-compatible label list which corresponds to dependencies on the module within the given ctx, and -// the excluded dependencies. -func BazelLabelForModuleDepsExcludesWithFn(ctx Bp2buildMutatorContext, modules, excludes []string, - moduleToLabelFn func(BazelConversionPathContext, blueprint.Module) string) bazel.LabelList { - moduleLabels := BazelLabelForModuleDepsWithFn(ctx, RemoveListFromList(modules, excludes), moduleToLabelFn, true) - if len(excludes) == 0 { - return moduleLabels - } - excludeLabels := BazelLabelForModuleDepsWithFn(ctx, excludes, moduleToLabelFn, false) - return bazel.LabelList{ - Includes: moduleLabels.Includes, - Excludes: excludeLabels.Includes, - } -} - -func BazelLabelForModuleSrcSingle(ctx Bp2buildMutatorContext, path string) bazel.Label { - if srcs := BazelLabelForModuleSrcExcludes(ctx, []string{path}, []string(nil)).Includes; len(srcs) > 0 { - return srcs[0] - } - return bazel.Label{} -} - -func BazelLabelForModuleDepSingle(ctx Bp2buildMutatorContext, path string) bazel.Label { - if srcs := BazelLabelForModuleDepsExcludes(ctx, []string{path}, []string(nil)).Includes; len(srcs) > 0 { - return srcs[0] - } - return bazel.Label{} -} - -// BazelLabelForModuleSrc expects a list of path (relative to local module directory) and module -// references (":") and returns a bazel.LabelList{} containing the resolved references in -// paths, relative to the local module, or Bazel-labels (absolute if in a different package or -// relative if within the same package). -// Properties must have been annotated with struct tag `android:"path"` so that dependencies modules -// will have already been handled by the pathdeps mutator. -func BazelLabelForModuleSrc(ctx Bp2buildMutatorContext, paths []string) bazel.LabelList { - return BazelLabelForModuleSrcExcludes(ctx, paths, []string(nil)) -} - -// BazelLabelForModuleSrc expects lists of path and excludes (relative to local module directory) -// and module references (":") and returns a bazel.LabelList{} containing the resolved -// references in paths, minus those in excludes, relative to the local module, or Bazel-labels -// (absolute if in a different package or relative if within the same package). -// Properties must have been annotated with struct tag `android:"path"` so that dependencies modules -// will have already been handled by the pathdeps mutator. -func BazelLabelForModuleSrcExcludes(ctx Bp2buildMutatorContext, paths, excludes []string) bazel.LabelList { - excludeLabels := expandSrcsForBazel(ctx, excludes, []string(nil), false) - excluded := make([]string, 0, len(excludeLabels.Includes)) - for _, e := range excludeLabels.Includes { - excluded = append(excluded, e.Label) - } - labels := expandSrcsForBazel(ctx, paths, excluded, true) - labels.Excludes = excludeLabels.Includes - labels = TransformSubpackagePaths(ctx.Config(), ctx.ModuleDir(), labels) - return labels -} - -func BazelLabelForSrcPatternExcludes(ctx BazelConversionPathContext, dir, pattern string, excludes []string) bazel.LabelList { - topRelPaths, err := ctx.GlobWithDeps(filepath.Join(dir, pattern), excludes) - if err != nil { - ctx.ModuleErrorf("Could not search dir: %s for pattern %s due to %v\n", dir, pattern, err) - } - // An intermediate list of labels relative to `dir` that assumes that there no subpacakges beneath `dir` - dirRelLabels := []bazel.Label{} - for _, topRelPath := range topRelPaths { - dirRelPath := Rel(ctx, dir, topRelPath) - dirRelLabels = append(dirRelLabels, bazel.Label{Label: "./" + dirRelPath}) - } - // Return the package boudary resolved labels - return TransformSubpackagePaths(ctx.Config(), dir, bazel.MakeLabelList(dirRelLabels)) -} - -// Returns true if a prefix + components[:i] is a package boundary. -// -// A package boundary is determined by a BUILD file in the directory. This can happen in 2 cases: -// -// 1. An Android.bp exists, which bp2build will always convert to a sibling BUILD file. -// 2. An Android.bp doesn't exist, but a checked-in BUILD/BUILD.bazel file exists, and that file -// is allowlisted by the bp2build configuration to be merged into the symlink forest workspace. -func isPackageBoundary(config Config, prefix string, components []string, componentIndex int) bool { - isSymlink := func(c Config, path string) bool { - f, err := c.fs.Lstat(path) - if err != nil { - // The file does not exist - return false - } - return f.Mode()&os.ModeSymlink == os.ModeSymlink - } - prefix = filepath.Join(prefix, filepath.Join(components[:componentIndex+1]...)) - if exists, _, _ := config.fs.Exists(filepath.Join(prefix, "Android.bp")); exists { - return true - } else if config.Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(prefix) || isSymlink(config, prefix) { - if exists, _, _ := config.fs.Exists(filepath.Join(prefix, "BUILD")); exists { - return true - } else if exists, _, _ := config.fs.Exists(filepath.Join(prefix, "BUILD.bazel")); exists { - return true - } - } - - return false -} - -// Transform a path (if necessary) to acknowledge package boundaries -// -// e.g. something like -// -// async_safe/include/async_safe/CHECK.h -// -// might become -// -// //bionic/libc/async_safe:include/async_safe/CHECK.h -// -// if the "async_safe" directory is actually a package and not just a directory. -// -// In particular, paths that extend into packages are transformed into absolute labels beginning with //. -func transformSubpackagePath(cfg Config, dir string, path bazel.Label) bazel.Label { - var newPath bazel.Label - - // Don't transform OriginalModuleName - newPath.OriginalModuleName = path.OriginalModuleName - // if it wasn't a module, store the original path. We may need the original path to replace - // references if it is actually in another package - if path.OriginalModuleName == "" { - newPath.OriginalModuleName = path.Label - } - - if strings.HasPrefix(path.Label, "//") { - // Assume absolute labels are already correct (e.g. //path/to/some/package:foo.h) - newPath.Label = path.Label - return newPath - } - if strings.HasPrefix(path.Label, "./") { - // Drop "./" for consistent handling of paths. - // Specifically, to not let "." be considered a package boundary. - // Say `inputPath` is `x/Android.bp` and that file has some module - // with `srcs=["y/a.c", "z/b.c"]`. - // And say the directory tree is: - // x - // ├── Android.bp - // ├── y - // │ ├── a.c - // │ └── Android.bp - // └── z - // └── b.c - // Then bazel equivalent labels in srcs should be: - // //x/y:a.c, x/z/b.c - // The above should still be the case if `x/Android.bp` had - // srcs=["./y/a.c", "./z/b.c"] - // However, if we didn't strip "./", we'd get - // //x/./y:a.c, //x/.:z/b.c - path.Label = strings.TrimPrefix(path.Label, "./") - } - pathComponents := strings.Split(path.Label, "/") - newLabel := "" - foundPackageBoundary := false - // Check the deepest subdirectory first and work upwards - for i := len(pathComponents) - 1; i >= 0; i-- { - pathComponent := pathComponents[i] - var sep string - if !foundPackageBoundary && isPackageBoundary(cfg, dir, pathComponents, i) { - sep = ":" - foundPackageBoundary = true - } else { - sep = "/" - } - if newLabel == "" { - newLabel = pathComponent - } else { - newLabel = pathComponent + sep + newLabel - } - } - if foundPackageBoundary { - // Ensure paths end up looking like //bionic/... instead of //./bionic/... - moduleDir := dir - if strings.HasPrefix(moduleDir, ".") { - moduleDir = moduleDir[1:] - } - // Make the path into an absolute label (e.g. //bionic/libc/foo:bar.h instead of just foo:bar.h) - if moduleDir == "" { - newLabel = "//" + newLabel - } else { - newLabel = "//" + moduleDir + "/" + newLabel - } - } - newPath.Label = newLabel - - return newPath -} - -// Transform paths to acknowledge package boundaries -// See transformSubpackagePath() for more information -func TransformSubpackagePaths(cfg Config, dir string, paths bazel.LabelList) bazel.LabelList { - var newPaths bazel.LabelList - for _, include := range paths.Includes { - newPaths.Includes = append(newPaths.Includes, transformSubpackagePath(cfg, dir, include)) - } - for _, exclude := range paths.Excludes { - newPaths.Excludes = append(newPaths.Excludes, transformSubpackagePath(cfg, dir, exclude)) - } - return newPaths -} - -// Converts root-relative Paths to a list of bazel.Label relative to the module in ctx. -func RootToModuleRelativePaths(ctx BazelConversionPathContext, paths Paths) []bazel.Label { - var newPaths []bazel.Label - for _, path := range PathsWithModuleSrcSubDir(ctx, paths, "") { - s := path.Rel() - newPaths = append(newPaths, bazel.Label{Label: s}) - } - return newPaths -} - -var Bp2buildDepTag bp2buildDepTag - -type bp2buildDepTag struct { - blueprint.BaseDependencyTag -} - -// expandSrcsForBazel returns bazel.LabelList with paths rooted from the module's local source -// directory and Bazel target labels, excluding those included in the excludes argument (which -// should already be expanded to resolve references to Soong-modules). Valid elements of paths -// include: -// - filepath, relative to local module directory, resolves as a filepath relative to the local -// source directory -// - glob, relative to the local module directory, resolves as filepath(s), relative to the local -// module directory. Because Soong does not have a concept of crossing package boundaries, the -// glob as computed by Soong may contain paths that cross package-boundaries that would be -// unknowingly omitted if the glob were handled by Bazel. To allow identification and detect -// (within Bazel) use of paths that cross package boundaries, we expand globs within Soong rather -// than converting Soong glob syntax to Bazel glob syntax. **Invalid for excludes.** -// - other modules using the ":name{.tag}" syntax. These modules must implement SourceFileProducer -// or OutputFileProducer. These resolve as a Bazel label for a target. If the Bazel target is in -// the local module directory, it will be returned relative to the current package (e.g. -// ":"). Otherwise, it will be returned as an absolute Bazel label (e.g. -// "//path/to/dir:"). If the reference to another module cannot be resolved,the function -// will panic. -// -// Properties passed as the paths or excludes argument must have been annotated with struct tag -// `android:"path"` so that dependencies on other modules will have already been handled by the -// pathdeps mutator. -func expandSrcsForBazel(ctx Bp2buildMutatorContext, paths, expandedExcludes []string, markAsDeps bool) bazel.LabelList { - if paths == nil { - return bazel.LabelList{} - } - labels := bazel.LabelList{ - Includes: []bazel.Label{}, - } - - // expandedExcludes contain module-dir relative paths, but root-relative paths - // are needed for GlobFiles later. - var rootRelativeExpandedExcludes []string - for _, e := range expandedExcludes { - rootRelativeExpandedExcludes = append(rootRelativeExpandedExcludes, filepath.Join(ctx.ModuleDir(), e)) - } - - for _, p := range paths { - if m, tag := SrcIsModuleWithTag(p); m != "" { - l := getOtherModuleLabel(ctx, m, tag, BazelModuleLabel, markAsDeps) - if l != nil && !InList(l.Label, expandedExcludes) { - if strings.HasPrefix(m, "//") { - // this is a module in a soong namespace - // It appears as //: in srcs, and not ://: - l.OriginalModuleName = m - } else { - l.OriginalModuleName = fmt.Sprintf(":%s", m) - } - labels.Includes = append(labels.Includes, *l) - } - } else { - var expandedPaths []bazel.Label - if pathtools.IsGlob(p) { - // e.g. turn "math/*.c" in - // external/arm-optimized-routines to external/arm-optimized-routines/math/*.c - rootRelativeGlobPath := pathForModuleSrc(ctx, p).String() - expandedPaths = RootToModuleRelativePaths(ctx, GlobFiles(ctx, rootRelativeGlobPath, rootRelativeExpandedExcludes)) - } else { - if !InList(p, expandedExcludes) { - expandedPaths = append(expandedPaths, bazel.Label{Label: p}) - } - } - labels.Includes = append(labels.Includes, expandedPaths...) - } - } - return labels -} - -// getOtherModuleLabel returns a bazel.Label for the given dependency/tag combination for the -// module. The label will be relative to the current directory if appropriate. The dependency must -// already be resolved by either deps mutator or path deps mutator. -func getOtherModuleLabel(ctx Bp2buildMutatorContext, dep, tag string, - labelFromModule func(BazelConversionPathContext, blueprint.Module) string, - markAsDep bool) *bazel.Label { - m, _ := ctx.ModuleFromName(dep) - // The module was not found in an Android.bp file, this is often due to: - // * a limited manifest - // * a required module not being converted from Android.mk - if m == nil { - ctx.AddMissingBp2buildDep(dep) - return &bazel.Label{ - Label: ":" + dep + "__BP2BUILD__MISSING__DEP", - } - } - // Returns true if a dependency from the current module to the target module - // should be skipped; doing so is a hack to circumvent certain problematic - // scenarios that will be addressed in the future. - shouldSkipDep := func(dep string) bool { - // Don't count dependencies of "libc". This is a hack to circumvent the - // fact that, in a variantless build graph, "libc" has a dependency on itself. - if ctx.ModuleName() == "libc" { - return true - } - - // TODO: b/303307672: Dependencies on this module happen to "work" because - // there is a source file with the same name as this module in the - // same directory. We should remove this hack and enforce the underlying - // module of this name is the actual one used. - if dep == "mke2fs.conf" { - return true - } - - // TODO: b/303310285: Remove this special-casing once all dependencies of - // crtbegin_dynamic are convertible - if ctx.ModuleName() == "crtbegin_dynamic" { - return true - } - - return false - } - if markAsDep && !shouldSkipDep(dep) { - ctx.AddDependency(ctx.Module(), Bp2buildDepTag, dep) - } - if !convertedToBazel(ctx, m) { - ctx.AddUnconvertedBp2buildDep(dep) - } - label := BazelModuleLabel(ctx, ctx.Module()) - otherLabel := labelFromModule(ctx, m) - - // TODO(b/165114590): Convert tag (":name{.tag}") to corresponding Bazel implicit output targets. - if (tag != "" && m.Name() == "framework-res") || - (tag == ".generated_srcjars" && ctx.OtherModuleType(m) == "java_aconfig_library") { - otherLabel += tag - } - - if samePackage(label, otherLabel) { - otherLabel = bazelShortLabel(otherLabel) - } - - return &bazel.Label{ - Label: otherLabel, - } -} - -func BazelModuleLabel(ctx BazelConversionPathContext, module blueprint.Module) string { - // TODO(b/165114590): Convert tag (":name{.tag}") to corresponding Bazel implicit output targets. - if !convertedToBazel(ctx, module) || isGoModule(module) { - return bp2buildModuleLabel(ctx, module) - } - b, _ := module.(Bazelable) - return b.GetBazelLabel(ctx, module) -} - -func bazelShortLabel(label string) string { - i := strings.Index(label, ":") - if i == -1 { - panic(fmt.Errorf("Could not find the ':' character in '%s', expected a fully qualified label.", label)) - } - return label[i:] -} - -func bazelPackage(label string) string { - i := strings.Index(label, ":") - if i == -1 { - panic(fmt.Errorf("Could not find the ':' character in '%s', expected a fully qualified label.", label)) - } - return label[0:i] -} - -func samePackage(label1, label2 string) bool { - return bazelPackage(label1) == bazelPackage(label2) -} - -func bp2buildModuleLabel(ctx BazelConversionContext, module blueprint.Module) string { - moduleName := moduleNameWithPossibleOverride(ctx, module, ctx.OtherModuleName(module)) - moduleDir := moduleDirWithPossibleOverride(ctx, module, ctx.OtherModuleDir(module)) - if moduleDir == Bp2BuildTopLevel { - moduleDir = "" - } - if a, ok := module.(Module); ok && IsModulePrebuilt(a) { - moduleName = RemoveOptionalPrebuiltPrefix(moduleName) - } - return fmt.Sprintf("//%s:%s", moduleDir, moduleName) -} - -// BazelOutPath is a Bazel output path compatible to be used for mixed builds within Soong/Ninja. -type BazelOutPath struct { - OutputPath -} - -// ensure BazelOutPath implements Path -var _ Path = BazelOutPath{} - -// ensure BazelOutPath implements genPathProvider -var _ genPathProvider = BazelOutPath{} - -// ensure BazelOutPath implements objPathProvider -var _ objPathProvider = BazelOutPath{} - -func (p BazelOutPath) genPathWithExt(ctx ModuleOutPathContext, subdir, ext string) ModuleGenPath { - return PathForModuleGen(ctx, subdir, pathtools.ReplaceExtension(p.path, ext)) -} - -func (p BazelOutPath) objPathWithExt(ctx ModuleOutPathContext, subdir, ext string) ModuleObjPath { - return PathForModuleObj(ctx, subdir, pathtools.ReplaceExtension(p.path, ext)) -} - -// PathForBazelOutRelative returns a BazelOutPath representing the path under an output directory dedicated to -// bazel-owned outputs. Calling .Rel() on the result will give the input path as relative to the given -// relativeRoot. -func PathForBazelOutRelative(ctx PathContext, relativeRoot string, path string) BazelOutPath { - validatedPath, err := validatePath(filepath.Join("execroot", "__main__", path)) - if err != nil { - reportPathError(ctx, err) - } - var relativeRootPath string - if pathComponents := strings.SplitN(path, "/", 4); len(pathComponents) >= 3 && - pathComponents[0] == "bazel-out" && pathComponents[2] == "bin" { - // If the path starts with something like: bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/ - // make it relative to that folder. bazel-out/volatile-status.txt is an example - // of something that starts with bazel-out but is not relative to the bin folder - relativeRootPath = filepath.Join("execroot", "__main__", pathComponents[0], pathComponents[1], pathComponents[2], relativeRoot) - } else { - relativeRootPath = filepath.Join("execroot", "__main__", relativeRoot) - } - - var relPath string - if relPath, err = filepath.Rel(relativeRootPath, validatedPath); err != nil || strings.HasPrefix(relPath, "../") { - // We failed to make this path relative to execroot/__main__, fall back to a non-relative path - // One case where this happens is when path is ../bazel_tools/something - relativeRootPath = "" - relPath = validatedPath - } - - outputPath := OutputPath{ - basePath{"", ""}, - ctx.Config().soongOutDir, - ctx.Config().BazelContext.OutputBase(), - } - - return BazelOutPath{ - // .withRel() appends its argument onto the current path, and only the most - // recently appended part is returned by outputPath.rel(). - // So outputPath.rel() will return relPath. - OutputPath: outputPath.withRel(relativeRootPath).withRel(relPath), - } -} - -// PathForBazelOut returns a BazelOutPath representing the path under an output directory dedicated to -// bazel-owned outputs. -func PathForBazelOut(ctx PathContext, path string) BazelOutPath { - return PathForBazelOutRelative(ctx, "", path) -} - -// PathsForBazelOut returns a list of paths representing the paths under an output directory -// dedicated to Bazel-owned outputs. -func PathsForBazelOut(ctx PathContext, paths []string) Paths { - outs := make(Paths, 0, len(paths)) - for _, p := range paths { - outs = append(outs, PathForBazelOut(ctx, p)) - } - return outs -} - -// BazelStringOrLabelFromProp splits a Soong module property that can be -// either a string literal, path (with android:path tag) or a module reference -// into separate bazel string or label attributes. Bazel treats string and label -// attributes as distinct types, so this function categorizes a string property -// into either one of them. -// -// e.g. apex.private_key = "foo.pem" can either refer to: -// -// 1. "foo.pem" in the current directory -> file target -// 2. "foo.pem" module -> rule target -// 3. "foo.pem" file in a different directory, prefixed by a product variable handled -// in a bazel macro. -> string literal -// -// For the first two cases, they are defined using the label attribute. For the third case, -// it's defined with the string attribute. -func BazelStringOrLabelFromProp( - ctx Bp2buildMutatorContext, - propToDistinguish *string) (bazel.LabelAttribute, bazel.StringAttribute) { - - var labelAttr bazel.LabelAttribute - var strAttr bazel.StringAttribute - - if propToDistinguish == nil { - // nil pointer - return labelAttr, strAttr - } - - prop := String(propToDistinguish) - if SrcIsModule(prop) != "" { - // If it's a module (SrcIsModule will return the module name), set the - // resolved label to the label attribute. - labelAttr.SetValue(BazelLabelForModuleDepSingle(ctx, prop)) - } else { - // Not a module name. This could be a string literal or a file target in - // the current dir. Check if the path exists: - path := ExistentPathForSource(ctx, ctx.ModuleDir(), prop) - - if path.Valid() && parentDir(path.String()) == ctx.ModuleDir() { - // If it exists and the path is relative to the current dir, resolve the bazel label - // for the _file target_ and set it to the label attribute. - // - // Resolution is necessary because this could be a file in a subpackage. - labelAttr.SetValue(BazelLabelForModuleSrcSingle(ctx, prop)) - } else { - // Otherwise, treat it as a string literal and assign to the string attribute. - strAttr.Value = propToDistinguish - } - } - - return labelAttr, strAttr -} diff --git a/android/bazel_paths_test.go b/android/bazel_paths_test.go deleted file mode 100644 index bed719cbb..000000000 --- a/android/bazel_paths_test.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2022 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "fmt" - "path/filepath" - "testing" - - "android/soong/bazel" - - "github.com/google/blueprint" - "github.com/google/blueprint/pathtools" -) - -type TestBazelPathContext struct{} - -func (*TestBazelPathContext) Config() Config { - cfg := NullConfig("out", "out/soong") - cfg.BazelContext = MockBazelContext{ - OutputBaseDir: "out/bazel", - } - return cfg -} - -func (*TestBazelPathContext) AddNinjaFileDeps(...string) { - panic("Unimplemented") -} - -func TestPathForBazelOut(t *testing.T) { - ctx := &TestBazelPathContext{} - out := PathForBazelOut(ctx, "foo/bar/baz/boq.txt") - expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/foo/bar/baz/boq.txt") - if out.String() != expectedPath { - t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) - } - - expectedRelPath := "foo/bar/baz/boq.txt" - if out.Rel() != expectedRelPath { - t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) - } -} - -func TestPathForBazelOutRelative(t *testing.T) { - ctx := &TestBazelPathContext{} - out := PathForBazelOutRelative(ctx, "foo/bar", "foo/bar/baz/boq.txt") - - expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/foo/bar/baz/boq.txt") - if out.String() != expectedPath { - t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) - } - - expectedRelPath := "baz/boq.txt" - if out.Rel() != expectedRelPath { - t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) - } -} - -func TestPathForBazelOutRelativeUnderBinFolder(t *testing.T) { - ctx := &TestBazelPathContext{} - out := PathForBazelOutRelative(ctx, "foo/bar", "bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/foo/bar/baz/boq.txt") - - expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/foo/bar/baz/boq.txt") - if out.String() != expectedPath { - t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) - } - - expectedRelPath := "baz/boq.txt" - if out.Rel() != expectedRelPath { - t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) - } -} - -func TestPathForBazelOutOutsideOfExecroot(t *testing.T) { - ctx := &TestBazelPathContext{} - out := PathForBazelOut(ctx, "../bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar") - - expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar") - if out.String() != expectedPath { - t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) - } - - expectedRelPath := "execroot/bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar" - if out.Rel() != expectedRelPath { - t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) - } -} - -func TestPathForBazelOutRelativeWithParentDirectoryRoot(t *testing.T) { - ctx := &TestBazelPathContext{} - out := PathForBazelOutRelative(ctx, "../bazel_tools", "../bazel_tools/foo/bar/baz.sh") - - expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/bazel_tools/foo/bar/baz.sh") - if out.String() != expectedPath { - t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) - } - - expectedRelPath := "foo/bar/baz.sh" - if out.Rel() != expectedRelPath { - t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) - } -} - -type TestBazelConversionPathContext struct { - TestBazelConversionContext - moduleDir string - cfg Config - mockGlobResults *[]string -} - -func (ctx *TestBazelConversionPathContext) AddNinjaFileDeps(...string) { - panic("Unimplemented") -} - -func (ctx *TestBazelConversionPathContext) GlobWithDeps(string, []string) ([]string, error) { - if ctx.mockGlobResults == nil { - return []string{}, fmt.Errorf("Set mock glob results first") - } - return *ctx.mockGlobResults, nil -} - -func (ctx *TestBazelConversionPathContext) PropertyErrorf(string, string, ...interface{}) { - panic("Unimplemented") -} - -func (ctx *TestBazelConversionPathContext) GetDirectDep(string) (blueprint.Module, blueprint.DependencyTag) { - panic("Unimplemented") -} - -func (ctx *TestBazelConversionPathContext) ModuleFromName(string) (blueprint.Module, bool) { - panic("Unimplemented") -} - -func (ctx *TestBazelConversionPathContext) AddUnconvertedBp2buildDep(string) { - panic("Unimplemented") -} - -func (ctx *TestBazelConversionPathContext) AddMissingBp2buildDep(string) { - panic("Unimplemented") -} - -func (ctx *TestBazelConversionPathContext) Module() Module { - panic("Unimplemented") -} - -func (ctx *TestBazelConversionPathContext) Config() Config { - return ctx.cfg -} - -func (ctx *TestBazelConversionPathContext) ModuleDir() string { - return ctx.moduleDir -} - -func (ctx *TestBazelConversionPathContext) ModuleName() string { - panic("Unimplemented") -} - -func (ctx *TestBazelConversionPathContext) ModuleType() string { - panic("Unimplemented") -} - -func TestTransformSubpackagePath(t *testing.T) { - cfg := NullConfig("out", "out/soong") - cfg.fs = pathtools.MockFs(map[string][]byte{ - "x/Android.bp": nil, - "x/y/Android.bp": nil, - }) - - var ctx BazelConversionPathContext = &TestBazelConversionPathContext{ - moduleDir: "x", - cfg: cfg, - } - pairs := map[string]string{ - "y/a.c": "//x/y:a.c", - "./y/a.c": "//x/y:a.c", - "z/b.c": "z/b.c", - "./z/b.c": "z/b.c", - } - for in, out := range pairs { - actual := transformSubpackagePath(ctx.Config(), ctx.ModuleDir(), bazel.Label{Label: in}).Label - if actual != out { - t.Errorf("expected:\n%v\nactual:\n%v", out, actual) - } - } -} - -// Check that the files in a specific directory are returned with labels that respect package boundaries -// Since the test uses a mock for GlobWithDeps, the params passed to BazelLabelForSrcPatternExcludes are no-ops -func TestBazelLabelForSrcPatternExcludes(t *testing.T) { - cfg := NullConfig("out", "out/soong") - cfg.fs = pathtools.MockFs(map[string][]byte{ - "x/Android.bp": nil, - "x/y/Android.bp": nil, - // .proto files - "foo.proto": nil, - "x/bar.proto": nil, - "x/baz.proto": nil, - "x/y/qux.proto": nil, - }) - - var ctx BazelConversionPathContext = &TestBazelConversionPathContext{ - cfg: cfg, - } - - // Root dir - ctx.(*TestBazelConversionPathContext).mockGlobResults = &[]string{"foo.proto", "x/bar.proto", "x/baz.proto", "x/y/qux.proto"} - actualLabelsFromRoot := BazelLabelForSrcPatternExcludes(ctx, ".", "**/*.proto", []string{}) - expectedLabelsAsString := []string{"foo.proto", "//x:bar.proto", "//x:baz.proto", "//x/y:qux.proto"} - for i, actual := range actualLabelsFromRoot.Includes { - AssertStringEquals(t, "Error in finding src labels relative to root directory", expectedLabelsAsString[i], actual.Label) - } - - // x dir - ctx.(*TestBazelConversionPathContext).mockGlobResults = &[]string{"x/bar.proto", "x/baz.proto", "x/y/qux.proto"} - actualLabelsFromRoot = BazelLabelForSrcPatternExcludes(ctx, "x", "**/*.proto", []string{}) - expectedLabelsAsString = []string{"bar.proto", "baz.proto", "//x/y:qux.proto"} - for i, actual := range actualLabelsFromRoot.Includes { - AssertStringEquals(t, "Error in finding src labels relative to x directory", expectedLabelsAsString[i], actual.Label) - } - - // y dir - ctx.(*TestBazelConversionPathContext).mockGlobResults = &[]string{"x/y/qux.proto"} - actualLabelsFromRoot = BazelLabelForSrcPatternExcludes(ctx, "x/y", "**/*.proto", []string{}) - expectedLabelsAsString = []string{"qux.proto"} - for i, actual := range actualLabelsFromRoot.Includes { - AssertStringEquals(t, "Error in finding src labels relative to x/y directory", expectedLabelsAsString[i], actual.Label) - } -} diff --git a/android/bazel_test.go b/android/bazel_test.go deleted file mode 100644 index e0145b527..000000000 --- a/android/bazel_test.go +++ /dev/null @@ -1,592 +0,0 @@ -// Copyright 2021 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package android - -import ( - "fmt" - "testing" - - "android/soong/android/allowlists" - "android/soong/bazel" - - "github.com/google/blueprint" - "github.com/google/blueprint/proptools" -) - -func TestConvertAllModulesInPackage(t *testing.T) { - testCases := []struct { - prefixes allowlists.Bp2BuildConfig - packageDir string - }{ - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a/b": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a/b", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a/b": allowlists.Bp2BuildDefaultTrueRecursively, - "a/b/c": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a/b", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultTrueRecursively, - "d/e/f": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a/b", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultFalse, - "a/b": allowlists.Bp2BuildDefaultTrueRecursively, - "a/b/c": allowlists.Bp2BuildDefaultFalse, - }, - packageDir: "a/b", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultTrueRecursively, - "a/b": allowlists.Bp2BuildDefaultFalse, - "a/b/c": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultFalseRecursively, - "a/b": allowlists.Bp2BuildDefaultTrue, - }, - packageDir: "a/b", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultFalseRecursively, - "a/b": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a/b/c", - }, - } - - for _, test := range testCases { - if ok, _ := bp2buildDefaultTrueRecursively(test.packageDir, test.prefixes); !ok { - t.Errorf("Expected to convert all modules in %s based on %v, but failed.", test.packageDir, test.prefixes) - } - } -} - -func TestModuleOptIn(t *testing.T) { - testCases := []struct { - prefixes allowlists.Bp2BuildConfig - packageDir string - }{ - { - prefixes: allowlists.Bp2BuildConfig{ - "a/b": allowlists.Bp2BuildDefaultFalse, - }, - packageDir: "a/b", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultFalse, - "a/b": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a/b": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a", // opt-in by default - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a/b/c": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a/b", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultTrueRecursively, - "d/e/f": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "foo/bar", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultTrueRecursively, - "a/b": allowlists.Bp2BuildDefaultFalse, - "a/b/c": allowlists.Bp2BuildDefaultTrueRecursively, - }, - packageDir: "a/b", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultFalse, - "a/b": allowlists.Bp2BuildDefaultTrueRecursively, - "a/b/c": allowlists.Bp2BuildDefaultFalse, - }, - packageDir: "a", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultFalseRecursively, - "a/b": allowlists.Bp2BuildDefaultTrue, - }, - packageDir: "a/b/c", - }, - { - prefixes: allowlists.Bp2BuildConfig{ - "a": allowlists.Bp2BuildDefaultTrueRecursively, - "a/b": allowlists.Bp2BuildDefaultFalseRecursively, - }, - packageDir: "a/b/c", - }, - } - - for _, test := range testCases { - if ok, _ := bp2buildDefaultTrueRecursively(test.packageDir, test.prefixes); ok { - t.Errorf("Expected to allow module opt-in in %s based on %v, but failed.", test.packageDir, test.prefixes) - } - } -} - -type TestBazelModule struct { - bazel.TestModuleInfo - BazelModuleBase -} - -var _ blueprint.Module = TestBazelModule{} - -func (m TestBazelModule) Name() string { - return m.TestModuleInfo.ModuleName -} - -func (m TestBazelModule) GenerateBuildActions(blueprint.ModuleContext) { -} - -type TestBazelConversionContext struct { - omc bazel.OtherModuleTestContext - allowlist Bp2BuildConversionAllowlist - errors []string -} - -var _ bazelOtherModuleContext = &TestBazelConversionContext{} - -func (bcc *TestBazelConversionContext) OtherModuleType(m blueprint.Module) string { - return bcc.omc.OtherModuleType(m) -} - -func (bcc *TestBazelConversionContext) OtherModuleName(m blueprint.Module) string { - return bcc.omc.OtherModuleName(m) -} - -func (bcc *TestBazelConversionContext) OtherModuleDir(m blueprint.Module) string { - return bcc.omc.OtherModuleDir(m) -} - -func (bcc *TestBazelConversionContext) ModuleErrorf(format string, args ...interface{}) { - bcc.errors = append(bcc.errors, fmt.Sprintf(format, args...)) -} - -func (bcc *TestBazelConversionContext) Config() Config { - return Config{ - &config{ - Bp2buildPackageConfig: bcc.allowlist, - }, - } -} - -var bazelableBazelModuleBase = BazelModuleBase{ - bazelProperties: properties{ - Bazel_module: BazelModuleProperties{ - CanConvertToBazel: true, - }, - }, -} - -func TestBp2BuildAllowlist(t *testing.T) { - testCases := []struct { - description string - shouldConvert bool - expectedErrors []string - module TestBazelModule - allowlist Bp2BuildConversionAllowlist - }{ - { - description: "allowlist enables module", - shouldConvert: true, - module: TestBazelModule{ - TestModuleInfo: bazel.TestModuleInfo{ - ModuleName: "foo", - Typ: "rule1", - Dir: "dir1", - }, - BazelModuleBase: bazelableBazelModuleBase, - }, - allowlist: Bp2BuildConversionAllowlist{ - moduleAlwaysConvert: map[string]bool{ - "foo": true, - }, - }, - }, - { - description: "module in name allowlist and type allowlist fails", - shouldConvert: false, - expectedErrors: []string{"A module \"foo\" of type \"rule1\" cannot be in moduleAlwaysConvert and also be in moduleTypeAlwaysConvert"}, - module: TestBazelModule{ - TestModuleInfo: bazel.TestModuleInfo{ - ModuleName: "foo", - Typ: "rule1", - Dir: "dir1", - }, - BazelModuleBase: bazelableBazelModuleBase, - }, - allowlist: Bp2BuildConversionAllowlist{ - moduleAlwaysConvert: map[string]bool{ - "foo": true, - }, - moduleTypeAlwaysConvert: map[string]bool{ - "rule1": true, - }, - }, - }, - { - description: "module in allowlist and denylist fails", - shouldConvert: false, - expectedErrors: []string{"a module \"foo\" cannot be in moduleDoNotConvert and also be in moduleAlwaysConvert"}, - module: TestBazelModule{ - TestModuleInfo: bazel.TestModuleInfo{ - ModuleName: "foo", - Typ: "rule1", - Dir: "dir1", - }, - BazelModuleBase: bazelableBazelModuleBase, - }, - allowlist: Bp2BuildConversionAllowlist{ - moduleAlwaysConvert: map[string]bool{ - "foo": true, - }, - moduleDoNotConvert: map[string]bool{ - "foo": true, - }, - }, - }, - { - description: "module allowlist and enabled directory", - shouldConvert: false, - expectedErrors: []string{"A module cannot be in a directory marked Bp2BuildDefaultTrue or Bp2BuildDefaultTrueRecursively and also be in moduleAlwaysConvert. Directory: 'existing/build/dir' Module: 'foo'"}, - module: TestBazelModule{ - TestModuleInfo: bazel.TestModuleInfo{ - ModuleName: "foo", - Typ: "rule1", - Dir: "existing/build/dir", - }, - BazelModuleBase: bazelableBazelModuleBase, - }, - allowlist: Bp2BuildConversionAllowlist{ - moduleAlwaysConvert: map[string]bool{ - "foo": true, - }, - defaultConfig: allowlists.Bp2BuildConfig{ - "existing/build/dir": allowlists.Bp2BuildDefaultTrue, - }, - }, - }, - { - description: "module allowlist and enabled subdirectory", - shouldConvert: false, - expectedErrors: []string{"A module cannot be in a directory marked Bp2BuildDefaultTrue or Bp2BuildDefaultTrueRecursively and also be in moduleAlwaysConvert. Directory: 'existing/build/dir' Module: 'foo'"}, - module: TestBazelModule{ - TestModuleInfo: bazel.TestModuleInfo{ - ModuleName: "foo", - Typ: "rule1", - Dir: "existing/build/dir/subdir", - }, - BazelModuleBase: bazelableBazelModuleBase, - }, - allowlist: Bp2BuildConversionAllowlist{ - moduleAlwaysConvert: map[string]bool{ - "foo": true, - }, - defaultConfig: allowlists.Bp2BuildConfig{ - "existing/build/dir": allowlists.Bp2BuildDefaultTrueRecursively, - }, - }, - }, - { - description: "module enabled in unit test short-circuits other allowlists", - shouldConvert: true, - module: TestBazelModule{ - TestModuleInfo: bazel.TestModuleInfo{ - ModuleName: "foo", - Typ: "rule1", - Dir: ".", - }, - BazelModuleBase: BazelModuleBase{ - bazelProperties: properties{ - Bazel_module: BazelModuleProperties{ - CanConvertToBazel: true, - Bp2build_available: proptools.BoolPtr(true), - }, - }, - }, - }, - allowlist: Bp2BuildConversionAllowlist{ - moduleAlwaysConvert: map[string]bool{ - "foo": true, - }, - moduleDoNotConvert: map[string]bool{ - "foo": true, - }, - }, - }, - } - - for _, test := range testCases { - t.Run(test.description, func(t *testing.T) { - bcc := &TestBazelConversionContext{ - omc: bazel.OtherModuleTestContext{ - Modules: []bazel.TestModuleInfo{ - test.module.TestModuleInfo, - }, - }, - allowlist: test.allowlist, - } - - shouldConvert := test.module.shouldConvertWithBp2build(bcc, - shouldConvertParams{ - module: test.module.TestModuleInfo, - moduleDir: test.module.TestModuleInfo.Dir, - moduleType: test.module.TestModuleInfo.Typ, - moduleName: test.module.TestModuleInfo.ModuleName, - }, - ) - if test.shouldConvert != shouldConvert { - t.Errorf("Module shouldConvert expected to be: %v, but was: %v", test.shouldConvert, shouldConvert) - } - - errorsMatch := true - if len(test.expectedErrors) != len(bcc.errors) { - errorsMatch = false - } else { - for i, err := range test.expectedErrors { - if err != bcc.errors[i] { - errorsMatch = false - } - } - } - if !errorsMatch { - t.Errorf("Expected errors to be: %v, but were: %v", test.expectedErrors, bcc.errors) - } - }) - } -} - -func TestBp2buildAllowList(t *testing.T) { - allowlist := GetBp2BuildAllowList() - for k, v := range allowlists.Bp2buildDefaultConfig { - if allowlist.defaultConfig[k] != v { - t.Errorf("bp2build default config of %s: expected: %v, got: %v", k, v, allowlist.defaultConfig[k]) - } - } - for k, v := range allowlists.Bp2buildKeepExistingBuildFile { - if allowlist.keepExistingBuildFile[k] != v { - t.Errorf("bp2build keep existing build file of %s: expected: %v, got: %v", k, v, allowlist.keepExistingBuildFile[k]) - } - } - for _, k := range allowlists.Bp2buildModuleTypeAlwaysConvertList { - if !allowlist.moduleTypeAlwaysConvert[k] { - t.Errorf("bp2build module type always convert of %s: expected: true, got: %v", k, allowlist.moduleTypeAlwaysConvert[k]) - } - } - for _, k := range allowlists.Bp2buildModuleDoNotConvertList { - if !allowlist.moduleDoNotConvert[k] { - t.Errorf("bp2build module do not convert of %s: expected: true, got: %v", k, allowlist.moduleDoNotConvert[k]) - } - } -} - -func TestShouldKeepExistingBuildFileForDir(t *testing.T) { - allowlist := NewBp2BuildAllowlist() - // entry "a/b2/c2" is moot because of its parent "a/b2" - allowlist.SetKeepExistingBuildFile(map[string]bool{"a": false, "a/b1": false, "a/b2": true, "a/b1/c1": true, "a/b2/c2": false}) - truths := []string{"a", "a/b1", "a/b2", "a/b1/c1", "a/b2/c", "a/b2/c2", "a/b2/c2/d"} - falsities := []string{"a1", "a/b", "a/b1/c"} - for _, dir := range truths { - if !allowlist.ShouldKeepExistingBuildFileForDir(dir) { - t.Errorf("%s expected TRUE but was FALSE", dir) - } - } - for _, dir := range falsities { - if allowlist.ShouldKeepExistingBuildFileForDir(dir) { - t.Errorf("%s expected FALSE but was TRUE", dir) - } - } -} - -type mixedBuildModule struct { - ModuleBase - BazelModuleBase - props struct { - Deps []string - Mixed_build_incompatible *bool - QueuedBazelCall bool `blueprint:"mutated"` - } -} - -type mixedBuildModuleInfo struct { - QueuedBazelCall bool -} - -var mixedBuildModuleProvider = blueprint.NewProvider(mixedBuildModuleInfo{}) - -func mixedBuildModuleFactory() Module { - m := &mixedBuildModule{} - m.AddProperties(&m.props) - InitAndroidArchModule(m, HostAndDeviceDefault, MultilibBoth) - InitBazelModule(m) - - return m -} - -func (m *mixedBuildModule) ConvertWithBp2build(ctx Bp2buildMutatorContext) { -} - -func (m *mixedBuildModule) DepsMutator(ctx BottomUpMutatorContext) { - ctx.AddDependency(ctx.Module(), installDepTag{}, m.props.Deps...) -} - -func (m *mixedBuildModule) GenerateAndroidBuildActions(ctx ModuleContext) { -} - -func (m *mixedBuildModule) IsMixedBuildSupported(ctx BaseModuleContext) bool { - return !proptools.Bool(m.props.Mixed_build_incompatible) -} - -func (m *mixedBuildModule) QueueBazelCall(ctx BaseModuleContext) { - m.props.QueuedBazelCall = true -} - -func (m *mixedBuildModule) ProcessBazelQueryResponse(ctx ModuleContext) { - ctx.SetProvider(mixedBuildModuleProvider, mixedBuildModuleInfo{ - QueuedBazelCall: m.props.QueuedBazelCall, - }) -} - -var prepareForMixedBuildTests = FixtureRegisterWithContext(func(ctx RegistrationContext) { - ctx.RegisterModuleType("deps", mixedBuildModuleFactory) - RegisterMixedBuildsMutator(ctx) -}) - -func TestMixedBuildsEnabledForType(t *testing.T) { - baseBp := ` - deps { - name: "foo", - deps: ["bar"], - target: { windows: { enabled: true } }, - %s - } -` - depBp := ` - deps { - name: "bar", - target: { - windows: { - enabled: true, - }, - }, - } -` - testCases := []struct { - desc string - variant *string - missingDeps bool - extraBpInfo string - mixedBuildsEnabled bool - }{ - { - desc: "mixed builds works", - mixedBuildsEnabled: true, - extraBpInfo: `bazel_module: { bp2build_available: true },`, - }, - { - desc: "missing deps", - missingDeps: true, - mixedBuildsEnabled: false, - extraBpInfo: `bazel_module: { bp2build_available: true },`, - }, - { - desc: "windows no mixed builds", - mixedBuildsEnabled: false, - variant: proptools.StringPtr("windows_x86"), - extraBpInfo: `bazel_module: { bp2build_available: true },`, - }, - { - desc: "mixed builds disabled by type", - mixedBuildsEnabled: false, - extraBpInfo: `mixed_build_incompatible: true, - bazel_module: { bp2build_available: true },`, - }, - { - desc: "mixed builds not bp2build available", - mixedBuildsEnabled: false, - extraBpInfo: `bazel_module: { bp2build_available: false },`, - }, - } - - for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { - handlers := GroupFixturePreparers( - prepareForMixedBuildTests, - PrepareForTestWithArchMutator, - FixtureModifyConfig(func(config Config) { - config.BazelContext = MockBazelContext{ - OutputBaseDir: "base", - } - config.Targets[Windows] = []Target{ - {Windows, Arch{ArchType: X86_64}, NativeBridgeDisabled, "", "", true}, - {Windows, Arch{ArchType: X86}, NativeBridgeDisabled, "", "", true}, - } - }), - ) - bp := fmt.Sprintf(baseBp, tc.extraBpInfo) - if tc.missingDeps { - handlers = GroupFixturePreparers( - handlers, - PrepareForTestWithAllowMissingDependencies, - ) - } else { - bp += depBp - } - result := handlers.RunTestWithBp(t, bp) - - variant := proptools.StringDefault(tc.variant, "android_arm64_armv8-a") - - m := result.ModuleForTests("foo", variant) - mixedBuildModuleInfo := result.TestContext.ModuleProvider(m.Module(), mixedBuildModuleProvider).(mixedBuildModuleInfo) - if w, g := tc.mixedBuildsEnabled, mixedBuildModuleInfo.QueuedBazelCall; w != g { - t.Errorf("Expected mixed builds enabled %t, got mixed builds enabled %t", w, g) - } - }) - } -} diff --git a/android/config.go b/android/config.go index 929350584..0fec79237 100644 --- a/android/config.go +++ b/android/config.go @@ -84,21 +84,13 @@ type CmdArgs struct { SoongOutDir string SoongVariables string - SymlinkForestMarker string - Bp2buildMarker string - BazelQueryViewDir string - ModuleGraphFile string - ModuleActionsFile string - DocFile string + BazelQueryViewDir string + ModuleGraphFile string + ModuleActionsFile string + DocFile string MultitreeBuild bool - BazelMode bool - BazelModeStaging bool - BazelForceEnabledModules string - - UseBazelProxy bool - BuildFromSourceStub bool EnsureAllowlistIntegrity bool @@ -109,12 +101,6 @@ const ( // Don't use bazel at all during module analysis. AnalysisNoBazel SoongBuildMode = iota - // Symlink fores mode: merge two directory trees into a symlink forest - SymlinkForest - - // Bp2build mode: Generate BUILD files from blueprint files and exit. - Bp2build - // Generate BUILD files which faithfully represent the dependency graph of // blueprint modules. Individual BUILD targets will not, however, faitfhully // express build semantics. @@ -125,15 +111,6 @@ const ( // Generate a documentation file for module type definitions and exit. GenerateDocFile - - // Use bazel during analysis of a few allowlisted build modules. The allowlist - // is considered "staging, as these are modules being prepared to be released - // into prod mode shortly after. - BazelStagingMode - - // Use bazel during analysis of build modules from an allowlist carefully - // curated by the build team to be proven stable. - BazelProdMode ) // SoongOutDir returns the build output directory for the configuration. @@ -265,10 +242,6 @@ type config struct { // Only available on configs created by TestConfig TestProductVariables *ProductVariables - // A specialized context object for Bazel/Soong mixed builds and migration - // purposes. - BazelContext BazelContext - ProductVariablesFileName string // BuildOS stores the OsType for the OS that the build is running on. @@ -310,9 +283,7 @@ type config struct { fs pathtools.FileSystem mockBpList string - BuildMode SoongBuildMode - Bp2buildPackageConfig Bp2BuildConversionAllowlist - Bp2buildSoongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions + BuildMode SoongBuildMode // If MultitreeBuild is true then this is one inner tree of a multitree // build directed by the multitree orchestrator. @@ -328,29 +299,6 @@ type config struct { OncePer - // These fields are only used for metrics collection. A module should be added - // to these maps only if its implementation supports Bazel handling in mixed - // builds. A module being in the "enabled" list indicates that there is a - // variant of that module for which bazel-handling actually took place. - // A module being in the "disabled" list indicates that there is a variant of - // that module for which bazel-handling was denied. - mixedBuildsLock sync.Mutex - mixedBuildEnabledModules map[string]struct{} - mixedBuildDisabledModules map[string]struct{} - - // These are modules to be built with Bazel beyond the allowlisted/build-mode - // specified modules. They are passed via the command-line flag - // "--bazel-force-enabled-modules" - bazelForceEnabledModules map[string]struct{} - - // Names of Bazel targets as defined by BUILD files in the source tree, - // keyed by the directory in which they are defined. - bazelTargetsByDir map[string][]string - - // If true, for any requests to Bazel, communicate with a Bazel proxy using - // unix sockets, instead of spawning Bazel as a subprocess. - UseBazelProxy bool - // If buildFromSourceStub is true then the Java API stubs are // built from the source Java files, not the signature text files. buildFromSourceStub bool @@ -561,14 +509,10 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) runGoTests: cmdArgs.RunGoTests, multilibConflicts: make(map[ArchType]bool), - moduleListFile: cmdArgs.ModuleListFile, - fs: pathtools.NewOsFs(absSrcDir), - mixedBuildDisabledModules: make(map[string]struct{}), - mixedBuildEnabledModules: make(map[string]struct{}), - bazelForceEnabledModules: make(map[string]struct{}), + moduleListFile: cmdArgs.ModuleListFile, + fs: pathtools.NewOsFs(absSrcDir), MultitreeBuild: cmdArgs.MultitreeBuild, - UseBazelProxy: cmdArgs.UseBazelProxy, buildFromSourceStub: cmdArgs.BuildFromSourceStub, } @@ -661,28 +605,9 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) config.BuildMode = mode } } - setBazelMode := func(arg bool, argName string, mode SoongBuildMode) { - if arg { - if config.BuildMode != AnalysisNoBazel { - fmt.Fprintf(os.Stderr, "buildMode is already set, illegal argument: %s", argName) - os.Exit(1) - } - config.BuildMode = mode - } - } - setBuildMode(cmdArgs.SymlinkForestMarker, SymlinkForest) - setBuildMode(cmdArgs.Bp2buildMarker, Bp2build) setBuildMode(cmdArgs.BazelQueryViewDir, GenerateQueryView) setBuildMode(cmdArgs.ModuleGraphFile, GenerateModuleGraph) setBuildMode(cmdArgs.DocFile, GenerateDocFile) - setBazelMode(cmdArgs.BazelMode, "--bazel-mode", BazelProdMode) - setBazelMode(cmdArgs.BazelModeStaging, "--bazel-mode-staging", BazelStagingMode) - - for _, module := range getForceEnabledModulesFromFlag(cmdArgs.BazelForceEnabledModules) { - config.bazelForceEnabledModules[module] = struct{}{} - } - config.BazelContext, err = NewBazelContext(config) - config.Bp2buildPackageConfig = GetBp2BuildAllowList() // TODO(b/276958307): Replace the hardcoded list to a sdk_library local prop. config.apiLibraries = map[string]struct{}{ @@ -719,13 +644,6 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) return Config{config}, err } -func getForceEnabledModulesFromFlag(forceEnabledFlag string) []string { - if forceEnabledFlag == "" { - return []string{} - } - return strings.Split(forceEnabledFlag, ",") -} - // mockFileSystem replaces all reads with accesses to the provided map of // filenames to contents stored as a byte slice. func (c *config) mockFileSystem(bp string, fs map[string][]byte) { @@ -756,41 +674,6 @@ func (c *config) mockFileSystem(bp string, fs map[string][]byte) { c.mockBpList = blueprint.MockModuleListFile } -// TODO(b/265062549): Add a field to our collected (and uploaded) metrics which -// describes a reason that we fell back to non-mixed builds. -// Returns true if "Bazel builds" is enabled. In this mode, part of build -// analysis is handled by Bazel. -func (c *config) IsMixedBuildsEnabled() bool { - globalMixedBuildsSupport := c.Once(OnceKey{"globalMixedBuildsSupport"}, func() interface{} { - if c.productVariables.DeviceArch != nil && *c.productVariables.DeviceArch == "riscv64" { - return false - } - // Disable Bazel when Kythe is running - if c.EmitXrefRules() { - return false - } - if c.IsEnvTrue("GLOBAL_THINLTO") { - return false - } - if len(c.productVariables.SanitizeHost) > 0 { - return false - } - if len(c.productVariables.SanitizeDevice) > 0 { - return false - } - if len(c.productVariables.SanitizeDeviceDiag) > 0 { - return false - } - if len(c.productVariables.SanitizeDeviceArch) > 0 { - return false - } - return true - }).(bool) - - bazelModeEnabled := c.BuildMode == BazelProdMode || c.BuildMode == BazelStagingMode - return globalMixedBuildsSupport && bazelModeEnabled -} - func (c *config) SetAllowMissingDependencies() { c.productVariables.Allow_missing_dependencies = proptools.BoolPtr(true) } @@ -1066,8 +949,6 @@ func (c *config) AllSupportedApiLevels() []ApiLevel { // DefaultAppTargetSdk returns the API level that platform apps are targeting. // This converts a codename to the exact ApiLevel it represents. func (c *config) DefaultAppTargetSdk(ctx EarlyModuleContext) ApiLevel { - // This logic is replicated in starlark, if changing logic here update starlark code too - // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=72;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061 if Bool(c.productVariables.Platform_sdk_final) { return c.PlatformSdkVersion() } @@ -1425,10 +1306,6 @@ func (c *config) PrebuiltHiddenApiDir(_ PathContext) string { return String(c.productVariables.PrebuiltHiddenApiDir) } -func (c *config) BazelModulesForceEnabledByFlag() map[string]struct{} { - return c.bazelForceEnabledModules -} - func (c *config) IsVndkDeprecated() bool { return !Bool(c.productVariables.KeepVndk) } @@ -2041,38 +1918,6 @@ func (c *config) UseHostMusl() bool { return Bool(c.productVariables.HostMusl) } -func (c *config) GetMixedBuildsEnabledModules() map[string]struct{} { - return c.mixedBuildEnabledModules -} - -func (c *config) GetMixedBuildsDisabledModules() map[string]struct{} { - return c.mixedBuildDisabledModules -} - -func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) { - moduleName := ctx.Module().Name() - c.mixedBuildsLock.Lock() - defer c.mixedBuildsLock.Unlock() - if useBazel { - c.mixedBuildEnabledModules[moduleName] = struct{}{} - } else { - c.mixedBuildDisabledModules[moduleName] = struct{}{} - } -} - -func (c *config) HasBazelBuildTargetInSource(dir string, target string) bool { - for _, existingTarget := range c.bazelTargetsByDir[dir] { - if target == existingTarget { - return true - } - } - return false -} - -func (c *config) SetBazelBuildFileTargets(bazelTargetsByDir map[string][]string) { - c.bazelTargetsByDir = bazelTargetsByDir -} - // ApiSurfaces directory returns the source path inside the api_surfaces repo // (relative to workspace root). func (c *config) ApiSurfacesDir(s ApiSurface, version string) string { @@ -2106,12 +1951,6 @@ func (c *config) SetBuildFromTextStub(b bool) { c.productVariables.Build_from_text_stub = boolPtr(b) } -func (c *config) AddForceEnabledModules(forceEnabled []string) { - for _, forceEnabledModule := range forceEnabled { - c.bazelForceEnabledModules[forceEnabledModule] = struct{}{} - } -} - func (c *config) SetApiLibraries(libs []string) { c.apiLibraries = make(map[string]struct{}) for _, lib := range libs { @@ -2123,11 +1962,6 @@ func (c *config) GetApiLibraries() map[string]struct{} { return c.apiLibraries } -// Bp2buildMode indicates whether the config is for bp2build mode of Soong -func (c *config) Bp2buildMode() bool { - return c.BuildMode == Bp2build -} - func (c *deviceConfig) CheckVendorSeappViolations() bool { return Bool(c.config.productVariables.CheckVendorSeappViolations) } diff --git a/android/metrics.go b/android/metrics.go index 35712727a..6834b1bde 100644 --- a/android/metrics.go +++ b/android/metrics.go @@ -19,7 +19,6 @@ import ( "io/ioutil" "os" "runtime" - "sort" "strconv" "time" @@ -93,23 +92,6 @@ func collectMetrics(config Config, eventHandler *metrics.EventHandler) *soong_me } metrics.Events = append(metrics.Events, &perfInfo) } - mixedBuildsInfo := soong_metrics_proto.MixedBuildsInfo{} - mixedBuildEnabledModules := make([]string, 0, len(config.mixedBuildEnabledModules)) - for module, _ := range config.mixedBuildEnabledModules { - mixedBuildEnabledModules = append(mixedBuildEnabledModules, module) - } - - mixedBuildDisabledModules := make([]string, 0, len(config.mixedBuildDisabledModules)) - for module, _ := range config.mixedBuildDisabledModules { - mixedBuildDisabledModules = append(mixedBuildDisabledModules, module) - } - // Sorted for deterministic output. - sort.Strings(mixedBuildEnabledModules) - sort.Strings(mixedBuildDisabledModules) - - mixedBuildsInfo.MixedBuildEnabledModules = mixedBuildEnabledModules - mixedBuildsInfo.MixedBuildDisabledModules = mixedBuildDisabledModules - metrics.MixedBuildsInfo = &mixedBuildsInfo return metrics } diff --git a/android/module.go b/android/module.go index 9d746426f..f57115799 100644 --- a/android/module.go +++ b/android/module.go @@ -16,7 +16,6 @@ package android import ( "android/soong/bazel" - "android/soong/ui/metrics/bp2build_metrics_proto" "crypto/md5" "encoding/hex" "encoding/json" @@ -96,16 +95,6 @@ type Module interface { AddProperties(props ...interface{}) GetProperties() []interface{} - // If this module should not have bazel BUILD definitions generated by bp2build, - // GetUnconvertedReason returns a reason this is the case. - GetUnconvertedReason() *UnconvertedReason - - // Bp2buildTargets returns the target(s) generated for Bazel via bp2build for this module - Bp2buildTargets() []bp2buildInfo - GetUnconvertedBp2buildDeps() []string - GetMissingBp2buildDeps() []string - GetPartitionForBp2build() string - BuildParamsForTests() []BuildParams RuleParamsForTests() map[blueprint.Rule]blueprint.RuleParams VariablesForTests() map[string]string @@ -520,9 +509,6 @@ type commonProperties struct { // constants in image.go, but can also be set to a custom value by individual module types. ImageVariation string `blueprint:"mutated"` - // Bazel conversion status - BazelConversionStatus BazelConversionStatus `blueprint:"mutated"` - // SoongConfigTrace records accesses to VendorVars (soong_config). The trace will be hashed // and used as a subdir of PathForModuleOut. Note that we mainly focus on incremental // builds among similar products (e.g. aosp_cf_x86_64_phone and aosp_cf_x86_64_foldable), @@ -532,41 +518,6 @@ type commonProperties struct { SoongConfigTraceHash string `blueprint:"mutated"` } -// CommonAttributes represents the common Bazel attributes from which properties -// in `commonProperties` are translated/mapped; such properties are annotated in -// a list their corresponding attribute. It is embedded within `bp2buildInfo`. -type CommonAttributes struct { - // Soong nameProperties -> Bazel name - Name string - - // Data mapped from: Required - Data bazel.LabelListAttribute - - // SkipData is neither a Soong nor Bazel target attribute - // If true, this will not fill the data attribute automatically - // This is useful for Soong modules that have 1:many Bazel targets - // Some of the generated Bazel targets might not have a data attribute - SkipData *bool - - Tags bazel.StringListAttribute - - Applicable_licenses bazel.LabelListAttribute - - Testonly *bool - - // Dir is neither a Soong nor Bazel target attribute - // If set, the bazel target will be created in this directory - // If unset, the bazel target will default to be created in the directory of the visited soong module - Dir *string -} - -// constraintAttributes represents Bazel attributes pertaining to build constraints, -// which make restrict building a Bazel target for some set of platforms. -type constraintAttributes struct { - // Constraint values this target can be built for. - Target_compatible_with bazel.LabelListAttribute -} - type distProperties struct { // configuration to distribute output files from this module to the distribution // directory (default: $OUT/dist, configurable with $DIST_DIR) @@ -804,234 +755,6 @@ func InitCommonOSAndroidMultiTargetsArchModule(m Module, hod HostOrDeviceSupport m.base().commonProperties.CreateCommonOSVariant = true } -func (attrs *CommonAttributes) getRequiredWithoutCycles(ctx *bottomUpMutatorContext, props *commonProperties) []string { - // Treat `required` as if it's empty if data should be skipped for this target, - // as `required` is only used for the `data` attribute at this time, and we want - // to avoid lookups of labels that won't actually be dependencies of this target. - // TODO: b/202299295 - Refactor this to use `required` dependencies, once they - // are handled other than passing to `data`. - if proptools.Bool(attrs.SkipData) { - return []string{} - } - // The required property can contain the module itself. This causes a cycle - // when generated as the 'data' label list attribute in Bazel. Remove it if - // it exists. See b/247985196. - _, requiredWithoutCycles := RemoveFromList(ctx.ModuleName(), props.Required) - return FirstUniqueStrings(requiredWithoutCycles) -} - -func (attrs *CommonAttributes) fillCommonBp2BuildModuleAttrs(ctx *bottomUpMutatorContext, - enabledPropertyOverrides bazel.BoolAttribute) constraintAttributes { - - mod := ctx.Module().base() - // Assert passed-in attributes include Name - if len(attrs.Name) == 0 { - if ctx.ModuleType() != "package" { - ctx.ModuleErrorf("CommonAttributes in fillCommonBp2BuildModuleAttrs expects a `.Name`!") - } - } - - depsToLabelList := func(deps []string) bazel.LabelListAttribute { - return bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, deps)) - } - - var enabledProperty bazel.BoolAttribute - - onlyAndroid := false - neitherHostNorDevice := false - - osSupport := map[string]bool{} - - // if the target is enabled and supports arch variance, determine the defaults based on the module - // type's host or device property and host_supported/device_supported properties - if mod.commonProperties.ArchSpecific { - moduleSupportsDevice := mod.DeviceSupported() - moduleSupportsHost := mod.HostSupported() - if moduleSupportsHost && !moduleSupportsDevice { - // for host only, we specify as unsupported on android rather than listing all host osSupport - // TODO(b/220874839): consider replacing this with a constraint that covers all host osSupport - // instead - enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(false)) - } else if moduleSupportsDevice && !moduleSupportsHost { - enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(true)) - // specify as a positive to ensure any target-specific enabled can be resolved - // also save that a target is only android, as if there is only the positive restriction on - // android, it'll be dropped, so we may need to add it back later - onlyAndroid = true - } else if !moduleSupportsHost && !moduleSupportsDevice { - neitherHostNorDevice = true - } - - for _, osType := range OsTypeList() { - if osType.Class == Host { - osSupport[osType.Name] = moduleSupportsHost - } else if osType.Class == Device { - osSupport[osType.Name] = moduleSupportsDevice - } - } - } - - if neitherHostNorDevice { - // we can't build this, disable - enabledProperty.Value = proptools.BoolPtr(false) - } else if mod.commonProperties.Enabled != nil { - enabledProperty.SetValue(mod.commonProperties.Enabled) - if !*mod.commonProperties.Enabled { - for oss, enabled := range osSupport { - if val := enabledProperty.SelectValue(bazel.OsConfigurationAxis, oss); enabled && val != nil && *val { - // if this should be disabled by default, clear out any enabling we've done - enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, oss, nil) - } - } - } - } - - attrs.Applicable_licenses = bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, mod.commonProperties.Licenses)) - - requiredWithoutCycles := attrs.getRequiredWithoutCycles(ctx, &mod.commonProperties) - required := depsToLabelList(requiredWithoutCycles) - archVariantProps := mod.GetArchVariantProperties(ctx, &commonProperties{}) - for axis, configToProps := range archVariantProps { - for config, _props := range configToProps { - if archProps, ok := _props.(*commonProperties); ok { - requiredWithoutCycles := attrs.getRequiredWithoutCycles(ctx, archProps) - required.SetSelectValue(axis, config, depsToLabelList(requiredWithoutCycles).Value) - if !neitherHostNorDevice { - if archProps.Enabled != nil { - if axis != bazel.OsConfigurationAxis || osSupport[config] { - enabledProperty.SetSelectValue(axis, config, archProps.Enabled) - } - } - } - } - } - } - - if !neitherHostNorDevice { - if enabledPropertyOverrides.Value != nil { - enabledProperty.Value = enabledPropertyOverrides.Value - } - for _, axis := range enabledPropertyOverrides.SortedConfigurationAxes() { - configToBools := enabledPropertyOverrides.ConfigurableValues[axis] - for cfg, val := range configToBools { - if axis != bazel.OsConfigurationAxis || osSupport[cfg] || val /*If enabled is explicitly requested via overrides */ { - enabledProperty.SetSelectValue(axis, cfg, &val) - } - } - } - } - - productConfigEnabledAttribute := bazel.LabelListAttribute{} - // TODO(b/234497586): Soong config variables and product variables have different overriding behavior, we - // should handle it correctly - if !proptools.BoolDefault(enabledProperty.Value, true) && !neitherHostNorDevice { - // If the module is not enabled by default, then we can check if a - // product variable enables it - productConfigEnabledAttribute = productVariableConfigEnableAttribute(ctx) - - if len(productConfigEnabledAttribute.ConfigurableValues) > 0 { - // In this case, an existing product variable configuration overrides any - // module-level `enable: false` definition - newValue := true - enabledProperty.Value = &newValue - } - } - - platformEnabledAttribute, err := enabledProperty.ToLabelListAttribute( - bazel.LabelList{[]bazel.Label{{Label: "@platforms//:incompatible"}}, nil}, - bazel.LabelList{[]bazel.Label{}, nil}) - if err != nil { - ctx.ModuleErrorf("Error processing platform enabled attribute: %s", err) - } - - // if android is the only arch/os enabled, then add a restriction to only be compatible with android - if platformEnabledAttribute.IsNil() && onlyAndroid { - l := bazel.LabelAttribute{} - l.SetValue(bazel.Label{Label: bazel.OsConfigurationAxis.SelectKey(Android.Name)}) - platformEnabledAttribute.Add(&l) - } - - attrs.Data.Append(required) - - // SkipData is not an attribute of any Bazel target - // Set this to nil so that it does not appear in the generated build file - attrs.SkipData = nil - - moduleEnableConstraints := bazel.LabelListAttribute{} - moduleEnableConstraints.Append(platformEnabledAttribute) - moduleEnableConstraints.Append(productConfigEnabledAttribute) - addCompatibilityConstraintForCompileMultilib(ctx, &moduleEnableConstraints) - - return constraintAttributes{Target_compatible_with: moduleEnableConstraints} -} - -var ( - incompatible = bazel.LabelList{[]bazel.Label{{Label: "@platforms//:incompatible"}}, nil} -) - -// If compile_mulitilib is set to -// 1. 32: Add an incompatibility constraint for non-32 arches -// 1. 64: Add an incompatibility constraint for non-64 arches -func addCompatibilityConstraintForCompileMultilib(ctx *bottomUpMutatorContext, enabled *bazel.LabelListAttribute) { - mod := ctx.Module().base() - multilib, _ := decodeMultilib(mod, mod.commonProperties.CompileOS, ctx.Config().IgnorePrefer32OnDevice()) - - switch multilib { - case "32": - // Add an incompatibility constraint for all known 64-bit arches - enabled.SetSelectValue(bazel.ArchConfigurationAxis, "arm64", incompatible) - enabled.SetSelectValue(bazel.ArchConfigurationAxis, "x86_64", incompatible) - enabled.SetSelectValue(bazel.ArchConfigurationAxis, "riscv64", incompatible) - case "64": - // Add an incompatibility constraint for all known 32-bit arches - enabled.SetSelectValue(bazel.ArchConfigurationAxis, "arm", incompatible) - enabled.SetSelectValue(bazel.ArchConfigurationAxis, "x86", incompatible) - case "both": - // Do nothing: "both" is trivially compatible with 32-bit and 64-bit - // The top level rule (e.g. apex/partition) will be responsible for building this module in both variants via an - // outgoing_transition. - default: // e.g. first, common - // TODO - b/299135307: Add bp2build support for these properties. - } - -} - -// Check product variables for `enabled: true` flag override. -// Returns a list of the constraint_value targets who enable this override. -func productVariableConfigEnableAttribute(ctx *bottomUpMutatorContext) bazel.LabelListAttribute { - result := bazel.LabelListAttribute{} - productVariableProps, errs := ProductVariableProperties(ctx, ctx.Module()) - for _, err := range errs { - ctx.ModuleErrorf("ProductVariableProperties error: %s", err) - } - if productConfigProps, exists := productVariableProps["Enabled"]; exists { - for productConfigProp, prop := range productConfigProps { - flag, ok := prop.(*bool) - if !ok { - ctx.ModuleErrorf("Could not convert product variable enabled property") - } - - if flag == nil { - // soong config var is not used to set `enabled`. nothing to do. - continue - } else if *flag { - axis := productConfigProp.ConfigurationAxis() - result.SetSelectValue(axis, bazel.ConditionsDefaultConfigKey, bazel.MakeLabelList([]bazel.Label{{Label: "@platforms//:incompatible"}})) - result.SetSelectValue(axis, productConfigProp.SelectKey(), bazel.LabelList{Includes: []bazel.Label{}}) - } else if scp, isSoongConfigProperty := productConfigProp.(SoongConfigProperty); isSoongConfigProperty && scp.value == bazel.ConditionsDefaultConfigKey { - // productVariableConfigEnableAttribute runs only if `enabled: false` is set at the top-level outside soong_config_variables - // conditions_default { enabled: false} is a no-op in this case - continue - } else { - // TODO(b/210546943): handle negative case where `enabled: false` - ctx.ModuleErrorf("`enabled: false` is not currently supported for configuration variables. See b/210546943") - } - } - } - - return result -} - // A ModuleBase object contains the properties that are common to all Android // modules. It should be included as an anonymous field in every module // struct definition. InitAndroidModule should then be called from the module's @@ -1146,81 +869,6 @@ type ModuleBase struct { licenseMetadataFile WritablePath } -// A struct containing all relevant information about a Bazel target converted via bp2build. -type bp2buildInfo struct { - Dir string - BazelProps bazel.BazelTargetModuleProperties - CommonAttrs CommonAttributes - ConstraintAttrs constraintAttributes - Attrs interface{} -} - -// TargetName returns the Bazel target name of a bp2build converted target. -func (b bp2buildInfo) TargetName() string { - return b.CommonAttrs.Name -} - -// TargetPackage returns the Bazel package of a bp2build converted target. -func (b bp2buildInfo) TargetPackage() string { - return b.Dir -} - -// BazelRuleClass returns the Bazel rule class of a bp2build converted target. -func (b bp2buildInfo) BazelRuleClass() string { - return b.BazelProps.Rule_class -} - -// BazelRuleLoadLocation returns the location of the Bazel rule of a bp2build converted target. -// This may be empty as native Bazel rules do not need to be loaded. -func (b bp2buildInfo) BazelRuleLoadLocation() string { - return b.BazelProps.Bzl_load_location -} - -// BazelAttributes returns the Bazel attributes of a bp2build converted target. -func (b bp2buildInfo) BazelAttributes() []interface{} { - return []interface{}{&b.CommonAttrs, &b.ConstraintAttrs, b.Attrs} -} - -func (m *ModuleBase) addBp2buildInfo(info bp2buildInfo) { - m.commonProperties.BazelConversionStatus.Bp2buildInfo = append(m.commonProperties.BazelConversionStatus.Bp2buildInfo, info) -} - -func (m *ModuleBase) setPartitionForBp2build(partition string) { - m.commonProperties.BazelConversionStatus.Partition = partition -} - -func (m *ModuleBase) setBp2buildUnconvertible(reasonType bp2build_metrics_proto.UnconvertedReasonType, detail string) { - m.commonProperties.BazelConversionStatus.UnconvertedReason = &UnconvertedReason{ - ReasonType: int(reasonType), - Detail: detail, - } -} - -func (m *ModuleBase) GetUnconvertedReason() *UnconvertedReason { - return m.commonProperties.BazelConversionStatus.UnconvertedReason -} - -// Bp2buildTargets returns the Bazel targets bp2build generated for this module. -func (m *ModuleBase) Bp2buildTargets() []bp2buildInfo { - return m.commonProperties.BazelConversionStatus.Bp2buildInfo -} - -// Bp2buildTargets returns the Bazel targets bp2build generated for this module. -func (m *ModuleBase) GetPartitionForBp2build() string { - return m.commonProperties.BazelConversionStatus.Partition -} - -// GetUnconvertedBp2buildDeps returns the list of module names of this module's direct dependencies that -// were not converted to Bazel. -func (m *ModuleBase) GetUnconvertedBp2buildDeps() []string { - return FirstUniqueStrings(m.commonProperties.BazelConversionStatus.UnconvertedDeps) -} - -// GetMissingBp2buildDeps returns the list of module names that were not found in Android.bp files. -func (m *ModuleBase) GetMissingBp2buildDeps() []string { - return FirstUniqueStrings(m.commonProperties.BazelConversionStatus.MissingDeps) -} - func (m *ModuleBase) AddJSONData(d *map[string]interface{}) { (*d)["Android"] = map[string]interface{}{ // Properties set in Blueprint or in blueprint of a defaults modules @@ -2031,11 +1679,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) return } - if mixedBuildMod, handled := m.isHandledByBazel(ctx); handled { - mixedBuildMod.ProcessBazelQueryResponse(ctx) - } else { - m.module.GenerateAndroidBuildActions(ctx) - } + m.module.GenerateAndroidBuildActions(ctx) if ctx.Failed() { return } @@ -2092,15 +1736,6 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) m.variables = ctx.variables } -func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) { - if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok { - if mixedBuildMod.IsMixedBuildSupported(ctx) && (MixedBuildsEnabled(ctx) == MixedBuildEnabled) { - return mixedBuildMod, true - } - } - return nil, false -} - // Check the supplied dist structure to make sure that it is valid. // // property - the base property, e.g. dist or dists[1], which is combined with the @@ -2193,6 +1828,18 @@ func (m *ModuleBase) IsNativeBridgeSupported() bool { return proptools.Bool(m.commonProperties.Native_bridge_supported) } +// ModuleNameWithPossibleOverride returns the name of the OverrideModule that overrides the current +// variant of this OverridableModule, or ctx.ModuleName() if this module is not an OverridableModule +// or if this variant is not overridden. +func ModuleNameWithPossibleOverride(ctx BaseModuleContext) string { + if overridable, ok := ctx.Module().(OverridableModule); ok { + if o := overridable.GetOverriddenBy(); o != "" { + return o + } + } + return ctx.ModuleName() +} + // SrcIsModule decodes module references in the format ":unqualified-name" or "//namespace:name" // into the module name, or empty string if the input was not a module reference. func SrcIsModule(s string) (module string) { @@ -2615,36 +2262,3 @@ func (s *soongConfigTraceSingleton) GenerateBuildActions(ctx SingletonContext) { WriteFileRule(ctx, outFile, string(j)) ctx.Phony("soong_config_trace", outFile) } - -// Interface implemented by xsd_config which has 1:many mappings in bp2build workspace -// This interface exists because we want to -// 1. Determine the name of the additional targets generated by the primary soong module -// 2. Enable distinguishing an xsd_config module from other Soong modules using type assertion -type XsdConfigBp2buildTargets interface { - CppBp2buildTargetName() string - JavaBp2buildTargetName() string -} - -// XsdModuleToTargetName is a function that takes an XsdConfigBp2buildTarget -type XsdModuleToTargetName func(xsd XsdConfigBp2buildTargets) string - -// XsdLabelMapper returns a bazel.LabelMapper for partitioning XSD sources/headers given an -// XsdModuleToTargetName function. -func XsdLabelMapper(targetName XsdModuleToTargetName) bazel.LabelMapper { - return func(ctx bazel.OtherModuleContext, label bazel.Label) (string, bool) { - mod, exists := ctx.ModuleFromName(label.OriginalModuleName) - if !exists { - return label.Label, false - } - xsdMod, isXsd := mod.(XsdConfigBp2buildTargets) - if !isXsd { - return label.Label, false - } - - // Remove the base module name - ret := strings.TrimSuffix(label.Label, mod.Name()) - // Append the language specific target name - ret += targetName(xsdMod) - return ret, true - } -} diff --git a/android/mutator.go b/android/mutator.go index 3ff9e6198..0d391a47b 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -15,11 +15,6 @@ package android import ( - "path/filepath" - - "android/soong/bazel" - "android/soong/ui/metrics/bp2build_metrics_proto" - "github.com/google/blueprint" ) @@ -32,40 +27,9 @@ import ( // run FinalDeps mutators (CreateVariations disallowed in this phase) // continue on to GenerateAndroidBuildActions -// RegisterMutatorsForBazelConversion is a alternate registration pipeline for bp2build. Exported for testing. -func RegisterMutatorsForBazelConversion(ctx *Context, preArchMutators []RegisterMutatorFunc) { - bp2buildMutators := append(preArchMutators, registerBp2buildConversionMutator) - registerMutatorsForBazelConversion(ctx, bp2buildMutators) -} - -func registerMutatorsForBazelConversion(ctx *Context, bp2buildMutators []RegisterMutatorFunc) { - mctx := ®isterMutatorsContext{ - bazelConversionMode: true, - } - - allMutators := append([]RegisterMutatorFunc{ - RegisterNamespaceMutator, - RegisterDefaultsPreArchMutators, - // TODO(b/165114590): this is required to resolve deps that are only prebuilts, but we should - // evaluate the impact on conversion. - RegisterPrebuiltsPreArchMutators, - RegisterPrebuiltsPostDepsMutators, - }, - bp2buildMutators...) - - // Register bp2build mutators - for _, f := range allMutators { - f(mctx) - } - - mctx.mutators.registerAll(ctx) -} - // collateGloballyRegisteredMutators constructs the list of mutators that have been registered // with the InitRegistrationContext and will be used at runtime. func collateGloballyRegisteredMutators() sortableComponents { - // ensure mixed builds mutator is the last mutator - finalDeps = append(finalDeps, registerMixedBuildsMutator) return collateRegisteredMutators(preArch, preDeps, postDeps, finalDeps) } @@ -94,9 +58,8 @@ func collateRegisteredMutators(preArch, preDeps, postDeps, finalDeps []RegisterM } type registerMutatorsContext struct { - mutators sortableComponents - finalPhase bool - bazelConversionMode bool + mutators sortableComponents + finalPhase bool } type RegisterMutatorsContext interface { @@ -219,58 +182,6 @@ func FinalDepsMutators(f RegisterMutatorFunc) { finalDeps = append(finalDeps, f) } -var bp2buildPreArchMutators = []RegisterMutatorFunc{} - -// A minimal context for Bp2build conversion -type Bp2buildMutatorContext interface { - BazelConversionPathContext - BaseMutatorContext - - // AddDependency adds a dependency to the given module. It returns a slice of modules for each - // dependency (some entries may be nil). - // - // If the mutator is parallel (see MutatorHandle.Parallel), this method will pause until the - // new dependencies have had the current mutator called on them. If the mutator is not - // parallel this method does not affect the ordering of the current mutator pass, but will - // be ordered correctly for all future mutator passes. - AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) []blueprint.Module - - // CreateBazelTargetModule creates a BazelTargetModule by calling the - // factory method, just like in CreateModule, but also requires - // BazelTargetModuleProperties containing additional metadata for the - // bp2build codegenerator. - CreateBazelTargetModule(bazel.BazelTargetModuleProperties, CommonAttributes, interface{}) - - // CreateBazelTargetModuleWithRestrictions creates a BazelTargetModule by calling the - // factory method, just like in CreateModule, but also requires - // BazelTargetModuleProperties containing additional metadata for the - // bp2build codegenerator. The generated target is restricted to only be buildable for certain - // platforms, as dictated by a given bool attribute: the target will not be buildable in - // any platform for which this bool attribute is false. - CreateBazelTargetModuleWithRestrictions(bazel.BazelTargetModuleProperties, CommonAttributes, interface{}, bazel.BoolAttribute) - - // MarkBp2buildUnconvertible registers the current module as "unconvertible to bp2build" for the - // given reason. - MarkBp2buildUnconvertible(reasonType bp2build_metrics_proto.UnconvertedReasonType, detail string) - - // CreateBazelTargetAliasInDir creates an alias definition in `dir` directory. - // This function can be used to create alias definitions in a directory that is different - // from the directory of the visited Soong module. - CreateBazelTargetAliasInDir(dir string, name string, actual bazel.Label) - - // CreateBazelConfigSetting creates a config_setting in /BUILD.bazel - // build/bazel has several static config_setting(s) that are used in Bazel builds. - // This function can be used to createa additional config_setting(s) based on the build graph - // (e.g. a config_setting specific to an apex variant) - CreateBazelConfigSetting(csa bazel.ConfigSettingAttributes, ca CommonAttributes, dir string) -} - -// PreArchBp2BuildMutators adds mutators to be register for converting Android Blueprint modules -// into Bazel BUILD targets that should run prior to deps and conversion. -func PreArchBp2BuildMutators(f RegisterMutatorFunc) { - bp2buildPreArchMutators = append(bp2buildPreArchMutators, f) -} - type BaseMutatorContext interface { BaseModuleContext @@ -301,7 +212,15 @@ type BottomUpMutator func(BottomUpMutatorContext) type BottomUpMutatorContext interface { BaseMutatorContext - Bp2buildMutatorContext + + // AddDependency adds a dependency to the given module. It returns a slice of modules for each + // dependency (some entries may be nil). + // + // If the mutator is parallel (see MutatorHandle.Parallel), this method will pause until the + // new dependencies have had the current mutator called on them. If the mutator is not + // parallel this method does not affect the ordering of the current mutator pass, but will + // be ordered correctly for all future mutator passes. + AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) []blueprint.Module // AddReverseDependency adds a dependency from the destination to the given module. // Does not affect the ordering of the current mutator pass, but will be ordered @@ -416,10 +335,9 @@ type bottomUpMutatorContext struct { } func bottomUpMutatorContextFactory(ctx blueprint.BottomUpMutatorContext, a Module, - finalPhase, bazelConversionMode bool) BottomUpMutatorContext { + finalPhase bool) BottomUpMutatorContext { moduleContext := a.base().baseModuleContextFactory(ctx) - moduleContext.bazelConversionMode = bazelConversionMode return &bottomUpMutatorContext{ bp: ctx, @@ -430,10 +348,9 @@ func bottomUpMutatorContextFactory(ctx blueprint.BottomUpMutatorContext, a Modul func (x *registerMutatorsContext) BottomUp(name string, m BottomUpMutator) MutatorHandle { finalPhase := x.finalPhase - bazelConversionMode := x.bazelConversionMode f := func(ctx blueprint.BottomUpMutatorContext) { if a, ok := ctx.Module().(Module); ok { - m(bottomUpMutatorContextFactory(ctx, a, finalPhase, bazelConversionMode)) + m(bottomUpMutatorContextFactory(ctx, a, finalPhase)) } } mutator := &mutator{name: x.mutatorName(name), bottomUpMutator: f} @@ -550,15 +467,13 @@ type TransitionMutator interface { } type androidTransitionMutator struct { - finalPhase bool - bazelConversionMode bool - mutator TransitionMutator + finalPhase bool + mutator TransitionMutator } func (a *androidTransitionMutator) Split(ctx blueprint.BaseModuleContext) []string { if m, ok := ctx.Module().(Module); ok { moduleContext := m.base().baseModuleContextFactory(ctx) - moduleContext.bazelConversionMode = a.bazelConversionMode return a.mutator.Split(&moduleContext) } else { return []string{""} @@ -607,15 +522,14 @@ func (a *androidTransitionMutator) IncomingTransition(ctx blueprint.IncomingTran func (a *androidTransitionMutator) Mutate(ctx blueprint.BottomUpMutatorContext, variation string) { if am, ok := ctx.Module().(Module); ok { - a.mutator.Mutate(bottomUpMutatorContextFactory(ctx, am, a.finalPhase, a.bazelConversionMode), variation) + a.mutator.Mutate(bottomUpMutatorContextFactory(ctx, am, a.finalPhase), variation) } } func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { atm := &androidTransitionMutator{ - finalPhase: x.finalPhase, - bazelConversionMode: x.bazelConversionMode, - mutator: m, + finalPhase: x.finalPhase, + mutator: m, } mutator := &mutator{ name: name, @@ -624,9 +538,6 @@ func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { } func (x *registerMutatorsContext) mutatorName(name string) string { - if x.bazelConversionMode { - return name + "_bp2build" - } return name } @@ -634,7 +545,6 @@ func (x *registerMutatorsContext) TopDown(name string, m TopDownMutator) Mutator f := func(ctx blueprint.TopDownMutatorContext) { if a, ok := ctx.Module().(Module); ok { moduleContext := a.base().baseModuleContextFactory(ctx) - moduleContext.bazelConversionMode = x.bazelConversionMode actx := &topDownMutatorContext{ bp: ctx, baseModuleContext: moduleContext, @@ -698,179 +608,6 @@ func registerDepsMutator(ctx RegisterMutatorsContext) { ctx.BottomUp("deps", depsMutator).Parallel() } -func (t *bottomUpMutatorContext) CreateBazelTargetModule( - bazelProps bazel.BazelTargetModuleProperties, - commonAttrs CommonAttributes, - attrs interface{}) { - t.createBazelTargetModule(bazelProps, commonAttrs, attrs, bazel.BoolAttribute{}) -} - -func (t *bottomUpMutatorContext) CreateBazelTargetModuleWithRestrictions( - bazelProps bazel.BazelTargetModuleProperties, - commonAttrs CommonAttributes, - attrs interface{}, - enabledProperty bazel.BoolAttribute) { - t.createBazelTargetModule(bazelProps, commonAttrs, attrs, enabledProperty) -} - -func (t *bottomUpMutatorContext) MarkBp2buildUnconvertible( - reasonType bp2build_metrics_proto.UnconvertedReasonType, detail string) { - mod := t.Module() - mod.base().setBp2buildUnconvertible(reasonType, detail) -} - -var ( - bazelAliasModuleProperties = bazel.BazelTargetModuleProperties{ - Rule_class: "alias", - } -) - -type bazelAliasAttributes struct { - Actual *bazel.LabelAttribute -} - -func (t *bottomUpMutatorContext) CreateBazelTargetAliasInDir( - dir string, - name string, - actual bazel.Label) { - mod := t.Module() - attrs := &bazelAliasAttributes{ - Actual: bazel.MakeLabelAttribute(actual.Label), - } - info := bp2buildInfo{ - Dir: dir, - BazelProps: bazelAliasModuleProperties, - CommonAttrs: CommonAttributes{Name: name}, - ConstraintAttrs: constraintAttributes{}, - Attrs: attrs, - } - mod.base().addBp2buildInfo(info) -} - -// Returns the directory in which the bazel target will be generated -// If ca.Dir is not nil, use that -// Otherwise default to the directory of the soong module -func dirForBazelTargetGeneration(t *bottomUpMutatorContext, ca *CommonAttributes) string { - dir := t.OtherModuleDir(t.Module()) - if ca.Dir != nil { - dir = *ca.Dir - // Restrict its use to dirs that contain an Android.bp file. - // There are several places in bp2build where we use the existence of Android.bp/BUILD on the filesystem - // to curate a compatible label for src files (e.g. headers for cc). - // If we arbritrarily create BUILD files, then it might render those curated labels incompatible. - if exists, _, _ := t.Config().fs.Exists(filepath.Join(dir, "Android.bp")); !exists { - t.ModuleErrorf("Cannot use ca.Dir to create a BazelTarget in dir: %v since it does not contain an Android.bp file", dir) - } - - // Set ca.Dir to nil so that it does not get emitted to the BUILD files - ca.Dir = nil - } - return dir -} - -func (t *bottomUpMutatorContext) CreateBazelConfigSetting( - csa bazel.ConfigSettingAttributes, - ca CommonAttributes, - dir string) { - mod := t.Module() - info := bp2buildInfo{ - Dir: dir, - BazelProps: bazel.BazelTargetModuleProperties{ - Rule_class: "config_setting", - }, - CommonAttrs: ca, - ConstraintAttrs: constraintAttributes{}, - Attrs: &csa, - } - mod.base().addBp2buildInfo(info) -} - -// ApexAvailableTags converts the apex_available property value of an ApexModule -// module and returns it as a list of keyed tags. -func ApexAvailableTags(mod Module) bazel.StringListAttribute { - attr := bazel.StringListAttribute{} - // Transform specific attributes into tags. - if am, ok := mod.(ApexModule); ok { - // TODO(b/218841706): hidl_interface has the apex_available prop, but it's - // defined directly as a prop and not via ApexModule, so this doesn't - // pick those props up. - apexAvailable := am.apexModuleBase().ApexAvailable() - // If a user does not specify apex_available in Android.bp, then soong provides a default. - // To avoid verbosity of BUILD files, remove this default from user-facing BUILD files. - if len(am.apexModuleBase().ApexProperties.Apex_available) == 0 { - apexAvailable = []string{} - } - attr.Value = ConvertApexAvailableToTags(apexAvailable) - } - return attr -} - -func ApexAvailableTagsWithoutTestApexes(ctx BaseModuleContext, mod Module) bazel.StringListAttribute { - attr := bazel.StringListAttribute{} - if am, ok := mod.(ApexModule); ok { - apexAvailableWithoutTestApexes := removeTestApexes(ctx, am.apexModuleBase().ApexAvailable()) - // If a user does not specify apex_available in Android.bp, then soong provides a default. - // To avoid verbosity of BUILD files, remove this default from user-facing BUILD files. - if len(am.apexModuleBase().ApexProperties.Apex_available) == 0 { - apexAvailableWithoutTestApexes = []string{} - } - attr.Value = ConvertApexAvailableToTags(apexAvailableWithoutTestApexes) - } - return attr -} - -func removeTestApexes(ctx BaseModuleContext, apex_available []string) []string { - testApexes := []string{} - for _, aa := range apex_available { - // ignore the wildcards - if InList(aa, AvailableToRecognziedWildcards) { - continue - } - mod, _ := ctx.ModuleFromName(aa) - if apex, ok := mod.(ApexTestInterface); ok && apex.IsTestApex() { - testApexes = append(testApexes, aa) - } - } - return RemoveListFromList(CopyOf(apex_available), testApexes) -} - -func ConvertApexAvailableToTags(apexAvailable []string) []string { - if len(apexAvailable) == 0 { - // We need nil specifically to make bp2build not add the tags property at all, - // instead of adding it with an empty list - return nil - } - result := make([]string, 0, len(apexAvailable)) - for _, a := range apexAvailable { - result = append(result, "apex_available="+a) - } - return result -} - -// ConvertApexAvailableToTagsWithoutTestApexes converts a list of apex names to a list of bazel tags -// This function drops any test apexes from the input. -func ConvertApexAvailableToTagsWithoutTestApexes(ctx BaseModuleContext, apexAvailable []string) []string { - noTestApexes := removeTestApexes(ctx, apexAvailable) - return ConvertApexAvailableToTags(noTestApexes) -} - -func (t *bottomUpMutatorContext) createBazelTargetModule( - bazelProps bazel.BazelTargetModuleProperties, - commonAttrs CommonAttributes, - attrs interface{}, - enabledProperty bazel.BoolAttribute) { - constraintAttributes := commonAttrs.fillCommonBp2BuildModuleAttrs(t, enabledProperty) - mod := t.Module() - info := bp2buildInfo{ - Dir: dirForBazelTargetGeneration(t, &commonAttrs), - BazelProps: bazelProps, - CommonAttrs: commonAttrs, - ConstraintAttrs: constraintAttributes, - Attrs: attrs, - } - mod.base().addBp2buildInfo(info) -} - // android.topDownMutatorContext either has to embed blueprint.TopDownMutatorContext, in which case every method that // has an overridden version in android.BaseModuleContext has to be manually forwarded to BaseModuleContext to avoid // ambiguous method errors, or it has to store a blueprint.TopDownMutatorContext non-embedded, in which case every diff --git a/android/mutator_test.go b/android/mutator_test.go index dbdfa3362..21eebd2c8 100644 --- a/android/mutator_test.go +++ b/android/mutator_test.go @@ -16,7 +16,6 @@ package android import ( "fmt" - "reflect" "strings" "testing" @@ -268,22 +267,3 @@ func TestNoCreateVariationsInFinalDeps(t *testing.T) { FixtureWithRootAndroidBp(`test {name: "foo"}`), ).RunTest(t) } - -func TestConvertApexAvailableToTags(t *testing.T) { - input := []string{ - "com.android.adbd", - "//apex_available:platform", - } - actual := ConvertApexAvailableToTags(input) - expected := []string{ - "apex_available=com.android.adbd", - "apex_available=//apex_available:platform", - } - if !reflect.DeepEqual(actual, expected) { - t.Errorf("Expected: %v, actual: %v", expected, actual) - } - - if ConvertApexAvailableToTags(nil) != nil { - t.Errorf("Expected providing nil to return nil") - } -} diff --git a/android/override_module.go b/android/override_module.go index 9e0de6fd0..1341f537c 100644 --- a/android/override_module.go +++ b/android/override_module.go @@ -214,17 +214,6 @@ func (b *OverridableModuleBase) override(ctx BaseModuleContext, m Module, o Over } b.overridableModuleProperties.OverriddenBy = o.Name() b.overridableModuleProperties.OverriddenByModuleDir = o.ModuleDir() - - if oBazelable, ok := o.base().module.(Bazelable); ok { - if bBazelable, ok := m.(Bazelable); ok { - oProps := oBazelable.bazelProps() - bProps := bBazelable.bazelProps() - bProps.Bazel_module.Bp2build_available = oProps.Bazel_module.Bp2build_available - bProps.Bazel_module.Label = oProps.Bazel_module.Label - } else { - ctx.ModuleErrorf("Override type cannot be Bazelable if original module type is not Bazelable %v %v.", o.Name(), m.Name()) - } - } } // GetOverriddenBy returns the name of the override module that has overridden this module. @@ -348,31 +337,3 @@ func replaceDepsOnOverridingModuleMutator(ctx BottomUpMutatorContext) { } } } - -// ModuleNameWithPossibleOverride returns the name of the OverrideModule that overrides the current -// variant of this OverridableModule, or ctx.ModuleName() if this module is not an OverridableModule -// or if this variant is not overridden. -func ModuleNameWithPossibleOverride(ctx BazelConversionContext) string { - return moduleNameWithPossibleOverride(ctx, ctx.Module(), ctx.OtherModuleName(ctx.Module())) -} - -func moduleNameWithPossibleOverride(ctx shouldConvertModuleContext, module blueprint.Module, name string) string { - if overridable, ok := module.(OverridableModule); ok { - if o := overridable.GetOverriddenBy(); o != "" { - return o - } - } - return name -} - -// moduleDirWithPossibleOverride returns the dir of the OverrideModule that overrides the current -// variant of the given OverridableModule, or ctx.OtherModuleName() if the module is not an -// OverridableModule or if the variant is not overridden. -func moduleDirWithPossibleOverride(ctx shouldConvertModuleContext, module blueprint.Module, dir string) string { - if overridable, ok := module.(OverridableModule); ok { - if o := overridable.GetOverriddenByModuleDir(); o != "" { - return o - } - } - return dir -} diff --git a/android/prebuilt.go b/android/prebuilt.go index 91c0aa182..a32a37d3a 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -461,10 +461,6 @@ func PrebuiltSelectModuleMutator(ctx BottomUpMutatorContext) { // Propagate the provider received from `all_apex_contributions` // to the source module ctx.VisitDirectDepsWithTag(acDepTag, func(am Module) { - if ctx.Config().Bp2buildMode() { - // This provider key is not applicable in bp2build - return - } psi := ctx.OtherModuleProvider(am, PrebuiltSelectionInfoProvider).(PrebuiltSelectionInfoMap) ctx.SetProvider(PrebuiltSelectionInfoProvider, psi) }) @@ -570,55 +566,20 @@ func (p *Prebuilt) usePrebuilt(ctx BaseMutatorContext, source Module, prebuilt M // fall back to the existing source vs prebuilt selection. // TODO: Drop the fallback mechanisms - if !ctx.Config().Bp2buildMode() { - if p.srcsSupplier != nil && len(p.srcsSupplier(ctx, prebuilt)) == 0 { - return false - } + if p.srcsSupplier != nil && len(p.srcsSupplier(ctx, prebuilt)) == 0 { + return false + } - // Skip prebuilt modules under unexported namespaces so that we won't - // end up shadowing non-prebuilt module when prebuilt module under same - // name happens to have a `Prefer` property set to true. - if ctx.Config().KatiEnabled() && !prebuilt.ExportedToMake() { - return false - } + // Skip prebuilt modules under unexported namespaces so that we won't + // end up shadowing non-prebuilt module when prebuilt module under same + // name happens to have a `Prefer` property set to true. + if ctx.Config().KatiEnabled() && !prebuilt.ExportedToMake() { + return false } // If source is not available or is disabled then always use the prebuilt. if source == nil || !source.Enabled() { - // If in bp2build mode, we need to check product variables & Soong config variables, which may - // have overridden the "enabled" property but have not been merged into the property value as - // they would in a non-bp2build mode invocation - if ctx.Config().Bp2buildMode() && source != nil { - productVariableProps, errs := ProductVariableProperties(ctx, source) - if productConfigProps, exists := productVariableProps["Enabled"]; len(errs) == 0 && exists && len(productConfigProps) == 1 { - var prop ProductConfigOrSoongConfigProperty - var value bool - for p, v := range productConfigProps { - prop = p - actual, ok := v.(*bool) - if ok { - value = proptools.Bool(actual) - } - } - if scv, ok := prop.(SoongConfigProperty); ok { - // If the product config var is enabled but the value of enabled is false still, the - // prebuilt is preferred. Otherwise, check if the prebulit is explicitly preferred - if ctx.Config().VendorConfig(scv.namespace).Bool(strings.ToLower(scv.name)) && !value { - return true - } - } else { - // TODO: b/300998219 - handle product vars - // We don't handle product variables yet, so return based on the non-product specific - // value of enabled - return true - } - } else { - // No "enabled" property override, return true since this module isn't enabled - return true - } - } else { - return true - } + return true } // If the use_source_config_var property is set then it overrides the prefer property setting. diff --git a/android/register.go b/android/register.go index de3135306..cd968cd02 100644 --- a/android/register.go +++ b/android/register.go @@ -15,15 +15,9 @@ package android import ( - "bufio" "fmt" - "path/filepath" - "reflect" - "regexp" - - "android/soong/shared" - "github.com/google/blueprint" + "reflect" ) // A sortable component is one whose registration order affects the order in which it is executed @@ -166,75 +160,6 @@ func NewContext(config Config) *Context { return ctx } -// Helper function to register the module types used in bp2build. -func registerModuleTypes(ctx *Context) { - for _, t := range moduleTypes { - t.register(ctx) - } - // Required for SingletonModule types, even though we are not using them. - for _, t := range singletons { - t.register(ctx) - } -} - -// RegisterForBazelConversion registers an alternate shadow pipeline of -// singletons, module types and mutators to register for converting Blueprint -// files to semantically equivalent BUILD files. -func (ctx *Context) RegisterForBazelConversion() { - registerModuleTypes(ctx) - RegisterMutatorsForBazelConversion(ctx, bp2buildPreArchMutators) -} - -// RegisterExistingBazelTargets reads Bazel BUILD.bazel and BUILD files under -// the workspace, and returns a map containing names of Bazel targets defined in -// these BUILD files. -// For example, maps "//foo/bar" to ["baz", "qux"] if `//foo/bar:{baz,qux}` exist. -func (c *Context) RegisterExistingBazelTargets(topDir string, existingBazelFiles []string) error { - result := map[string][]string{} - - // Search for instances of `name = "$NAME"` (with arbitrary spacing). - targetNameRegex := regexp.MustCompile(`(?m)^\s*name\s*=\s*\"([^\"]+)\"`) - - parseBuildFile := func(path string) error { - fullPath := shared.JoinPath(topDir, path) - sourceDir := filepath.Dir(path) - - fileInfo, err := c.Config().fs.Stat(fullPath) - if err != nil { - return fmt.Errorf("Error accessing Bazel file '%s': %s", path, err) - } - if !fileInfo.IsDir() && - (fileInfo.Name() == "BUILD" || fileInfo.Name() == "BUILD.bazel") { - f, err := c.Config().fs.Open(fullPath) - if err != nil { - return fmt.Errorf("Error reading Bazel file '%s': %s", path, err) - } - defer f.Close() - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - matches := targetNameRegex.FindAllStringSubmatch(line, -1) - for _, match := range matches { - result[sourceDir] = append(result[sourceDir], match[1]) - } - } - } - return nil - } - - for _, path := range existingBazelFiles { - if !c.Config().Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(filepath.Dir(path)) { - continue - } - err := parseBuildFile(path) - if err != nil { - return err - } - } - c.Config().SetBazelBuildFileTargets(result) - return nil -} - // Register the pipeline of singletons, module types, and mutators for // generating build.ninja and other files for Kati, from Android.bp files. func (ctx *Context) Register() { @@ -260,8 +185,6 @@ func (ctx *Context) registerSingletonMakeVarsProvider(makevars SingletonMakeVars func collateGloballyRegisteredSingletons() sortableComponents { allSingletons := append(sortableComponents(nil), singletons...) allSingletons = append(allSingletons, - singleton{parallel: true, name: "bazeldeps", factory: BazelSingleton}, - // Register phony just before makevars so it can write out its phony rules as Make rules singleton{parallel: false, name: "phony", factory: phonySingletonFactory}, diff --git a/android/test_config.go b/android/test_config.go index 2a59d9228..9e1ac70df 100644 --- a/android/test_config.go +++ b/android/test_config.go @@ -61,11 +61,7 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string // passed to PathForSource or PathForModuleSrc. TestAllowNonExistentPaths: true, - BazelContext: noopBazelContext{}, - BuildMode: BazelProdMode, - mixedBuildDisabledModules: make(map[string]struct{}), - mixedBuildEnabledModules: make(map[string]struct{}), - bazelForceEnabledModules: make(map[string]struct{}), + BuildMode: AnalysisNoBazel, } config.deviceConfig = &deviceConfig{ config: config, diff --git a/android/testing.go b/android/testing.go index c596468c3..fa4dffd9b 100644 --- a/android/testing.go +++ b/android/testing.go @@ -219,10 +219,6 @@ func (ctx *TestContext) FinalDepsMutators(f RegisterMutatorFunc) { ctx.finalDeps = append(ctx.finalDeps, f) } -func (ctx *TestContext) RegisterBp2BuildConfig(config Bp2BuildConversionAllowlist) { - ctx.config.Bp2buildPackageConfig = config -} - // PreArchBp2BuildMutators adds mutators to be register for converting Android Blueprint modules // into Bazel BUILD targets that should run prior to deps and conversion. func (ctx *TestContext) PreArchBp2BuildMutators(f RegisterMutatorFunc) { @@ -449,12 +445,6 @@ func (ctx *TestContext) Register() { ctx.singletonOrder = componentsToNames(singletons) } -// RegisterForBazelConversion prepares a test context for bp2build conversion. -func (ctx *TestContext) RegisterForBazelConversion() { - ctx.config.BuildMode = Bp2build - RegisterMutatorsForBazelConversion(ctx.Context, ctx.bp2buildPreArch) -} - func (ctx *TestContext) ParseFileList(rootDir string, filePaths []string) (deps []string, errs []error) { // This function adapts the old style ParseFileList calls that are spread throughout the tests // to the new style that takes a config. diff --git a/android/util.go b/android/util.go index f687bca28..ae1c65756 100644 --- a/android/util.go +++ b/android/util.go @@ -591,3 +591,9 @@ func CheckDuplicate(values []string) (duplicate string, found bool) { } return "", false } + +func AddToStringSet(set map[string]bool, items []string) { + for _, item := range items { + set[item] = true + } +} diff --git a/android/variable.go b/android/variable.go index aecad3b84..fa4cfc1fd 100644 --- a/android/variable.go +++ b/android/variable.go @@ -20,7 +20,6 @@ import ( "runtime" "strings" - "android/soong/android/soongconfig" "android/soong/bazel" "github.com/google/blueprint/proptools" @@ -746,44 +745,6 @@ func (p SoongConfigProperty) SelectKey() string { // property, like ["-DDEFINES"] for cflags. type ProductConfigProperties map[string]map[ProductConfigOrSoongConfigProperty]interface{} -// ProductVariableProperties returns a ProductConfigProperties containing only the properties which -// have been set for the given module. -func ProductVariableProperties(ctx ArchVariantContext, module Module) (ProductConfigProperties, []error) { - var errs []error - moduleBase := module.base() - - productConfigProperties := ProductConfigProperties{} - - if moduleBase.variableProperties != nil { - productVariablesProperty := proptools.FieldNameForProperty("product_variables") - if moduleBase.ArchSpecific() { - for /* axis */ _, configToProps := range moduleBase.GetArchVariantProperties(ctx, moduleBase.variableProperties) { - for config, props := range configToProps { - variableValues := reflect.ValueOf(props).Elem().FieldByName(productVariablesProperty) - productConfigProperties.AddProductConfigProperties(variableValues, config) - } - } - } else { - variableValues := reflect.ValueOf(moduleBase.variableProperties).Elem().FieldByName(productVariablesProperty) - productConfigProperties.AddProductConfigProperties(variableValues, "") - } - } - - if m, ok := module.(Bazelable); ok && m.namespacedVariableProps() != nil { - for namespace, namespacedVariableProps := range m.namespacedVariableProps() { - for _, namespacedVariableProp := range namespacedVariableProps { - variableValues := reflect.ValueOf(namespacedVariableProp).Elem().FieldByName(soongconfig.SoongConfigProperty) - err := productConfigProperties.AddSoongConfigProperties(namespace, variableValues) - if err != nil { - errs = append(errs, err) - } - } - } - } - - return productConfigProperties, errs -} - func (p *ProductConfigProperties) AddProductConfigProperty( propertyName, productVariableName, arch string, propertyValue interface{}) { @@ -833,10 +794,6 @@ func (p *ProductConfigProperties) AddEitherProperty( } } -var ( - conditionsDefaultField string = proptools.FieldNameForProperty(bazel.ConditionsDefaultConfigKey) -) - // maybeExtractConfigVarProp attempts to read this value as a config var struct // wrapped by interfaces and ptrs. If it's not the right type, the second return // value is false. diff --git a/bp2build/Android.bp b/bp2build/Android.bp index aff703df0..ba1268214 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -7,16 +7,11 @@ bootstrap_go_package { pkgPath: "android/soong/bp2build", srcs: [ "androidbp_to_build_templates.go", - "bp2build.go", - "bp2build_product_config.go", "build_conversion.go", "bzl_conversion.go", "configurability.go", "constants.go", "conversion.go", - "metrics.go", - "symlink_forest.go", - "testing.go", ], deps: [ "blueprint-bootstrap", @@ -41,7 +36,6 @@ bootstrap_go_package { ], testSrcs: [ "conversion_test.go", - "performance_test.go", ], pluginFor: [ "soong_build", diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go deleted file mode 100644 index 1496ca74b..000000000 --- a/bp2build/bp2build.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bp2build - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "android/soong/android" - "android/soong/bazel" - "android/soong/shared" - "android/soong/starlark_import" -) - -func deleteFilesExcept(ctx *CodegenContext, rootOutputPath android.OutputPath, except []BazelFile) { - // Delete files that should no longer be present. - bp2buildDirAbs := shared.JoinPath(ctx.topDir, rootOutputPath.String()) - - filesToDelete := make(map[string]struct{}) - err := filepath.Walk(bp2buildDirAbs, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - relPath, err := filepath.Rel(bp2buildDirAbs, path) - if err != nil { - return err - } - filesToDelete[relPath] = struct{}{} - } - return nil - }) - if err != nil { - fmt.Printf("ERROR reading %s: %s", bp2buildDirAbs, err) - os.Exit(1) - } - - for _, bazelFile := range except { - filePath := filepath.Join(bazelFile.Dir, bazelFile.Basename) - delete(filesToDelete, filePath) - } - for f, _ := range filesToDelete { - absPath := shared.JoinPath(bp2buildDirAbs, f) - if err := os.RemoveAll(absPath); err != nil { - fmt.Printf("ERROR deleting %s: %s", absPath, err) - os.Exit(1) - } - } -} - -// Codegen is the backend of bp2build. The code generator is responsible for -// writing .bzl files that are equivalent to Android.bp files that are capable -// of being built with Bazel. -func Codegen(ctx *CodegenContext) *CodegenMetrics { - ctx.Context().BeginEvent("Codegen") - defer ctx.Context().EndEvent("Codegen") - // This directory stores BUILD files that could be eventually checked-in. - bp2buildDir := android.PathForOutput(ctx, "bp2build") - - res, errs := GenerateBazelTargets(ctx, true) - if len(errs) > 0 { - errMsgs := make([]string, len(errs)) - for i, err := range errs { - errMsgs[i] = fmt.Sprintf("%q", err) - } - fmt.Printf("ERROR: Encountered %d error(s): \nERROR: %s", len(errs), strings.Join(errMsgs, "\n")) - os.Exit(1) - } - var bp2buildFiles []BazelFile - productConfig, err := createProductConfigFiles(ctx, res.moduleNameToPartition, res.metrics.convertedModulePathMap) - ctx.Context().EventHandler.Do("CreateBazelFile", func() { - allTargets := make(map[string]BazelTargets) - for k, v := range res.buildFileToTargets { - allTargets[k] = append(allTargets[k], v...) - } - for k, v := range productConfig.bp2buildTargets { - allTargets[k] = append(allTargets[k], v...) - } - bp2buildFiles = CreateBazelFiles(nil, allTargets, ctx.mode) - }) - bp2buildFiles = append(bp2buildFiles, productConfig.bp2buildFiles...) - injectionFiles, err := createSoongInjectionDirFiles(ctx, res.metrics) - if err != nil { - fmt.Printf("%s\n", err.Error()) - os.Exit(1) - } - injectionFiles = append(injectionFiles, productConfig.injectionFiles...) - - writeFiles(ctx, bp2buildDir, bp2buildFiles) - // Delete files under the bp2build root which weren't just written. An - // alternative would have been to delete the whole directory and write these - // files. However, this would regenerate files which were otherwise unchanged - // since the last bp2build run, which would have negative incremental - // performance implications. - deleteFilesExcept(ctx, bp2buildDir, bp2buildFiles) - - writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles) - starlarkDeps, err := starlark_import.GetNinjaDeps() - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - os.Exit(1) - } - ctx.AddNinjaFileDeps(starlarkDeps...) - return &res.metrics -} - -// Get the output directory and create it if it doesn't exist. -func getOrCreateOutputDir(outputDir android.OutputPath, ctx android.PathContext, dir string) android.OutputPath { - dirPath := outputDir.Join(ctx, dir) - if err := android.CreateOutputDirIfNonexistent(dirPath, os.ModePerm); err != nil { - fmt.Printf("ERROR: path %s: %s", dirPath, err.Error()) - } - return dirPath -} - -// writeFiles materializes a list of BazelFile rooted at outputDir. -func writeFiles(ctx android.PathContext, outputDir android.OutputPath, files []BazelFile) { - for _, f := range files { - p := getOrCreateOutputDir(outputDir, ctx, f.Dir).Join(ctx, f.Basename) - if err := writeFile(p, f.Contents); err != nil { - panic(fmt.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err)) - } - } -} - -func writeFile(pathToFile android.OutputPath, content string) error { - // These files are made editable to allow users to modify and iterate on them - // in the source tree. - return android.WriteFileToOutputDir(pathToFile, []byte(content), 0644) -} diff --git a/bp2build/bp2build_product_config.go b/bp2build/bp2build_product_config.go deleted file mode 100644 index 4c09d6728..000000000 --- a/bp2build/bp2build_product_config.go +++ /dev/null @@ -1,885 +0,0 @@ -package bp2build - -import ( - "encoding/json" - "fmt" - "path/filepath" - "reflect" - "sort" - "strings" - - "android/soong/android" - "android/soong/android/soongconfig" - "android/soong/starlark_import" - - "github.com/google/blueprint/proptools" - "go.starlark.net/starlark" -) - -type createProductConfigFilesResult struct { - injectionFiles []BazelFile - bp2buildFiles []BazelFile - bp2buildTargets map[string]BazelTargets -} - -type bazelLabel struct { - repo string - pkg string - target string -} - -const releaseAconfigValueSetsName = "release_aconfig_value_sets" - -func (l *bazelLabel) Less(other *bazelLabel) bool { - if l.repo < other.repo { - return true - } - if l.repo > other.repo { - return false - } - if l.pkg < other.pkg { - return true - } - if l.pkg > other.pkg { - return false - } - return l.target < other.target -} - -func (l *bazelLabel) String() string { - return fmt.Sprintf("@%s//%s:%s", l.repo, l.pkg, l.target) -} - -func createProductConfigFiles( - ctx *CodegenContext, - moduleNameToPartition map[string]string, - convertedModulePathMap map[string]string) (createProductConfigFilesResult, error) { - cfg := &ctx.config - targetProduct := "unknown" - if cfg.HasDeviceProduct() { - targetProduct = cfg.DeviceProduct() - } - targetBuildVariant := "user" - if cfg.Eng() { - targetBuildVariant = "eng" - } else if cfg.Debuggable() { - targetBuildVariant = "userdebug" - } - - var res createProductConfigFilesResult - - productVariables := ctx.Config().ProductVariables() - // TODO(b/306243251): For some reason, using the real value of native_coverage makes some select - // statements ambiguous - productVariables.Native_coverage = nil - productVariablesBytes, err := json.Marshal(productVariables) - if err != nil { - return res, err - } - - currentProductFolder := fmt.Sprintf("build/bazel/products/%s", targetProduct) - if len(productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory) > 0 { - currentProductFolder = fmt.Sprintf("%s%s", productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory, targetProduct) - } - - productReplacer := strings.NewReplacer( - "{PRODUCT}", targetProduct, - "{VARIANT}", targetBuildVariant, - "{PRODUCT_FOLDER}", currentProductFolder) - - productsForTestingMap, err := starlark_import.GetStarlarkValue[map[string]map[string]starlark.Value]("products_for_testing") - if err != nil { - return res, err - } - productsForTesting := android.SortedKeys(productsForTestingMap) - for i := range productsForTesting { - productsForTesting[i] = fmt.Sprintf(" \"@//build/bazel/tests/products:%s\",", productsForTesting[i]) - } - - productLabelsToVariables := make(map[bazelLabel]*android.ProductVariables) - productLabelsToVariables[bazelLabel{ - repo: "", - pkg: currentProductFolder, - target: targetProduct, - }] = &productVariables - for product, productVariablesStarlark := range productsForTestingMap { - productVariables, err := starlarkMapToProductVariables(productVariablesStarlark) - if err != nil { - return res, err - } - productLabelsToVariables[bazelLabel{ - repo: "", - pkg: "build/bazel/tests/products", - target: product, - }] = &productVariables - } - - res.bp2buildTargets = make(map[string]BazelTargets) - res.bp2buildTargets[currentProductFolder] = append(res.bp2buildTargets[currentProductFolder], BazelTarget{ - name: productReplacer.Replace("{PRODUCT}"), - packageName: currentProductFolder, - content: productReplacer.Replace(`android_product( - name = "{PRODUCT}", - soong_variables = _soong_variables, -)`), - ruleClass: "android_product", - loads: []BazelLoad{ - { - file: ":soong.variables.bzl", - symbols: []BazelLoadSymbol{{ - symbol: "variables", - alias: "_soong_variables", - }}, - }, - { - file: "//build/bazel/product_config:android_product.bzl", - symbols: []BazelLoadSymbol{{symbol: "android_product"}}, - }, - }, - }) - createTargets(ctx, productLabelsToVariables, moduleNameToPartition, convertedModulePathMap, res.bp2buildTargets) - - platformMappingContent, err := platformMappingContent( - productLabelsToVariables, - ctx.Config().Bp2buildSoongConfigDefinitions, - convertedModulePathMap) - if err != nil { - return res, err - } - - res.injectionFiles = []BazelFile{ - newFile( - "product_config_platforms", - "BUILD.bazel", - productReplacer.Replace(` -package(default_visibility = [ - "@//build/bazel/product_config:__subpackages__", - "@soong_injection//product_config_platforms:__subpackages__", -]) - -load("@//{PRODUCT_FOLDER}:soong.variables.bzl", _soong_variables = "variables") -load("@//build/bazel/product_config:android_product.bzl", "android_product") - -# Bazel will qualify its outputs by the platform name. When switching between products, this -# means that soong-built files that depend on bazel-built files will suddenly get different -# dependency files, because the path changes, and they will be rebuilt. In order to avoid this -# extra rebuilding, make mixed builds always use a single platform so that the bazel artifacts -# are always under the same path. -android_product( - name = "mixed_builds_product", - soong_variables = _soong_variables, - extra_constraints = ["@//build/bazel/platforms:mixed_builds"], -) -`)), - newFile( - "product_config_platforms", - "product_labels.bzl", - productReplacer.Replace(` -# This file keeps a list of all the products in the android source tree, because they're -# discovered as part of a preprocessing step before bazel runs. -# TODO: When we start generating the platforms for more than just the -# currently lunched product, they should all be listed here -product_labels = [ - "@soong_injection//product_config_platforms:mixed_builds_product", - "@//{PRODUCT_FOLDER}:{PRODUCT}", -`)+strings.Join(productsForTesting, "\n")+"\n]\n"), - newFile( - "product_config_platforms", - "common.bazelrc", - productReplacer.Replace(` -build --platform_mappings=platform_mappings -build --platforms @//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64 -build --//build/bazel/product_config:target_build_variant={VARIANT} - -build:android --platforms=@//{PRODUCT_FOLDER}:{PRODUCT} -build:linux_x86 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86 -build:linux_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64 -build:linux_bionic_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_bionic_x86_64 -build:linux_musl_x86 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_musl_x86 -build:linux_musl_x86_64 --platforms=@//{PRODUCT_FOLDER}:{PRODUCT}_linux_musl_x86_64 -`)), - newFile( - "product_config_platforms", - "linux.bazelrc", - productReplacer.Replace(` -build --host_platform @//{PRODUCT_FOLDER}:{PRODUCT}_linux_x86_64 -`)), - newFile( - "product_config_platforms", - "darwin.bazelrc", - productReplacer.Replace(` -build --host_platform @//{PRODUCT_FOLDER}:{PRODUCT}_darwin_x86_64 -`)), - } - res.bp2buildFiles = []BazelFile{ - newFile( - "", - "platform_mappings", - platformMappingContent), - newFile( - currentProductFolder, - "soong.variables.bzl", - `variables = json.decode("""`+strings.ReplaceAll(string(productVariablesBytes), "\\", "\\\\")+`""")`), - } - - return res, nil -} - -func platformMappingContent( - productLabelToVariables map[bazelLabel]*android.ProductVariables, - soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions, - convertedModulePathMap map[string]string) (string, error) { - var result strings.Builder - - mergedConvertedModulePathMap := make(map[string]string) - for k, v := range convertedModulePathMap { - mergedConvertedModulePathMap[k] = v - } - additionalModuleNamesToPackages, err := starlark_import.GetStarlarkValue[map[string]string]("additional_module_names_to_packages") - if err != nil { - return "", err - } - for k, v := range additionalModuleNamesToPackages { - mergedConvertedModulePathMap[k] = v - } - - productLabels := make([]bazelLabel, 0, len(productLabelToVariables)) - for k := range productLabelToVariables { - productLabels = append(productLabels, k) - } - sort.Slice(productLabels, func(i, j int) bool { - return productLabels[i].Less(&productLabels[j]) - }) - result.WriteString("platforms:\n") - for _, productLabel := range productLabels { - platformMappingSingleProduct(productLabel, productLabelToVariables[productLabel], soongConfigDefinitions, mergedConvertedModulePathMap, &result) - } - return result.String(), nil -} - -var bazelPlatformSuffixes = []string{ - "", - "_darwin_arm64", - "_darwin_x86_64", - "_linux_bionic_arm64", - "_linux_bionic_x86_64", - "_linux_musl_x86", - "_linux_musl_x86_64", - "_linux_x86", - "_linux_x86_64", - "_windows_x86", - "_windows_x86_64", -} - -func platformMappingSingleProduct( - label bazelLabel, - productVariables *android.ProductVariables, - soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions, - convertedModulePathMap map[string]string, - result *strings.Builder) { - - platform_sdk_version := -1 - if productVariables.Platform_sdk_version != nil { - platform_sdk_version = *productVariables.Platform_sdk_version - } - - defaultAppCertificateFilegroup := "//build/bazel/utils:empty_filegroup" - if proptools.String(productVariables.DefaultAppCertificate) != "" { - defaultAppCertificateFilegroup = "@//" + filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) + ":generated_android_certificate_directory" - } - - // TODO: b/301598690 - commas can't be escaped in a string-list passed in a platform mapping, - // so commas are switched for ":" here, and must be back-substituted into commas - // wherever the AAPTCharacteristics product config variable is used. - AAPTConfig := []string{} - for _, conf := range productVariables.AAPTConfig { - AAPTConfig = append(AAPTConfig, strings.Replace(conf, ",", ":", -1)) - } - - for _, suffix := range bazelPlatformSuffixes { - result.WriteString(" ") - result.WriteString(label.String()) - result.WriteString(suffix) - result.WriteString("\n") - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_characteristics=%s\n", proptools.String(productVariables.AAPTCharacteristics))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_config=%s\n", strings.Join(AAPTConfig, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:aapt_preferred_config=%s\n", proptools.String(productVariables.AAPTPreferredConfig))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:always_use_prebuilt_sdks=%t\n", proptools.Bool(productVariables.Always_use_prebuilt_sdks))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:arc=%t\n", proptools.Bool(productVariables.Arc))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:apex_global_min_sdk_version_override=%s\n", proptools.String(productVariables.ApexGlobalMinSdkVersionOverride))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:binder32bit=%t\n", proptools.Bool(productVariables.Binder32bit))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_from_text_stub=%t\n", proptools.Bool(productVariables.Build_from_text_stub))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_broken_incorrect_partition_images=%t\n", productVariables.BuildBrokenIncorrectPartitionImages)) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_id=%s\n", proptools.String(productVariables.BuildId))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:build_version_tags=%s\n", strings.Join(productVariables.BuildVersionTags, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:cfi_exclude_paths=%s\n", strings.Join(productVariables.CFIExcludePaths, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:cfi_include_paths=%s\n", strings.Join(productVariables.CFIIncludePaths, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:compressed_apex=%t\n", proptools.Bool(productVariables.CompressedApex))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:default_app_certificate=%s\n", proptools.String(productVariables.DefaultAppCertificate))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:default_app_certificate_filegroup=%s\n", defaultAppCertificateFilegroup)) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_abi=%s\n", strings.Join(productVariables.DeviceAbi, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_max_page_size_supported=%s\n", proptools.String(productVariables.DeviceMaxPageSizeSupported))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_name=%s\n", proptools.String(productVariables.DeviceName))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_no_bionic_page_size_macro=%t\n", proptools.Bool(productVariables.DeviceNoBionicPageSizeMacro))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_product=%s\n", proptools.String(productVariables.DeviceProduct))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_platform=%s\n", label.String())) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:enable_cfi=%t\n", proptools.BoolDefault(productVariables.EnableCFI, true))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:enforce_vintf_manifest=%t\n", proptools.Bool(productVariables.Enforce_vintf_manifest))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_not_svelte=%t\n", proptools.Bool(productVariables.Malloc_not_svelte))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_pattern_fill_contents=%t\n", proptools.Bool(productVariables.Malloc_pattern_fill_contents))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_zero_contents=%t\n", proptools.Bool(productVariables.Malloc_zero_contents))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_exclude_paths=%s\n", strings.Join(productVariables.MemtagHeapExcludePaths, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_async_include_paths=%s\n", strings.Join(productVariables.MemtagHeapAsyncIncludePaths, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:memtag_heap_sync_include_paths=%s\n", strings.Join(productVariables.MemtagHeapSyncIncludePaths, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:manifest_package_name_overrides=%s\n", strings.Join(productVariables.ManifestPackageNameOverrides, ","))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:native_coverage=%t\n", proptools.Bool(productVariables.Native_coverage))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_sdk_final=%t\n", proptools.Bool(productVariables.Platform_sdk_final))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_security_patch=%s\n", proptools.String(productVariables.Platform_security_patch))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_version_last_stable=%s\n", proptools.String(productVariables.Platform_version_last_stable))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_version_name=%s\n", proptools.String(productVariables.Platform_version_name))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:product_brand=%s\n", productVariables.ProductBrand)) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:product_manufacturer=%s\n", productVariables.ProductManufacturer)) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_aconfig_flag_default_permission=%s\n", productVariables.ReleaseAconfigFlagDefaultPermission)) - releaseAconfigValueSets := "//build/bazel/product_config:empty_aconfig_value_sets" - if len(productVariables.ReleaseAconfigValueSets) > 0 { - releaseAconfigValueSets = "@//" + label.pkg + ":" + releaseAconfigValueSetsName + "_" + label.target - } - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_aconfig_value_sets=%s\n", releaseAconfigValueSets)) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:release_version=%s\n", productVariables.ReleaseVersion)) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:platform_sdk_version=%d\n", platform_sdk_version)) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:safestack=%t\n", proptools.Bool(productVariables.Safestack))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:treble_linker_namespaces=%t\n", proptools.Bool(productVariables.Treble_linker_namespaces))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:tidy_checks=%s\n", proptools.String(productVariables.TidyChecks))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:uml=%t\n", proptools.Bool(productVariables.Uml))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:unbundled_build=%t\n", proptools.Bool(productVariables.Unbundled_build))) - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:unbundled_build_apps=%s\n", strings.Join(productVariables.Unbundled_build_apps, ","))) - - for _, override := range productVariables.CertificateOverrides { - parts := strings.SplitN(override, ":", 2) - if apexPath, ok := convertedModulePathMap[parts[0]]; ok { - if overrideCertPath, ok := convertedModulePathMap[parts[1]]; ok { - result.WriteString(fmt.Sprintf(" --%s:%s_certificate_override=%s:%s\n", apexPath, parts[0], overrideCertPath, parts[1])) - } - } - } - - for _, namespace := range android.SortedKeys(productVariables.VendorVars) { - for _, variable := range android.SortedKeys(productVariables.VendorVars[namespace]) { - value := productVariables.VendorVars[namespace][variable] - key := namespace + "__" + variable - _, hasBool := soongConfigDefinitions.BoolVars[key] - _, hasString := soongConfigDefinitions.StringVars[key] - _, hasValue := soongConfigDefinitions.ValueVars[key] - if !hasBool && !hasString && !hasValue { - // Not all soong config variables are defined in Android.bp files. For example, - // prebuilt_bootclasspath_fragment uses soong config variables in a nonstandard - // way, that causes them to be present in the soong.variables file but not - // defined in an Android.bp file. There's also nothing stopping you from setting - // a variable in make that doesn't exist in soong. We only generate build - // settings for the ones that exist in soong, so skip all others. - continue - } - if hasBool && hasString || hasBool && hasValue || hasString && hasValue { - panic(fmt.Sprintf("Soong config variable %s:%s appears to be of multiple types. bool? %t, string? %t, value? %t", namespace, variable, hasBool, hasString, hasValue)) - } - if hasBool { - // Logic copied from soongConfig.Bool() - value = strings.ToLower(value) - if value == "1" || value == "y" || value == "yes" || value == "on" || value == "true" { - value = "true" - } else { - value = "false" - } - } - result.WriteString(fmt.Sprintf(" --//build/bazel/product_config/soong_config_variables:%s=%s\n", strings.ToLower(key), value)) - } - } - } -} - -func starlarkMapToProductVariables(in map[string]starlark.Value) (android.ProductVariables, error) { - result := android.ProductVariables{} - productVarsReflect := reflect.ValueOf(&result).Elem() - for i := 0; i < productVarsReflect.NumField(); i++ { - field := productVarsReflect.Field(i) - fieldType := productVarsReflect.Type().Field(i) - name := fieldType.Name - if name == "BootJars" || name == "ApexBootJars" || name == "VendorSnapshotModules" || - name == "RecoverySnapshotModules" { - // These variables have more complicated types, and we don't need them right now - continue - } - if _, ok := in[name]; ok { - if name == "VendorVars" { - vendorVars, err := starlark_import.Unmarshal[map[string]map[string]string](in[name]) - if err != nil { - return result, err - } - field.Set(reflect.ValueOf(vendorVars)) - continue - } - switch field.Type().Kind() { - case reflect.Bool: - val, err := starlark_import.Unmarshal[bool](in[name]) - if err != nil { - return result, err - } - field.SetBool(val) - case reflect.String: - val, err := starlark_import.Unmarshal[string](in[name]) - if err != nil { - return result, err - } - field.SetString(val) - case reflect.Slice: - if field.Type().Elem().Kind() != reflect.String { - return result, fmt.Errorf("slices of types other than strings are unimplemented") - } - val, err := starlark_import.UnmarshalReflect(in[name], field.Type()) - if err != nil { - return result, err - } - field.Set(val) - case reflect.Pointer: - switch field.Type().Elem().Kind() { - case reflect.Bool: - val, err := starlark_import.UnmarshalNoneable[bool](in[name]) - if err != nil { - return result, err - } - field.Set(reflect.ValueOf(val)) - case reflect.String: - val, err := starlark_import.UnmarshalNoneable[string](in[name]) - if err != nil { - return result, err - } - field.Set(reflect.ValueOf(val)) - case reflect.Int: - val, err := starlark_import.UnmarshalNoneable[int](in[name]) - if err != nil { - return result, err - } - field.Set(reflect.ValueOf(val)) - default: - return result, fmt.Errorf("pointers of types other than strings/bools are unimplemented: %s", field.Type().Elem().Kind().String()) - } - default: - return result, fmt.Errorf("unimplemented type: %s", field.Type().String()) - } - } - } - - result.Native_coverage = proptools.BoolPtr( - proptools.Bool(result.GcovCoverage) || - proptools.Bool(result.ClangCoverage)) - - return result, nil -} - -func createTargets( - ctx *CodegenContext, - productLabelsToVariables map[bazelLabel]*android.ProductVariables, - moduleNameToPartition map[string]string, - convertedModulePathMap map[string]string, - res map[string]BazelTargets) { - createGeneratedAndroidCertificateDirectories(productLabelsToVariables, res) - createAvbKeyFilegroups(productLabelsToVariables, res) - createReleaseAconfigValueSetsFilegroup(productLabelsToVariables, res) - for label, variables := range productLabelsToVariables { - createSystemPartition(ctx, label, &variables.PartitionVarsForBazelMigrationOnlyDoNotUse, moduleNameToPartition, convertedModulePathMap, res) - } -} - -func createGeneratedAndroidCertificateDirectories(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) { - var allDefaultAppCertificateDirs []string - for _, productVariables := range productLabelsToVariables { - if proptools.String(productVariables.DefaultAppCertificate) != "" { - d := filepath.Dir(proptools.String(productVariables.DefaultAppCertificate)) - if !android.InList(d, allDefaultAppCertificateDirs) { - allDefaultAppCertificateDirs = append(allDefaultAppCertificateDirs, d) - } - } - } - for _, dir := range allDefaultAppCertificateDirs { - content := `filegroup( - name = "generated_android_certificate_directory", - srcs = glob([ - "*.pk8", - "*.pem", - "*.avbpubkey", - ]), - visibility = ["//visibility:public"], -)` - targets[dir] = append(targets[dir], BazelTarget{ - name: "generated_android_certificate_directory", - packageName: dir, - content: content, - ruleClass: "filegroup", - }) - } -} - -func createReleaseAconfigValueSetsFilegroup(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) { - for label, productVariables := range productLabelsToVariables { - if len(productVariables.ReleaseAconfigValueSets) > 0 { - key := label.target - dir := label.pkg - var value_sets strings.Builder - for _, value_set := range productVariables.ReleaseAconfigValueSets { - value_sets.WriteString(" \"" + value_set + "\",\n") - } - - name := releaseAconfigValueSetsName + "_" + key - content := "aconfig_value_sets(\n" + - " name = \"" + name + "\",\n" + - " value_sets = [\n" + - value_sets.String() + - " ],\n" + - " visibility = [\"//visibility:public\"],\n" + - ")" - targets[dir] = append(targets[dir], BazelTarget{ - name: name, - packageName: dir, - content: content, - ruleClass: "aconfig_value_sets", - loads: []BazelLoad{{ - file: "//build/bazel/rules/aconfig:aconfig_value_sets.bzl", - symbols: []BazelLoadSymbol{{ - symbol: "aconfig_value_sets", - }}, - }}, - }) - } - } -} - -func createAvbKeyFilegroups(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) { - var allAvbKeys []string - for _, productVariables := range productLabelsToVariables { - for _, partitionVariables := range productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.PartitionQualifiedVariables { - if partitionVariables.BoardAvbKeyPath != "" { - if !android.InList(partitionVariables.BoardAvbKeyPath, allAvbKeys) { - allAvbKeys = append(allAvbKeys, partitionVariables.BoardAvbKeyPath) - } - } - } - } - for _, key := range allAvbKeys { - dir := filepath.Dir(key) - name := filepath.Base(key) - content := fmt.Sprintf(`filegroup( - name = "%s_filegroup", - srcs = ["%s"], - visibility = ["//visibility:public"], -)`, name, name) - targets[dir] = append(targets[dir], BazelTarget{ - name: name + "_filegroup", - packageName: dir, - content: content, - ruleClass: "filegroup", - }) - } -} - -func createSystemPartition( - ctx *CodegenContext, - platformLabel bazelLabel, - variables *android.PartitionVariables, - moduleNameToPartition map[string]string, - convertedModulePathMap map[string]string, - targets map[string]BazelTargets) { - if !variables.PartitionQualifiedVariables["system"].BuildingImage { - return - } - qualifiedVariables := variables.PartitionQualifiedVariables["system"] - - imageProps := generateImagePropDictionary(variables, "system") - imageProps["skip_fsck"] = "true" - - var properties strings.Builder - for _, prop := range android.SortedKeys(imageProps) { - properties.WriteString(prop) - properties.WriteRune('=') - properties.WriteString(imageProps[prop]) - properties.WriteRune('\n') - } - - var extraProperties strings.Builder - if variables.BoardAvbEnable { - extraProperties.WriteString(" avb_enable = True,\n") - extraProperties.WriteString(fmt.Sprintf(" avb_add_hashtree_footer_args = %q,\n", qualifiedVariables.BoardAvbAddHashtreeFooterArgs)) - keypath := qualifiedVariables.BoardAvbKeyPath - if keypath != "" { - extraProperties.WriteString(fmt.Sprintf(" avb_key = \"//%s:%s\",\n", filepath.Dir(keypath), filepath.Base(keypath)+"_filegroup")) - extraProperties.WriteString(fmt.Sprintf(" avb_algorithm = %q,\n", qualifiedVariables.BoardAvbAlgorithm)) - extraProperties.WriteString(fmt.Sprintf(" avb_rollback_index = %s,\n", qualifiedVariables.BoardAvbRollbackIndex)) - extraProperties.WriteString(fmt.Sprintf(" avb_rollback_index_location = %s,\n", qualifiedVariables.BoardAvbRollbackIndexLocation)) - } - } - - var deps []string - for _, mod := range variables.ProductPackages { - if path, ok := convertedModulePathMap[mod]; ok && ctx.Config().BazelContext.IsModuleNameAllowed(mod, false) { - if partition, ok := moduleNameToPartition[mod]; ok && partition == "system" { - if path == "//." { - path = "//" - } - deps = append(deps, fmt.Sprintf(" \"%s:%s\",\n", path, mod)) - } - } - } - if len(deps) > 0 { - sort.Strings(deps) - extraProperties.WriteString(" deps = [\n") - for _, dep := range deps { - extraProperties.WriteString(dep) - } - extraProperties.WriteString(" ],\n") - } - - targets[platformLabel.pkg] = append(targets[platformLabel.pkg], BazelTarget{ - name: "system_image", - packageName: platformLabel.pkg, - content: fmt.Sprintf(`partition( - name = "system_image", - base_staging_dir = "//build/bazel/bazel_sandwich:system_staging_dir", - base_staging_dir_file_list = "//build/bazel/bazel_sandwich:system_staging_dir_file_list", - root_dir = "//build/bazel/bazel_sandwich:root_staging_dir", - selinux_file_contexts = "//build/bazel/bazel_sandwich:selinux_file_contexts", - image_properties = """ -%s -""", -%s - type = "system", -)`, properties.String(), extraProperties.String()), - ruleClass: "partition", - loads: []BazelLoad{{ - file: "//build/bazel/rules/partitions:partition.bzl", - symbols: []BazelLoadSymbol{{ - symbol: "partition", - }}, - }}, - }, BazelTarget{ - name: "system_image_test", - packageName: platformLabel.pkg, - content: `partition_diff_test( - name = "system_image_test", - partition1 = "//build/bazel/bazel_sandwich:make_system_image", - partition2 = ":system_image", -)`, - ruleClass: "partition_diff_test", - loads: []BazelLoad{{ - file: "//build/bazel/rules/partitions/diff:partition_diff.bzl", - symbols: []BazelLoadSymbol{{ - symbol: "partition_diff_test", - }}, - }}, - }, BazelTarget{ - name: "run_system_image_test", - packageName: platformLabel.pkg, - content: `run_test_in_build( - name = "run_system_image_test", - test = ":system_image_test", -)`, - ruleClass: "run_test_in_build", - loads: []BazelLoad{{ - file: "//build/bazel/bazel_sandwich:run_test_in_build.bzl", - symbols: []BazelLoadSymbol{{ - symbol: "run_test_in_build", - }}, - }}, - }) -} - -var allPartitionTypes = []string{ - "system", - "vendor", - "cache", - "userdata", - "product", - "system_ext", - "oem", - "odm", - "vendor_dlkm", - "odm_dlkm", - "system_dlkm", -} - -// An equivalent of make's generate-image-prop-dictionary function -func generateImagePropDictionary(variables *android.PartitionVariables, partitionType string) map[string]string { - partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType] - if !ok { - panic("Unknown partitionType: " + partitionType) - } - ret := map[string]string{} - if partitionType == "system" { - if len(variables.PartitionQualifiedVariables["system_other"].BoardPartitionSize) > 0 { - ret["system_other_size"] = variables.PartitionQualifiedVariables["system_other"].BoardPartitionSize - } - if len(partitionQualifiedVariables.ProductHeadroom) > 0 { - ret["system_headroom"] = partitionQualifiedVariables.ProductHeadroom - } - addCommonRoFlagsToImageProps(variables, partitionType, ret) - } - // TODO: other partition-specific logic - if variables.TargetUserimagesUseExt2 { - ret["fs_type"] = "ext2" - } else if variables.TargetUserimagesUseExt3 { - ret["fs_type"] = "ext3" - } else if variables.TargetUserimagesUseExt4 { - ret["fs_type"] = "ext4" - } - - if !variables.TargetUserimagesSparseExtDisabled { - ret["extfs_sparse_flag"] = "-s" - } - if !variables.TargetUserimagesSparseErofsDisabled { - ret["erofs_sparse_flag"] = "-s" - } - if !variables.TargetUserimagesSparseSquashfsDisabled { - ret["squashfs_sparse_flag"] = "-s" - } - if !variables.TargetUserimagesSparseF2fsDisabled { - ret["f2fs_sparse_flag"] = "-S" - } - erofsCompressor := variables.BoardErofsCompressor - if len(erofsCompressor) == 0 && hasErofsPartition(variables) { - if len(variables.BoardErofsUseLegacyCompression) > 0 { - erofsCompressor = "lz4" - } else { - erofsCompressor = "lz4hc,9" - } - } - if len(erofsCompressor) > 0 { - ret["erofs_default_compressor"] = erofsCompressor - } - if len(variables.BoardErofsCompressorHints) > 0 { - ret["erofs_default_compress_hints"] = variables.BoardErofsCompressorHints - } - if len(variables.BoardErofsCompressorHints) > 0 { - ret["erofs_default_compress_hints"] = variables.BoardErofsCompressorHints - } - if len(variables.BoardErofsPclusterSize) > 0 { - ret["erofs_pcluster_size"] = variables.BoardErofsPclusterSize - } - if len(variables.BoardErofsShareDupBlocks) > 0 { - ret["erofs_share_dup_blocks"] = variables.BoardErofsShareDupBlocks - } - if len(variables.BoardErofsUseLegacyCompression) > 0 { - ret["erofs_use_legacy_compression"] = variables.BoardErofsUseLegacyCompression - } - if len(variables.BoardExt4ShareDupBlocks) > 0 { - ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks - } - if len(variables.BoardFlashLogicalBlockSize) > 0 { - ret["flash_logical_block_size"] = variables.BoardFlashLogicalBlockSize - } - if len(variables.BoardFlashEraseBlockSize) > 0 { - ret["flash_erase_block_size"] = variables.BoardFlashEraseBlockSize - } - if len(variables.BoardExt4ShareDupBlocks) > 0 { - ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks - } - if len(variables.BoardExt4ShareDupBlocks) > 0 { - ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks - } - for _, partitionType := range allPartitionTypes { - if qualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]; ok && len(qualifiedVariables.ProductVerityPartition) > 0 { - ret[partitionType+"_verity_block_device"] = qualifiedVariables.ProductVerityPartition - } - } - // TODO: Vboot - // TODO: AVB - if variables.BoardUsesRecoveryAsBoot { - ret["recovery_as_boot"] = "true" - } - if variables.ProductUseDynamicPartitionSize { - ret["use_dynamic_partition_size"] = "true" - } - if variables.CopyImagesForTargetFilesZip { - ret["use_fixed_timestamp"] = "true" - } - return ret -} - -// Soong equivalent of make's add-common-ro-flags-to-image-props -func addCommonRoFlagsToImageProps(variables *android.PartitionVariables, partitionType string, ret map[string]string) { - partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType] - if !ok { - panic("Unknown partitionType: " + partitionType) - } - if len(partitionQualifiedVariables.BoardErofsCompressor) > 0 { - ret[partitionType+"_erofs_compressor"] = partitionQualifiedVariables.BoardErofsCompressor - } - if len(partitionQualifiedVariables.BoardErofsCompressHints) > 0 { - ret[partitionType+"_erofs_compress_hints"] = partitionQualifiedVariables.BoardErofsCompressHints - } - if len(partitionQualifiedVariables.BoardErofsPclusterSize) > 0 { - ret[partitionType+"_erofs_pcluster_size"] = partitionQualifiedVariables.BoardErofsPclusterSize - } - if len(partitionQualifiedVariables.BoardExtfsRsvPct) > 0 { - ret[partitionType+"_extfs_rsv_pct"] = partitionQualifiedVariables.BoardExtfsRsvPct - } - if len(partitionQualifiedVariables.BoardF2fsSloadCompressFlags) > 0 { - ret[partitionType+"_f2fs_sldc_flags"] = partitionQualifiedVariables.BoardF2fsSloadCompressFlags - } - if len(partitionQualifiedVariables.BoardFileSystemCompress) > 0 { - ret[partitionType+"_f2fs_compress"] = partitionQualifiedVariables.BoardFileSystemCompress - } - if len(partitionQualifiedVariables.BoardFileSystemType) > 0 { - ret[partitionType+"_fs_type"] = partitionQualifiedVariables.BoardFileSystemType - } - if len(partitionQualifiedVariables.BoardJournalSize) > 0 { - ret[partitionType+"_journal_size"] = partitionQualifiedVariables.BoardJournalSize - } - if len(partitionQualifiedVariables.BoardPartitionReservedSize) > 0 { - ret[partitionType+"_reserved_size"] = partitionQualifiedVariables.BoardPartitionReservedSize - } - if len(partitionQualifiedVariables.BoardPartitionSize) > 0 { - ret[partitionType+"_size"] = partitionQualifiedVariables.BoardPartitionSize - } - if len(partitionQualifiedVariables.BoardSquashfsBlockSize) > 0 { - ret[partitionType+"_squashfs_block_size"] = partitionQualifiedVariables.BoardSquashfsBlockSize - } - if len(partitionQualifiedVariables.BoardSquashfsCompressor) > 0 { - ret[partitionType+"_squashfs_compressor"] = partitionQualifiedVariables.BoardSquashfsCompressor - } - if len(partitionQualifiedVariables.BoardSquashfsCompressorOpt) > 0 { - ret[partitionType+"_squashfs_compressor_opt"] = partitionQualifiedVariables.BoardSquashfsCompressorOpt - } - if len(partitionQualifiedVariables.BoardSquashfsDisable4kAlign) > 0 { - ret[partitionType+"_squashfs_disable_4k_align"] = partitionQualifiedVariables.BoardSquashfsDisable4kAlign - } - if len(partitionQualifiedVariables.BoardPartitionSize) == 0 && len(partitionQualifiedVariables.BoardPartitionReservedSize) == 0 && len(partitionQualifiedVariables.ProductHeadroom) == 0 { - ret[partitionType+"_disable_sparse"] = "true" - } - addCommonFlagsToImageProps(variables, partitionType, ret) -} - -func hasErofsPartition(variables *android.PartitionVariables) bool { - return variables.PartitionQualifiedVariables["product"].BoardFileSystemType == "erofs" || - variables.PartitionQualifiedVariables["system_ext"].BoardFileSystemType == "erofs" || - variables.PartitionQualifiedVariables["odm"].BoardFileSystemType == "erofs" || - variables.PartitionQualifiedVariables["vendor"].BoardFileSystemType == "erofs" || - variables.PartitionQualifiedVariables["system"].BoardFileSystemType == "erofs" || - variables.PartitionQualifiedVariables["vendor_dlkm"].BoardFileSystemType == "erofs" || - variables.PartitionQualifiedVariables["odm_dlkm"].BoardFileSystemType == "erofs" || - variables.PartitionQualifiedVariables["system_dlkm"].BoardFileSystemType == "erofs" -} - -// Soong equivalent of make's add-common-flags-to-image-props -func addCommonFlagsToImageProps(variables *android.PartitionVariables, partitionType string, ret map[string]string) { - // The selinux_fc will be handled separately - partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType] - if !ok { - panic("Unknown partitionType: " + partitionType) - } - ret["building_"+partitionType+"_image"] = boolToMakeString(partitionQualifiedVariables.BuildingImage) -} - -func boolToMakeString(b bool) string { - if b { - return "true" - } - return "" -} diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go index d2187ffa6..af2f55019 100644 --- a/bp2build/build_conversion.go +++ b/bp2build/build_conversion.go @@ -28,10 +28,7 @@ import ( "android/soong/android" "android/soong/bazel" "android/soong/starlark_fmt" - "android/soong/ui/metrics/bp2build_metrics_proto" - "github.com/google/blueprint" - "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/proptools" ) @@ -201,18 +198,13 @@ func (ctx *CodegenContext) Mode() CodegenMode { type CodegenMode int const ( - // Bp2Build - generate BUILD files with targets buildable by Bazel directly. - // - // This mode is used for the Soong->Bazel build definition conversion. - Bp2Build CodegenMode = iota - // QueryView - generate BUILD files with targets representing fully mutated // Soong modules, representing the fully configured Soong module graph with // variants and dependency edges. // // This mode is used for discovering and introspecting the existing Soong // module graph. - QueryView + QueryView CodegenMode = iota ) type unconvertedDepsMode int @@ -227,8 +219,6 @@ const ( func (mode CodegenMode) String() string { switch mode { - case Bp2Build: - return "Bp2Build" case QueryView: return "QueryView" default: @@ -256,9 +246,6 @@ func (ctx *CodegenContext) Context() *android.Context { return ctx.context } // writing BUILD files in the output directory. func NewCodegenContext(config android.Config, context *android.Context, mode CodegenMode, topDir string) *CodegenContext { var unconvertedDeps unconvertedDepsMode - if config.IsEnvTrue("BP2BUILD_ERROR_UNCONVERTED") { - unconvertedDeps = errorModulesUnconvertedDeps - } return &CodegenContext{ context: context, config: config, @@ -281,526 +268,30 @@ func propsToAttributes(props map[string]string) string { type conversionResults struct { buildFileToTargets map[string]BazelTargets moduleNameToPartition map[string]string - metrics CodegenMetrics } func (r conversionResults) BuildDirToTargets() map[string]BazelTargets { return r.buildFileToTargets } -// struct to store state of b bazel targets (e.g. go targets which do not implement android.Module) -// this implements bp2buildModule interface and is passed to generateBazelTargets -type bTarget struct { - targetName string - targetPackage string - bazelRuleClass string - bazelRuleLoadLocation string - bazelAttributes []interface{} -} - -var _ bp2buildModule = (*bTarget)(nil) - -func (b bTarget) TargetName() string { - return b.targetName -} - -func (b bTarget) TargetPackage() string { - return b.targetPackage -} - -func (b bTarget) BazelRuleClass() string { - return b.bazelRuleClass -} - -func (b bTarget) BazelRuleLoadLocation() string { - return b.bazelRuleLoadLocation -} - -func (b bTarget) BazelAttributes() []interface{} { - return b.bazelAttributes -} - -// Creates a target_compatible_with entry that is *not* compatible with android -func targetNotCompatibleWithAndroid() bazel.LabelListAttribute { - ret := bazel.LabelListAttribute{} - ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsAndroid, - bazel.MakeLabelList( - []bazel.Label{ - bazel.Label{ - Label: "@platforms//:incompatible", - }, - }, - ), - ) - return ret -} - -// helper function to return labels for srcs used in bootstrap_go_package and bootstrap_go_binary -// this function has the following limitations which make it unsuitable for widespread use -// - wildcard patterns in srcs -// This is ok for go since build/blueprint does not support it. -// -// Prefer to use `BazelLabelForModuleSrc` instead -func goSrcLabels(cfg android.Config, moduleDir string, srcs []string, linuxSrcs, darwinSrcs []string) bazel.LabelListAttribute { - labels := func(srcs []string) bazel.LabelList { - ret := []bazel.Label{} - for _, src := range srcs { - srcLabel := bazel.Label{ - Label: src, - } - ret = append(ret, srcLabel) - } - // Respect package boundaries - return android.TransformSubpackagePaths( - cfg, - moduleDir, - bazel.MakeLabelList(ret), - ) - } - - ret := bazel.LabelListAttribute{} - // common - ret.SetSelectValue(bazel.NoConfigAxis, "", labels(srcs)) - // linux - ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsLinux, labels(linuxSrcs)) - // darwin - ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsDarwin, labels(darwinSrcs)) - return ret -} - -func goDepLabels(deps []string, goModulesMap nameToGoLibraryModule) bazel.LabelListAttribute { - labels := []bazel.Label{} - for _, dep := range deps { - moduleDir := goModulesMap[dep].Dir - if moduleDir == "." { - moduleDir = "" - } - label := bazel.Label{ - Label: fmt.Sprintf("//%s:%s", moduleDir, dep), - } - labels = append(labels, label) - } - return bazel.MakeLabelListAttribute(bazel.MakeLabelList(labels)) -} - -// attributes common to blueprint_go_binary and bootstap_go_package -type goAttributes struct { - Importpath bazel.StringAttribute - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - Data bazel.LabelListAttribute - Target_compatible_with bazel.LabelListAttribute - - // attributes for the dynamically generated go_test target - Embed bazel.LabelListAttribute -} - -type goTestProperties struct { - name string - dir string - testSrcs []string - linuxTestSrcs []string - darwinTestSrcs []string - testData []string - // Name of the target that should be compiled together with the test - embedName string -} - -// Creates a go_test target for bootstrap_go_package / blueprint_go_binary -func generateBazelTargetsGoTest(ctx *android.Context, goModulesMap nameToGoLibraryModule, gp goTestProperties) (BazelTarget, error) { - ca := android.CommonAttributes{ - Name: gp.name, - } - ga := goAttributes{ - Srcs: goSrcLabels(ctx.Config(), gp.dir, gp.testSrcs, gp.linuxTestSrcs, gp.darwinTestSrcs), - Data: goSrcLabels(ctx.Config(), gp.dir, gp.testData, []string{}, []string{}), - Embed: bazel.MakeLabelListAttribute( - bazel.MakeLabelList( - []bazel.Label{bazel.Label{Label: ":" + gp.embedName}}, - ), - ), - Target_compatible_with: targetNotCompatibleWithAndroid(), - } - - libTest := bTarget{ - targetName: gp.name, - targetPackage: gp.dir, - bazelRuleClass: "go_test", - bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl", - bazelAttributes: []interface{}{&ca, &ga}, - } - return generateBazelTarget(ctx, libTest) -} - -// TODO - b/288491147: testSrcs of certain bootstrap_go_package/blueprint_go_binary are not hermetic and depend on -// testdata checked into the filesystem. -// Denylist the generation of go_test targets for these Soong modules. -// The go_library/go_binary will still be generated, since those are hermitic. -var ( - goTestsDenylist = []string{ - "android-archive-zip", - "bazel_notice_gen", - "blueprint-bootstrap-bpdoc", - "blueprint-microfactory", - "blueprint-pathtools", - "bssl_ar", - "compliance_checkmetadata", - "compliance_checkshare", - "compliance_dumpgraph", - "compliance_dumpresolutions", - "compliance_listshare", - "compliance-module", - "compliancenotice_bom", - "compliancenotice_shippedlibs", - "compliance_rtrace", - "compliance_sbom", - "golang-protobuf-internal-fuzz-jsonfuzz", - "golang-protobuf-internal-fuzz-textfuzz", - "golang-protobuf-internal-fuzz-wirefuzz", - "htmlnotice", - "protoc-gen-go", - "rbcrun-module", - "spdx-tools-builder", - "spdx-tools-builder2v1", - "spdx-tools-builder2v2", - "spdx-tools-builder2v3", - "spdx-tools-idsearcher", - "spdx-tools-spdx-json", - "spdx-tools-utils", - "soong-ui-build", - "textnotice", - "xmlnotice", - } -) - -func testOfGoPackageIsIncompatible(g *bootstrap.GoPackage) bool { - return android.InList(g.Name(), goTestsDenylist) || - // Denylist tests of soong_build - // Theses tests have a guard that prevent usage outside a test environment - // The guard (`ensureTestOnly`) looks for a `-test` in os.Args, which is present in soong's gotestrunner, but missing in `b test` - g.IsPluginFor("soong_build") || - // soong-android is a dep of soong_build - // This dependency is created by soong_build by listing it in its deps explicitly in Android.bp, and not via `plugin_for` in `soong-android` - g.Name() == "soong-android" -} - -func testOfGoBinaryIsIncompatible(g *bootstrap.GoBinary) bool { - return android.InList(g.Name(), goTestsDenylist) -} - -func generateBazelTargetsGoPackage(ctx *android.Context, g *bootstrap.GoPackage, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) { - ca := android.CommonAttributes{ - Name: g.Name(), - } - - // For this bootstrap_go_package dep chain, - // A --> B --> C ( ---> depends on) - // Soong provides the convenience of only listing B as deps of A even if a src file of A imports C - // Bazel OTOH - // 1. requires C to be listed in `deps` expllicity. - // 2. does not require C to be listed if src of A does not import C - // - // bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps - transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap) - - ga := goAttributes{ - Importpath: bazel.StringAttribute{ - Value: proptools.StringPtr(g.GoPkgPath()), - }, - Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()), - Deps: goDepLabels( - android.FirstUniqueStrings(transitiveDeps), - goModulesMap, - ), - Target_compatible_with: targetNotCompatibleWithAndroid(), - } - - lib := bTarget{ - targetName: g.Name(), - targetPackage: ctx.ModuleDir(g), - bazelRuleClass: "go_library", - bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl", - bazelAttributes: []interface{}{&ca, &ga}, - } - retTargets := []BazelTarget{} - var retErrs []error - if libTarget, err := generateBazelTarget(ctx, lib); err == nil { - retTargets = append(retTargets, libTarget) - } else { - retErrs = []error{err} - } - - // If the library contains test srcs, create an additional go_test target - if !testOfGoPackageIsIncompatible(g) && (len(g.TestSrcs()) > 0 || len(g.LinuxTestSrcs()) > 0 || len(g.DarwinTestSrcs()) > 0) { - gp := goTestProperties{ - name: g.Name() + "-test", - dir: ctx.ModuleDir(g), - testSrcs: g.TestSrcs(), - linuxTestSrcs: g.LinuxTestSrcs(), - darwinTestSrcs: g.DarwinTestSrcs(), - testData: g.TestData(), - embedName: g.Name(), // embed the source go_library in the test so that its .go files are included in the compilation unit - } - if libTestTarget, err := generateBazelTargetsGoTest(ctx, goModulesMap, gp); err == nil { - retTargets = append(retTargets, libTestTarget) - } else { - retErrs = append(retErrs, err) - } - } - - return retTargets, retErrs -} - -type goLibraryModule struct { - Dir string - Deps []string -} - -type buildConversionMetadata struct { - nameToGoLibraryModule nameToGoLibraryModule - ndkHeaders []blueprint.Module -} - -type nameToGoLibraryModule map[string]goLibraryModule - -// Visit each module in the graph, and collect metadata about the build graph -// If a module is of type `bootstrap_go_package`, return a map containing metadata like its dir and deps -// If a module is of type `ndk_headers`, add it to a list and return the list -func createBuildConversionMetadata(ctx *android.Context) buildConversionMetadata { - goMap := nameToGoLibraryModule{} - ndkHeaders := []blueprint.Module{} - ctx.VisitAllModules(func(m blueprint.Module) { - moduleType := ctx.ModuleType(m) - // We do not need to store information about blueprint_go_binary since it does not have any rdeps - if moduleType == "bootstrap_go_package" { - goMap[m.Name()] = goLibraryModule{ - Dir: ctx.ModuleDir(m), - Deps: m.(*bootstrap.GoPackage).Deps(), - } - } else if moduleType == "ndk_headers" || moduleType == "versioned_ndk_headers" { - ndkHeaders = append(ndkHeaders, m) - } - }) - return buildConversionMetadata{ - nameToGoLibraryModule: goMap, - ndkHeaders: ndkHeaders, - } -} - -// Returns the deps in the transitive closure of a go target -func transitiveGoDeps(directDeps []string, goModulesMap nameToGoLibraryModule) []string { - allDeps := directDeps - i := 0 - for i < len(allDeps) { - curr := allDeps[i] - allDeps = append(allDeps, goModulesMap[curr].Deps...) - i += 1 - } - allDeps = android.SortedUniqueStrings(allDeps) - return allDeps -} - -func generateBazelTargetsGoBinary(ctx *android.Context, g *bootstrap.GoBinary, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) { - ca := android.CommonAttributes{ - Name: g.Name(), - } - - retTargets := []BazelTarget{} - var retErrs []error - - // For this bootstrap_go_package dep chain, - // A --> B --> C ( ---> depends on) - // Soong provides the convenience of only listing B as deps of A even if a src file of A imports C - // Bazel OTOH - // 1. requires C to be listed in `deps` expllicity. - // 2. does not require C to be listed if src of A does not import C - // - // bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps - transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap) - - goSource := "" - // If the library contains test srcs, create an additional go_test target - // The go_test target will embed a go_source containining the source .go files it tests - if !testOfGoBinaryIsIncompatible(g) && (len(g.TestSrcs()) > 0 || len(g.LinuxTestSrcs()) > 0 || len(g.DarwinTestSrcs()) > 0) { - // Create a go_source containing the source .go files of go_library - // This target will be an `embed` of the go_binary and go_test - goSource = g.Name() + "-source" - ca := android.CommonAttributes{ - Name: goSource, - } - ga := goAttributes{ - Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()), - Deps: goDepLabels(transitiveDeps, goModulesMap), - Target_compatible_with: targetNotCompatibleWithAndroid(), - } - libTestSource := bTarget{ - targetName: goSource, - targetPackage: ctx.ModuleDir(g), - bazelRuleClass: "go_source", - bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl", - bazelAttributes: []interface{}{&ca, &ga}, - } - if libSourceTarget, err := generateBazelTarget(ctx, libTestSource); err == nil { - retTargets = append(retTargets, libSourceTarget) - } else { - retErrs = append(retErrs, err) - } - - // Create a go_test target - gp := goTestProperties{ - name: g.Name() + "-test", - dir: ctx.ModuleDir(g), - testSrcs: g.TestSrcs(), - linuxTestSrcs: g.LinuxTestSrcs(), - darwinTestSrcs: g.DarwinTestSrcs(), - testData: g.TestData(), - // embed the go_source in the test - embedName: g.Name() + "-source", - } - if libTestTarget, err := generateBazelTargetsGoTest(ctx, goModulesMap, gp); err == nil { - retTargets = append(retTargets, libTestTarget) - } else { - retErrs = append(retErrs, err) - } - - } - - // Create a go_binary target - ga := goAttributes{ - Deps: goDepLabels(transitiveDeps, goModulesMap), - Target_compatible_with: targetNotCompatibleWithAndroid(), - } - - // If the binary has testSrcs, embed the common `go_source` - if goSource != "" { - ga.Embed = bazel.MakeLabelListAttribute( - bazel.MakeLabelList( - []bazel.Label{bazel.Label{Label: ":" + goSource}}, - ), - ) - } else { - ga.Srcs = goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()) - } - - bin := bTarget{ - targetName: g.Name(), - targetPackage: ctx.ModuleDir(g), - bazelRuleClass: "go_binary", - bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl", - bazelAttributes: []interface{}{&ca, &ga}, - } - - if binTarget, err := generateBazelTarget(ctx, bin); err == nil { - retTargets = append(retTargets, binTarget) - } else { - retErrs = []error{err} - } - - return retTargets, retErrs -} - func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (conversionResults, []error) { ctx.Context().BeginEvent("GenerateBazelTargets") defer ctx.Context().EndEvent("GenerateBazelTargets") buildFileToTargets := make(map[string]BazelTargets) - // Simple metrics tracking for bp2build - metrics := CreateCodegenMetrics() - dirs := make(map[string]bool) moduleNameToPartition := make(map[string]string) var errs []error - // Visit go libraries in a pre-run and store its state in a map - // The time complexity remains O(N), and this does not add significant wall time. - meta := createBuildConversionMetadata(ctx.Context()) - nameToGoLibMap := meta.nameToGoLibraryModule - ndkHeaders := meta.ndkHeaders - bpCtx := ctx.Context() bpCtx.VisitAllModules(func(m blueprint.Module) { dir := bpCtx.ModuleDir(m) - moduleType := bpCtx.ModuleType(m) dirs[dir] = true var targets []BazelTarget - var targetErrs []error switch ctx.Mode() { - case Bp2Build: - if aModule, ok := m.(android.Module); ok { - reason := aModule.GetUnconvertedReason() - if reason != nil { - // If this module was force-enabled, cause an error. - if _, ok := ctx.Config().BazelModulesForceEnabledByFlag()[m.Name()]; ok && m.Name() != "" { - err := fmt.Errorf("Force Enabled Module %s not converted", m.Name()) - errs = append(errs, err) - } - - // Log the module isn't to be converted by bp2build. - // TODO: b/291598248 - Log handcrafted modules differently than other unconverted modules. - metrics.AddUnconvertedModule(m, moduleType, dir, *reason) - return - } - if len(aModule.Bp2buildTargets()) == 0 { - panic(fmt.Errorf("illegal bp2build invariant: module '%s' was neither converted nor marked unconvertible", aModule.Name())) - } - - // Handle modules converted to generated targets. - targets, targetErrs = generateBazelTargets(bpCtx, aModule) - errs = append(errs, targetErrs...) - for _, t := range targets { - // A module can potentially generate more than 1 Bazel - // target, each of a different rule class. - metrics.IncrementRuleClassCount(t.ruleClass) - } - - // record the partition - moduleNameToPartition[android.RemoveOptionalPrebuiltPrefix(aModule.Name())] = aModule.GetPartitionForBp2build() - - // Log the module. - metrics.AddConvertedModule(aModule, moduleType, dir) - - // Handle modules with unconverted deps. By default, emit a warning. - if unconvertedDeps := aModule.GetUnconvertedBp2buildDeps(); len(unconvertedDeps) > 0 { - msg := fmt.Sprintf("%s %s:%s depends on unconverted modules: %s", - moduleType, bpCtx.ModuleDir(m), m.Name(), strings.Join(unconvertedDeps, ", ")) - switch ctx.unconvertedDepMode { - case warnUnconvertedDeps: - metrics.moduleWithUnconvertedDepsMsgs = append(metrics.moduleWithUnconvertedDepsMsgs, msg) - case errorModulesUnconvertedDeps: - errs = append(errs, fmt.Errorf(msg)) - return - } - } - if unconvertedDeps := aModule.GetMissingBp2buildDeps(); len(unconvertedDeps) > 0 { - msg := fmt.Sprintf("%s %s:%s depends on missing modules: %s", - moduleType, bpCtx.ModuleDir(m), m.Name(), strings.Join(unconvertedDeps, ", ")) - switch ctx.unconvertedDepMode { - case warnUnconvertedDeps: - metrics.moduleWithMissingDepsMsgs = append(metrics.moduleWithMissingDepsMsgs, msg) - case errorModulesUnconvertedDeps: - errs = append(errs, fmt.Errorf(msg)) - return - } - } - } else if glib, ok := m.(*bootstrap.GoPackage); ok { - targets, targetErrs = generateBazelTargetsGoPackage(bpCtx, glib, nameToGoLibMap) - errs = append(errs, targetErrs...) - metrics.IncrementRuleClassCount("bootstrap_go_package") - metrics.AddConvertedModule(glib, "bootstrap_go_package", dir) - } else if gbin, ok := m.(*bootstrap.GoBinary); ok { - targets, targetErrs = generateBazelTargetsGoBinary(bpCtx, gbin, nameToGoLibMap) - errs = append(errs, targetErrs...) - metrics.IncrementRuleClassCount("blueprint_go_binary") - metrics.AddConvertedModule(gbin, "blueprint_go_binary", dir) - } else { - metrics.AddUnconvertedModule(m, moduleType, dir, android.UnconvertedReason{ - ReasonType: int(bp2build_metrics_proto.UnconvertedReasonType_TYPE_UNSUPPORTED), - }) - return - } case QueryView: // Blocklist certain module types from being generated. if canonicalizeModuleType(bpCtx.ModuleType(m)) == "package" { @@ -824,37 +315,6 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers } }) - // Create an ndk_sysroot target that has a dependency edge on every target corresponding to Soong's ndk_headers - // This root target will provide headers to sdk variants of jni libraries - if ctx.Mode() == Bp2Build { - var depLabels bazel.LabelList - for _, ndkHeader := range ndkHeaders { - depLabel := bazel.Label{ - Label: "//" + bpCtx.ModuleDir(ndkHeader) + ":" + ndkHeader.Name(), - } - depLabels.Add(&depLabel) - } - a := struct { - Deps bazel.LabelListAttribute - }{ - Deps: bazel.MakeLabelListAttribute(bazel.UniqueSortedBazelLabelList(depLabels)), - } - ndkSysroot := bTarget{ - targetName: "ndk_sysroot", - targetPackage: "build/bazel/rules/cc", // The location is subject to change, use build/bazel for now - bazelRuleClass: "cc_library_headers", - bazelRuleLoadLocation: "//build/bazel/rules/cc:cc_library_headers.bzl", - bazelAttributes: []interface{}{&a}, - } - - if t, err := generateBazelTarget(bpCtx, ndkSysroot); err == nil { - dir := ndkSysroot.targetPackage - buildFileToTargets[dir] = append(buildFileToTargets[dir], t) - } else { - errs = append(errs, err) - } - } - if len(errs) > 0 { return conversionResults{}, errs } @@ -883,72 +343,9 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers return conversionResults{ buildFileToTargets: buildFileToTargets, moduleNameToPartition: moduleNameToPartition, - metrics: metrics, }, errs } -func generateBazelTargets(ctx bpToBuildContext, m android.Module) ([]BazelTarget, []error) { - var targets []BazelTarget - var errs []error - for _, m := range m.Bp2buildTargets() { - target, err := generateBazelTarget(ctx, m) - if err != nil { - errs = append(errs, err) - return targets, errs - } - targets = append(targets, target) - } - return targets, errs -} - -type bp2buildModule interface { - TargetName() string - TargetPackage() string - BazelRuleClass() string - BazelRuleLoadLocation() string - BazelAttributes() []interface{} -} - -func generateBazelTarget(ctx bpToBuildContext, m bp2buildModule) (BazelTarget, error) { - ruleClass := m.BazelRuleClass() - bzlLoadLocation := m.BazelRuleLoadLocation() - - // extract the bazel attributes from the module. - attrs := m.BazelAttributes() - props, err := extractModuleProperties(attrs, true) - if err != nil { - return BazelTarget{}, err - } - - // name is handled in a special manner - delete(props.Attrs, "name") - - // Return the Bazel target with rule class and attributes, ready to be - // code-generated. - attributes := propsToAttributes(props.Attrs) - var content string - targetName := m.TargetName() - if targetName != "" { - content = fmt.Sprintf(ruleTargetTemplate, ruleClass, targetName, attributes) - } else { - content = fmt.Sprintf(unnamedRuleTargetTemplate, ruleClass, attributes) - } - var loads []BazelLoad - if bzlLoadLocation != "" { - loads = append(loads, BazelLoad{ - file: bzlLoadLocation, - symbols: []BazelLoadSymbol{{symbol: ruleClass}}, - }) - } - return BazelTarget{ - name: targetName, - packageName: m.TargetPackage(), - ruleClass: ruleClass, - loads: loads, - content: content, - }, nil -} - // Convert a module and its deps and props into a Bazel macro/rule // representation in the BUILD file. func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) (BazelTarget, error) { diff --git a/bp2build/conversion.go b/bp2build/conversion.go index be3c7ff38..9f1aa09c6 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -1,15 +1,10 @@ package bp2build import ( - "encoding/json" - "fmt" "reflect" - "strconv" "strings" "android/soong/android" - "android/soong/starlark_fmt" - "github.com/google/blueprint/proptools" ) @@ -19,79 +14,6 @@ type BazelFile struct { Contents string } -// createSoongInjectionDirFiles returns most of the files to write to the soong_injection directory. -// Some other files also come from CreateProductConfigFiles -func createSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) ([]BazelFile, error) { - cfg := ctx.Config() - var files []BazelFile - - files = append(files, newFile("android", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package. - files = append(files, newFile("android", "constants.bzl", android.BazelCcToolchainVars(cfg))) - - if buf, err := json.MarshalIndent(metrics.convertedModuleWithType, "", " "); err != nil { - return []BazelFile{}, err - } else { - files = append(files, newFile("metrics", "converted_modules.json", string(buf))) - } - - convertedModulePathMap, err := json.MarshalIndent(metrics.convertedModulePathMap, "", "\t") - if err != nil { - panic(err) - } - files = append(files, newFile("metrics", GeneratedBuildFileName, "")) // Creates a //metrics package. - files = append(files, newFile("metrics", "converted_modules_path_map.json", string(convertedModulePathMap))) - files = append(files, newFile("metrics", "converted_modules_path_map.bzl", "modules = "+strings.ReplaceAll(string(convertedModulePathMap), "\\", "\\\\"))) - - files = append(files, newFile("product_config", "soong_config_variables.bzl", cfg.Bp2buildSoongConfigDefinitions.String())) - - files = append(files, newFile("product_config", "arch_configuration.bzl", android.StarlarkArchConfigurations())) - - apiLevelsMap, err := android.GetApiLevelsMap(cfg) - if err != nil { - return nil, err - } - apiLevelsContent, err := json.Marshal(apiLevelsMap) - if err != nil { - return nil, err - } - files = append(files, newFile("api_levels", GeneratedBuildFileName, `exports_files(["api_levels.json"])`)) - // TODO(b/269691302) value of apiLevelsContent is product variable dependent and should be avoided for soong injection - files = append(files, newFile("api_levels", "api_levels.json", string(apiLevelsContent))) - files = append(files, newFile("api_levels", "platform_versions.bzl", platformVersionContents(cfg))) - - files = append(files, newFile("allowlists", GeneratedBuildFileName, "")) - // TODO(b/262781701): Create an alternate soong_build entrypoint for writing out these files only when requested - files = append(files, newFile("allowlists", "mixed_build_prod_allowlist.txt", strings.Join(android.GetBazelEnabledModules(android.BazelProdMode), "\n")+"\n")) - files = append(files, newFile("allowlists", "mixed_build_staging_allowlist.txt", strings.Join(android.GetBazelEnabledModules(android.BazelStagingMode), "\n")+"\n")) - - return files, nil -} - -func platformVersionContents(cfg android.Config) string { - // Despite these coming from cfg.productVariables, they are actually hardcoded in global - // makefiles, not set in individual product config makesfiles, so they're safe to just export - // and load() directly. - - platformVersionActiveCodenames := make([]string, 0, len(cfg.PlatformVersionActiveCodenames())) - for _, codename := range cfg.PlatformVersionActiveCodenames() { - platformVersionActiveCodenames = append(platformVersionActiveCodenames, fmt.Sprintf("%q", codename)) - } - - platformSdkVersion := "None" - if cfg.RawPlatformSdkVersion() != nil { - platformSdkVersion = strconv.Itoa(*cfg.RawPlatformSdkVersion()) - } - - return fmt.Sprintf(` -platform_versions = struct( - platform_sdk_final = %s, - platform_sdk_version = %s, - platform_sdk_codename = %q, - platform_version_active_codenames = [%s], -) -`, starlark_fmt.PrintBool(cfg.PlatformSdkFinal()), platformSdkVersion, cfg.PlatformSdkCodename(), strings.Join(platformVersionActiveCodenames, ", ")) -} - func CreateBazelFiles(ruleShims map[string]RuleShim, buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile { var files []BazelFile @@ -125,20 +47,7 @@ func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode) targets.sort() var content string - if mode == Bp2Build { - content = `# READ THIS FIRST: -# This file was automatically generated by bp2build for the Bazel migration project. -# Feel free to edit or test it, but do *not* check it into your version control system. -` - content += targets.LoadStatements() - content += "\n\n" - // Get package rule from the handcrafted BUILD file, otherwise emit the default one. - prText := "package(default_visibility = [\"//visibility:public\"])\n" - if pr := targets.packageRule(); pr != nil { - prText = pr.content - } - content += prText - } else if mode == QueryView { + if mode == QueryView { content = soongModuleLoad } if content != "" { @@ -161,14 +70,6 @@ func newFile(dir, basename, content string) BazelFile { const ( bazelRulesSubDir = "build/bazel/queryview_rules" - - // additional files: - // * workspace file - // * base BUILD file - // * rules BUILD file - // * rules providers.bzl file - // * rules soong_module.bzl file - numAdditionalFiles = 5 ) var ( diff --git a/bp2build/metrics.go b/bp2build/metrics.go deleted file mode 100644 index 20002c67f..000000000 --- a/bp2build/metrics.go +++ /dev/null @@ -1,231 +0,0 @@ -package bp2build - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "android/soong/android" - "android/soong/shared" - "android/soong/ui/metrics/bp2build_metrics_proto" - - "google.golang.org/protobuf/proto" - - "github.com/google/blueprint" -) - -type moduleInfo struct { - Name string `json:"name"` - Type string `json:"type"` -} - -// CodegenMetrics represents information about the Blueprint-to-BUILD -// conversion process. -// Use CreateCodegenMetrics() to get a properly initialized instance -type CodegenMetrics struct { - serialized *bp2build_metrics_proto.Bp2BuildMetrics - // List of modules with unconverted deps - // NOTE: NOT in the .proto - moduleWithUnconvertedDepsMsgs []string - - // List of modules with missing deps - // NOTE: NOT in the .proto - moduleWithMissingDepsMsgs []string - - // Map of converted modules and paths to call - // NOTE: NOT in the .proto - convertedModulePathMap map[string]string - - // Name and type of converted modules - convertedModuleWithType []moduleInfo -} - -func CreateCodegenMetrics() CodegenMetrics { - return CodegenMetrics{ - serialized: &bp2build_metrics_proto.Bp2BuildMetrics{ - RuleClassCount: make(map[string]uint64), - ConvertedModuleTypeCount: make(map[string]uint64), - TotalModuleTypeCount: make(map[string]uint64), - UnconvertedModules: make(map[string]*bp2build_metrics_proto.UnconvertedReason), - }, - convertedModulePathMap: make(map[string]string), - } -} - -// Serialize returns the protoized version of CodegenMetrics: bp2build_metrics_proto.Bp2BuildMetrics -func (metrics *CodegenMetrics) Serialize() *bp2build_metrics_proto.Bp2BuildMetrics { - return metrics.serialized -} - -// Print the codegen metrics to stdout. -func (metrics *CodegenMetrics) Print() { - generatedTargetCount := uint64(0) - for _, ruleClass := range android.SortedKeys(metrics.serialized.RuleClassCount) { - count := metrics.serialized.RuleClassCount[ruleClass] - fmt.Printf("[bp2build] %s: %d targets\n", ruleClass, count) - generatedTargetCount += count - } - fmt.Printf( - `[bp2build] Converted %d Android.bp modules to %d total generated BUILD targets. Included %d handcrafted BUILD targets. There are %d total Android.bp modules. -%d converted modules have unconverted deps: - %s -%d converted modules have missing deps: - %s -`, - metrics.serialized.GeneratedModuleCount, - generatedTargetCount, - metrics.serialized.HandCraftedModuleCount, - metrics.TotalModuleCount(), - len(metrics.moduleWithUnconvertedDepsMsgs), - strings.Join(metrics.moduleWithUnconvertedDepsMsgs, "\n\t"), - len(metrics.moduleWithMissingDepsMsgs), - strings.Join(metrics.moduleWithMissingDepsMsgs, "\n\t"), - ) -} - -const bp2buildMetricsFilename = "bp2build_metrics.pb" - -// fail prints $PWD to stderr, followed by the given printf string and args (vals), -// then the given alert, and then exits with 1 for failure -func fail(err error, alertFmt string, vals ...interface{}) { - cwd, wderr := os.Getwd() - if wderr != nil { - cwd = "FAILED TO GET $PWD: " + wderr.Error() - } - fmt.Fprintf(os.Stderr, "\nIn "+cwd+":\n"+alertFmt+"\n"+err.Error()+"\n", vals...) - os.Exit(1) -} - -// Write the bp2build-protoized codegen metrics into the given directory -func (metrics *CodegenMetrics) Write(dir string) { - if _, err := os.Stat(dir); os.IsNotExist(err) { - // The metrics dir doesn't already exist, so create it (and parents) - if err := os.MkdirAll(dir, 0755); err != nil { // rx for all; w for user - fail(err, "Failed to `mkdir -p` %s", dir) - } - } else if err != nil { - fail(err, "Failed to `stat` %s", dir) - } - metricsFile := filepath.Join(dir, bp2buildMetricsFilename) - if err := metrics.dump(metricsFile); err != nil { - fail(err, "Error outputting %s", metricsFile) - } - if _, err := os.Stat(metricsFile); err != nil { - if os.IsNotExist(err) { - fail(err, "MISSING BP2BUILD METRICS OUTPUT: %s", metricsFile) - } else { - fail(err, "FAILED TO `stat` BP2BUILD METRICS OUTPUT: %s", metricsFile) - } - } -} - -// ReadCodegenMetrics loads CodegenMetrics from `dir` -// returns a nil pointer if the file doesn't exist -func ReadCodegenMetrics(dir string) *CodegenMetrics { - metricsFile := filepath.Join(dir, bp2buildMetricsFilename) - if _, err := os.Stat(metricsFile); err != nil { - if os.IsNotExist(err) { - return nil - } else { - fail(err, "FAILED TO `stat` BP2BUILD METRICS OUTPUT: %s", metricsFile) - panic("unreachable after fail") - } - } - if buf, err := os.ReadFile(metricsFile); err != nil { - fail(err, "FAILED TO READ BP2BUILD METRICS OUTPUT: %s", metricsFile) - panic("unreachable after fail") - } else { - bp2BuildMetrics := bp2build_metrics_proto.Bp2BuildMetrics{ - RuleClassCount: make(map[string]uint64), - ConvertedModuleTypeCount: make(map[string]uint64), - TotalModuleTypeCount: make(map[string]uint64), - } - if err := proto.Unmarshal(buf, &bp2BuildMetrics); err != nil { - fail(err, "FAILED TO PARSE BP2BUILD METRICS OUTPUT: %s", metricsFile) - } - return &CodegenMetrics{ - serialized: &bp2BuildMetrics, - convertedModulePathMap: make(map[string]string), - } - } -} - -func (metrics *CodegenMetrics) IncrementRuleClassCount(ruleClass string) { - metrics.serialized.RuleClassCount[ruleClass] += 1 -} - -func (metrics *CodegenMetrics) AddEvent(event *bp2build_metrics_proto.Event) { - metrics.serialized.Events = append(metrics.serialized.Events, event) -} - -func (metrics *CodegenMetrics) SetSymlinkCount(n uint64) { - if m := metrics.serialized.WorkspaceSymlinkCount; m != 0 { - fmt.Fprintf(os.Stderr, "unexpected non-zero workspaceSymlinkCount of %d", m) - } - metrics.serialized.WorkspaceSymlinkCount = n -} - -func (metrics *CodegenMetrics) SetMkDirCount(n uint64) { - if m := metrics.serialized.WorkspaceMkDirCount; m != 0 { - fmt.Fprintf(os.Stderr, "unexpected non-zero workspaceDirCount of %d", m) - } - metrics.serialized.WorkspaceMkDirCount = n -} - -func (metrics *CodegenMetrics) TotalModuleCount() uint64 { - return metrics.serialized.HandCraftedModuleCount + - metrics.serialized.GeneratedModuleCount + - metrics.serialized.UnconvertedModuleCount -} - -// Dump serializes the metrics to the given filename -func (metrics *CodegenMetrics) dump(filename string) (err error) { - ser := metrics.Serialize() - return shared.Save(ser, filename) -} - -type ConversionType int - -const ( - Generated ConversionType = iota - Handcrafted -) - -func (metrics *CodegenMetrics) AddConvertedModule(m blueprint.Module, moduleType string, dir string) { - //a package module has empty name - if moduleType == "package" { - return - } - // Undo prebuilt_ module name prefix modifications - moduleName := android.RemoveOptionalPrebuiltPrefix(m.Name()) - metrics.serialized.ConvertedModules = append(metrics.serialized.ConvertedModules, moduleName) - metrics.convertedModuleWithType = append(metrics.convertedModuleWithType, moduleInfo{ - moduleName, - moduleType, - }) - metrics.convertedModulePathMap[moduleName] = "//" + dir - metrics.serialized.ConvertedModuleTypeCount[moduleType] += 1 - metrics.serialized.TotalModuleTypeCount[moduleType] += 1 - metrics.serialized.GeneratedModuleCount += 1 -} - -func (metrics *CodegenMetrics) AddUnconvertedModule(m blueprint.Module, moduleType string, dir string, - reason android.UnconvertedReason) { - //a package module has empty name - if moduleType == "package" { - return - } - // Undo prebuilt_ module name prefix modifications - moduleName := android.RemoveOptionalPrebuiltPrefix(m.Name()) - metrics.serialized.UnconvertedModules[moduleName] = &bp2build_metrics_proto.UnconvertedReason{ - Type: bp2build_metrics_proto.UnconvertedReasonType(reason.ReasonType), - Detail: reason.Detail, - } - metrics.serialized.UnconvertedModuleCount += 1 - metrics.serialized.TotalModuleTypeCount[moduleType] += 1 - - if reason.ReasonType == int(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE) { - metrics.serialized.HandCraftedModuleCount += 1 - } -} diff --git a/bp2build/performance_test.go b/bp2build/performance_test.go deleted file mode 100644 index 5f80b8398..000000000 --- a/bp2build/performance_test.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2021 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bp2build - -// to run the benchmarks in this file, you must run go test with the -bench. -// The benchmarked portion will run for the specified time (can be set via -benchtime) -// This can mean if you are benchmarking a faster portion of a larger operation, it will take -// longer. -// If you are seeing a small number of iterations for a specific run, the data is less reliable, to -// run for longer, set -benchtime to a larger value. - -import ( - "fmt" - "math" - "strings" - "testing" - - "android/soong/android" -) - -const ( - performance_test_dir = "." -) - -func genCustomModule(i int, convert bool) string { - var conversionString string - if convert { - conversionString = `bazel_module: { bp2build_available: true },` - } - return fmt.Sprintf(` -custom { - name: "arch_paths_%[1]d", - string_list_prop: ["\t", "\n"], - string_prop: "a\t\n\r", - arch_paths: ["outer", ":outer_dep_%[1]d"], - arch: { - x86: { - arch_paths: ["abc", ":x86_dep_%[1]d"], - }, - x86_64: { - arch_paths: ["64bit"], - arch_paths_exclude: ["outer"], - }, - }, - %[2]s -} - -custom { - name: "outer_dep_%[1]d", - %[2]s -} - -custom { - name: "x86_dep_%[1]d", - %[2]s -} -`, i, conversionString) -} - -func genCustomModuleBp(pctConverted float64) string { - modules := 100 - - bp := make([]string, 0, modules) - toConvert := int(math.Round(float64(modules) * pctConverted)) - - for i := 0; i < modules; i++ { - bp = append(bp, genCustomModule(i, i < toConvert)) - } - return strings.Join(bp, "\n\n") -} - -type testConfig struct { - config android.Config - ctx *android.TestContext - codegenCtx *CodegenContext -} - -func (tc testConfig) parse() []error { - _, errs := tc.ctx.ParseFileList(performance_test_dir, []string{"Android.bp"}) - return errs -} - -func (tc testConfig) resolveDependencies() []error { - _, errs := tc.ctx.ResolveDependencies(tc.config) - return errs -} - -func (tc testConfig) convert() { - generateBazelTargetsForDir(tc.codegenCtx, performance_test_dir) -} - -func setup(builddir string, tcSize float64) testConfig { - config := android.TestConfig(buildDir, nil, genCustomModuleBp(tcSize), nil) - ctx := android.NewTestContext(config) - - registerCustomModuleForBp2buildConversion(ctx) - codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "") - return testConfig{ - config, - ctx, - codegenCtx, - } -} - -var pctToConvert = []float64{0.0, 0.01, 0.05, 0.10, 0.25, 0.5, 0.75, 1.0} - -// This is not intended to test performance, but to verify performance infra continues to work -func TestConvertManyModulesFull(t *testing.T) { - for _, tcSize := range pctToConvert { - - t.Run(fmt.Sprintf("pctConverted %f", tcSize), func(t *testing.T) { - testConfig := setup(buildDir, tcSize) - - errs := testConfig.parse() - if len(errs) > 0 { - t.Fatalf("Unexpected errors: %s", errs) - } - - errs = testConfig.resolveDependencies() - if len(errs) > 0 { - t.Fatalf("Unexpected errors: %s", errs) - } - - testConfig.convert() - }) - } -} - -func BenchmarkManyModulesFull(b *testing.B) { - for _, tcSize := range pctToConvert { - - b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) { - for n := 0; n < b.N; n++ { - b.StopTimer() - testConfig := setup(buildDir, tcSize) - - b.StartTimer() - errs := testConfig.parse() - if len(errs) > 0 { - b.Fatalf("Unexpected errors: %s", errs) - } - - errs = testConfig.resolveDependencies() - if len(errs) > 0 { - b.Fatalf("Unexpected errors: %s", errs) - } - - testConfig.convert() - b.StopTimer() - } - }) - } -} - -func BenchmarkManyModulesResolveDependencies(b *testing.B) { - for _, tcSize := range pctToConvert { - - b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) { - for n := 0; n < b.N; n++ { - b.StopTimer() - // setup we don't want to measure - testConfig := setup(buildDir, tcSize) - - errs := testConfig.parse() - if len(errs) > 0 { - b.Fatalf("Unexpected errors: %s", errs) - } - - b.StartTimer() - errs = testConfig.resolveDependencies() - b.StopTimer() - if len(errs) > 0 { - b.Fatalf("Unexpected errors: %s", errs) - } - - testConfig.convert() - } - }) - } -} - -func BenchmarkManyModulesGenerateBazelTargetsForDir(b *testing.B) { - for _, tcSize := range pctToConvert { - - b.Run(fmt.Sprintf("pctConverted %f", tcSize), func(b *testing.B) { - for n := 0; n < b.N; n++ { - b.StopTimer() - // setup we don't want to measure - testConfig := setup(buildDir, tcSize) - - errs := testConfig.parse() - if len(errs) > 0 { - b.Fatalf("Unexpected errors: %s", errs) - } - - errs = testConfig.resolveDependencies() - if len(errs) > 0 { - b.Fatalf("Unexpected errors: %s", errs) - } - - b.StartTimer() - testConfig.convert() - b.StopTimer() - } - }) - } -} diff --git a/bp2build/symlink_forest.go b/bp2build/symlink_forest.go deleted file mode 100644 index 15a6df03a..000000000 --- a/bp2build/symlink_forest.go +++ /dev/null @@ -1,511 +0,0 @@ -// Copyright 2022 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bp2build - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "sort" - "strconv" - "sync" - "sync/atomic" - - "android/soong/shared" - - "github.com/google/blueprint/pathtools" -) - -// A tree structure that describes what to do at each directory in the created -// symlink tree. Currently, it is used to enumerate which files/directories -// should be excluded from symlinking. Each instance of "node" represents a file -// or a directory. If excluded is true, then that file/directory should be -// excluded from symlinking. Otherwise, the node is not excluded, but one of its -// descendants is (otherwise the node in question would not exist) - -type instructionsNode struct { - name string - excluded bool // If false, this is just an intermediate node - children map[string]*instructionsNode -} - -type symlinkForestContext struct { - verbose bool - topdir string // $TOPDIR - - // State - wg sync.WaitGroup - depCh chan string - mkdirCount atomic.Uint64 - symlinkCount atomic.Uint64 -} - -// Ensures that the node for the given path exists in the tree and returns it. -func ensureNodeExists(root *instructionsNode, path string) *instructionsNode { - if path == "" { - return root - } - - if path[len(path)-1] == '/' { - path = path[:len(path)-1] // filepath.Split() leaves a trailing slash - } - - dir, base := filepath.Split(path) - - // First compute the parent node... - dn := ensureNodeExists(root, dir) - - // then create the requested node as its direct child, if needed. - if child, ok := dn.children[base]; ok { - return child - } else { - dn.children[base] = &instructionsNode{base, false, make(map[string]*instructionsNode)} - return dn.children[base] - } -} - -// Turns a list of paths to be excluded into a tree -func instructionsFromExcludePathList(paths []string) *instructionsNode { - result := &instructionsNode{"", false, make(map[string]*instructionsNode)} - - for _, p := range paths { - ensureNodeExists(result, p).excluded = true - } - - return result -} - -func mergeBuildFiles(output string, srcBuildFile string, generatedBuildFile string, verbose bool) error { - - srcBuildFileContent, err := os.ReadFile(srcBuildFile) - if err != nil { - return err - } - - generatedBuildFileContent, err := os.ReadFile(generatedBuildFile) - if err != nil { - return err - } - - // There can't be a package() call in both the source and generated BUILD files. - // bp2build will generate a package() call for licensing information, but if - // there's no licensing information, it will still generate a package() call - // that just sets default_visibility=public. If the handcrafted build file - // also has a package() call, we'll allow it to override the bp2build - // generated one if it doesn't have any licensing information. If the bp2build - // one has licensing information and the handcrafted one exists, we'll leave - // them both in for bazel to throw an error. - packageRegex := regexp.MustCompile(`(?m)^package\s*\(`) - packageDefaultVisibilityRegex := regexp.MustCompile(`(?m)^package\s*\(\s*default_visibility\s*=\s*\[\s*"//visibility:public",?\s*]\s*\)`) - if packageRegex.Find(srcBuildFileContent) != nil { - if verbose && packageDefaultVisibilityRegex.Find(generatedBuildFileContent) != nil { - fmt.Fprintf(os.Stderr, "Both '%s' and '%s' have a package() target, removing the first one\n", - generatedBuildFile, srcBuildFile) - } - generatedBuildFileContent = packageDefaultVisibilityRegex.ReplaceAll(generatedBuildFileContent, []byte{}) - } - - newContents := generatedBuildFileContent - if newContents[len(newContents)-1] != '\n' { - newContents = append(newContents, '\n') - } - newContents = append(newContents, srcBuildFileContent...) - - // Say you run bp2build 4 times: - // - The first time there's only an Android.bp file. bp2build will convert it to a build file - // under out/soong/bp2build, then symlink from the forest to that generated file - // - Then you add a handcrafted BUILD file in the same directory. bp2build will merge this with - // the generated one, and write the result to the output file in the forest. But the output - // file was a symlink to out/soong/bp2build from the previous step! So we erroneously update - // the file in out/soong/bp2build instead. So far this doesn't cause any problems... - // - You run a 3rd bp2build with no relevant changes. Everything continues to work. - // - You then add a comment to the handcrafted BUILD file. This causes a merge with the - // generated file again. But since we wrote to the generated file in step 2, the generated - // file has an old copy of the handcrafted file in it! This probably causes duplicate bazel - // targets. - // To solve this, if we see that the output file is a symlink from a previous build, remove it. - stat, err := os.Lstat(output) - if err != nil && !os.IsNotExist(err) { - return err - } else if err == nil { - if stat.Mode()&os.ModeSymlink == os.ModeSymlink { - if verbose { - fmt.Fprintf(os.Stderr, "Removing symlink so that we can replace it with a merged file: %s\n", output) - } - err = os.Remove(output) - if err != nil { - return err - } - } - } - - return pathtools.WriteFileIfChanged(output, newContents, 0666) -} - -// Calls readdir() and returns it as a map from the basename of the files in dir -// to os.FileInfo. -func readdirToMap(dir string) map[string]os.FileInfo { - entryList, err := ioutil.ReadDir(dir) - result := make(map[string]os.FileInfo) - - if err != nil { - if os.IsNotExist(err) { - // It's okay if a directory doesn't exist; it just means that one of the - // trees to be merged contains parts the other doesn't - return result - } else { - fmt.Fprintf(os.Stderr, "Cannot readdir '%s': %s\n", dir, err) - os.Exit(1) - } - } - - for _, fi := range entryList { - result[fi.Name()] = fi - } - - return result -} - -// Creates a symbolic link at dst pointing to src -func symlinkIntoForest(topdir, dst, src string) uint64 { - srcPath := shared.JoinPath(topdir, src) - dstPath := shared.JoinPath(topdir, dst) - - // Check whether a symlink already exists. - if dstInfo, err := os.Lstat(dstPath); err != nil { - if !os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "Failed to lstat '%s': %s", dst, err) - os.Exit(1) - } - } else { - if dstInfo.Mode()&os.ModeSymlink != 0 { - // Assume that the link's target is correct, i.e. no manual tampering. - // E.g. OUT_DIR could have been previously used with a different source tree check-out! - return 0 - } else { - if err := os.RemoveAll(dstPath); err != nil { - fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", dst, err) - os.Exit(1) - } - } - } - - // Create symlink. - if err := os.Symlink(srcPath, dstPath); err != nil { - fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err) - os.Exit(1) - } - return 1 -} - -func isDir(path string, fi os.FileInfo) bool { - if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { - return fi.IsDir() - } - - fi2, statErr := os.Stat(path) - if statErr == nil { - return fi2.IsDir() - } - - // Check if this is a dangling symlink. If so, treat it like a file, not a dir. - _, lstatErr := os.Lstat(path) - if lstatErr != nil { - fmt.Fprintf(os.Stderr, "Cannot stat or lstat '%s': %s\n%s\n", path, statErr, lstatErr) - os.Exit(1) - } - - return false -} - -// Returns the mtime of the soong_build binary to determine whether we should -// force symlink_forest to re-execute -func getSoongBuildMTime() (int64, error) { - binaryPath, err := os.Executable() - if err != nil { - return 0, err - } - - info, err := os.Stat(binaryPath) - if err != nil { - return 0, err - } - - return info.ModTime().UnixMilli(), nil -} - -// cleanSymlinkForest will remove the whole symlink forest directory -func cleanSymlinkForest(topdir, forest string) error { - return os.RemoveAll(shared.JoinPath(topdir, forest)) -} - -// This returns whether symlink forest should clean and replant symlinks. -// It compares the mtime of this executable with the mtime of the last-run -// soong_build binary. If they differ, then we should clean and replant. -func shouldCleanSymlinkForest(topdir string, forest string, soongBuildMTime int64) (bool, error) { - mtimeFilePath := shared.JoinPath(topdir, forest, "soong_build_mtime") - mtimeFileContents, err := os.ReadFile(mtimeFilePath) - if err != nil { - if os.IsNotExist(err) { - // This is likely the first time this has run with this functionality - clean away! - return true, nil - } else { - return false, err - } - } - return strconv.FormatInt(soongBuildMTime, 10) != string(mtimeFileContents), nil -} - -func writeSoongBuildMTimeFile(topdir, forest string, mtime int64) error { - mtimeFilePath := shared.JoinPath(topdir, forest, "soong_build_mtime") - contents := []byte(strconv.FormatInt(mtime, 10)) - - return os.WriteFile(mtimeFilePath, contents, 0666) -} - -// Recursively plants a symlink forest at forestDir. The symlink tree will -// contain every file in buildFilesDir and srcDir excluding the files in -// instructions. Collects every directory encountered during the traversal of -// srcDir . -func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *instructionsNode, forestDir string, buildFilesDir string, srcDir string) { - defer context.wg.Done() - - if instructions != nil && instructions.excluded { - // Excluded paths are skipped at the level of the non-excluded parent. - fmt.Fprintf(os.Stderr, "may not specify a root-level exclude directory '%s'", srcDir) - os.Exit(1) - } - - // We don't add buildFilesDir here because the bp2build files marker files is - // already a dependency which covers it. If we ever wanted to turn this into - // a generic symlink forest creation tool, we'd need to add it, too. - context.depCh <- srcDir - - srcDirMap := readdirToMap(shared.JoinPath(context.topdir, srcDir)) - buildFilesMap := readdirToMap(shared.JoinPath(context.topdir, buildFilesDir)) - - renamingBuildFile := false - if _, ok := srcDirMap["BUILD"]; ok { - if _, ok := srcDirMap["BUILD.bazel"]; !ok { - if _, ok := buildFilesMap["BUILD.bazel"]; ok { - renamingBuildFile = true - srcDirMap["BUILD.bazel"] = srcDirMap["BUILD"] - delete(srcDirMap, "BUILD") - if instructions != nil { - if _, ok := instructions.children["BUILD"]; ok { - instructions.children["BUILD.bazel"] = instructions.children["BUILD"] - delete(instructions.children, "BUILD") - } - } - } - } - } - - allEntries := make([]string, 0, len(srcDirMap)+len(buildFilesMap)) - for n := range srcDirMap { - allEntries = append(allEntries, n) - } - for n := range buildFilesMap { - if _, ok := srcDirMap[n]; !ok { - allEntries = append(allEntries, n) - } - } - // Tests read the error messages generated, so ensure their order is deterministic - sort.Strings(allEntries) - - fullForestPath := shared.JoinPath(context.topdir, forestDir) - createForestDir := false - if fi, err := os.Lstat(fullForestPath); err != nil { - if os.IsNotExist(err) { - createForestDir = true - } else { - fmt.Fprintf(os.Stderr, "Could not read info for '%s': %s\n", forestDir, err) - } - } else if fi.Mode()&os.ModeDir == 0 { - if err := os.RemoveAll(fullForestPath); err != nil { - fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", forestDir, err) - os.Exit(1) - } - createForestDir = true - } - if createForestDir { - if err := os.MkdirAll(fullForestPath, 0777); err != nil { - fmt.Fprintf(os.Stderr, "Could not mkdir '%s': %s\n", forestDir, err) - os.Exit(1) - } - context.mkdirCount.Add(1) - } - - // Start with a list of items that already exist in the forest, and remove - // each element as it is processed in allEntries. Any remaining items in - // forestMapForDeletion must be removed. (This handles files which were - // removed since the previous forest generation). - forestMapForDeletion := readdirToMap(shared.JoinPath(context.topdir, forestDir)) - - for _, f := range allEntries { - if f[0] == '.' { - continue // Ignore dotfiles - } - delete(forestMapForDeletion, f) - // todo add deletionCount metric - - // The full paths of children in the input trees and in the output tree - forestChild := shared.JoinPath(forestDir, f) - srcChild := shared.JoinPath(srcDir, f) - if f == "BUILD.bazel" && renamingBuildFile { - srcChild = shared.JoinPath(srcDir, "BUILD") - } - buildFilesChild := shared.JoinPath(buildFilesDir, f) - - // Descend in the instruction tree if it exists - var instructionsChild *instructionsNode - if instructions != nil { - instructionsChild = instructions.children[f] - } - - srcChildEntry, sExists := srcDirMap[f] - buildFilesChildEntry, bExists := buildFilesMap[f] - - if instructionsChild != nil && instructionsChild.excluded { - if bExists { - context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild)) - } - continue - } - - sDir := sExists && isDir(shared.JoinPath(context.topdir, srcChild), srcChildEntry) - bDir := bExists && isDir(shared.JoinPath(context.topdir, buildFilesChild), buildFilesChildEntry) - - if !sExists { - if bDir && instructionsChild != nil { - // Not in the source tree, but we have to exclude something from under - // this subtree, so descend - context.wg.Add(1) - go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild) - } else { - // Not in the source tree, symlink BUILD file - context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild)) - } - } else if !bExists { - if sDir && instructionsChild != nil { - // Not in the build file tree, but we have to exclude something from - // under this subtree, so descend - context.wg.Add(1) - go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild) - } else { - // Not in the build file tree, symlink source tree, carry on - context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, srcChild)) - } - } else if sDir && bDir { - // Both are directories. Descend. - context.wg.Add(1) - go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild) - } else if !sDir && !bDir { - // Neither is a directory. Merge them. - srcBuildFile := shared.JoinPath(context.topdir, srcChild) - generatedBuildFile := shared.JoinPath(context.topdir, buildFilesChild) - // The Android.bp file that codegen used to produce `buildFilesChild` is - // already a dependency, we can ignore `buildFilesChild`. - context.depCh <- srcChild - if err := mergeBuildFiles(shared.JoinPath(context.topdir, forestChild), srcBuildFile, generatedBuildFile, context.verbose); err != nil { - fmt.Fprintf(os.Stderr, "Error merging %s and %s: %s", - srcBuildFile, generatedBuildFile, err) - os.Exit(1) - } - } else { - // Both exist and one is a file. This is an error. - fmt.Fprintf(os.Stderr, - "Conflict in workspace symlink tree creation: both '%s' and '%s' exist and exactly one is a directory\n", - srcChild, buildFilesChild) - os.Exit(1) - } - } - - // Remove all files in the forest that exist in neither the source - // tree nor the build files tree. (This handles files which were removed - // since the previous forest generation). - for f := range forestMapForDeletion { - var instructionsChild *instructionsNode - if instructions != nil { - instructionsChild = instructions.children[f] - } - - if instructionsChild != nil && instructionsChild.excluded { - // This directory may be excluded because bazel writes to it under the - // forest root. Thus this path is intentionally left alone. - continue - } - forestChild := shared.JoinPath(context.topdir, forestDir, f) - if err := os.RemoveAll(forestChild); err != nil { - fmt.Fprintf(os.Stderr, "Failed to remove '%s/%s': %s", forestDir, f, err) - os.Exit(1) - } - } -} - -// PlantSymlinkForest Creates a symlink forest by merging the directory tree at "buildFiles" and -// "srcDir" while excluding paths listed in "exclude". Returns the set of paths -// under srcDir on which readdir() had to be called to produce the symlink -// forest. -func PlantSymlinkForest(verbose bool, topdir string, forest string, buildFiles string, exclude []string) (deps []string, mkdirCount, symlinkCount uint64) { - context := &symlinkForestContext{ - verbose: verbose, - topdir: topdir, - depCh: make(chan string), - mkdirCount: atomic.Uint64{}, - symlinkCount: atomic.Uint64{}, - } - - // Check whether soong_build has been modified since the last run - soongBuildMTime, err := getSoongBuildMTime() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - - shouldClean, err := shouldCleanSymlinkForest(topdir, forest, soongBuildMTime) - - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } else if shouldClean { - err = cleanSymlinkForest(topdir, forest) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - } - - instructions := instructionsFromExcludePathList(exclude) - go func() { - context.wg.Add(1) - plantSymlinkForestRecursive(context, instructions, forest, buildFiles, ".") - context.wg.Wait() - close(context.depCh) - }() - - for dep := range context.depCh { - deps = append(deps, dep) - } - - err = writeSoongBuildMTimeFile(topdir, forest, soongBuildMTime) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - return deps, context.mkdirCount.Load(), context.symlinkCount.Load() -} diff --git a/bp2build/testing.go b/bp2build/testing.go deleted file mode 100644 index c9781647b..000000000 --- a/bp2build/testing.go +++ /dev/null @@ -1,785 +0,0 @@ -// Copyright 2021 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bp2build - -/* -For shareable/common bp2build testing functionality and dumping ground for -specific-but-shared functionality among tests in package -*/ - -import ( - "fmt" - "path/filepath" - "regexp" - "sort" - "strings" - "testing" - - "android/soong/ui/metrics/bp2build_metrics_proto" - - "github.com/google/blueprint/proptools" - - "android/soong/android" - "android/soong/android/allowlists" - "android/soong/bazel" -) - -var ( - buildDir string -) - -var labelRegex = regexp.MustCompile(`^//([^: ]+):([^ ]+)$`) -var simpleModuleNameRegex = regexp.MustCompile(`^[^: /]+$`) - -func checkError(t *testing.T, errs []error, expectedErr error) bool { - t.Helper() - - if len(errs) != 1 { - return false - } - if strings.Contains(errs[0].Error(), expectedErr.Error()) { - return true - } - - return false -} - -func errored(t *testing.T, tc Bp2buildTestCase, errs []error) bool { - t.Helper() - if tc.ExpectedErr != nil { - // Rely on checkErrors, as this test case is expected to have an error. - return false - } - - if len(errs) > 0 { - for _, err := range errs { - t.Errorf("%s: %s", tc.Description, err) - } - return true - } - - // All good, continue execution. - return false -} - -func RunBp2BuildTestCaseSimple(t *testing.T, tc Bp2buildTestCase) { - t.Helper() - RunBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, tc) -} - -type Bp2buildTestCase struct { - Description string - ModuleTypeUnderTest string - ModuleTypeUnderTestFactory android.ModuleFactory - // Text to add to the toplevel, root Android.bp file. If Dir is not set, all - // ExpectedBazelTargets are assumed to be generated by this file. - Blueprint string - // ExpectedBazelTargets compares the BazelTargets generated in `Dir` (if not empty). - // Otherwise, it checks the BazelTargets generated by `Blueprint` in the root directory. - ExpectedBazelTargets []string - // ExpectedConvertedModules asserts that modules in this list are labeled as "converted - // by bp2build" in the metrics reported by bp2build. - ExpectedConvertedModules []string - // ExpectedHandcraftedModules asserts that modules in this list are labeled as "handcrafted - // in build files" in the metrics reported by bp2build. Such modules are either explicitly - // defined in a BUILD file (by name), or registered as "otherwise implicitly handled" - // by bp2build (for example, by macros owned by other modules). - ExpectedHandcraftedModules []string - - // AlreadyExistingBuildContents, if non-empty, simulates an already-present source BUILD file - // in the directory under test. The BUILD file has the given contents. This BUILD file - // will also be treated as "BUILD file to keep" by the simulated bp2build environment. - AlreadyExistingBuildContents string - - // StubbedBuildDefinitions, if non-empty, adds stub definitions to already-present source - // BUILD files for each bazel label given. The BUILD files with these stub definitions - // are added to the BUILD file given in AlreadyExistingBuildContents. - // Labels may be of the form //pkg/to:target_name (which would be defined in pkg/to/BUILD.bazel) - // or `target_name` (which would be defined in ./BUILD.bazel). - StubbedBuildDefinitions []string - - Filesystem map[string]string - // Dir sets the directory which will be compared against the targets in ExpectedBazelTargets. - // This should used in conjunction with the Filesystem property to check for targets - // generated from a directory that is not the root. - // If not set, all ExpectedBazelTargets are assumed to be generated by the text in the - // Blueprint property. - Dir string - // An error with a string contained within the string of the expected error - ExpectedErr error - UnconvertedDepsMode unconvertedDepsMode - - // For every directory listed here, the BUILD file for that directory will - // be merged with the generated BUILD file. This allows custom BUILD targets - // to be used in tests, or use BUILD files to draw package boundaries. - KeepBuildFileForDirs []string - - // An extra FixturePreparer to use when running the test. If you need multiple extra - // FixturePreparers, use android.GroupFixturePreparers() - ExtraFixturePreparer android.FixturePreparer - - // If bp2build_product_config.go should run as part of the test. - RunBp2buildProductConfig bool -} - -func RunBp2BuildTestCase(t *testing.T, registerModuleTypes func(ctx android.RegistrationContext), tc Bp2buildTestCase) { - t.Helper() - preparers := []android.FixturePreparer{ - android.FixtureRegisterWithContext(registerModuleTypes), - } - if tc.ExtraFixturePreparer != nil { - preparers = append(preparers, tc.ExtraFixturePreparer) - } - preparers = append(preparers, android.FixtureSetTestRunner(&bazelTestRunner{generateProductConfigTargets: tc.RunBp2buildProductConfig})) - bp2buildSetup := android.GroupFixturePreparers( - preparers..., - ) - runBp2BuildTestCaseWithSetup(t, bp2buildSetup, tc) -} - -func runBp2BuildTestCaseWithSetup(t *testing.T, extraPreparer android.FixturePreparer, tc Bp2buildTestCase) { - t.Helper() - if tc.Filesystem == nil { - tc.Filesystem = map[string]string{} - } - checkDir := "." - if tc.Dir != "" { - checkDir = tc.Dir - } - keepExistingBuildDirs := tc.KeepBuildFileForDirs - buildFilesToParse := []string{} - - if len(tc.StubbedBuildDefinitions) > 0 { - for _, buildDef := range tc.StubbedBuildDefinitions { - globalLabelMatch := labelRegex.FindStringSubmatch(buildDef) - var dir, targetName string - if len(globalLabelMatch) > 0 { - dir = globalLabelMatch[1] - targetName = globalLabelMatch[2] - } else { - if !simpleModuleNameRegex.MatchString(buildDef) { - t.Errorf("Stubbed build definition '%s' must be either a simple module name or of global target syntax (//foo/bar:baz).", buildDef) - return - } - dir = "." - targetName = buildDef - } - buildFilePath := filepath.Join(dir, "BUILD") - tc.Filesystem[buildFilePath] += - MakeBazelTarget( - "bp2build_test_stub", - targetName, - AttrNameToString{}) - keepExistingBuildDirs = append(keepExistingBuildDirs, dir) - buildFilesToParse = append(buildFilesToParse, buildFilePath) - } - } - if len(tc.AlreadyExistingBuildContents) > 0 { - buildFilePath := filepath.Join(checkDir, "BUILD") - tc.Filesystem[buildFilePath] += tc.AlreadyExistingBuildContents - keepExistingBuildDirs = append(keepExistingBuildDirs, checkDir) - buildFilesToParse = append(buildFilesToParse, buildFilePath) - } - filesystem := make(map[string][]byte) - for f, content := range tc.Filesystem { - filesystem[f] = []byte(content) - } - preparers := []android.FixturePreparer{ - extraPreparer, - android.FixtureMergeMockFs(filesystem), - android.FixtureWithRootAndroidBp(tc.Blueprint), - android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { - ctx.RegisterModuleType(tc.ModuleTypeUnderTest, tc.ModuleTypeUnderTestFactory) - }), - android.FixtureModifyContextWithMockFs(func(ctx *android.TestContext) { - // A default configuration for tests to not have to specify bp2build_available on top level - // targets. - bp2buildConfig := android.NewBp2BuildAllowlist().SetDefaultConfig( - allowlists.Bp2BuildConfig{ - android.Bp2BuildTopLevel: allowlists.Bp2BuildDefaultTrueRecursively, - }, - ) - for _, f := range keepExistingBuildDirs { - bp2buildConfig.SetKeepExistingBuildFile(map[string]bool{ - f: /*recursive=*/ false, - }) - } - ctx.RegisterBp2BuildConfig(bp2buildConfig) - // This setting is added to bp2build invocations. It prevents bp2build - // from cloning modules to their original state after mutators run. This - // would lose some data intentionally set by these mutators. - ctx.SkipCloneModulesAfterMutators = true - err := ctx.RegisterExistingBazelTargets(".", buildFilesToParse) - if err != nil { - t.Errorf("error parsing build files in test setup: %s", err) - } - }), - android.FixtureModifyEnv(func(env map[string]string) { - if tc.UnconvertedDepsMode == errorModulesUnconvertedDeps { - env["BP2BUILD_ERROR_UNCONVERTED"] = "true" - } - }), - } - - preparer := android.GroupFixturePreparers(preparers...) - if tc.ExpectedErr != nil { - pattern := "\\Q" + tc.ExpectedErr.Error() + "\\E" - preparer = preparer.ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern(pattern)) - } - result := preparer.RunTestWithCustomResult(t).(*BazelTestResult) - if len(result.Errs) > 0 { - return - } - - expectedTargets := map[string][]string{ - checkDir: tc.ExpectedBazelTargets, - } - - result.CompareAllBazelTargets(t, tc, expectedTargets, true) -} - -// bazelTestRunner customizes the test fixture mechanism to run tests of the bp2build build mode. -type bazelTestRunner struct { - generateProductConfigTargets bool -} - -func (b *bazelTestRunner) FinalPreparer(result *android.TestResult) android.CustomTestResult { - ctx := result.TestContext - ctx.RegisterForBazelConversion() - - return &BazelTestResult{TestResult: result} -} - -func (b *bazelTestRunner) PostParseProcessor(result android.CustomTestResult) { - bazelResult := result.(*BazelTestResult) - ctx := bazelResult.TestContext - config := bazelResult.Config - _, errs := ctx.ResolveDependencies(config) - if bazelResult.CollateErrs(errs) { - return - } - - codegenCtx := NewCodegenContext(config, ctx.Context, Bp2Build, "") - res, errs := GenerateBazelTargets(codegenCtx, false) - if bazelResult.CollateErrs(errs) { - return - } - if b.generateProductConfigTargets { - productConfig, err := createProductConfigFiles(codegenCtx, res.moduleNameToPartition, res.metrics.convertedModulePathMap) - if err != nil { - bazelResult.CollateErrs([]error{err}) - return - } - for k, v := range productConfig.bp2buildTargets { - res.buildFileToTargets[k] = append(res.buildFileToTargets[k], v...) - } - } - - // Store additional data for access by tests. - bazelResult.conversionResults = res -} - -// BazelTestResult is a wrapper around android.TestResult to provide type safe access to the bazel -// specific data stored by the bazelTestRunner. -type BazelTestResult struct { - *android.TestResult - - // The result returned by the GenerateBazelTargets function. - conversionResults -} - -// CompareAllBazelTargets compares the BazelTargets produced by the test for all the directories -// with the supplied set of expected targets. -// -// If ignoreUnexpected=false then this enforces an exact match where every BazelTarget produced must -// have a corresponding expected BazelTarget. -// -// If ignoreUnexpected=true then it will ignore directories for which there are no expected targets. -func (b BazelTestResult) CompareAllBazelTargets(t *testing.T, tc Bp2buildTestCase, expectedTargets map[string][]string, ignoreUnexpected bool) { - t.Helper() - actualTargets := b.buildFileToTargets - - // Generate the sorted set of directories to check. - dirsToCheck := android.SortedKeys(expectedTargets) - if !ignoreUnexpected { - // This needs to perform an exact match so add the directories in which targets were - // produced to the list of directories to check. - dirsToCheck = append(dirsToCheck, android.SortedKeys(actualTargets)...) - dirsToCheck = android.SortedUniqueStrings(dirsToCheck) - } - - for _, dir := range dirsToCheck { - expected := expectedTargets[dir] - actual := actualTargets[dir] - - if expected == nil { - if actual != nil { - t.Errorf("did not expect any bazel modules in %q but found %d", dir, len(actual)) - } - } else if actual == nil { - expectedCount := len(expected) - if expectedCount > 0 { - t.Errorf("expected %d bazel modules in %q but did not find any", expectedCount, dir) - } - } else { - b.CompareBazelTargets(t, tc.Description, expected, actual) - } - } - - for _, module := range tc.ExpectedConvertedModules { - if _, found := b.metrics.convertedModulePathMap[module]; !found { - t.Errorf("expected %s to be generated by bp2build, but was not. Map of converted modules: %s", module, b.metrics.convertedModulePathMap) - } - } - - for _, module := range tc.ExpectedHandcraftedModules { - if reason, found := b.metrics.serialized.UnconvertedModules[module]; !found { - t.Errorf("expected %s to be marked 'unconverted' by bp2build, but was not found. Full list: %s", - module, b.metrics.serialized.UnconvertedModules) - } else { - if reason.Type != bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE { - t.Errorf("expected %s to be marked 'handcrafted' by bp2build, but was disabled for another reason: %s", module, reason) - } - } - } -} - -func (b BazelTestResult) CompareBazelTargets(t *testing.T, description string, expectedContents []string, actualTargets BazelTargets) { - t.Helper() - if actualCount, expectedCount := len(actualTargets), len(expectedContents); actualCount != expectedCount { - t.Errorf("%s: Expected %d bazel target (%s), got %d (%s)", - description, expectedCount, expectedContents, actualCount, actualTargets) - } else { - sort.SliceStable(actualTargets, func(i, j int) bool { - return actualTargets[i].name < actualTargets[j].name - }) - sort.SliceStable(expectedContents, func(i, j int) bool { - return getTargetName(expectedContents[i]) < getTargetName(expectedContents[j]) - }) - for i, actualTarget := range actualTargets { - if w, g := expectedContents[i], actualTarget.content; w != g { - t.Errorf( - "%s[%d]: Expected generated Bazel target to be `%s`, got `%s`", - description, i, w, g) - } - } - } -} - -type nestedProps struct { - Nested_prop *string -} - -type EmbeddedProps struct { - Embedded_prop *string -} - -type OtherEmbeddedProps struct { - Other_embedded_prop *string -} - -type customProps struct { - EmbeddedProps - *OtherEmbeddedProps - - Bool_prop bool - Bool_ptr_prop *bool - // Ensure that properties tagged `blueprint:mutated` are omitted - Int_prop int `blueprint:"mutated"` - Int64_ptr_prop *int64 - String_prop string - String_literal_prop *string `android:"arch_variant"` - String_ptr_prop *string - String_list_prop []string - - Nested_props nestedProps - Nested_props_ptr *nestedProps - - Arch_paths []string `android:"path,arch_variant"` - Arch_paths_exclude []string `android:"path,arch_variant"` - - // Prop used to indicate this conversion should be 1 module -> multiple targets - One_to_many_prop *bool - - // Prop used to simulate an unsupported property in bp2build conversion. If this - // is true, this module should be treated as "unconvertible" via bp2build. - Does_not_convert_to_bazel *bool - - Api *string // File describing the APIs of this module - - Test_config_setting *bool // Used to test generation of config_setting targets - - Dir *string // Dir in which the Bazel Target will be created -} - -type customModule struct { - android.ModuleBase - android.BazelModuleBase - - props customProps -} - -// OutputFiles is needed because some instances of this module use dist with a -// tag property which requires the module implements OutputFileProducer. -func (m *customModule) OutputFiles(tag string) (android.Paths, error) { - return android.PathsForTesting("path" + tag), nil -} - -func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { - // nothing for now. -} - -func customModuleFactoryBase() android.Module { - module := &customModule{} - module.AddProperties(&module.props) - android.InitBazelModule(module) - return module -} - -func customModuleFactoryHostAndDevice() android.Module { - m := customModuleFactoryBase() - android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibBoth) - return m -} - -func customModuleFactoryDeviceSupported() android.Module { - m := customModuleFactoryBase() - android.InitAndroidArchModule(m, android.DeviceSupported, android.MultilibBoth) - return m -} - -func customModuleFactoryHostSupported() android.Module { - m := customModuleFactoryBase() - android.InitAndroidArchModule(m, android.HostSupported, android.MultilibBoth) - return m -} - -func customModuleFactoryHostAndDeviceDefault() android.Module { - m := customModuleFactoryBase() - android.InitAndroidArchModule(m, android.HostAndDeviceDefault, android.MultilibBoth) - return m -} - -func customModuleFactoryNeitherHostNorDeviceSupported() android.Module { - m := customModuleFactoryBase() - android.InitAndroidArchModule(m, android.NeitherHostNorDeviceSupported, android.MultilibBoth) - return m -} - -type testProps struct { - Test_prop struct { - Test_string_prop string - } -} - -type customTestModule struct { - android.ModuleBase - - props customProps - test_props testProps -} - -func (m *customTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { - // nothing for now. -} - -func customTestModuleFactoryBase() android.Module { - m := &customTestModule{} - m.AddProperties(&m.props) - m.AddProperties(&m.test_props) - return m -} - -func customTestModuleFactory() android.Module { - m := customTestModuleFactoryBase() - android.InitAndroidModule(m) - return m -} - -type customDefaultsModule struct { - android.ModuleBase - android.DefaultsModuleBase -} - -func customDefaultsModuleFactoryBase() android.DefaultsModule { - module := &customDefaultsModule{} - module.AddProperties(&customProps{}) - return module -} - -func customDefaultsModuleFactoryBasic() android.Module { - return customDefaultsModuleFactoryBase() -} - -func customDefaultsModuleFactory() android.Module { - m := customDefaultsModuleFactoryBase() - android.InitDefaultsModule(m) - return m -} - -type EmbeddedAttr struct { - Embedded_attr *string -} - -type OtherEmbeddedAttr struct { - Other_embedded_attr *string -} - -type customBazelModuleAttributes struct { - EmbeddedAttr - *OtherEmbeddedAttr - String_literal_prop bazel.StringAttribute - String_ptr_prop *string - String_list_prop []string - Arch_paths bazel.LabelListAttribute - Api bazel.LabelAttribute -} - -func (m *customModule) dir() *string { - return m.props.Dir -} - -func (m *customModule) ConvertWithBp2build(ctx android.Bp2buildMutatorContext) { - if p := m.props.Does_not_convert_to_bazel; p != nil && *p { - ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_PROPERTY_UNSUPPORTED, "") - return - } - if p := m.props.One_to_many_prop; p != nil && *p { - customBp2buildOneToMany(ctx, m) - return - } - - paths := bazel.LabelListAttribute{} - strAttr := bazel.StringAttribute{} - for axis, configToProps := range m.GetArchVariantProperties(ctx, &customProps{}) { - for config, props := range configToProps { - if custProps, ok := props.(*customProps); ok { - if custProps.Arch_paths != nil { - paths.SetSelectValue(axis, config, android.BazelLabelForModuleSrcExcludes(ctx, custProps.Arch_paths, custProps.Arch_paths_exclude)) - } - if custProps.String_literal_prop != nil { - strAttr.SetSelectValue(axis, config, custProps.String_literal_prop) - } - } - } - } - productVariableProps, errs := android.ProductVariableProperties(ctx, ctx.Module()) - for _, err := range errs { - ctx.ModuleErrorf("ProductVariableProperties error: %s", err) - } - if props, ok := productVariableProps["String_literal_prop"]; ok { - for c, p := range props { - if val, ok := p.(*string); ok { - strAttr.SetSelectValue(c.ConfigurationAxis(), c.SelectKey(), val) - } - } - } - - paths.ResolveExcludes() - - attrs := &customBazelModuleAttributes{ - String_literal_prop: strAttr, - String_ptr_prop: m.props.String_ptr_prop, - String_list_prop: m.props.String_list_prop, - Arch_paths: paths, - } - - attrs.Embedded_attr = m.props.Embedded_prop - if m.props.OtherEmbeddedProps != nil { - attrs.OtherEmbeddedAttr = &OtherEmbeddedAttr{Other_embedded_attr: m.props.OtherEmbeddedProps.Other_embedded_prop} - } - - props := bazel.BazelTargetModuleProperties{ - Rule_class: "custom", - } - - ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name(), Dir: m.dir()}, attrs) - - if proptools.Bool(m.props.Test_config_setting) { - m.createConfigSetting(ctx) - } - -} - -func (m *customModule) createConfigSetting(ctx android.Bp2buildMutatorContext) { - csa := bazel.ConfigSettingAttributes{ - Flag_values: bazel.StringMapAttribute{ - "//build/bazel/rules/my_string_setting": m.Name(), - }, - } - ca := android.CommonAttributes{ - Name: m.Name() + "_config_setting", - } - ctx.CreateBazelConfigSetting( - csa, - ca, - ctx.ModuleDir(), - ) -} - -// A bp2build mutator that uses load statements and creates a 1:M mapping from -// module to target. -func customBp2buildOneToMany(ctx android.Bp2buildMutatorContext, m *customModule) { - - baseName := m.Name() - attrs := &customBazelModuleAttributes{} - - myLibraryProps := bazel.BazelTargetModuleProperties{ - Rule_class: "my_library", - Bzl_load_location: "//build/bazel/rules:rules.bzl", - } - ctx.CreateBazelTargetModule(myLibraryProps, android.CommonAttributes{Name: baseName}, attrs) - - protoLibraryProps := bazel.BazelTargetModuleProperties{ - Rule_class: "proto_library", - Bzl_load_location: "//build/bazel/rules:proto.bzl", - } - ctx.CreateBazelTargetModule(protoLibraryProps, android.CommonAttributes{Name: baseName + "_proto_library_deps"}, attrs) - - myProtoLibraryProps := bazel.BazelTargetModuleProperties{ - Rule_class: "my_proto_library", - Bzl_load_location: "//build/bazel/rules:proto.bzl", - } - ctx.CreateBazelTargetModule(myProtoLibraryProps, android.CommonAttributes{Name: baseName + "_my_proto_library_deps"}, attrs) -} - -// Helper method for tests to easily access the targets in a dir. -func generateBazelTargetsForDir(codegenCtx *CodegenContext, dir string) (BazelTargets, []error) { - // TODO: Set generateFilegroups to true and/or remove the generateFilegroups argument completely - res, err := GenerateBazelTargets(codegenCtx, false) - if err != nil { - return BazelTargets{}, err - } - return res.buildFileToTargets[dir], err -} - -func registerCustomModuleForBp2buildConversion(ctx *android.TestContext) { - ctx.RegisterModuleType("custom", customModuleFactoryHostAndDevice) - ctx.RegisterForBazelConversion() -} - -func simpleModule(typ, name string) string { - return fmt.Sprintf(` -%s { - name: "%s", -}`, typ, name) -} - -type AttrNameToString map[string]string - -func (a AttrNameToString) clone() AttrNameToString { - newAttrs := make(AttrNameToString, len(a)) - for k, v := range a { - newAttrs[k] = v - } - return newAttrs -} - -// makeBazelTargetNoRestrictions returns bazel target build file definition that can be host or -// device specific, or independent of host/device. -func makeBazelTargetHostOrDevice(typ, name string, attrs AttrNameToString, hod android.HostOrDeviceSupported) string { - if _, ok := attrs["target_compatible_with"]; !ok { - switch hod { - case android.HostSupported: - attrs["target_compatible_with"] = `select({ - "//build/bazel_common_rules/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - case android.DeviceSupported: - attrs["target_compatible_with"] = `["//build/bazel_common_rules/platforms/os:android"]` - } - } - - attrStrings := make([]string, 0, len(attrs)+1) - if name != "" { - attrStrings = append(attrStrings, fmt.Sprintf(` name = "%s",`, name)) - } - for _, k := range android.SortedKeys(attrs) { - attrStrings = append(attrStrings, fmt.Sprintf(" %s = %s,", k, attrs[k])) - } - return fmt.Sprintf(`%s( -%s -)`, typ, strings.Join(attrStrings, "\n")) -} - -// MakeBazelTargetNoRestrictions returns bazel target build file definition that does not add a -// target_compatible_with. This is useful for module types like filegroup and genrule that arch not -// arch variant -func MakeBazelTargetNoRestrictions(typ, name string, attrs AttrNameToString) string { - return makeBazelTargetHostOrDevice(typ, name, attrs, android.HostAndDeviceDefault) -} - -// makeBazelTargetNoRestrictions returns bazel target build file definition that is device specific -// as this is the most common default in Soong. -func MakeBazelTarget(typ, name string, attrs AttrNameToString) string { - return makeBazelTargetHostOrDevice(typ, name, attrs, android.DeviceSupported) -} - -type ExpectedRuleTarget struct { - Rule string - Name string - Attrs AttrNameToString - Hod android.HostOrDeviceSupported -} - -func (ebr ExpectedRuleTarget) String() string { - return makeBazelTargetHostOrDevice(ebr.Rule, ebr.Name, ebr.Attrs, ebr.Hod) -} - -func makeCcStubSuiteTargets(name string, attrs AttrNameToString) string { - if _, hasStubs := attrs["stubs_symbol_file"]; !hasStubs { - return "" - } - STUB_SUITE_ATTRS := map[string]string{ - "api_surface": "api_surface", - "stubs_symbol_file": "symbol_file", - "stubs_versions": "versions", - "soname": "soname", - "source_library_label": "source_library_label", - } - - stubSuiteAttrs := AttrNameToString{} - for key, _ := range attrs { - if _, stubSuiteAttr := STUB_SUITE_ATTRS[key]; stubSuiteAttr { - stubSuiteAttrs[STUB_SUITE_ATTRS[key]] = attrs[key] - } else { - panic(fmt.Sprintf("unused cc_stub_suite attr %q\n", key)) - } - } - return MakeBazelTarget("cc_stub_suite", name+"_stub_libs", stubSuiteAttrs) -} - -func MakeNeverlinkDuplicateTarget(moduleType string, name string) string { - return MakeNeverlinkDuplicateTargetWithAttrs(moduleType, name, AttrNameToString{ - "sdk_version": `"current"`, // use as default - }) -} - -func MakeNeverlinkDuplicateTargetWithAttrs(moduleType string, name string, extraAttrs AttrNameToString) string { - attrs := extraAttrs - attrs["neverlink"] = `True` - attrs["exports"] = `[":` + name + `"]` - return MakeBazelTarget(moduleType, name+"-neverlink", attrs) -} - -func getTargetName(targetContent string) string { - data := strings.Split(targetContent, "name = \"") - if len(data) < 2 { - return "" - } else { - endIndex := strings.Index(data[1], "\"") - return data[1][:endIndex] - } -} diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 5abbdb7d5..7742492bf 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -28,8 +28,6 @@ import ( "android/soong/android/allowlists" "android/soong/bp2build" "android/soong/shared" - "android/soong/ui/metrics/bp2build_metrics_proto" - "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/deptools" @@ -74,16 +72,10 @@ func init() { flag.StringVar(&cmdlineArgs.ModuleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules") flag.StringVar(&cmdlineArgs.DocFile, "soong_docs", "", "build documentation file to output") flag.StringVar(&cmdlineArgs.BazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top") - flag.StringVar(&cmdlineArgs.Bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit") - flag.StringVar(&cmdlineArgs.SymlinkForestMarker, "symlink_forest_marker", "", "If set, create the bp2build symlink forest, touch the specified marker file, then exit") flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output") flag.StringVar(&cmdlineArgs.SoongVariables, "soong_variables", "soong.variables", "the file contains all build variables") - flag.StringVar(&cmdlineArgs.BazelForceEnabledModules, "bazel-force-enabled-modules", "", "additional modules to build with Bazel. Comma-delimited") flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file") flag.BoolVar(&cmdlineArgs.MultitreeBuild, "multitree-build", false, "this is a multitree build") - flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules") - flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules") - flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses") flag.BoolVar(&cmdlineArgs.BuildFromSourceStub, "build-from-source-stub", false, "build Java stubs from source files instead of API text files") flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built") // Flags that probably shouldn't be flags of soong_build, but we haven't found @@ -110,40 +102,6 @@ func newContext(configuration android.Config) *android.Context { return ctx } -// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a -// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata -// for modules that should be handled by Bazel. -func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string { - ctx.EventHandler.Begin("mixed_build") - defer ctx.EventHandler.End("mixed_build") - - bazelHook := func() error { - err := ctx.Config().BazelContext.QueueBazelSandwichCqueryRequests(ctx.Config()) - if err != nil { - return err - } - return ctx.Config().BazelContext.InvokeBazel(ctx.Config(), ctx) - } - ctx.SetBeforePrepareBuildActionsHook(bazelHook) - ninjaDeps, err := bootstrap.RunBlueprint(cmdlineArgs.Args, bootstrap.DoEverything, ctx.Context, ctx.Config()) - maybeQuit(err, "") - ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - - bazelPaths, err := readFileLines(ctx.Config().Getenv("BAZEL_DEPS_FILE")) - if err != nil { - panic("Bazel deps file not found: " + err.Error()) - } - ninjaDeps = append(ninjaDeps, bazelPaths...) - ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...) - - writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps) - - if needToWriteNinjaHint(ctx) { - writeNinjaHint(ctx) - } - return cmdlineArgs.OutFile -} - func needToWriteNinjaHint(ctx *android.Context) bool { switch ctx.Config().GetenvWithDefault("SOONG_GENERATES_NINJA_HINT", "") { case "always": @@ -228,67 +186,6 @@ func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandl maybeQuit(err, "error writing soong_build metrics %s", metricsFile) } -// Errors out if any modules expected to be mixed_built were not, unless -// the modules did not exist. -func checkForAllowlistIntegrityError(configuration android.Config, isStagingMode bool) error { - modules := findMisconfiguredModules(configuration, isStagingMode) - if len(modules) == 0 { - return nil - } - - return fmt.Errorf("Error: expected the following modules to be mixed_built: %s", modules) -} - -// Returns true if the given module has all of the following true: -// 1. Is allowlisted to be built with Bazel. -// 2. Has a variant which is *not* built with Bazel. -// 3. Has no variant which is built with Bazel. -// -// This indicates the allowlisting of this variant had no effect. -// TODO(b/280457637): Return true for nonexistent modules. -func isAllowlistMisconfiguredForModule(module string, mixedBuildsEnabled map[string]struct{}, mixedBuildsDisabled map[string]struct{}) bool { - _, enabled := mixedBuildsEnabled[module] - - if enabled { - return false - } - - _, disabled := mixedBuildsDisabled[module] - return disabled - -} - -// Returns the list of modules that should have been mixed_built (per the -// allowlists and cmdline flags) but were not. -// Note: nonexistent modules are excluded from the list. See b/280457637 -func findMisconfiguredModules(configuration android.Config, isStagingMode bool) []string { - retval := []string{} - forceEnabledModules := configuration.BazelModulesForceEnabledByFlag() - - mixedBuildsEnabled := configuration.GetMixedBuildsEnabledModules() - mixedBuildsDisabled := configuration.GetMixedBuildsDisabledModules() - for _, module := range allowlists.ProdMixedBuildsEnabledList { - if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) { - retval = append(retval, module) - } - } - - if isStagingMode { - for _, module := range allowlists.StagingMixedBuildsEnabledList { - if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) { - retval = append(retval, module) - } - } - } - - for module, _ := range forceEnabledModules { - if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) { - retval = append(retval, module) - } - } - return retval -} - func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) { graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile)) maybeQuit(graphErr, "graph err") @@ -424,37 +321,9 @@ func main() { ctx := newContext(configuration) android.StartBackgroundMetrics(configuration) - var finalOutputFile string - - // Run Soong for a specific activity, like bp2build, queryview - // or the actual Soong build for the build.ninja file. - switch configuration.BuildMode { - case android.SymlinkForest: - finalOutputFile = runSymlinkForestCreation(ctx, extraNinjaDeps, metricsDir) - case android.Bp2build: - // Run the alternate pipeline of bp2build mutators and singleton to convert - // Blueprint to BUILD files before everything else. - finalOutputFile = runBp2Build(ctx, extraNinjaDeps, metricsDir) - default: - ctx.Register() - isMixedBuildsEnabled := configuration.IsMixedBuildsEnabled() - if isMixedBuildsEnabled { - finalOutputFile = runMixedModeBuild(ctx, extraNinjaDeps) - if cmdlineArgs.EnsureAllowlistIntegrity { - if err := checkForAllowlistIntegrityError(configuration, cmdlineArgs.BazelModeStaging); err != nil { - maybeQuit(err, "") - } - } - } else { - finalOutputFile = runSoongOnlyBuild(ctx, extraNinjaDeps) - } - writeMetrics(configuration, ctx.EventHandler, metricsDir) - } - - // Register this environment variablesas being an implicit dependencies of - // soong_build. Changes to this environment variable will result in - // retriggering soong_build. - configuration.Getenv("USE_BAZEL_VERSION") + ctx.Register() + finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps) + writeMetrics(configuration, ctx.EventHandler, metricsDir) writeUsedEnvironmentFile(configuration) @@ -497,213 +366,6 @@ func touch(path string) { maybeQuit(err, "error touching '%s'", path) } -// Read the bazel.list file that the Soong Finder already dumped earlier (hopefully) -// It contains the locations of BUILD files, BUILD.bazel files, etc. in the source dir -func getExistingBazelRelatedFiles(topDir string) ([]string, error) { - bazelFinderFile := filepath.Join(filepath.Dir(cmdlineArgs.ModuleListFile), "bazel.list") - if !filepath.IsAbs(bazelFinderFile) { - // Assume this was a relative path under topDir - bazelFinderFile = filepath.Join(topDir, bazelFinderFile) - } - return readFileLines(bazelFinderFile) -} - -func bazelArtifacts() []string { - return []string{ - "bazel-bin", - "bazel-genfiles", - "bazel-out", - "bazel-testlogs", - "bazel-workspace", - "bazel-" + filepath.Base(topDir), - } -} - -// This could in theory easily be separated into a binary that generically -// merges two directories into a symlink tree. The main obstacle is that this -// function currently depends on both Bazel-specific knowledge (the existence -// of bazel-* symlinks) and configuration (the set of BUILD.bazel files that -// should and should not be kept) -// -// Ideally, bp2build would write a file that contains instructions to the -// symlink tree creation binary. Then the latter would not need to depend on -// the very heavy-weight machinery of soong_build . -func runSymlinkForestCreation(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string { - var ninjaDeps []string - var mkdirCount, symlinkCount uint64 - - ctx.EventHandler.Do("symlink_forest", func() { - ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - verbose := ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE") - - // PlantSymlinkForest() returns all the directories that were readdir()'ed. - // Such a directory SHOULD be added to `ninjaDeps` so that a child directory - // or file created/deleted under it would trigger an update of the symlink forest. - generatedRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "bp2build") - workspaceRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "workspace") - var symlinkForestDeps []string - ctx.EventHandler.Do("plant", func() { - symlinkForestDeps, mkdirCount, symlinkCount = bp2build.PlantSymlinkForest( - verbose, topDir, workspaceRoot, generatedRoot, excludedFromSymlinkForest(ctx, verbose)) - }) - ninjaDeps = append(ninjaDeps, symlinkForestDeps...) - }) - - writeDepFile(cmdlineArgs.SymlinkForestMarker, ctx.EventHandler, ninjaDeps) - touch(shared.JoinPath(topDir, cmdlineArgs.SymlinkForestMarker)) - codegenMetrics := bp2build.ReadCodegenMetrics(metricsDir) - if codegenMetrics == nil { - m := bp2build.CreateCodegenMetrics() - codegenMetrics = &m - } else { - //TODO (usta) we cannot determine if we loaded a stale file, i.e. from an unrelated prior - //invocation of codegen. We should simply use a separate .pb file - } - codegenMetrics.SetSymlinkCount(symlinkCount) - codegenMetrics.SetMkDirCount(mkdirCount) - writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir) - return cmdlineArgs.SymlinkForestMarker -} - -func excludedFromSymlinkForest(ctx *android.Context, verbose bool) []string { - excluded := bazelArtifacts() - if cmdlineArgs.OutDir[0] != '/' { - excluded = append(excluded, cmdlineArgs.OutDir) - } - - // Find BUILD files in the srcDir which are not in the allowlist - // (android.Bp2BuildConversionAllowlist#ShouldKeepExistingBuildFileForDir) - // and return their paths so they can be left out of the Bazel workspace dir (i.e. ignored) - existingBazelFiles, err := getExistingBazelRelatedFiles(topDir) - maybeQuit(err, "Error determining existing Bazel-related files") - - for _, path := range existingBazelFiles { - fullPath := shared.JoinPath(topDir, path) - fileInfo, err2 := os.Stat(fullPath) - if err2 != nil { - // Warn about error, but continue trying to check files - fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fullPath, err2) - continue - } - // Exclude only files named 'BUILD' or 'BUILD.bazel' and unless forcibly kept - if fileInfo.IsDir() || - (fileInfo.Name() != "BUILD" && fileInfo.Name() != "BUILD.bazel") || - ctx.Config().Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(filepath.Dir(path)) { - // Don't ignore this existing build file - continue - } - if verbose { - fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", path) - } - excluded = append(excluded, path) - } - - // Temporarily exclude stuff to make `bazel build //external/...` (and `bazel build //frameworks/...`) work - excluded = append(excluded, - // FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite - // symlink expansion error for Bazel - "external/autotest/venv/autotest_lib", - "external/autotest/autotest_lib", - "external/autotest/client/autotest_lib/client", - - // FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison - // It contains several symlinks back to real source dirs, and those source dirs contain - // BUILD files we want to ignore - "external/google-fruit/extras/bazel_root/third_party/fruit", - - // FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue - "frameworks/compile/slang", - ) - return excluded -} - -// Run Soong in the bp2build mode. This creates a standalone context that registers -// an alternate pipeline of mutators and singletons specifically for generating -// Bazel BUILD files instead of Ninja files. -func runBp2Build(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string { - var codegenMetrics *bp2build.CodegenMetrics - ctx.EventHandler.Do("bp2build", func() { - - ctx.EventHandler.Do("read_build", func() { - existingBazelFiles, err := getExistingBazelRelatedFiles(topDir) - maybeQuit(err, "Error determining existing Bazel-related files") - - err = ctx.RegisterExistingBazelTargets(topDir, existingBazelFiles) - maybeQuit(err, "Error parsing existing Bazel-related files") - }) - - // Propagate "allow misssing dependencies" bit. This is normally set in - // newContext(), but we create ctx without calling that method. - ctx.SetAllowMissingDependencies(ctx.Config().AllowMissingDependencies()) - ctx.SetNameInterface(newNameResolver(ctx.Config())) - ctx.RegisterForBazelConversion() - ctx.SetModuleListFile(cmdlineArgs.ModuleListFile) - // Skip cloning modules during bp2build's blueprint run. Some mutators set - // bp2build-related module values which should be preserved during codegen. - ctx.SkipCloneModulesAfterMutators = true - - var ninjaDeps []string - ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - - // Run the loading and analysis pipeline to prepare the graph of regular - // Modules parsed from Android.bp files, and the BazelTargetModules mapped - // from the regular Modules. - ctx.EventHandler.Do("bootstrap", func() { - blueprintArgs := cmdlineArgs - bootstrapDeps, err := bootstrap.RunBlueprint(blueprintArgs.Args, - bootstrap.StopBeforePrepareBuildActions, ctx.Context, ctx.Config()) - maybeQuit(err, "") - ninjaDeps = append(ninjaDeps, bootstrapDeps...) - }) - - globListFiles := writeBuildGlobsNinjaFile(ctx) - ninjaDeps = append(ninjaDeps, globListFiles...) - - // Run the code-generation phase to convert BazelTargetModules to BUILD files - // and print conversion codegenMetrics to the user. - codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build, topDir) - codegenMetrics = bp2build.Codegen(codegenContext) - - ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) - - writeDepFile(cmdlineArgs.Bp2buildMarker, ctx.EventHandler, ninjaDeps) - touch(shared.JoinPath(topDir, cmdlineArgs.Bp2buildMarker)) - }) - - // Only report metrics when in bp2build mode. The metrics aren't relevant - // for queryview, since that's a total repo-wide conversion and there's a - // 1:1 mapping for each module. - if ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE") { - codegenMetrics.Print() - } - writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir) - return cmdlineArgs.Bp2buildMarker -} - -// Write Bp2Build metrics into $LOG_DIR -func writeBp2BuildMetrics(codegenMetrics *bp2build.CodegenMetrics, eventHandler *metrics.EventHandler, metricsDir string) { - for _, event := range eventHandler.CompletedEvents() { - codegenMetrics.AddEvent(&bp2build_metrics_proto.Event{ - Name: event.Id, - StartTime: uint64(event.Start.UnixNano()), - RealTime: event.RuntimeNanoseconds(), - }) - } - if len(metricsDir) < 1 { - fmt.Fprintf(os.Stderr, "\nMissing required env var for generating bp2build metrics: LOG_DIR\n") - os.Exit(1) - } - codegenMetrics.Write(metricsDir) -} - -func readFileLines(path string) ([]string, error) { - data, err := os.ReadFile(path) - if err == nil { - return strings.Split(strings.TrimSpace(string(data)), "\n"), nil - } - return nil, err - -} func maybeQuit(err error, format string, args ...interface{}) { if err == nil { return