From c13f785061be5c3445cbdb3554b528d5483f2aae Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Wed, 17 May 2023 13:01:48 -0400 Subject: [PATCH] Handle missing dependencies in mixed builds Bazel will fail on queries with missing dependencies. Instead, we check for missing dependencies in mixed builds and we fall back to Soong when we are aware of missing dependencies in a module. Test: go test soong tests Change-Id: I4f83752704970d8b43650d5b55ff35799c7bc625 --- android/bazel.go | 22 ++++-- android/bazel_handler.go | 9 ++- android/bazel_test.go | 147 +++++++++++++++++++++++++++++++++++++++ android/module.go | 21 +++++- android/mutator.go | 20 ++++++ 5 files changed, 209 insertions(+), 10 deletions(-) diff --git a/android/bazel.go b/android/bazel.go index 114b1f5c2..d32663483 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -46,6 +46,10 @@ const ( // that is not a platform incompatibility. Example: the module-type is not // enabled, or is not bp2build-converted. ModuleIncompatibility + + // Missing dependencies. We can't query Bazel for modules if it has missing dependencies, there + // will be failures. + ModuleMissingDeps ) // FileGroupAsLibrary describes a filegroup module that is converted to some library @@ -367,16 +371,26 @@ func GetBp2BuildAllowList() Bp2BuildConversionAllowlist { // As a side effect, calling this method will also log whether this module is // mixed build enabled for metrics reporting. func MixedBuildsEnabled(ctx BaseModuleContext) MixedBuildEnabledStatus { - module := ctx.Module() - apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) - withinApex := !apexInfo.IsForPlatform() - platformIncompatible := isPlatformIncompatible(ctx.Os(), ctx.Arch().ArchType) if platformIncompatible { ctx.Config().LogMixedBuild(ctx, false) return TechnicalIncompatibility } + if ctx.Config().AllowMissingDependencies() { + missingDeps := ctx.getMissingDependencies() + // If there are missing dependencies, querying Bazel will fail. Soong instead fails at execution + // time, not loading/analysis. disable mixed builds and fall back to Soong to maintain that + // behavior. + if len(missingDeps) > 0 { + ctx.Config().LogMixedBuild(ctx, false) + return ModuleMissingDeps + } + } + + module := ctx.Module() + apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) + withinApex := !apexInfo.IsForPlatform() mixedBuildEnabled := ctx.Config().IsMixedBuildsEnabled() && module.Enabled() && convertedToBazel(ctx, module) && diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 5291ecae9..a3c51c290 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -29,6 +29,7 @@ import ( "android/soong/bazel/cquery" "android/soong/shared" "android/soong/starlark_fmt" + "github.com/google/blueprint" "github.com/google/blueprint/metrics" @@ -73,14 +74,12 @@ var ( } ) -func init() { - RegisterMixedBuildsMutator(InitRegistrationContext) +func registerMixedBuildsMutator(ctx RegisterMutatorsContext) { + ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() } func RegisterMixedBuildsMutator(ctx RegistrationContext) { - ctx.FinalDepsMutators(func(ctx RegisterMutatorsContext) { - ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() - }) + ctx.FinalDepsMutators(registerMixedBuildsMutator) } func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) { diff --git a/android/bazel_test.go b/android/bazel_test.go index 77e251575..13fd40849 100644 --- a/android/bazel_test.go +++ b/android/bazel_test.go @@ -436,3 +436,150 @@ func TestShouldKeepExistingBuildFileForDir(t *testing.T) { } } } + +type mixedBuildModule struct { + ModuleBase + BazelModuleBase + props struct { + Deps []string + Mixed_build_incompatible *bool + QueuedBazelCall bool `blueprint:"mutated"` + } +} + +type mixedBuildModuleInfo struct { + QueuedBazelCall bool +} + +var mixedBuildModuleProvider = blueprint.NewProvider(mixedBuildModuleInfo{}) + +func mixedBuildModuleFactory() Module { + m := &mixedBuildModule{} + m.AddProperties(&m.props) + InitAndroidArchModule(m, HostAndDeviceDefault, MultilibBoth) + InitBazelModule(m) + + return m +} + +func (m *mixedBuildModule) ConvertWithBp2build(ctx TopDownMutatorContext) { +} + +func (m *mixedBuildModule) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddDependency(ctx.Module(), installDepTag{}, m.props.Deps...) +} + +func (m *mixedBuildModule) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +func (m *mixedBuildModule) IsMixedBuildSupported(ctx BaseModuleContext) bool { + return !proptools.Bool(m.props.Mixed_build_incompatible) +} + +func (m *mixedBuildModule) QueueBazelCall(ctx BaseModuleContext) { + m.props.QueuedBazelCall = true +} + +func (m *mixedBuildModule) ProcessBazelQueryResponse(ctx ModuleContext) { + ctx.SetProvider(mixedBuildModuleProvider, mixedBuildModuleInfo{ + QueuedBazelCall: m.props.QueuedBazelCall, + }) +} + +var prepareForMixedBuildTests = FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.RegisterModuleType("deps", mixedBuildModuleFactory) + RegisterMixedBuildsMutator(ctx) +}) + +func TestMixedBuildsEnabledForType(t *testing.T) { + baseBp := ` + deps { + name: "foo", + deps: ["bar"], + target: { windows: { enabled: true } }, + %s + } +` + depBp := ` + deps { + name: "bar", + target: { + windows: { + enabled: true, + }, + }, + } +` + testCases := []struct { + desc string + variant *string + missingDeps bool + extraBpInfo string + mixedBuildsEnabled bool + }{ + { + desc: "mixed builds works", + mixedBuildsEnabled: true, + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "missing deps", + missingDeps: true, + mixedBuildsEnabled: false, + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "windows no mixed builds", + mixedBuildsEnabled: false, + variant: proptools.StringPtr("windows_x86"), + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "mixed builds disabled by type", + mixedBuildsEnabled: false, + extraBpInfo: `mixed_build_incompatible: true, + bazel_module: { bp2build_available: true },`, + }, + { + desc: "mixed builds not bp2build available", + mixedBuildsEnabled: false, + extraBpInfo: `bazel_module: { bp2build_available: false },`, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + handlers := GroupFixturePreparers( + prepareForMixedBuildTests, + PrepareForTestWithArchMutator, + FixtureModifyConfig(func(config Config) { + config.BazelContext = MockBazelContext{ + OutputBaseDir: "base", + } + config.Targets[Windows] = []Target{ + {Windows, Arch{ArchType: X86_64}, NativeBridgeDisabled, "", "", true}, + {Windows, Arch{ArchType: X86}, NativeBridgeDisabled, "", "", true}, + } + }), + ) + bp := fmt.Sprintf(baseBp, tc.extraBpInfo) + if tc.missingDeps { + handlers = GroupFixturePreparers( + handlers, + PrepareForTestWithAllowMissingDependencies, + ) + } else { + bp += depBp + } + result := handlers.RunTestWithBp(t, bp) + + variant := proptools.StringDefault(tc.variant, "android_arm64_armv8-a") + + m := result.ModuleForTests("foo", variant) + mixedBuildModuleInfo := result.TestContext.ModuleProvider(m.Module(), mixedBuildModuleProvider).(mixedBuildModuleInfo) + if w, g := tc.mixedBuildsEnabled, mixedBuildModuleInfo.QueuedBazelCall; w != g { + t.Errorf("Expected mixed builds enabled %t, got mixed builds enabled %t", w, g) + } + }) + } +} diff --git a/android/module.go b/android/module.go index db602a0aa..d8f4f77c5 100644 --- a/android/module.go +++ b/android/module.go @@ -354,6 +354,10 @@ type BaseModuleContext interface { AddMissingDependencies(missingDeps []string) + // getMissingDependencies returns the list of missing dependencies. + // Calling this function prevents adding new dependencies. + getMissingDependencies() []string + // AddUnconvertedBp2buildDep stores module name of a direct dependency that was not converted via bp2build AddUnconvertedBp2buildDep(dep string) @@ -939,7 +943,8 @@ type commonProperties struct { NamespaceExportedToMake bool `blueprint:"mutated"` - MissingDeps []string `blueprint:"mutated"` + MissingDeps []string `blueprint:"mutated"` + CheckedMissingDeps bool `blueprint:"mutated"` // Name and variant strings stored by mutators to enable Module.String() DebugName string `blueprint:"mutated"` @@ -2862,6 +2867,20 @@ func (b *baseModuleContext) AddMissingDependencies(deps []string) { } } +func (b *baseModuleContext) checkedMissingDeps() bool { + return b.Module().base().commonProperties.CheckedMissingDeps +} + +func (b *baseModuleContext) getMissingDependencies() []string { + checked := &b.Module().base().commonProperties.CheckedMissingDeps + *checked = true + var missingDeps []string + missingDeps = append(missingDeps, b.Module().base().commonProperties.MissingDeps...) + missingDeps = append(missingDeps, b.bp.EarlyGetMissingDependencies()...) + missingDeps = FirstUniqueStrings(missingDeps) + return missingDeps +} + type AllowDisabledModuleDependency interface { blueprint.DependencyTag AllowDisabledModuleDependency(target Module) bool diff --git a/android/mutator.go b/android/mutator.go index 0a091eb6f..41853154f 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -67,6 +67,8 @@ func registerMutatorsForBazelConversion(ctx *Context, bp2buildMutators []Registe // collateGloballyRegisteredMutators constructs the list of mutators that have been registered // with the InitRegistrationContext and will be used at runtime. func collateGloballyRegisteredMutators() sortableComponents { + // ensure mixed builds mutator is the last mutator + finalDeps = append(finalDeps, registerMixedBuildsMutator) return collateRegisteredMutators(preArch, preDeps, postDeps, finalDeps) } @@ -885,10 +887,16 @@ func (b *bottomUpMutatorContext) Rename(name string) { } func (b *bottomUpMutatorContext) AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddDependency(module, tag, name...) } func (b *bottomUpMutatorContext) AddReverseDependency(module blueprint.Module, tag blueprint.DependencyTag, name string) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.AddReverseDependency(module, tag, name) } @@ -938,11 +946,17 @@ func (b *bottomUpMutatorContext) SetDefaultDependencyVariation(variation *string func (b *bottomUpMutatorContext) AddVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddVariationDependencies(variations, tag, names...) } func (b *bottomUpMutatorContext) AddFarVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddFarVariationDependencies(variations, tag, names...) } @@ -952,10 +966,16 @@ func (b *bottomUpMutatorContext) AddInterVariantDependency(tag blueprint.Depende } func (b *bottomUpMutatorContext) ReplaceDependencies(name string) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.ReplaceDependencies(name) } func (b *bottomUpMutatorContext) ReplaceDependenciesIf(name string, predicate blueprint.ReplaceDependencyPredicate) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.ReplaceDependenciesIf(name, predicate) }