From 7385067640a3a50c9315e040947067c9996df418 Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Mon, 14 Dec 2020 08:25:34 -0500 Subject: [PATCH] bp2build: framework for generating BazelTargetModules. This CL creates the framework necessary for generating BazelTargetModules from regular Soong Android modules. BazelTargetModules are code-generated into Bazel targets in BUILD files. See the follow-up CL for examples of creating filegroup/genrule BazelTargetModules. Test: GENERATE_BAZEL_FILES=true m nothing # creates out/soong/bp2build with no BUILD files, because there are no BazelTargetModules in the module graph. Change-Id: I33a96365bd439043b13af6db9e439592e9983188 --- android/module.go | 24 +++++++ android/mutator.go | 28 +++++++- android/register.go | 2 +- android/testing.go | 16 +++++ bazel/properties.go | 9 ++- bp2build/Android.bp | 1 + bp2build/androidbp_to_build_templates.go | 6 +- bp2build/bp2build.go | 4 +- bp2build/build_conversion.go | 43 ++++++++++- bp2build/build_conversion_test.go | 55 +++++++++++++- bp2build/bzl_conversion_test.go | 2 +- bp2build/conversion.go | 23 ++++-- bp2build/conversion_test.go | 92 ++++++++++++++++-------- bp2build/testing.go | 37 ++++++++++ cmd/soong_build/queryview.go | 4 +- 15 files changed, 295 insertions(+), 51 deletions(-) diff --git a/android/module.go b/android/module.go index 17035bb88..a0ee33a2d 100644 --- a/android/module.go +++ b/android/module.go @@ -15,6 +15,7 @@ package android import ( + "android/soong/bazel" "fmt" "os" "path" @@ -491,6 +492,26 @@ type Module interface { TransitivePackagingSpecs() []PackagingSpec } +type BazelTargetModule interface { + Module + + BazelTargetModuleProperties() *bazel.BazelTargetModuleProperties +} + +func InitBazelTargetModule(module BazelTargetModule) { + module.AddProperties(module.BazelTargetModuleProperties()) + InitAndroidModule(module) +} + +type BazelTargetModuleBase struct { + ModuleBase + Properties bazel.BazelTargetModuleProperties +} + +func (btmb *BazelTargetModuleBase) BazelTargetModuleProperties() *bazel.BazelTargetModuleProperties { + return &btmb.Properties +} + // Qualified id for a module type qualifiedModuleName struct { // The package (i.e. directory) in which the module is defined, without trailing / @@ -1069,6 +1090,9 @@ type ModuleBase struct { archProperties [][]interface{} customizableProperties []interface{} + // Properties specific to the Blueprint to BUILD migration. + bazelTargetModuleProperties bazel.BazelTargetModuleProperties + // Information about all the properties on the module that contains visibility rules that need // checking. visibilityPropertyInfo []visibilityProperty diff --git a/android/mutator.go b/android/mutator.go index 72c68b2a4..2a2be6c33 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -44,9 +44,16 @@ func registerMutatorsToContext(ctx *blueprint.Context, mutators []*mutator) { } } -func registerMutatorsForBazelConversion(ctx *blueprint.Context) { - // FIXME(b/171263886): Start bringing in mutators to make the Bionic - // module subgraph suitable for automated conversion. +// RegisterMutatorsForBazelConversion is a alternate registration pipeline for bp2build. Exported for testing. +func RegisterMutatorsForBazelConversion(ctx *blueprint.Context, bp2buildMutators []RegisterMutatorFunc) { + mctx := ®isterMutatorsContext{} + + // Register bp2build mutators + for _, f := range bp2buildMutators { + f(mctx) + } + + registerMutatorsToContext(ctx, mctx.mutators) } func registerMutators(ctx *blueprint.Context, preArch, preDeps, postDeps, finalDeps []RegisterMutatorFunc) { @@ -196,6 +203,21 @@ func FinalDepsMutators(f RegisterMutatorFunc) { finalDeps = append(finalDeps, f) } +var bp2buildMutators = []RegisterMutatorFunc{} + +// RegisterBp2BuildMutator registers specially crafted mutators for +// converting Blueprint/Android modules into special modules that can +// be code-generated into Bazel BUILD targets. +// +// TODO(b/178068862): bring this into TestContext. +func RegisterBp2BuildMutator(moduleType string, m func(TopDownMutatorContext)) { + mutatorName := moduleType + "_bp2build" + f := func(ctx RegisterMutatorsContext) { + ctx.TopDown(mutatorName, m) + } + bp2buildMutators = append(bp2buildMutators, f) +} + type BaseMutatorContext interface { BaseModuleContext diff --git a/android/register.go b/android/register.go index f84acad3b..02fc97e2b 100644 --- a/android/register.go +++ b/android/register.go @@ -115,7 +115,7 @@ func (ctx *Context) RegisterForBazelConversion() { ctx.RegisterSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory)) } - registerMutatorsForBazelConversion(ctx.Context) + RegisterMutatorsForBazelConversion(ctx.Context, bp2buildMutators) } // Register the pipeline of singletons, module types, and mutators for diff --git a/android/testing.go b/android/testing.go index 470cfd63f..76172d18e 100644 --- a/android/testing.go +++ b/android/testing.go @@ -57,6 +57,7 @@ func NewTestArchContext(config Config) *TestContext { type TestContext struct { *Context preArch, preDeps, postDeps, finalDeps []RegisterMutatorFunc + bp2buildMutators []RegisterMutatorFunc NameResolver *NameResolver } @@ -81,12 +82,27 @@ func (ctx *TestContext) FinalDepsMutators(f RegisterMutatorFunc) { ctx.finalDeps = append(ctx.finalDeps, f) } +// RegisterBp2BuildMutator registers a BazelTargetModule mutator for converting a module +// type to the equivalent Bazel target. +func (ctx *TestContext) RegisterBp2BuildMutator(moduleType string, m func(TopDownMutatorContext)) { + mutatorName := moduleType + "_bp2build" + f := func(ctx RegisterMutatorsContext) { + ctx.TopDown(mutatorName, m) + } + bp2buildMutators = append(bp2buildMutators, f) +} + func (ctx *TestContext) Register() { registerMutators(ctx.Context.Context, ctx.preArch, ctx.preDeps, ctx.postDeps, ctx.finalDeps) ctx.RegisterSingletonType("env", EnvSingleton) } +// RegisterForBazelConversion prepares a test context for bp2build conversion. +func (ctx *TestContext) RegisterForBazelConversion() { + RegisterMutatorsForBazelConversion(ctx.Context.Context, bp2buildMutators) +} + func (ctx *TestContext) ParseFileList(rootDir string, filePaths []string) (deps []string, errs []error) { // This function adapts the old style ParseFileList calls that are spread throughout the tests // to the new style that takes a config. diff --git a/bazel/properties.go b/bazel/properties.go index 8bb195644..ac0047b49 100644 --- a/bazel/properties.go +++ b/bazel/properties.go @@ -19,9 +19,16 @@ type bazelModuleProperties struct { Label string } -// Properties contains common module properties for migration purposes. +// Properties contains common module properties for Bazel migration purposes. type Properties struct { // In USE_BAZEL_ANALYSIS=1 mode, this represents the Bazel target replacing // this Soong module. Bazel_module bazelModuleProperties } + +// BazelTargetModuleProperties contain properties and metadata used for +// Blueprint to BUILD file conversion. +type BazelTargetModuleProperties struct { + // The Bazel rule class for this target. + Rule_class string +} diff --git a/bp2build/Android.bp b/bp2build/Android.bp index 49587f488..800757446 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -10,6 +10,7 @@ bootstrap_go_package { ], deps: [ "soong-android", + "soong-bazel", ], testSrcs: [ "build_conversion_test.go", diff --git a/bp2build/androidbp_to_build_templates.go b/bp2build/androidbp_to_build_templates.go index 75c3ccb9a..9bac86ba8 100644 --- a/bp2build/androidbp_to_build_templates.go +++ b/bp2build/androidbp_to_build_templates.go @@ -15,7 +15,7 @@ package bp2build const ( - // The default `load` preamble for every generated BUILD file. + // The default `load` preamble for every generated queryview BUILD file. soongModuleLoad = `package(default_visibility = ["//visibility:public"]) load("//build/bazel/queryview_rules:soong_module.bzl", "soong_module") @@ -31,6 +31,10 @@ load("//build/bazel/queryview_rules:soong_module.bzl", "soong_module") module_deps = %s, %s)` + bazelTarget = `%s( + name = "%s", +%s)` + // A simple provider to mark and differentiate Soong module rule shims from // regular Bazel rules. Every Soong module rule shim returns a // SoongModuleInfo provider, and can only depend on rules returning diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go index 49729e02a..75b60977a 100644 --- a/bp2build/bp2build.go +++ b/bp2build/bp2build.go @@ -28,9 +28,9 @@ func Codegen(ctx CodegenContext) { ruleShims := CreateRuleShims(android.ModuleTypeFactories()) - buildToTargets := GenerateSoongModuleTargets(ctx.Context()) + buildToTargets := GenerateSoongModuleTargets(ctx.Context(), true) - filesToWrite := CreateBazelFiles(ruleShims, buildToTargets) + filesToWrite := CreateBazelFiles(ruleShims, buildToTargets, true) for _, f := range filesToWrite { if err := writeFile(outputDir, ctx, f); err != nil { fmt.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err) diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go index bece8f65c..da2fb7f89 100644 --- a/bp2build/build_conversion.go +++ b/bp2build/build_conversion.go @@ -73,16 +73,51 @@ func propsToAttributes(props map[string]string) string { return attributes } -func GenerateSoongModuleTargets(ctx bpToBuildContext) map[string][]BazelTarget { +func GenerateSoongModuleTargets(ctx bpToBuildContext, bp2buildEnabled bool) map[string][]BazelTarget { buildFileToTargets := make(map[string][]BazelTarget) ctx.VisitAllModules(func(m blueprint.Module) { dir := ctx.ModuleDir(m) - t := generateSoongModuleTarget(ctx, m) + var t BazelTarget + + if bp2buildEnabled { + if _, ok := m.(android.BazelTargetModule); !ok { + return + } + t = generateBazelTarget(ctx, m) + } else { + t = generateSoongModuleTarget(ctx, m) + } + buildFileToTargets[ctx.ModuleDir(m)] = append(buildFileToTargets[dir], t) }) return buildFileToTargets } +func generateBazelTarget(ctx bpToBuildContext, m blueprint.Module) BazelTarget { + // extract the bazel attributes from the module. + props := getBuildProperties(ctx, m) + + // extract the rule class name from the attributes. Since the string value + // will be string-quoted, remove the quotes here. + ruleClass := strings.Replace(props.Attrs["rule_class"], "\"", "", 2) + // Delete it from being generated in the BUILD file. + delete(props.Attrs, "rule_class") + + // Return the Bazel target with rule class and attributes, ready to be + // code-generated. + attributes := propsToAttributes(props.Attrs) + targetName := targetNameForBp2Build(ctx, m) + return BazelTarget{ + name: targetName, + content: fmt.Sprintf( + bazelTarget, + ruleClass, + targetName, + attributes, + ), + } +} + // Convert a module and its deps and props into a Bazel macro/rule // representation in the BUILD file. func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) BazelTarget { @@ -306,6 +341,10 @@ func makeIndent(indent int) string { return strings.Repeat(" ", indent) } +func targetNameForBp2Build(c bpToBuildContext, logicModule blueprint.Module) string { + return strings.Replace(c.ModuleName(logicModule), "__bp2build__", "", 1) +} + func targetNameWithVariant(c bpToBuildContext, logicModule blueprint.Module) string { name := "" if c.ModuleSubDir(logicModule) != "" { diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go index 4e31aa78a..02175802b 100644 --- a/bp2build/build_conversion_test.go +++ b/bp2build/build_conversion_test.go @@ -200,7 +200,7 @@ func TestGenerateSoongModuleTargets(t *testing.T) { _, errs = ctx.PrepareBuildActions(config) android.FailIfErrored(t, errs) - bazelTargets := GenerateSoongModuleTargets(ctx.Context.Context)[dir] + bazelTargets := GenerateSoongModuleTargets(ctx.Context.Context, false)[dir] if g, w := len(bazelTargets), 1; g != w { t.Fatalf("Expected %d bazel target, got %d", w, g) } @@ -210,7 +210,58 @@ func TestGenerateSoongModuleTargets(t *testing.T) { t.Errorf( "Expected generated Bazel target to be '%s', got '%s'", testCase.expectedBazelTarget, - actualBazelTarget, + actualBazelTarget.content, + ) + } + } +} + +func TestGenerateBazelTargetModules(t *testing.T) { + testCases := []struct { + bp string + expectedBazelTarget string + }{ + { + bp: `custom { + name: "foo", + string_list_prop: ["a", "b"], + string_prop: "a", +}`, + expectedBazelTarget: `custom( + name = "foo", + string_list_prop = [ + "a", + "b", + ], + string_prop = "a", +)`, + }, + } + + dir := "." + for _, testCase := range testCases { + config := android.TestConfig(buildDir, nil, testCase.bp, nil) + ctx := android.NewTestContext(config) + ctx.RegisterModuleType("custom", customModuleFactory) + ctx.RegisterBp2BuildMutator("custom", customBp2BuildMutator) + ctx.RegisterForBazelConversion() + + _, errs := ctx.ParseFileList(dir, []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.ResolveDependencies(config) + android.FailIfErrored(t, errs) + + bazelTargets := GenerateSoongModuleTargets(ctx.Context.Context, true)[dir] + if g, w := len(bazelTargets), 1; g != w { + t.Fatalf("Expected %d bazel target, got %d", w, g) + } + + actualBazelTarget := bazelTargets[0] + if actualBazelTarget.content != testCase.expectedBazelTarget { + t.Errorf( + "Expected generated Bazel target to be '%s', got '%s'", + testCase.expectedBazelTarget, + actualBazelTarget.content, ) } } diff --git a/bp2build/bzl_conversion_test.go b/bp2build/bzl_conversion_test.go index 8bea3f6cf..01c727123 100644 --- a/bp2build/bzl_conversion_test.go +++ b/bp2build/bzl_conversion_test.go @@ -172,7 +172,7 @@ func TestGenerateSoongModuleBzl(t *testing.T) { content: "irrelevant", }, } - files := CreateBazelFiles(ruleShims, make(map[string][]BazelTarget)) + files := CreateBazelFiles(ruleShims, make(map[string][]BazelTarget), false) var actualSoongModuleBzl BazelFile for _, f := range files { diff --git a/bp2build/conversion.go b/bp2build/conversion.go index cdfb38b14..cccc47400 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -17,7 +17,8 @@ type BazelFile struct { func CreateBazelFiles( ruleShims map[string]RuleShim, - buildToTargets map[string][]BazelTarget) []BazelFile { + buildToTargets map[string][]BazelTarget, + bp2buildEnabled bool) []BazelFile { files := make([]BazelFile, 0, len(ruleShims)+len(buildToTargets)+numAdditionalFiles) // Write top level files: WORKSPACE and BUILD. These files are empty. @@ -26,22 +27,30 @@ func CreateBazelFiles( files = append(files, newFile("", "BUILD", "")) files = append(files, newFile(bazelRulesSubDir, "BUILD", "")) - files = append(files, newFile(bazelRulesSubDir, "providers.bzl", providersBzl)) - for bzlFileName, ruleShim := range ruleShims { - files = append(files, newFile(bazelRulesSubDir, bzlFileName+".bzl", ruleShim.content)) + if !bp2buildEnabled { + // These files are only used for queryview. + files = append(files, newFile(bazelRulesSubDir, "providers.bzl", providersBzl)) + + for bzlFileName, ruleShim := range ruleShims { + files = append(files, newFile(bazelRulesSubDir, bzlFileName+".bzl", ruleShim.content)) + } + files = append(files, newFile(bazelRulesSubDir, "soong_module.bzl", generateSoongModuleBzl(ruleShims))) } - files = append(files, newFile(bazelRulesSubDir, "soong_module.bzl", generateSoongModuleBzl(ruleShims))) - files = append(files, createBuildFiles(buildToTargets)...) + files = append(files, createBuildFiles(buildToTargets, bp2buildEnabled)...) return files } -func createBuildFiles(buildToTargets map[string][]BazelTarget) []BazelFile { +func createBuildFiles(buildToTargets map[string][]BazelTarget, bp2buildEnabled bool) []BazelFile { files := make([]BazelFile, 0, len(buildToTargets)) for _, dir := range android.SortedStringKeys(buildToTargets) { content := soongModuleLoad + if bp2buildEnabled { + // No need to load soong_module for bp2build BUILD files. + content = "" + } targets := buildToTargets[dir] sort.Slice(targets, func(i, j int) bool { return targets[i].name < targets[j].name }) for _, t := range targets { diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index a38fa6a55..b40aa1b25 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -19,12 +19,44 @@ import ( "testing" ) -func TestCreateBazelFiles_AddsTopLevelFiles(t *testing.T) { - files := CreateBazelFiles(map[string]RuleShim{}, map[string][]BazelTarget{}) - expectedFilePaths := []struct { - dir string - basename string - }{ +type filepath struct { + dir string + basename string +} + +func assertFilecountsAreEqual(t *testing.T, actual []BazelFile, expected []filepath) { + if a, e := len(actual), len(expected); a != e { + t.Errorf("Expected %d files, got %d", e, a) + } +} + +func assertFileContent(t *testing.T, actual []BazelFile, expected []filepath) { + for i := range actual { + if g, w := actual[i], expected[i]; g.Dir != w.dir || g.Basename != w.basename { + t.Errorf("Did not find expected file %s/%s", g.Dir, g.Basename) + } else if g.Basename == "BUILD" || g.Basename == "WORKSPACE" { + if g.Contents != "" { + t.Errorf("Expected %s to have no content.", g) + } + } else if g.Contents == "" { + t.Errorf("Contents of %s unexpected empty.", g) + } + } +} + +func sortFiles(files []BazelFile) { + sort.Slice(files, func(i, j int) bool { + if dir1, dir2 := files[i].Dir, files[j].Dir; dir1 == dir2 { + return files[i].Basename < files[j].Basename + } else { + return dir1 < dir2 + } + }) +} + +func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) { + files := CreateBazelFiles(map[string]RuleShim{}, map[string][]BazelTarget{}, false) + expectedFilePaths := []filepath{ { dir: "", basename: "BUILD", @@ -47,27 +79,29 @@ func TestCreateBazelFiles_AddsTopLevelFiles(t *testing.T) { }, } - if g, w := len(files), len(expectedFilePaths); g != w { - t.Errorf("Expected %d files, got %d", w, g) - } - - sort.Slice(files, func(i, j int) bool { - if dir1, dir2 := files[i].Dir, files[j].Dir; dir1 == dir2 { - return files[i].Basename < files[j].Basename - } else { - return dir1 < dir2 - } - }) - - for i := range files { - if g, w := files[i], expectedFilePaths[i]; g.Dir != w.dir || g.Basename != w.basename { - t.Errorf("Did not find expected file %s/%s", g.Dir, g.Basename) - } else if g.Basename == "BUILD" || g.Basename == "WORKSPACE" { - if g.Contents != "" { - t.Errorf("Expected %s to have no content.", g) - } - } else if g.Contents == "" { - t.Errorf("Contents of %s unexpected empty.", g) - } - } + assertFilecountsAreEqual(t, files, expectedFilePaths) + sortFiles(files) + assertFileContent(t, files, expectedFilePaths) +} + +func TestCreateBazelFiles_Bp2Build_AddsTopLevelFiles(t *testing.T) { + files := CreateBazelFiles(map[string]RuleShim{}, map[string][]BazelTarget{}, true) + expectedFilePaths := []filepath{ + { + dir: "", + basename: "BUILD", + }, + { + dir: "", + basename: "WORKSPACE", + }, + { + dir: bazelRulesSubDir, + basename: "BUILD", + }, + } + + assertFilecountsAreEqual(t, files, expectedFilePaths) + sortFiles(files) + assertFileContent(t, files, expectedFilePaths) } diff --git a/bp2build/testing.go b/bp2build/testing.go index 2da32c660..4c31d2d02 100644 --- a/bp2build/testing.go +++ b/bp2build/testing.go @@ -2,6 +2,9 @@ package bp2build import ( "android/soong/android" + "android/soong/bazel" + + "github.com/google/blueprint/proptools" ) type nestedProps struct { @@ -100,3 +103,37 @@ func customDefaultsModuleFactory() android.Module { android.InitDefaultsModule(m) return m } + +type customBazelModuleAttributes struct { + Name *string + String_prop string + String_list_prop []string +} + +type customBazelModule struct { + android.BazelTargetModuleBase + customBazelModuleAttributes +} + +func customBazelModuleFactory() android.Module { + module := &customBazelModule{} + module.AddProperties(&module.customBazelModuleAttributes) + android.InitBazelTargetModule(module) + return module +} + +func (m *customBazelModule) Name() string { return m.BaseModuleName() } +func (m *customBazelModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {} + +func customBp2BuildMutator(ctx android.TopDownMutatorContext) { + if m, ok := ctx.Module().(*customModule); ok { + name := "__bp2build__" + m.Name() + ctx.CreateModule(customBazelModuleFactory, &customBazelModuleAttributes{ + Name: proptools.StringPtr(name), + String_prop: m.props.String_prop, + String_list_prop: m.props.String_list_prop, + }, &bazel.BazelTargetModuleProperties{ + Rule_class: "custom", + }) + } +} diff --git a/cmd/soong_build/queryview.go b/cmd/soong_build/queryview.go index 79ea94af7..305a22440 100644 --- a/cmd/soong_build/queryview.go +++ b/cmd/soong_build/queryview.go @@ -24,9 +24,9 @@ import ( func createBazelQueryView(ctx *android.Context, bazelQueryViewDir string) error { ruleShims := bp2build.CreateRuleShims(android.ModuleTypeFactories()) - buildToTargets := bp2build.GenerateSoongModuleTargets(*ctx) + buildToTargets := bp2build.GenerateSoongModuleTargets(*ctx, false) - filesToWrite := bp2build.CreateBazelFiles(ruleShims, buildToTargets) + filesToWrite := bp2build.CreateBazelFiles(ruleShims, buildToTargets, false) for _, f := range filesToWrite { if err := writeReadOnlyFile(bazelQueryViewDir, f); err != nil { return err