Merge changes from topic "go_bp2build" into main am: 06f3c9bb42 am: c16d84cd70 am: 0541f6d7ba am: dd4d7a25f8

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/2655342

Change-Id: I03a3a78a9e1c0dc23a3c3d9b387785da8fea3b59
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

android/bazel_paths.go

@@ -198,7 +198,7 @@ func BazelLabelForModuleSrcExcludes(ctx BazelConversionPathContext, paths, exclu
 	}
 	labels := expandSrcsForBazel(ctx, paths, excluded)
 	labels.Excludes = excludeLabels.Includes
-	labels = transformSubpackagePaths(ctx, labels)
+	labels = TransformSubpackagePaths(ctx.Config(), ctx.ModuleDir(), labels)
 	return labels
 }
 
@@ -237,7 +237,7 @@ func isPackageBoundary(config Config, prefix string, components []string, compon
 // if the "async_safe" directory is actually a package and not just a directory.
 //
 // In particular, paths that extend into packages are transformed into absolute labels beginning with //.
-func transformSubpackagePath(ctx BazelConversionPathContext, path bazel.Label) bazel.Label {
+func transformSubpackagePath(cfg Config, dir string, path bazel.Label) bazel.Label {
 	var newPath bazel.Label
 
 	// Don't transform OriginalModuleName
@@ -281,7 +281,7 @@ func transformSubpackagePath(ctx BazelConversionPathContext, path bazel.Label) b
 	for i := len(pathComponents) - 1; i >= 0; i-- {
 		pathComponent := pathComponents[i]
 		var sep string
-		if !foundPackageBoundary && isPackageBoundary(ctx.Config(), ctx.ModuleDir(), pathComponents, i) {
+		if !foundPackageBoundary && isPackageBoundary(cfg, dir, pathComponents, i) {
 			sep = ":"
 			foundPackageBoundary = true
 		} else {
@@ -295,7 +295,7 @@ func transformSubpackagePath(ctx BazelConversionPathContext, path bazel.Label) b
 	}
 	if foundPackageBoundary {
 		// Ensure paths end up looking like //bionic/... instead of //./bionic/...
-		moduleDir := ctx.ModuleDir()
+		moduleDir := dir
 		if strings.HasPrefix(moduleDir, ".") {
 			moduleDir = moduleDir[1:]
 		}
@@ -313,13 +313,13 @@ func transformSubpackagePath(ctx BazelConversionPathContext, path bazel.Label) b
 
 // Transform paths to acknowledge package boundaries
 // See transformSubpackagePath() for more information
-func transformSubpackagePaths(ctx BazelConversionPathContext, paths bazel.LabelList) bazel.LabelList {
+func TransformSubpackagePaths(cfg Config, dir string, paths bazel.LabelList) bazel.LabelList {
 	var newPaths bazel.LabelList
 	for _, include := range paths.Includes {
-		newPaths.Includes = append(newPaths.Includes, transformSubpackagePath(ctx, include))
+		newPaths.Includes = append(newPaths.Includes, transformSubpackagePath(cfg, dir, include))
 	}
 	for _, exclude := range paths.Excludes {
-		newPaths.Excludes = append(newPaths.Excludes, transformSubpackagePath(ctx, exclude))
+		newPaths.Excludes = append(newPaths.Excludes, transformSubpackagePath(cfg, dir, exclude))
 	}
 	return newPaths
 }

android/bazel_paths_test.go

@@ -175,7 +175,7 @@ func TestTransformSubpackagePath(t *testing.T) {
 		"./z/b.c": "z/b.c",
 	}
 	for in, out := range pairs {
-		actual := transformSubpackagePath(ctx, bazel.Label{Label: in}).Label
+		actual := transformSubpackagePath(ctx.Config(), ctx.ModuleDir(), bazel.Label{Label: in}).Label
 		if actual != out {
 			t.Errorf("expected:\n%v\nactual:\n%v", out, actual)
 		}

bp2build/Android.bp

@@ -19,6 +19,7 @@ bootstrap_go_package {
         "testing.go",
     ],
     deps: [
+        "blueprint-bootstrap",
         "soong-aidl-library",
         "soong-android",
         "soong-android-allowlists",
@@ -37,6 +38,7 @@ bootstrap_go_package {
         "soong-ui-metrics",
     ],
     testSrcs: [
+        "go_conversion_test.go",
         "aar_conversion_test.go",
         "aidl_library_conversion_test.go",
         "android_app_certificate_conversion_test.go",

bp2build/build_conversion.go

@@ -30,6 +30,7 @@ import (
 	"android/soong/starlark_fmt"
 	"android/soong/ui/metrics/bp2build_metrics_proto"
 	"github.com/google/blueprint"
+	"github.com/google/blueprint/bootstrap"
 	"github.com/google/blueprint/proptools"
 )
 
@@ -252,6 +253,235 @@ func (r conversionResults) BuildDirToTargets() map[string]BazelTargets {
 	return r.buildFileToTargets
 }
 
+// struct to store state of go bazel targets
+// this implements bp2buildModule interface and is passed to generateBazelTargets
+type goBazelTarget struct {
+	targetName            string
+	targetPackage         string
+	bazelRuleClass        string
+	bazelRuleLoadLocation string
+	bazelAttributes       []interface{}
+}
+
+var _ bp2buildModule = (*goBazelTarget)(nil)
+
+func (g goBazelTarget) TargetName() string {
+	return g.targetName
+}
+
+func (g goBazelTarget) TargetPackage() string {
+	return g.targetPackage
+}
+
+func (g goBazelTarget) BazelRuleClass() string {
+	return g.bazelRuleClass
+}
+
+func (g goBazelTarget) BazelRuleLoadLocation() string {
+	return g.bazelRuleLoadLocation
+}
+
+func (g goBazelTarget) BazelAttributes() []interface{} {
+	return g.bazelAttributes
+}
+
+// Creates a target_compatible_with entry that is *not* compatible with android
+func targetNotCompatibleWithAndroid() bazel.LabelListAttribute {
+	ret := bazel.LabelListAttribute{}
+	ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsAndroid,
+		bazel.MakeLabelList(
+			[]bazel.Label{
+				bazel.Label{
+					Label: "@platforms//:incompatible",
+				},
+			},
+		),
+	)
+	return ret
+}
+
+// helper function to return labels for srcs used in bootstrap_go_package and bootstrap_go_binary
+// this function has the following limitations which make it unsuitable for widespread use
+// - wildcard patterns in srcs
+// This is ok for go since build/blueprint does not support it.
+//
+// Prefer to use `BazelLabelForModuleSrc` instead
+func goSrcLabels(cfg android.Config, moduleDir string, srcs []string, linuxSrcs, darwinSrcs []string) bazel.LabelListAttribute {
+	labels := func(srcs []string) bazel.LabelList {
+		ret := []bazel.Label{}
+		for _, src := range srcs {
+			srcLabel := bazel.Label{
+				Label: src,
+			}
+			ret = append(ret, srcLabel)
+		}
+		// Respect package boundaries
+		return android.TransformSubpackagePaths(
+			cfg,
+			moduleDir,
+			bazel.MakeLabelList(ret),
+		)
+	}
+
+	ret := bazel.LabelListAttribute{}
+	// common
+	ret.SetSelectValue(bazel.NoConfigAxis, "", labels(srcs))
+	// linux
+	ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsLinux, labels(linuxSrcs))
+	// darwin
+	ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsDarwin, labels(darwinSrcs))
+	return ret
+}
+
+func goDepLabels(deps []string, goModulesMap nameToGoLibraryModule) bazel.LabelListAttribute {
+	labels := []bazel.Label{}
+	for _, dep := range deps {
+		moduleDir := goModulesMap[dep].Dir
+		if moduleDir == "." {
+			moduleDir = ""
+		}
+		label := bazel.Label{
+			Label: fmt.Sprintf("//%s:%s", moduleDir, dep),
+		}
+		labels = append(labels, label)
+	}
+	return bazel.MakeLabelListAttribute(bazel.MakeLabelList(labels))
+}
+
+// attributes common to blueprint_go_binary and bootstap_go_package
+type goAttributes struct {
+	Importpath             bazel.StringAttribute
+	Srcs                   bazel.LabelListAttribute
+	Deps                   bazel.LabelListAttribute
+	Target_compatible_with bazel.LabelListAttribute
+}
+
+func generateBazelTargetsGoPackage(ctx *android.Context, g *bootstrap.GoPackage, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) {
+	ca := android.CommonAttributes{
+		Name: g.Name(),
+	}
+
+	// For this bootstrap_go_package dep chain,
+	// A --> B --> C ( ---> depends on)
+	// Soong provides the convenience of only listing B as deps of A even if a src file of A imports C
+	// Bazel OTOH
+	// 1. requires C to be listed in `deps` expllicity.
+	// 2. does not require C to be listed if src of A does not import C
+	//
+	// bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps
+	transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap)
+
+	ga := goAttributes{
+		Importpath: bazel.StringAttribute{
+			Value: proptools.StringPtr(g.GoPkgPath()),
+		},
+		Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()),
+		Deps: goDepLabels(
+			android.FirstUniqueStrings(transitiveDeps),
+			goModulesMap,
+		),
+		Target_compatible_with: targetNotCompatibleWithAndroid(),
+	}
+
+	lib := goBazelTarget{
+		targetName:            g.Name(),
+		targetPackage:         ctx.ModuleDir(g),
+		bazelRuleClass:        "go_library",
+		bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
+		bazelAttributes:       []interface{}{&ca, &ga},
+	}
+	// TODO - b/284483729: Create go_test target from testSrcs
+	libTarget, err := generateBazelTarget(ctx, lib)
+	if err != nil {
+		return []BazelTarget{}, []error{err}
+	}
+	return []BazelTarget{libTarget}, nil
+}
+
+type goLibraryModule struct {
+	Dir  string
+	Deps []string
+}
+
+type nameToGoLibraryModule map[string]goLibraryModule
+
+// Visit each module in the graph
+// If a module is of type `bootstrap_go_package`, return a map containing metadata like its dir and deps
+func createGoLibraryModuleMap(ctx *android.Context) nameToGoLibraryModule {
+	ret := nameToGoLibraryModule{}
+	ctx.VisitAllModules(func(m blueprint.Module) {
+		moduleType := ctx.ModuleType(m)
+		// We do not need to store information about blueprint_go_binary since it does not have any rdeps
+		if moduleType == "bootstrap_go_package" {
+			ret[m.Name()] = goLibraryModule{
+				Dir:  ctx.ModuleDir(m),
+				Deps: m.(*bootstrap.GoPackage).Deps(),
+			}
+		}
+	})
+	return ret
+}
+
+// Returns the deps in the transitive closure of a go target
+func transitiveGoDeps(directDeps []string, goModulesMap nameToGoLibraryModule) []string {
+	allDeps := directDeps
+	i := 0
+	for i < len(allDeps) {
+		curr := allDeps[i]
+		allDeps = append(allDeps, goModulesMap[curr].Deps...)
+		i += 1
+	}
+	allDeps = android.SortedUniqueStrings(allDeps)
+	return allDeps
+}
+
+func generateBazelTargetsGoBinary(ctx *android.Context, g *bootstrap.GoBinary, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) {
+	ca := android.CommonAttributes{
+		Name: g.Name(),
+	}
+
+	// For this bootstrap_go_package dep chain,
+	// A --> B --> C ( ---> depends on)
+	// Soong provides the convenience of only listing B as deps of A even if a src file of A imports C
+	// Bazel OTOH
+	// 1. requires C to be listed in `deps` expllicity.
+	// 2. does not require C to be listed if src of A does not import C
+	//
+	// bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps
+	transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap)
+
+	ga := goAttributes{
+		Srcs:                   goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()),
+		Deps:                   goDepLabels(transitiveDeps, goModulesMap),
+		Target_compatible_with: targetNotCompatibleWithAndroid(),
+	}
+
+	bin := goBazelTarget{
+		targetName:            g.Name(),
+		targetPackage:         ctx.ModuleDir(g),
+		bazelRuleClass:        "go_binary",
+		bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
+		bazelAttributes:       []interface{}{&ca, &ga},
+	}
+	// TODO - b/284483729: Create go_test target from testSrcs
+	binTarget, err := generateBazelTarget(ctx, bin)
+	if err != nil {
+		return []BazelTarget{}, []error{err}
+	}
+	return []BazelTarget{binTarget}, nil
+}
+
+var (
+	// TODO - b/284483729: Remove this denyilst
+	// Temporary denylist of go binaries that are currently used in mixed builds
+	// This denylist allows us to rollout bp2build converters for go targets without affecting mixed builds
+	goBinaryDenylist = []string{
+		"soong_zip",
+		"zip2zip",
+		"bazel_notice_gen",
+	}
+)
+
 func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (conversionResults, []error) {
 	buildFileToTargets := make(map[string]BazelTargets)
 
@@ -262,6 +492,10 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
 
 	var errs []error
 
+	// Visit go libraries in a pre-run and store its state in a map
+	// The time complexity remains O(N), and this does not add significant wall time.
+	nameToGoLibMap := createGoLibraryModuleMap(ctx.Context())
+
 	bpCtx := ctx.Context()
 	bpCtx.VisitAllModules(func(m blueprint.Module) {
 		dir := bpCtx.ModuleDir(m)
@@ -269,6 +503,7 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
 		dirs[dir] = true
 
 		var targets []BazelTarget
+		var targetErrs []error
 
 		switch ctx.Mode() {
 		case Bp2Build:
@@ -317,7 +552,6 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
 						return
 					}
 				}
-				var targetErrs []error
 				targets, targetErrs = generateBazelTargets(bpCtx, aModule)
 				errs = append(errs, targetErrs...)
 				for _, t := range targets {
@@ -336,6 +570,14 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
 					metrics.AddUnconvertedModule(m, moduleType, dir, *reason)
 				}
 				return
+			} else if glib, ok := m.(*bootstrap.GoPackage); ok {
+				targets, targetErrs = generateBazelTargetsGoPackage(bpCtx, glib, nameToGoLibMap)
+				errs = append(errs, targetErrs...)
+				metrics.IncrementRuleClassCount("go_library")
+			} else if gbin, ok := m.(*bootstrap.GoBinary); ok && !android.InList(m.Name(), goBinaryDenylist) {
+				targets, targetErrs = generateBazelTargetsGoBinary(bpCtx, gbin, nameToGoLibMap)
+				errs = append(errs, targetErrs...)
+				metrics.IncrementRuleClassCount("go_binary")
 			} else {
 				metrics.AddUnconvertedModule(m, moduleType, dir, android.UnconvertedReason{
 					ReasonType: int(bp2build_metrics_proto.UnconvertedReasonType_TYPE_UNSUPPORTED),

bp2build/go_conversion_test.go

@@ -0,0 +1,150 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bp2build
+
+import (
+	"testing"
+
+	"github.com/google/blueprint/bootstrap"
+
+	"android/soong/android"
+)
+
+func runGoTests(t *testing.T, tc Bp2buildTestCase) {
+	RunBp2BuildTestCase(t, func(ctx android.RegistrationContext) {
+		tCtx := ctx.(*android.TestContext)
+		bootstrap.RegisterGoModuleTypes(tCtx.Context.Context) // android.TestContext --> android.Context --> blueprint.Context
+	}, tc)
+}
+
+func TestConvertGoPackage(t *testing.T) {
+	bp := `
+bootstrap_go_package {
+	name: "foo",
+	pkgPath: "android/foo",
+	deps: [
+		"bar",
+	],
+	srcs: [
+		"foo1.go",
+		"foo2.go",
+	],
+	linux: {
+		srcs: [
+			"foo_linux.go",
+		],
+	},
+	darwin: {
+		srcs: [
+			"foo_darwin.go",
+		],
+	},
+	testSrcs: [
+		"foo1_test.go",
+		"foo2_test.go",
+	],
+}
+`
+	depBp := `
+bootstrap_go_package {
+	name: "bar",
+}
+`
+	t.Parallel()
+	runGoTests(t, Bp2buildTestCase{
+		Description:         "Convert bootstrap_go_package to go_library",
+		ModuleTypeUnderTest: "bootrstap_go_package",
+		Blueprint:           bp,
+		Filesystem: map[string]string{
+			"bar/Android.bp": depBp, // Put dep in Android.bp to reduce boilerplate in ExpectedBazelTargets
+		},
+		ExpectedBazelTargets: []string{makeBazelTargetHostOrDevice("go_library", "foo",
+			AttrNameToString{
+				"deps":       `["//bar:bar"]`,
+				"importpath": `"android/foo"`,
+				"srcs": `[
+        "foo1.go",
+        "foo2.go",
+    ] + select({
+        "//build/bazel/platforms/os:darwin": ["foo_darwin.go"],
+        "//build/bazel/platforms/os:linux_glibc": ["foo_linux.go"],
+        "//conditions:default": [],
+    })`,
+			},
+			android.HostSupported,
+		)},
+	})
+}
+
+func TestConvertGoBinaryWithTransitiveDeps(t *testing.T) {
+	bp := `
+blueprint_go_binary {
+	name: "foo",
+	srcs: ["main.go"],
+	deps: ["bar"],
+}
+`
+	depBp := `
+bootstrap_go_package {
+	name: "bar",
+	deps: ["baz"],
+}
+bootstrap_go_package {
+	name: "baz",
+}
+`
+	t.Parallel()
+	runGoTests(t, Bp2buildTestCase{
+		Description: "Convert blueprint_go_binary to go_binary",
+		Blueprint:   bp,
+		Filesystem: map[string]string{
+			"bar/Android.bp": depBp, // Put dep in Android.bp to reduce boilerplate in ExpectedBazelTargets
+		},
+		ExpectedBazelTargets: []string{makeBazelTargetHostOrDevice("go_binary", "foo",
+			AttrNameToString{
+				"deps": `[
+        "//bar:bar",
+        "//bar:baz",
+    ]`,
+				"srcs": `["main.go"]`,
+			},
+			android.HostSupported,
+		)},
+	})
+}
+
+func TestConvertGoBinaryWithSrcInDifferentPackage(t *testing.T) {
+	bp := `
+blueprint_go_binary {
+	name: "foo",
+	srcs: ["subdir/main.go"],
+}
+`
+	t.Parallel()
+	runGoTests(t, Bp2buildTestCase{
+		Description: "Convert blueprint_go_binary with src in different package",
+		Blueprint:   bp,
+		Filesystem: map[string]string{
+			"subdir/Android.bp": "",
+		},
+		ExpectedBazelTargets: []string{makeBazelTargetHostOrDevice("go_binary", "foo",
+			AttrNameToString{
+				"deps": `[]`,
+				"srcs": `["//subdir:main.go"]`,
+			},
+			android.HostSupported,
+		)},
+	})
+}