Merge "Handle proto.include_dirs in bp2build for CC" into main

android/proto.go

@@ -156,8 +156,9 @@ func ProtoRule(rule *RuleBuilder, protoFile Path, flags ProtoFlags, deps Paths,
 
 // Bp2buildProtoInfo contains information necessary to pass on to language specific conversion.
 type Bp2buildProtoInfo struct {
-	Type       *string
+	Type                  *string
-	Proto_libs bazel.LabelList
+	Proto_libs            bazel.LabelList
+	Transitive_proto_libs bazel.LabelList
 }
 
 type ProtoAttrs struct {
@@ -211,6 +212,7 @@ func Bp2buildProtoProperties(ctx Bp2buildMutatorContext, m *ModuleBase, srcs baz
 	}
 
 	var protoLibraries bazel.LabelList
+	var transitiveProtoLibraries bazel.LabelList
 	var directProtoSrcs bazel.LabelList
 
 	// For filegroups that should be converted to proto_library just collect the
@@ -234,6 +236,7 @@ func Bp2buildProtoProperties(ctx Bp2buildMutatorContext, m *ModuleBase, srcs baz
 
 	if len(directProtoSrcs.Includes) > 0 {
 		pkgToSrcs := partitionSrcsByPackage(ctx.ModuleDir(), directProtoSrcs)
+		protoIncludeDirs := []string{}
 		for _, pkg := range SortedStringKeys(pkgToSrcs) {
 			srcs := pkgToSrcs[pkg]
 			attrs := ProtoAttrs{
@@ -262,7 +265,7 @@ func Bp2buildProtoProperties(ctx Bp2buildMutatorContext, m *ModuleBase, srcs baz
 							if dep, ok := includeDirsToProtoDeps[dir]; ok {
 								attrs.Deps.Add(bazel.MakeLabelAttribute(dep))
 							} else {
-								ctx.PropertyErrorf("Could not find the proto_library target for include dir", dir)
+								protoIncludeDirs = append(protoIncludeDirs, dir)
 							}
 						}
 					} else if props.Proto.Type != info.Type && props.Proto.Type != nil {
@@ -308,9 +311,84 @@ func Bp2buildProtoProperties(ctx Bp2buildMutatorContext, m *ModuleBase, srcs baz
 				Label: l,
 			})
 		}
+		protoLibrariesInIncludeDir := createProtoLibraryTargetsForIncludeDirs(ctx, protoIncludeDirs)
+		transitiveProtoLibraries.Append(protoLibrariesInIncludeDir)
 	}
 
 	info.Proto_libs = protoLibraries
+	info.Transitive_proto_libs = transitiveProtoLibraries
 
 	return info, true
 }
+
+var (
+	protoIncludeDirGeneratedSuffix = ".include_dir_bp2build_generated_proto"
+	protoIncludeDirsBp2buildKey    = NewOnceKey("protoIncludeDirsBp2build")
+)
+
+func getProtoIncludeDirsBp2build(config Config) *map[protoIncludeDirKey]bool {
+	return config.Once(protoIncludeDirsBp2buildKey, func() interface{} {
+		return &map[protoIncludeDirKey]bool{}
+	}).(*map[protoIncludeDirKey]bool)
+}
+
+// key for dynamically creating proto_library per proto.include_dirs
+type protoIncludeDirKey struct {
+	dir            string
+	subpackgeInDir string
+}
+
+// createProtoLibraryTargetsForIncludeDirs creates additional proto_library targets for .proto files in includeDirs
+// Since Bazel imposes a constratint that the proto_library must be in the same package as the .proto file, this function
+// might create the targets in a subdirectory of `includeDir`
+// Returns the labels of the proto_library targets
+func createProtoLibraryTargetsForIncludeDirs(ctx Bp2buildMutatorContext, includeDirs []string) bazel.LabelList {
+	var ret bazel.LabelList
+	for _, dir := range includeDirs {
+		if exists, _, _ := ctx.Config().fs.Exists(filepath.Join(dir, "Android.bp")); !exists {
+			ctx.ModuleErrorf("TODO: Add support for proto.include_dir: %v. This directory does not contain an Android.bp file", dir)
+		}
+		dirMap := getProtoIncludeDirsBp2build(ctx.Config())
+		// Find all proto file targets in this dir
+		protoLabelsInDir := BazelLabelForSrcPatternExcludes(ctx, dir, "**/*.proto", []string{})
+		// Partition the labels by package and subpackage(s)
+		protoLabelelsPartitionedByPkg := partitionSrcsByPackage(dir, protoLabelsInDir)
+		for _, pkg := range SortedStringKeys(protoLabelelsPartitionedByPkg) {
+			label := strings.ReplaceAll(dir, "/", ".") + protoIncludeDirGeneratedSuffix
+			ret.Add(&bazel.Label{
+				Label: "//" + pkg + ":" + label,
+			})
+			key := protoIncludeDirKey{dir: dir, subpackgeInDir: pkg}
+			if _, exists := (*dirMap)[key]; exists {
+				// A proto_library has already been created for this package relative to this include dir
+				continue
+			}
+			(*dirMap)[key] = true
+			srcs := protoLabelelsPartitionedByPkg[pkg]
+			rel, err := filepath.Rel(dir, pkg)
+			if err != nil {
+				ctx.ModuleErrorf("Could not create a proto_library in pkg %v due to %v\n", pkg, err)
+			}
+			// Create proto_library
+			attrs := ProtoAttrs{
+				Srcs:                bazel.MakeLabelListAttribute(srcs),
+				Strip_import_prefix: proptools.StringPtr(""),
+			}
+			if rel != "." {
+				attrs.Import_prefix = proptools.StringPtr(rel)
+			}
+			ctx.CreateBazelTargetModule(
+				bazel.BazelTargetModuleProperties{Rule_class: "proto_library"},
+				CommonAttributes{
+					Name: label,
+					Dir:  proptools.StringPtr(pkg),
+					// This proto_library is used to construct a ProtoInfo
+					// But it might not be buildable on its own
+					Tags: bazel.MakeStringListAttribute([]string{"manual"}),
+				},
+				&attrs,
+			)
+		}
+	}
+	return ret
+}

bp2build/cc_library_conversion_test.go

@@ -2394,7 +2394,7 @@ func TestCcLibraryProtoIncludeDirsUnknown(t *testing.T) {
 	},
 	include_build_directory: false,
 }`,
-		ExpectedErr: fmt.Errorf("module \"foo\": Could not find the proto_library target for include dir: external/protobuf/abc"),
+		ExpectedErr: fmt.Errorf("module \"foo\": TODO: Add support for proto.include_dir: external/protobuf/abc. This directory does not contain an Android.bp file"),
 	})
 }
 
@@ -5068,3 +5068,72 @@ cc_library_static {
 	}
 	runCcLibraryTestCase(t, tc)
 }
+
+func TestProtoIncludeDirs(t *testing.T) {
+	tc := Bp2buildTestCase{
+		Description:                "cc_library depends on .proto files using proto.include_dirs",
+		ModuleTypeUnderTest:        "cc_library",
+		ModuleTypeUnderTestFactory: cc.LibraryFactory,
+		Blueprint: `
+cc_library_static {
+	name: "foo",
+	srcs: [
+	   "foo.proto",
+	],
+	proto: {
+		include_dirs: ["bar"],
+	}
+}
+` + simpleModuleDoNotConvertBp2build("cc_library", "libprotobuf-cpp-lite"),
+		Filesystem: map[string]string{
+			"bar/Android.bp":     "",
+			"bar/bar.proto":      "",
+			"bar/baz/Android.bp": "",
+			"bar/baz/baz.proto":  "",
+		},
+	}
+
+	// We will run the test 3 times and check in the root, bar and bar/baz directories
+	// Root dir
+	tc.ExpectedBazelTargets = []string{
+		MakeBazelTarget("cc_library_static", "foo", AttrNameToString{
+			"local_includes":                    `["."]`,
+			"deps":                              `[":libprotobuf-cpp-lite"]`,
+			"implementation_whole_archive_deps": `[":foo_cc_proto_lite"]`,
+		}),
+		MakeBazelTarget("proto_library", "foo_proto", AttrNameToString{
+			"srcs": `["foo.proto"]`,
+		}),
+		MakeBazelTarget("cc_lite_proto_library", "foo_cc_proto_lite", AttrNameToString{
+			"deps": `[":foo_proto"]`,
+			"transitive_deps": `[
+        "//bar:bar.include_dir_bp2build_generated_proto",
+        "//bar/baz:bar.include_dir_bp2build_generated_proto",
+    ]`,
+		}),
+	}
+	runCcLibraryTestCase(t, tc)
+
+	// bar dir
+	tc.Dir = "bar"
+	tc.ExpectedBazelTargets = []string{
+		MakeBazelTarget("proto_library", "bar.include_dir_bp2build_generated_proto", AttrNameToString{
+			"srcs":                `["bar.proto"]`,
+			"strip_import_prefix": `""`,
+			"tags":                `["manual"]`,
+		}),
+	}
+	runCcLibraryTestCase(t, tc)
+
+	// bar/baz dir
+	tc.Dir = "bar/baz"
+	tc.ExpectedBazelTargets = []string{
+		MakeBazelTarget("proto_library", "bar.include_dir_bp2build_generated_proto", AttrNameToString{
+			"srcs":                `["//bar/baz:baz.proto"]`,
+			"strip_import_prefix": `""`,
+			"import_prefix":       `"baz"`,
+			"tags":                `["manual"]`,
+		}),
+	}
+	runCcLibraryTestCase(t, tc)
+}

cc/bp2build.go

@@ -960,7 +960,7 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module)
 	(&linkerAttrs).wholeArchiveDeps.Append(compilerAttrs.exportXsdSrcs)
 	(&linkerAttrs).implementationWholeArchiveDeps.Append(compilerAttrs.xsdSrcs)
 
-	protoDep := bp2buildProto(ctx, module, compilerAttrs.protoSrcs)
+	protoDep := bp2buildProto(ctx, module, compilerAttrs.protoSrcs, linkerAttrs)
 
 	// bp2buildProto will only set wholeStaticLib or implementationWholeStaticLib, but we don't know
 	// which. This will add the newly generated proto library to the appropriate attribute and nothing

cc/proto.go

@@ -165,7 +165,17 @@ func protoFlags(ctx ModuleContext, flags Flags, p *android.ProtoProperties) Flag
 }
 
 type protoAttributes struct {
-	Deps            bazel.LabelListAttribute
+	Deps bazel.LabelListAttribute
+
+	// A list of proto_library targets that that the proto_library in `deps` depends on
+	// This list is overestimation.
+	// Overestimation is necessary since Soong includes other protos via proto.include_dirs and not
+	// a specific .proto file module explicitly.
+	Transitive_deps bazel.LabelListAttribute
+
+	// A list of cc_library_* targets that the generated cpp code depends on
+	Cc_deps bazel.LabelListAttribute
+
 	Min_sdk_version *string
 }
 
@@ -175,7 +185,7 @@ type bp2buildProtoDeps struct {
 	protoDep                     *bazel.LabelAttribute
 }
 
-func bp2buildProto(ctx android.Bp2buildMutatorContext, m *Module, protoSrcs bazel.LabelListAttribute) bp2buildProtoDeps {
+func bp2buildProto(ctx android.Bp2buildMutatorContext, m *Module, protoSrcs bazel.LabelListAttribute, la linkerAttributes) bp2buildProtoDeps {
 	var ret bp2buildProtoDeps
 
 	protoInfo, ok := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, protoSrcs)
@@ -204,6 +214,35 @@ func bp2buildProto(ctx android.Bp2buildMutatorContext, m *Module, protoSrcs baze
 
 	var protoAttrs protoAttributes
 	protoAttrs.Deps.SetValue(protoInfo.Proto_libs)
+	protoAttrs.Transitive_deps.SetValue(protoInfo.Transitive_proto_libs)
+
+	// Add the implementation deps of the top-level cc_library_static
+	// This is necessary to compile the internal root of cc_proto_library.
+	// Without this, clang might not be able to find .h files that the generated cpp files depends on
+	protoAttrs.Cc_deps = *la.implementationDeps.Clone()
+	protoAttrs.Cc_deps.Append(la.implementationDynamicDeps)
+	protoAttrs.Cc_deps.Append(la.implementationWholeArchiveDeps)
+	protoAttrs.Cc_deps.Append(la.wholeArchiveDeps)
+	// Subtract myself to prevent possible circular dep
+	protoAttrs.Cc_deps = bazel.SubtractBazelLabelListAttribute(
+		protoAttrs.Cc_deps,
+		bazel.MakeLabelListAttribute(
+			bazel.MakeLabelList([]bazel.Label{
+				bazel.Label{Label: ":" + m.Name() + suffix},
+			}),
+		),
+	)
+	// Subtract the protobuf libraries since cc_proto_library implicitly adds them
+	protoAttrs.Cc_deps = bazel.SubtractBazelLabelListAttribute(
+		protoAttrs.Cc_deps,
+		bazel.MakeLabelListAttribute(
+			bazel.MakeLabelList([]bazel.Label{
+				bazel.Label{Label: "//external/protobuf:libprotobuf-cpp-full", OriginalModuleName: "libprotobuf-cpp-full"},
+				bazel.Label{Label: "//external/protobuf:libprotobuf-cpp-lite", OriginalModuleName: "libprotobuf-cpp-lite"},
+			}),
+		),
+	)
+
 	protoAttrs.Min_sdk_version = m.Properties.Min_sdk_version
 
 	name := m.Name() + suffix