Move proto compilation to RuleBuilder

Using blueprint.Rule for protoc commands was causing code duplication
because there was no good way to run the same protoc for cc, java and
python but then run custom source packaging steps for java and python.
Move most of the code into a common function that returns a
RuleBuilder, and then let java and python add their own commands at
the end of the rule.

Bug: 70706119
Test: All Soong tests
Test: m checkbuild
Change-Id: Ic692136775d273bcc4f4de99620ab4878667c83a
This commit is contained in:
Colin Cross
2019-03-28 14:45:07 -07:00
parent 6e1c3faed5
commit 19878da6a0
12 changed files with 170 additions and 199 deletions

View File

@@ -193,6 +193,7 @@ bootstrap_go_package {
"cc/gen_test.go",
"cc/genrule_test.go",
"cc/library_test.go",
"cc/proto_test.go",
"cc/test_data_test.go",
"cc/util_test.go",
],

View File

@@ -14,6 +14,12 @@
package android
import (
"strings"
"github.com/google/blueprint/proptools"
)
// TODO(ccross): protos are often used to communicate between multiple modules. If the only
// way to convert a proto to source is to reference it as a source file, and external modules cannot
// reference source files in other modules, then every module that owns a proto file will need to
@@ -22,9 +28,17 @@ package android
// and then external modules could depend on the proto module but use their own settings to
// generate the source.
func ProtoFlags(ctx ModuleContext, p *ProtoProperties) []string {
protoFlags := []string{}
type ProtoFlags struct {
Flags []string
CanonicalPathFromRoot bool
Dir ModuleGenPath
SubDir ModuleGenPath
OutTypeFlag string
OutParams []string
}
func GetProtoFlags(ctx ModuleContext, p *ProtoProperties) ProtoFlags {
var protoFlags []string
if len(p.Proto.Local_include_dirs) > 0 {
localProtoIncludeDirs := PathsForModuleSrc(ctx, p.Proto.Local_include_dirs)
protoFlags = append(protoFlags, JoinWithPrefix(localProtoIncludeDirs.Strings(), "-I"))
@@ -34,24 +48,12 @@ func ProtoFlags(ctx ModuleContext, p *ProtoProperties) []string {
protoFlags = append(protoFlags, JoinWithPrefix(rootProtoIncludeDirs.Strings(), "-I"))
}
return protoFlags
}
func ProtoCanonicalPathFromRoot(ctx ModuleContext, p *ProtoProperties) bool {
if p.Proto.Canonical_path_from_root == nil {
return true
return ProtoFlags{
Flags: protoFlags,
CanonicalPathFromRoot: proptools.BoolDefault(p.Proto.Canonical_path_from_root, true),
Dir: PathForModuleGen(ctx, "proto"),
SubDir: PathForModuleGen(ctx, "proto", ctx.ModuleDir()),
}
return *p.Proto.Canonical_path_from_root
}
// ProtoDir returns the module's "gen/proto" directory
func ProtoDir(ctx ModuleContext) ModuleGenPath {
return PathForModuleGen(ctx, "proto")
}
// ProtoSubDir returns the module's "gen/proto/path/to/module" directory
func ProtoSubDir(ctx ModuleContext) ModuleGenPath {
return PathForModuleGen(ctx, "proto", ctx.ModuleDir())
}
type ProtoProperties struct {
@@ -76,3 +78,28 @@ type ProtoProperties struct {
Canonical_path_from_root *bool
} `android:"arch_variant"`
}
func ProtoRule(ctx ModuleContext, rule *RuleBuilder, protoFile Path, flags ProtoFlags, deps Paths,
outDir WritablePath, depFile WritablePath, outputs WritablePaths) {
var protoBase string
if flags.CanonicalPathFromRoot {
protoBase = "."
} else {
rel := protoFile.Rel()
protoBase = strings.TrimSuffix(protoFile.String(), rel)
}
rule.Command().
Tool(ctx.Config().HostToolPath(ctx, "aprotoc")).
FlagWithArg(flags.OutTypeFlag+"=", strings.Join(flags.OutParams, ",")+":"+outDir.String()).
FlagWithDepFile("--dependency_out=", depFile).
FlagWithArg("-I ", protoBase).
Flags(flags.Flags).
Input(protoFile).
Implicits(deps).
ImplicitOutputs(outputs)
rule.Command().
Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")).Flag(depFile.String())
}

View File

@@ -261,12 +261,9 @@ type builderFlags struct {
stripUseGnuStrip bool
protoDeps android.Paths
protoFlags string
protoOutTypeFlag string
protoOutParams string
proto android.ProtoFlags
protoC bool
protoOptionsFile bool
protoRoot bool
}
type Objects struct {

View File

@@ -162,13 +162,10 @@ type Flags struct {
GroupStaticLibs bool
proto android.ProtoFlags
protoDeps android.Paths
protoFlags []string // Flags that apply to proto source files
protoOutTypeFlag string // The output type, --cpp_out for example
protoOutParams []string // Flags that modify the output of proto generated files
protoC bool // Whether to use C instead of C++
protoOptionsFile bool // Whether to look for a .options file next to the .proto
ProtoRoot bool
protoC bool // Whether to use C instead of C++
protoOptionsFile bool // Whether to look for a .options file next to the .proto
}
type ObjectLinkerProperties struct {

View File

@@ -848,10 +848,10 @@ func (library *libraryDecorator) link(ctx ModuleContext,
if Bool(library.Properties.Proto.Export_proto_headers) {
if library.baseCompiler.hasSrcExt(".proto") {
includes := []string{}
if flags.ProtoRoot {
includes = append(includes, "-I"+android.ProtoSubDir(ctx).String())
if flags.proto.CanonicalPathFromRoot {
includes = append(includes, "-I"+flags.proto.SubDir.String())
}
includes = append(includes, "-I"+android.ProtoDir(ctx).String())
includes = append(includes, "-I"+flags.proto.Dir.String())
library.reexportFlags(includes)
library.reuseExportedFlags = append(library.reuseExportedFlags, includes...)
library.reexportDeps(library.baseCompiler.pathDeps) // TODO: restrict to proto deps

View File

@@ -15,47 +15,26 @@
package cc
import (
"strings"
"github.com/google/blueprint"
"github.com/google/blueprint/pathtools"
"android/soong/android"
)
func init() {
pctx.HostBinToolVariable("protocCmd", "aprotoc")
pctx.HostBinToolVariable("depFixCmd", "dep_fixer")
}
var (
proto = pctx.AndroidStaticRule("protoc",
blueprint.RuleParams{
Command: "$protocCmd $protoOut=$protoOutParams:$outDir --dependency_out=$out.d -I $protoBase $protoFlags $in && " +
`$depFixCmd $out.d`,
CommandDeps: []string{"$protocCmd", "$depFixCmd"},
Depfile: "${out}.d",
Deps: blueprint.DepsGCC,
}, "protoFlags", "protoOut", "protoOutParams", "protoBase", "outDir")
)
// genProto creates a rule to convert a .proto file to generated .pb.cc and .pb.h files and returns
// the paths to the generated files.
func genProto(ctx android.ModuleContext, protoFile android.Path, flags builderFlags) (ccFile, headerFile android.WritablePath) {
func genProto(ctx android.ModuleContext, protoFile android.Path, flags builderFlags) (cc, header android.WritablePath) {
var ccFile, headerFile android.ModuleGenPath
srcSuffix := ".cc"
if flags.protoC {
srcSuffix = ".c"
}
var protoBase string
if flags.protoRoot {
protoBase = "."
if flags.proto.CanonicalPathFromRoot {
ccFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb"+srcSuffix)
headerFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb.h")
} else {
rel := protoFile.Rel()
protoBase = strings.TrimSuffix(protoFile.String(), rel)
ccFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, "pb"+srcSuffix))
headerFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, "pb.h"))
}
@@ -63,27 +42,19 @@ func genProto(ctx android.ModuleContext, protoFile android.Path, flags builderFl
protoDeps := flags.protoDeps
if flags.protoOptionsFile {
optionsFile := pathtools.ReplaceExtension(protoFile.String(), "options")
optionsPath := android.ExistentPathForSource(ctx, optionsFile)
if optionsPath.Valid() {
protoDeps = append(android.Paths{optionsPath.Path()}, protoDeps...)
}
optionsPath := android.PathForSource(ctx, optionsFile)
protoDeps = append(android.Paths{optionsPath}, protoDeps...)
}
ctx.Build(pctx, android.BuildParams{
Rule: proto,
Description: "protoc " + protoFile.Rel(),
Output: ccFile,
ImplicitOutput: headerFile,
Input: protoFile,
Implicits: protoDeps,
Args: map[string]string{
"outDir": android.ProtoDir(ctx).String(),
"protoFlags": flags.protoFlags,
"protoOut": flags.protoOutTypeFlag,
"protoOutParams": flags.protoOutParams,
"protoBase": protoBase,
},
})
outDir := flags.proto.Dir
depFile := ccFile.ReplaceExtension(ctx, "d")
outputs := android.WritablePaths{ccFile, headerFile}
rule := android.NewRuleBuilder()
android.ProtoRule(ctx, rule, protoFile, flags.proto, protoDeps, outDir, depFile, outputs)
rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel())
return ccFile, headerFile
}
@@ -143,13 +114,11 @@ func protoDeps(ctx BaseModuleContext, deps Deps, p *android.ProtoProperties, sta
func protoFlags(ctx ModuleContext, flags Flags, p *android.ProtoProperties) Flags {
flags.CFlags = append(flags.CFlags, "-DGOOGLE_PROTOBUF_NO_RTTI")
flags.ProtoRoot = android.ProtoCanonicalPathFromRoot(ctx, p)
if flags.ProtoRoot {
flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.ProtoSubDir(ctx).String())
flags.proto = android.GetProtoFlags(ctx, p)
if flags.proto.CanonicalPathFromRoot {
flags.GlobalFlags = append(flags.GlobalFlags, "-I"+flags.proto.SubDir.String())
}
flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.ProtoDir(ctx).String())
flags.protoFlags = android.ProtoFlags(ctx, p)
flags.GlobalFlags = append(flags.GlobalFlags, "-I"+flags.proto.Dir.String())
var plugin string
@@ -157,18 +126,18 @@ func protoFlags(ctx ModuleContext, flags Flags, p *android.ProtoProperties) Flag
case "nanopb-c", "nanopb-c-enable_malloc", "nanopb-c-16bit", "nanopb-c-enable_malloc-16bit", "nanopb-c-32bit", "nanopb-c-enable_malloc-32bit":
flags.protoC = true
flags.protoOptionsFile = true
flags.protoOutTypeFlag = "--nanopb_out"
flags.proto.OutTypeFlag = "--nanopb_out"
plugin = "protoc-gen-nanopb"
case "full":
flags.protoOutTypeFlag = "--cpp_out"
flags.proto.OutTypeFlag = "--cpp_out"
case "lite":
flags.protoOutTypeFlag = "--cpp_out"
flags.protoOutParams = append(flags.protoOutParams, "lite")
flags.proto.OutTypeFlag = "--cpp_out"
flags.proto.OutParams = append(flags.proto.OutParams, "lite")
case "":
// TODO(b/119714316): this should be equivalent to "lite" in
// order to match protoDeps, but some modules are depending on
// this behavior
flags.protoOutTypeFlag = "--cpp_out"
flags.proto.OutTypeFlag = "--cpp_out"
default:
ctx.PropertyErrorf("proto.type", "unknown proto type %q",
String(p.Proto.Type))
@@ -177,7 +146,7 @@ func protoFlags(ctx ModuleContext, flags Flags, p *android.ProtoProperties) Flag
if plugin != "" {
path := ctx.Config().HostToolPath(ctx, plugin)
flags.protoDeps = append(flags.protoDeps, path)
flags.protoFlags = append(flags.protoFlags, "--plugin="+path.String())
flags.proto.Flags = append(flags.proto.Flags, "--plugin="+path.String())
}
return flags

36
cc/proto_test.go Normal file
View File

@@ -0,0 +1,36 @@
// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cc
import (
"strings"
"testing"
)
func TestProto(t *testing.T) {
t.Run("simple", func(t *testing.T) {
ctx := testCc(t, `
cc_library_shared {
name: "libfoo",
srcs: ["a.proto"],
}`)
proto := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Output("proto/a.pb.cc")
if cmd := proto.RuleParams.Command; !strings.Contains(cmd, "--cpp_out=") {
t.Errorf("expected '--cpp_out' in %q", cmd)
}
})
}

View File

@@ -85,12 +85,9 @@ func flagsToBuilderFlags(in Flags) builderFlags {
groupStaticLibs: in.GroupStaticLibs,
protoDeps: in.protoDeps,
protoFlags: strings.Join(in.protoFlags, " "),
protoOutTypeFlag: in.protoOutTypeFlag,
protoOutParams: strings.Join(in.protoOutParams, ","),
proto: in.proto,
protoC: in.protoC,
protoOptionsFile: in.protoOptionsFile,
protoRoot: in.ProtoRoot,
}
}

View File

@@ -153,10 +153,7 @@ type javaBuilderFlags struct {
kotlincFlags string
kotlincClasspath classpath
protoFlags []string
protoOutTypeFlag string // The flag itself: --java_out
protoOutParams string // Parameters to that flag: --java_out=$protoOutParams:$outDir
protoRoot bool
proto android.ProtoFlags
}
func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, shardIdx int,

View File

@@ -15,57 +15,34 @@
package java
import (
"strings"
"github.com/google/blueprint"
"android/soong/android"
)
func init() {
pctx.HostBinToolVariable("protocCmd", "aprotoc")
pctx.HostBinToolVariable("depFixCmd", "dep_fixer")
}
var (
proto = pctx.AndroidStaticRule("protoc",
blueprint.RuleParams{
Command: `rm -rf $out.tmp && mkdir -p $out.tmp && ` +
`$protocCmd $protoOut=$protoOutParams:$out.tmp --dependency_out=$out.d -I $protoBase $protoFlags $in && ` +
`$depFixCmd $out.d && ` +
`${config.SoongZipCmd} -jar -o $out -C $out.tmp -D $out.tmp && rm -rf $out.tmp`,
CommandDeps: []string{
"$protocCmd",
"$depFixCmd",
"${config.SoongZipCmd}",
},
Depfile: "${out}.d",
Deps: blueprint.DepsGCC,
}, "protoBase", "protoFlags", "protoOut", "protoOutParams")
)
func genProto(ctx android.ModuleContext, protoFile android.Path, flags javaBuilderFlags) android.Path {
srcJarFile := android.GenPathWithExt(ctx, "proto", protoFile, "srcjar")
var protoBase string
if flags.protoRoot {
protoBase = "."
} else {
protoBase = strings.TrimSuffix(protoFile.String(), protoFile.Rel())
}
outDir := srcJarFile.ReplaceExtension(ctx, "tmp")
depFile := srcJarFile.ReplaceExtension(ctx, "srcjar.d")
ctx.Build(pctx, android.BuildParams{
Rule: proto,
Description: "protoc " + protoFile.Rel(),
Output: srcJarFile,
Input: protoFile,
Args: map[string]string{
"protoBase": protoBase,
"protoOut": flags.protoOutTypeFlag,
"protoOutParams": flags.protoOutParams,
"protoFlags": strings.Join(flags.protoFlags, " "),
},
})
rule := android.NewRuleBuilder()
rule.Command().Text("rm -rf").Flag(outDir.String())
rule.Command().Text("mkdir -p").Flag(outDir.String())
android.ProtoRule(ctx, rule, protoFile, flags.proto, nil, outDir, depFile, nil)
// Proto generated java files have an unknown package name in the path, so package the entire output directory
// into a srcjar.
rule.Command().
Tool(ctx.Config().HostToolPath(ctx, "soong_zip")).
Flag("-jar").
FlagWithOutput("-o ", srcJarFile).
FlagWithArg("-C ", outDir.String()).
FlagWithArg("-D ", outDir.String())
rule.Command().Text("rm -rf").Flag(outDir.String())
rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel())
return srcJarFile
}
@@ -93,30 +70,24 @@ func protoDeps(ctx android.BottomUpMutatorContext, p *android.ProtoProperties) {
func protoFlags(ctx android.ModuleContext, j *CompilerProperties, p *android.ProtoProperties,
flags javaBuilderFlags) javaBuilderFlags {
flags.proto = android.GetProtoFlags(ctx, p)
switch String(p.Proto.Type) {
case "micro":
flags.protoOutTypeFlag = "--javamicro_out"
flags.proto.OutTypeFlag = "--javamicro_out"
case "nano":
flags.protoOutTypeFlag = "--javanano_out"
flags.proto.OutTypeFlag = "--javanano_out"
case "lite":
flags.protoOutTypeFlag = "--java_out"
flags.protoOutParams = "lite"
flags.proto.OutTypeFlag = "--java_out"
flags.proto.OutParams = append(flags.proto.OutParams, "lite")
case "full", "":
flags.protoOutTypeFlag = "--java_out"
flags.proto.OutTypeFlag = "--java_out"
default:
ctx.PropertyErrorf("proto.type", "unknown proto type %q",
String(p.Proto.Type))
}
if len(j.Proto.Output_params) > 0 {
if flags.protoOutParams != "" {
flags.protoOutParams += ","
}
flags.protoOutParams += strings.Join(j.Proto.Output_params, ",")
}
flags.protoFlags = android.ProtoFlags(ctx, p)
flags.protoRoot = android.ProtoCanonicalPathFromRoot(ctx, p)
flags.proto.OutParams = append(flags.proto.OutParams, j.Proto.Output_params...)
return flags
}

View File

@@ -16,58 +16,35 @@ package python
import (
"android/soong/android"
"strings"
"github.com/google/blueprint"
)
func init() {
pctx.HostBinToolVariable("protocCmd", "aprotoc")
}
func genProto(ctx android.ModuleContext, protoFile android.Path, flags android.ProtoFlags, pkgPath string) android.Path {
srcsZipFile := android.PathForModuleGen(ctx, protoFile.Base()+".srcszip")
var (
proto = pctx.AndroidStaticRule("protoc",
blueprint.RuleParams{
Command: `rm -rf $out.tmp && mkdir -p $out.tmp && ` +
`$protocCmd --python_out=$out.tmp --dependency_out=$out.d -I $protoBase $protoFlags $in && ` +
`$parCmd -o $out $pkgPathArgs -C $out.tmp -D $out.tmp && rm -rf $out.tmp`,
CommandDeps: []string{
"$protocCmd",
"$parCmd",
},
Depfile: "${out}.d",
Deps: blueprint.DepsGCC,
}, "protoBase", "protoFlags", "pkgPathArgs")
)
outDir := srcsZipFile.ReplaceExtension(ctx, "tmp")
depFile := srcsZipFile.ReplaceExtension(ctx, "srcszip.d")
func genProto(ctx android.ModuleContext, p *android.ProtoProperties,
protoFile android.Path, protoFlags []string, pkgPath string) android.Path {
srcJarFile := android.PathForModuleGen(ctx, protoFile.Base()+".srcszip")
rule := android.NewRuleBuilder()
protoRoot := android.ProtoCanonicalPathFromRoot(ctx, p)
rule.Command().Text("rm -rf").Flag(outDir.String())
rule.Command().Text("mkdir -p").Flag(outDir.String())
var protoBase string
if protoRoot {
protoBase = "."
} else {
protoBase = strings.TrimSuffix(protoFile.String(), protoFile.Rel())
}
android.ProtoRule(ctx, rule, protoFile, flags, nil, outDir, depFile, nil)
var pkgPathArgs string
// Proto generated python files have an unknown package name in the path, so package the entire output directory
// into a srcszip.
zipCmd := rule.Command().
Tool(ctx.Config().HostToolPath(ctx, "soong_zip")).
FlagWithOutput("-o ", srcsZipFile).
FlagWithArg("-C ", outDir.String()).
FlagWithArg("-D ", outDir.String())
if pkgPath != "" {
pkgPathArgs = "-P " + pkgPath
zipCmd.FlagWithArg("-P ", pkgPath)
}
ctx.Build(pctx, android.BuildParams{
Rule: proto,
Description: "protoc " + protoFile.Rel(),
Output: srcJarFile,
Input: protoFile,
Args: map[string]string{
"protoBase": protoBase,
"protoFlags": strings.Join(protoFlags, " "),
"pkgPathArgs": pkgPathArgs,
},
})
return srcJarFile
rule.Command().Text("rm -rf").Flag(outDir.String())
rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel())
return srcsZipFile
}

View File

@@ -516,9 +516,11 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi
}
var zips android.Paths
if len(protoSrcs) > 0 {
protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
protoFlags.OutTypeFlag = "--python_out"
for _, srcFile := range protoSrcs {
zip := genProto(ctx, &p.protoProperties, srcFile,
android.ProtoFlags(ctx, &p.protoProperties), pkgPath)
zip := genProto(ctx, srcFile, protoFlags, pkgPath)
zips = append(zips, zip)
}
}