From 2f33635542b5fe97b6779cb2c6b3f5d2277c51b5 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 26 Oct 2016 10:03:47 -0700 Subject: [PATCH 1/3] Simplify compileObjs None of the direct users of compileObjs were using any of its complexity (generated sources, excluded sources, extra sources). Move the complexity back in to baseCompiler.compile. Test: m -j Change-Id: I2e59d216682c00dd12a1395cf2448827d1c48023 --- cc/compiler.go | 33 ++++++++++++++++++--------------- cc/library.go | 26 +++++++++++--------------- cc/ndk_library.go | 8 ++------ 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/cc/compiler.go b/cc/compiler.go index a0068ada3..32d98e4a8 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -100,6 +100,7 @@ func NewBaseCompiler() *baseCompiler { type baseCompiler struct { Properties BaseCompilerProperties + deps android.Paths } var _ compiler = (*baseCompiler)(nil) @@ -337,10 +338,21 @@ func ndkPathDeps(ctx ModuleContext) android.Paths { func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Paths { pathDeps := deps.GeneratedHeaders pathDeps = append(pathDeps, ndkPathDeps(ctx)...) + + srcs := ctx.ExpandSources(compiler.Properties.Srcs, compiler.Properties.Exclude_srcs) + srcs = append(srcs, deps.GeneratedSources...) + + buildFlags := flagsToBuilderFlags(flags) + + srcs, genDeps := genSources(ctx, srcs, buildFlags) + + pathDeps = append(pathDeps, genDeps...) + pathDeps = append(pathDeps, flags.CFlagsDeps...) + + compiler.deps = pathDeps + // Compile files listed in c.Properties.Srcs into objects - objFiles := compileObjs(ctx, flags, "", - compiler.Properties.Srcs, compiler.Properties.Exclude_srcs, - deps.GeneratedSources, pathDeps) + objFiles := compileObjs(ctx, buildFlags, "", srcs, compiler.deps) if ctx.Failed() { return nil @@ -350,17 +362,8 @@ func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathD } // Compile a list of source files into objects a specified subdirectory -func compileObjs(ctx android.ModuleContext, flags Flags, - subdir string, srcFiles, excludes []string, extraSrcs, deps android.Paths) android.Paths { +func compileObjs(ctx android.ModuleContext, flags builderFlags, + subdir string, srcFiles, deps android.Paths) android.Paths { - buildFlags := flagsToBuilderFlags(flags) - - inputFiles := ctx.ExpandSources(srcFiles, excludes) - inputFiles = append(inputFiles, extraSrcs...) - srcPaths, gendeps := genSources(ctx, inputFiles, buildFlags) - - deps = append(deps, gendeps...) - deps = append(deps, flags.CFlagsDeps...) - - return TransformSourceToObj(ctx, subdir, srcPaths, buildFlags, deps) + return TransformSourceToObj(ctx, subdir, srcFiles, flags, deps) } diff --git a/cc/library.go b/cc/library.go index a61e6a108..d3d142e51 100644 --- a/cc/library.go +++ b/cc/library.go @@ -25,9 +25,8 @@ import ( type LibraryProperties struct { Static struct { - Srcs []string `android:"arch_variant"` - Exclude_srcs []string `android:"arch_variant"` - Cflags []string `android:"arch_variant"` + Srcs []string `android:"arch_variant"` + Cflags []string `android:"arch_variant"` Enabled *bool `android:"arch_variant"` Whole_static_libs []string `android:"arch_variant"` @@ -35,9 +34,8 @@ type LibraryProperties struct { Shared_libs []string `android:"arch_variant"` } `android:"arch_variant"` Shared struct { - Srcs []string `android:"arch_variant"` - Exclude_srcs []string `android:"arch_variant"` - Cflags []string `android:"arch_variant"` + Srcs []string `android:"arch_variant"` + Cflags []string `android:"arch_variant"` Enabled *bool `android:"arch_variant"` Whole_static_libs []string `android:"arch_variant"` @@ -253,18 +251,16 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa objFiles = library.baseCompiler.compile(ctx, flags, deps) library.reuseObjFiles = objFiles - - pathDeps := deps.GeneratedHeaders - pathDeps = append(pathDeps, ndkPathDeps(ctx)...) + buildFlags := flagsToBuilderFlags(flags) if library.static() { - objFiles = append(objFiles, compileObjs(ctx, flags, android.DeviceStaticLibrary, - library.Properties.Static.Srcs, library.Properties.Static.Exclude_srcs, - nil, pathDeps)...) + srcs := android.PathsForModuleSrc(ctx, library.Properties.Static.Srcs) + objFiles = append(objFiles, compileObjs(ctx, buildFlags, android.DeviceStaticLibrary, + srcs, library.baseCompiler.deps)...) } else { - objFiles = append(objFiles, compileObjs(ctx, flags, android.DeviceSharedLibrary, - library.Properties.Shared.Srcs, library.Properties.Shared.Exclude_srcs, - nil, pathDeps)...) + srcs := android.PathsForModuleSrc(ctx, library.Properties.Shared.Srcs) + objFiles = append(objFiles, compileObjs(ctx, buildFlags, android.DeviceSharedLibrary, + srcs, library.baseCompiler.deps)...) } return objFiles diff --git a/cc/ndk_library.go b/cc/ndk_library.go index 37ce2687b..6b0c325ed 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -227,12 +227,8 @@ func (c *stubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) a ) subdir := "" - srcs := []string{} - excludeSrcs := []string{} - extraSrcs := []android.Path{stubSrcPath} - extraDeps := []android.Path{} - return compileObjs(ctx, flags, subdir, srcs, excludeSrcs, - extraSrcs, extraDeps) + srcs := []android.Path{stubSrcPath} + return compileObjs(ctx, flagsToBuilderFlags(flags), subdir, srcs, nil) } func (linker *stubDecorator) linkerDeps(ctx BaseModuleContext, deps Deps) Deps { From 4f6fc9c1d81687914cc77bf06b10af8022f64198 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 26 Oct 2016 10:05:25 -0700 Subject: [PATCH 2/3] Add Path.Base helper Add Path.Base to return the basename of a Path object. Test: builds Change-Id: I2bf1519dc5039bedb1586e7ec18297a293e32e11 --- android/paths.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/android/paths.go b/android/paths.go index 35ed18010..56c3715f0 100644 --- a/android/paths.go +++ b/android/paths.go @@ -79,8 +79,11 @@ type Path interface { // Returns the path in string form String() string - // Returns the current file extension of the path + // Ext returns the extension of the last element of the path Ext() string + + // Base returns the last element of the path + Base() string } // WritablePath is a type of path that can be used as an output for build rules. @@ -284,6 +287,10 @@ func (p basePath) Ext() string { return filepath.Ext(p.path) } +func (p basePath) Base() string { + return filepath.Base(p.path) +} + // SourcePath is a Path representing a file path rooted from SrcDir type SourcePath struct { basePath From 0c461f1f6e2a1663b426eec2247e5a26f2822970 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 20 Oct 2016 16:11:43 -0700 Subject: [PATCH 3/3] Add support for .proto files .proto files are translated to .pb.cc and .pb.h files, which are then compiled normally. Bug: 32286026 Test: mmma -j system/extras/perfprofd Change-Id: I538071424d667aacf35b4b8bfebe217f5f092726 --- Android.bp | 1 + androidmk/cmd/androidmk/android.go | 1 + cc/builder.go | 1 + cc/cc.go | 1 + cc/compiler.go | 25 ++++-- cc/gen.go | 4 + cc/library.go | 15 ++++ cc/proto.go | 131 +++++++++++++++++++++++++++++ cc/util.go | 1 + 9 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 cc/proto.go diff --git a/Android.bp b/Android.bp index 88210983a..f9ba2c22b 100644 --- a/Android.bp +++ b/Android.bp @@ -129,6 +129,7 @@ bootstrap_go_package { "cc/gen.go", "cc/makevars.go", "cc/prebuilt.go", + "cc/proto.go", "cc/relocation_packer.go", "cc/sanitize.go", "cc/stl.go", diff --git a/androidmk/cmd/androidmk/android.go b/androidmk/cmd/androidmk/android.go index 6646c9db0..1fac4bf88 100644 --- a/androidmk/cmd/androidmk/android.go +++ b/androidmk/cmd/androidmk/android.go @@ -30,6 +30,7 @@ var standardProperties = map[string]struct { "LOCAL_CERTIFICATE": {"certificate", bpparser.StringType}, "LOCAL_PACKAGE_NAME": {"name", bpparser.StringType}, "LOCAL_MODULE_RELATIVE_PATH": {"relative_install_path", bpparser.StringType}, + "LOCAL_PROTOC_OPTIMIZE_TYPE": {"proto.type", bpparser.StringType}, // List properties "LOCAL_SRC_FILES_EXCLUDE": {"exclude_srcs", bpparser.ListType}, diff --git a/cc/builder.go b/cc/builder.go index b813783de..60aad0b49 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -173,6 +173,7 @@ type builderFlags struct { ldFlags string libFlags string yaccFlags string + protoFlags string toolchain config.Toolchain clang bool diff --git a/cc/cc.go b/cc/cc.go index 529a32972..33c9611c1 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -97,6 +97,7 @@ type Flags struct { ConlyFlags []string // Flags that apply to C source files CppFlags []string // Flags that apply to C++ source files YaccFlags []string // Flags that apply to Yacc source files + protoFlags []string // Flags that apply to proto source files LdFlags []string // Flags that apply to linker command lines libFlags []string // Flags to add libraries early to the link order diff --git a/cc/compiler.go b/cc/compiler.go index 32d98e4a8..0184ee9f4 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -100,6 +100,7 @@ func NewBaseCompiler() *baseCompiler { type baseCompiler struct { Properties BaseCompilerProperties + Proto ProtoProperties deps android.Paths } @@ -123,6 +124,10 @@ func (compiler *baseCompiler) compilerDeps(ctx BaseModuleContext, deps Deps) Dep deps.GeneratedSources = append(deps.GeneratedSources, compiler.Properties.Generated_sources...) deps.GeneratedHeaders = append(deps.GeneratedHeaders, compiler.Properties.Generated_headers...) + if compiler.hasProto() { + deps = protoDeps(ctx, deps, &compiler.Proto) + } + return deps } @@ -160,11 +165,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flag "${config.CommonNativehelperInclude}") } - flags.GlobalFlags = append(flags.GlobalFlags, []string{ - "-I" + android.PathForModuleSrc(ctx).String(), - "-I" + android.PathForModuleOut(ctx).String(), - "-I" + android.PathForModuleGen(ctx).String(), - }...) + flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.PathForModuleSrc(ctx).String()) } if ctx.sdk() { @@ -321,9 +322,23 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flag flags.CFlags = append(flags.CFlags, "-DANDROID_STRICT") } + if compiler.hasProto() { + flags = protoFlags(ctx, flags, &compiler.Proto) + } + return flags } +func (compiler *baseCompiler) hasProto() bool { + for _, src := range compiler.Properties.Srcs { + if filepath.Ext(src) == ".proto" { + return true + } + } + + return false +} + var gnuToCReplacer = strings.NewReplacer("gnu", "c") func ndkPathDeps(ctx ModuleContext) android.Paths { diff --git a/cc/gen.go b/cc/gen.go index b35eee5df..145189562 100644 --- a/cc/gen.go +++ b/cc/gen.go @@ -95,6 +95,10 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths, cppFile := android.GenPathWithExt(ctx, srcFile, "cpp") srcFiles[i] = cppFile genLex(ctx, srcFile, cppFile) + case ".proto": + cppFile, headerFile := genProto(ctx, srcFile, buildFlags.protoFlags) + srcFiles[i] = cppFile + deps = append(deps, headerFile) } } diff --git a/cc/library.go b/cc/library.go index d3d142e51..99a9b4886 100644 --- a/cc/library.go +++ b/cc/library.go @@ -55,6 +55,11 @@ type LibraryProperties struct { // rename host libraries to prevent overlap with system installed libraries Unique_host_soname *bool + Proto struct { + // export headers generated from .proto sources + Export_proto_headers bool + } + VariantName string `blueprint:"mutated"` // Build a static variant @@ -473,6 +478,16 @@ func (library *libraryDecorator) link(ctx ModuleContext, library.reexportFlags(deps.ReexportedFlags) library.reexportDeps(deps.ReexportedFlagsDeps) + if library.baseCompiler.hasProto() { + if library.Properties.Proto.Export_proto_headers { + library.reexportFlags([]string{ + "-I" + protoSubDir(ctx).String(), + "-I" + protoDir(ctx).String(), + }) + library.reexportDeps(library.baseCompiler.deps) // TODO: restrict to proto deps + } + } + return out } diff --git a/cc/proto.go b/cc/proto.go new file mode 100644 index 000000000..3d3ca59f7 --- /dev/null +++ b/cc/proto.go @@ -0,0 +1,131 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "strings" + + "github.com/google/blueprint" + + "android/soong/android" +) + +func init() { + pctx.HostBinToolVariable("protocCmd", "aprotoc") +} + +var ( + proto = pctx.AndroidStaticRule("protoc", + blueprint.RuleParams{ + Command: "$protocCmd --cpp_out=$outDir $protoFlags $in", + CommandDeps: []string{"$protocCmd"}, + Description: "protoc $out", + }, "protoFlags", "outDir") +) + +// TODO(ccross): protos are often used to communicate between multiple modules. If the only +// way to convert a proto to source is to reference it as a source file, and external modules cannot +// reference source files in other modules, then every module that owns a proto file will need to +// export a library for every type of external user (lite vs. full, c vs. c++ vs. java). It would +// be better to support a proto module type that exported a proto file along with some include dirs, +// and then external modules could depend on the proto module but use their own settings to +// generate the source. + +func genProto(ctx android.ModuleContext, protoFile android.Path, + protoFlags string) (android.ModuleGenPath, android.ModuleGenPath) { + + outDir := android.PathForModuleGen(ctx, "proto") + baseName := strings.TrimSuffix(protoFile.Base(), protoFile.Ext()) + + outFile := android.PathForModuleGen(ctx, "proto", ctx.ModuleDir(), baseName+".pb.cc") + headerFile := android.PathForModuleGen(ctx, "proto", ctx.ModuleDir(), baseName+".pb.h") + ctx.ModuleBuild(pctx, android.ModuleBuildParams{ + Rule: proto, + Outputs: android.WritablePaths{outFile, headerFile}, + Input: protoFile, + Args: map[string]string{ + "outDir": outDir.String(), + "protoFlags": protoFlags, + }, + }) + + return outFile, headerFile +} + +// protoDir returns the module's "gen/proto" directory +func protoDir(ctx android.ModuleContext) android.ModuleGenPath { + return android.PathForModuleGen(ctx, "proto") +} + +// protoSubDir returns the module's "gen/proto/path/to/module" directory +func protoSubDir(ctx android.ModuleContext) android.ModuleGenPath { + return android.PathForModuleGen(ctx, "proto", ctx.ModuleDir()) +} + +type ProtoProperties struct { + Proto struct { + // Proto generator type (full, lite) + Type string + // Link statically against the protobuf runtime + Static bool + } +} + +func protoDeps(ctx BaseModuleContext, deps Deps, p *ProtoProperties) Deps { + var lib string + var static bool + + switch p.Proto.Type { + case "full": + if ctx.sdk() { + lib = "libprotobuf-cpp-full-ndk" + static = true + } else { + lib = "libprotobuf-cpp-full" + } + case "lite", "": + if ctx.sdk() { + lib = "libprotobuf-cpp-lite-ndk" + static = true + } else { + lib = "libprotobuf-cpp-lite" + if p.Proto.Static { + static = true + } + } + default: + ctx.PropertyErrorf("proto.type", "unknown proto type %q", p.Proto.Type) + } + + if static { + deps.StaticLibs = append(deps.StaticLibs, lib) + deps.ReexportStaticLibHeaders = append(deps.ReexportStaticLibHeaders, lib) + } else { + deps.SharedLibs = append(deps.SharedLibs, lib) + deps.ReexportSharedLibHeaders = append(deps.ReexportSharedLibHeaders, lib) + } + + return deps +} + +func protoFlags(ctx ModuleContext, flags Flags, p *ProtoProperties) Flags { + flags.CFlags = append(flags.CFlags, "-DGOOGLE_PROTOBUF_NO_RTTI") + flags.GlobalFlags = append(flags.GlobalFlags, + "-I"+protoSubDir(ctx).String(), + "-I"+protoDir(ctx).String(), + ) + + return flags +} diff --git a/cc/util.go b/cc/util.go index 9ab74c3eb..07296b9e9 100644 --- a/cc/util.go +++ b/cc/util.go @@ -93,6 +93,7 @@ func flagsToBuilderFlags(in Flags) builderFlags { conlyFlags: strings.Join(in.ConlyFlags, " "), cppFlags: strings.Join(in.CppFlags, " "), yaccFlags: strings.Join(in.YaccFlags, " "), + protoFlags: strings.Join(in.protoFlags, " "), ldFlags: strings.Join(in.LdFlags, " "), libFlags: strings.Join(in.libFlags, " "), toolchain: in.Toolchain,