Merge changes from topics "prebuilt_build_tool", "prebuilt_build_tool_make"

* changes:
  Support per-module MakeVars
  Switch cc's use of bison and flex to prebuilt_build_tool
  Add prebuilt_build_tool to allow genrules to use prebuilt tools

android/Android.bp

@@ -39,6 +39,7 @@ bootstrap_go_package {
         "paths.go",
         "phony.go",
         "prebuilt.go",
+        "prebuilt_build_tool.go",
         "proto.go",
         "register.go",
         "rule_builder.go",

android/defs.go

@@ -69,7 +69,7 @@ var (
 	// A symlink rule.
 	Symlink = pctx.AndroidStaticRule("Symlink",
 		blueprint.RuleParams{
-			Command:     "ln -f -s $fromPath $out",
+			Command:     "rm -f $out && ln -f -s $fromPath $out",
 			Description: "symlink $out",
 		},
 		"fromPath")

android/filegroup.go

@@ -15,9 +15,7 @@
 package android
 
 import (
-	"io"
 	"strings"
-	"text/template"
 )
 
 func init() {
@@ -71,23 +69,8 @@ func (fg *fileGroup) Srcs() Paths {
 	return append(Paths{}, fg.srcs...)
 }
 
-var androidMkTemplate = template.Must(template.New("filegroup").Parse(`
+func (fg *fileGroup) MakeVars(ctx MakeVarsModuleContext) {
-ifdef {{.makeVar}}
-  $(error variable {{.makeVar}} set by soong module is already set in make)
-endif
-{{.makeVar}} := {{.value}}
-.KATI_READONLY := {{.makeVar}}
-`))
-
-func (fg *fileGroup) AndroidMk() AndroidMkData {
-	return AndroidMkData{
-		Custom: func(w io.Writer, name, prefix, moduleDir string, data AndroidMkData) {
 	if makeVar := String(fg.properties.Export_to_make_var); makeVar != "" {
-				androidMkTemplate.Execute(w, map[string]string{
+		ctx.StrictRaw(makeVar, strings.Join(fg.srcs.Strings(), " "))
-					"makeVar": makeVar,
-					"value":   strings.Join(fg.srcs.Strings(), " "),
-				})
-			}
-		},
 	}
 }

android/makevars.go

@@ -17,6 +17,7 @@ package android
 import (
 	"bytes"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -34,43 +35,16 @@ func androidMakeVarsProvider(ctx MakeVarsContext) {
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-// Interface for other packages to use to declare make variables
+
-type MakeVarsContext interface {
+// BaseMakeVarsContext contains the common functions for other packages to use
+// to declare make variables
+type BaseMakeVarsContext interface {
 	Config() Config
 	DeviceConfig() DeviceConfig
 	AddNinjaFileDeps(deps ...string)
 
-	ModuleName(module blueprint.Module) string
-	ModuleDir(module blueprint.Module) string
-	ModuleSubDir(module blueprint.Module) string
-	ModuleType(module blueprint.Module) string
-	BlueprintFile(module blueprint.Module) string
-
-	ModuleErrorf(module blueprint.Module, format string, args ...interface{})
-	Errorf(format string, args ...interface{})
 	Failed() bool
 
-	VisitAllModules(visit func(Module))
-	VisitAllModulesIf(pred func(Module) bool, visit func(Module))
-
-	// Verify the make variable matches the Soong version, fail the build
-	// if it does not. If the make variable is empty, just set it.
-	Strict(name, ninjaStr string)
-	// Check to see if the make variable matches the Soong version, warn if
-	// it does not. If the make variable is empty, just set it.
-	Check(name, ninjaStr string)
-
-	// These are equivalent to the above, but sort the make and soong
-	// variables before comparing them. They also show the unique entries
-	// in each list when displaying the difference, instead of the entire
-	// string.
-	StrictSorted(name, ninjaStr string)
-	CheckSorted(name, ninjaStr string)
-
-	// Evaluates a ninja string and returns the result. Used if more
-	// complicated modification needs to happen before giving it to Make.
-	Eval(ninjaStr string) (string, error)
-
 	// These are equivalent to Strict and Check, but do not attempt to
 	// evaluate the values before writing them to the Makefile. They can
 	// be used when all ninja variables have already been evaluated through
@@ -108,6 +82,48 @@ type MakeVarsContext interface {
 	DistForGoalsWithFilename(goals []string, path Path, filename string)
 }
 
+// MakeVarsContext contains the set of functions available for MakeVarsProvider
+// and SingletonMakeVarsProvider implementations.
+type MakeVarsContext interface {
+	BaseMakeVarsContext
+
+	ModuleName(module blueprint.Module) string
+	ModuleDir(module blueprint.Module) string
+	ModuleSubDir(module blueprint.Module) string
+	ModuleType(module blueprint.Module) string
+	BlueprintFile(module blueprint.Module) string
+
+	ModuleErrorf(module blueprint.Module, format string, args ...interface{})
+	Errorf(format string, args ...interface{})
+
+	VisitAllModules(visit func(Module))
+	VisitAllModulesIf(pred func(Module) bool, visit func(Module))
+
+	// Verify the make variable matches the Soong version, fail the build
+	// if it does not. If the make variable is empty, just set it.
+	Strict(name, ninjaStr string)
+	// Check to see if the make variable matches the Soong version, warn if
+	// it does not. If the make variable is empty, just set it.
+	Check(name, ninjaStr string)
+
+	// These are equivalent to the above, but sort the make and soong
+	// variables before comparing them. They also show the unique entries
+	// in each list when displaying the difference, instead of the entire
+	// string.
+	StrictSorted(name, ninjaStr string)
+	CheckSorted(name, ninjaStr string)
+
+	// Evaluates a ninja string and returns the result. Used if more
+	// complicated modification needs to happen before giving it to Make.
+	Eval(ninjaStr string) (string, error)
+}
+
+// MakeVarsModuleContext contains the set of functions available for modules
+// implementing the ModuleMakeVarsProvider interface.
+type MakeVarsModuleContext interface {
+	BaseMakeVarsContext
+}
+
 var _ PathContext = MakeVarsContext(nil)
 
 type MakeVarsProvider func(ctx MakeVarsContext)
@@ -135,6 +151,14 @@ func SingletonmakeVarsProviderAdapter(singleton SingletonMakeVarsProvider) MakeV
 	return func(ctx MakeVarsContext) { singleton.MakeVars(ctx) }
 }
 
+// ModuleMakeVarsProvider is a Module with an extra method to provide extra values to be exported to Make.
+type ModuleMakeVarsProvider interface {
+	Module
+
+	// MakeVars uses a MakeVarsModuleContext to provide extra values to be exported to Make.
+	MakeVars(ctx MakeVarsModuleContext)
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 
 func makeVarsSingletonFunc() Singleton {
@@ -209,10 +233,45 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) {
 		dists = append(dists, mctx.dists...)
 	}
 
+	ctx.VisitAllModules(func(m Module) {
+		if provider, ok := m.(ModuleMakeVarsProvider); ok {
+			mctx := &makeVarsContext{
+				SingletonContext: ctx,
+			}
+
+			provider.MakeVars(mctx)
+
+			vars = append(vars, mctx.vars...)
+			phonies = append(phonies, mctx.phonies...)
+			dists = append(dists, mctx.dists...)
+		}
+	})
+
 	if ctx.Failed() {
 		return
 	}
 
+	sort.Slice(vars, func(i, j int) bool {
+		return vars[i].name < vars[j].name
+	})
+	sort.Slice(phonies, func(i, j int) bool {
+		return phonies[i].name < phonies[j].name
+	})
+	lessArr := func(a, b []string) bool {
+		if len(a) == len(b) {
+			for i := range a {
+				if a[i] < b[i] {
+					return true
+				}
+			}
+			return false
+		}
+		return len(a) < len(b)
+	}
+	sort.Slice(dists, func(i, j int) bool {
+		return lessArr(dists[i].goals, dists[j].goals) || lessArr(dists[i].paths, dists[j].paths)
+	})
+
 	outBytes := s.writeVars(vars)
 
 	if err := pathtools.WriteFileIfChanged(outFile, outBytes, 0666); err != nil {

android/prebuilt_build_tool.go

@@ -0,0 +1,104 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+	"path"
+	"path/filepath"
+)
+
+func init() {
+	RegisterModuleType("prebuilt_build_tool", prebuiltBuildToolFactory)
+}
+
+type prebuiltBuildToolProperties struct {
+	// Source file to be executed for this build tool
+	Src *string `android:"path,arch_variant"`
+
+	// Extra files that should trigger rules using this tool to rebuild
+	Deps []string `android:"path,arch_variant"`
+
+	// Create a make variable with the specified name that contains the path to
+	// this prebuilt built tool, relative to the root of the source tree.
+	Export_to_make_var *string
+}
+
+type prebuiltBuildTool struct {
+	ModuleBase
+	prebuilt Prebuilt
+
+	properties prebuiltBuildToolProperties
+
+	toolPath OptionalPath
+}
+
+func (t *prebuiltBuildTool) Name() string {
+	return t.prebuilt.Name(t.ModuleBase.Name())
+}
+
+func (t *prebuiltBuildTool) Prebuilt() *Prebuilt {
+	return &t.prebuilt
+}
+
+func (t *prebuiltBuildTool) DepsMutator(ctx BottomUpMutatorContext) {
+	if t.properties.Src == nil {
+		ctx.PropertyErrorf("src", "missing prebuilt source file")
+	}
+}
+
+func (t *prebuiltBuildTool) GenerateAndroidBuildActions(ctx ModuleContext) {
+	sourcePath := t.prebuilt.SingleSourcePath(ctx)
+	installedPath := PathForModuleOut(ctx, t.ModuleBase.Name())
+	deps := PathsForModuleSrc(ctx, t.properties.Deps)
+
+	relPath, err := filepath.Rel(path.Dir(installedPath.String()), sourcePath.String())
+	if err != nil {
+		ctx.ModuleErrorf("Unabled to generate symlink between %q and %q: %s", installedPath.String(), sourcePath.String())
+	}
+
+	ctx.Build(pctx, BuildParams{
+		Rule:      Symlink,
+		Output:    installedPath,
+		Input:     sourcePath,
+		Implicits: deps,
+		Args: map[string]string{
+			"fromPath": relPath,
+		},
+	})
+
+	t.toolPath = OptionalPathForPath(installedPath)
+}
+
+func (t *prebuiltBuildTool) HostToolPath() OptionalPath {
+	return t.toolPath
+}
+
+func (t *prebuiltBuildTool) MakeVars(ctx MakeVarsModuleContext) {
+	if makeVar := String(t.properties.Export_to_make_var); makeVar != "" {
+		ctx.StrictRaw(makeVar, t.toolPath.String())
+	}
+}
+
+var _ HostToolProvider = &prebuiltBuildTool{}
+
+// prebuilt_build_tool is to declare prebuilts to be used during the build, particularly for use
+// in genrules with the "tools" property.
+func prebuiltBuildToolFactory() Module {
+	module := &prebuiltBuildTool{}
+	module.AddProperties(&module.properties)
+	InitSingleSourcePrebuiltModule(module, &module.properties, "Src")
+	InitAndroidArchModule(module, HostSupportedNoCross, MultilibFirst)
+	return module
+}

cc/cc.go

@@ -116,6 +116,8 @@ type Deps struct {
 	// Used for host bionic
 	LinkerFlagsFile string
 	DynamicLinker   string
+
+	Tools []string
 }
 
 type PathDeps struct {
@@ -158,6 +160,8 @@ type PathDeps struct {
 
 	// Path to the dynamic linker binary
 	DynamicLinker android.OptionalPath
+
+	Tools map[string]android.Path
 }
 
 // LocalOrGlobalFlags contains flags that need to have values set globally by the build system or locally by the module
@@ -425,6 +429,12 @@ type xref interface {
 	XrefCcFiles() android.Paths
 }
 
+type ToolDependencyTag struct {
+	blueprint.BaseDependencyTag
+
+	Name string
+}
+
 var (
 	dataLibDepTag         = DependencyTag{Name: "data_lib", Library: true, Shared: true}
 	sharedExportDepTag    = DependencyTag{Name: "shared", Library: true, Shared: true, ReexportFlags: true}
@@ -1694,6 +1704,7 @@ func (c *Module) deps(ctx DepsContext) Deps {
 	deps.LateSharedLibs = android.LastUniqueStrings(deps.LateSharedLibs)
 	deps.HeaderLibs = android.LastUniqueStrings(deps.HeaderLibs)
 	deps.RuntimeLibs = android.LastUniqueStrings(deps.RuntimeLibs)
+	deps.Tools = android.LastUniqueStrings(deps.Tools)
 
 	for _, lib := range deps.ReexportSharedLibHeaders {
 		if !inList(lib, deps.SharedLibs) {
@@ -2037,6 +2048,11 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) {
 			}, vndkExtDepTag, vndkdep.getVndkExtendsModuleName())
 		}
 	}
+
+	for _, tool := range deps.Tools {
+		actx.AddFarVariationDependencies(ctx.Config().BuildOSTarget.Variations(),
+			ToolDependencyTag{Name: tool}, tool)
+	}
 }
 
 func BeginMutator(ctx android.BottomUpMutatorContext) {
@@ -2219,6 +2235,21 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps {
 		depName := ctx.OtherModuleName(dep)
 		depTag := ctx.OtherModuleDependencyTag(dep)
 
+		if toolDep, ok := depTag.(ToolDependencyTag); ok {
+			if toolMod, ok := dep.(android.HostToolProvider); ok {
+				if depPaths.Tools == nil {
+					depPaths.Tools = make(map[string]android.Path)
+				}
+				toolPath := toolMod.HostToolPath()
+				if !toolPath.Valid() {
+					ctx.ModuleErrorf("Failed to find path for host tool %q", toolDep.Name)
+				}
+				depPaths.Tools[toolDep.Name] = toolPath.Path()
+			} else {
+				ctx.ModuleErrorf("Found module, but not host tool for %q", toolDep.Name)
+			}
+		}
+
 		ccDep, ok := dep.(LinkableInterface)
 		if !ok {
 

cc/compiler.go

@@ -251,6 +251,14 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps {
 		deps.StaticLibs = append(deps.StaticLibs, "libomp")
 	}
 
+	if compiler.hasSrcExt(".y") || compiler.hasSrcExt(".yy") {
+		deps.Tools = append(deps.Tools, "bison", "m4")
+	}
+
+	if compiler.hasSrcExt(".l") || compiler.hasSrcExt(".ll") {
+		deps.Tools = append(deps.Tools, "flex", "m4")
+	}
+
 	return deps
 }
 
@@ -581,7 +589,7 @@ func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathD
 
 	srcs := append(android.Paths(nil), compiler.srcsBeforeGen...)
 
-	srcs, genDeps := genSources(ctx, srcs, buildFlags)
+	srcs, genDeps := genSources(ctx, srcs, buildFlags, deps.Tools)
 	pathDeps = append(pathDeps, genDeps...)
 
 	compiler.pathDeps = pathDeps

cc/gen.go

@@ -24,20 +24,11 @@ import (
 )
 
 func init() {
-	pctx.SourcePathVariable("lexCmd", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/flex")
-	pctx.SourcePathVariable("m4Cmd", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/m4")
-
 	pctx.HostBinToolVariable("aidlCmd", "aidl-cpp")
 	pctx.HostBinToolVariable("syspropCmd", "sysprop_cpp")
 }
 
 var (
-	lex = pctx.AndroidStaticRule("lex",
-		blueprint.RuleParams{
-			Command:     "M4=$m4Cmd $lexCmd -o$out $in",
-			CommandDeps: []string{"$lexCmd", "$m4Cmd"},
-		})
-
 	sysprop = pctx.AndroidStaticRule("sysprop",
 		blueprint.RuleParams{
 			Command: "$syspropCmd --header-dir=$headerOutDir --public-header-dir=$publicOutDir " +
@@ -66,7 +57,8 @@ type YaccProperties struct {
 }
 
 func genYacc(ctx android.ModuleContext, rule *android.RuleBuilder, yaccFile android.Path,
-	outFile android.ModuleGenPath, props *YaccProperties) (headerFiles android.Paths) {
+	outFile android.ModuleGenPath, props *YaccProperties,
+	tools map[string]android.Path) (headerFiles android.Paths) {
 
 	outDir := android.PathForModuleGen(ctx, "yacc")
 	headerFile := android.GenPathWithExt(ctx, "yacc", yaccFile, "h")
@@ -97,9 +89,17 @@ func genYacc(ctx android.ModuleContext, rule *android.RuleBuilder, yaccFile andr
 		}
 	}
 
-	cmd.Text("BISON_PKGDATADIR=prebuilts/build-tools/common/bison").
+	bison, ok := tools["bison"]
-		FlagWithInput("M4=", ctx.Config().PrebuiltBuildTool(ctx, "m4")).
+	if !ok {
-		PrebuiltBuildTool(ctx, "bison").
+		ctx.ModuleErrorf("Unable to find bison")
+	}
+	m4, ok := tools["m4"]
+	if !ok {
+		ctx.ModuleErrorf("Unable to find m4")
+	}
+
+	cmd.FlagWithInput("M4=", m4).
+		Tool(bison).
 		Flag("-d").
 		Flags(flags).
 		FlagWithOutput("--defines=", headerFile).
@@ -153,13 +153,23 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr
 	}
 }
 
-func genLex(ctx android.ModuleContext, lexFile android.Path, outFile android.ModuleGenPath) {
+func genLex(ctx android.ModuleContext, rule *android.RuleBuilder, lexFile android.Path,
-	ctx.Build(pctx, android.BuildParams{
+	outFile android.ModuleGenPath, tools map[string]android.Path) {
-		Rule:        lex,
+
-		Description: "lex " + lexFile.Rel(),
+	flex, ok := tools["flex"]
-		Output:      outFile,
+	if !ok {
-		Input:       lexFile,
+		ctx.ModuleErrorf("Unable to find flex")
-	})
+	}
+	m4, ok := tools["m4"]
+	if !ok {
+		ctx.ModuleErrorf("Unable to find m4")
+	}
+
+	rule.Command().
+		FlagWithInput("M4=", m4).
+		Tool(flex).
+		FlagWithOutput("-o", outFile).
+		Input(lexFile)
 }
 
 func genSysprop(ctx android.ModuleContext, syspropFile android.Path) (android.Path, android.Paths) {
@@ -206,14 +216,22 @@ func genWinMsg(ctx android.ModuleContext, srcFile android.Path, flags builderFla
 	return rcFile, headerFile
 }
 
-func genSources(ctx android.ModuleContext, srcFiles android.Paths,
+func genSources(ctx android.ModuleContext, srcFiles android.Paths, buildFlags builderFlags,
-	buildFlags builderFlags) (android.Paths, android.Paths) {
+	tools map[string]android.Path) (android.Paths, android.Paths) {
 
 	var deps android.Paths
 	var rsFiles android.Paths
 
 	var aidlRule *android.RuleBuilder
 
+	var lexRule_ *android.RuleBuilder
+	lexRule := func() *android.RuleBuilder {
+		if lexRule_ == nil {
+			lexRule_ = android.NewRuleBuilder().Sbox(android.PathForModuleGen(ctx, "lex"))
+		}
+		return lexRule_
+	}
+
 	var yaccRule_ *android.RuleBuilder
 	yaccRule := func() *android.RuleBuilder {
 		if yaccRule_ == nil {
@@ -227,19 +245,19 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths,
 		case ".y":
 			cFile := android.GenPathWithExt(ctx, "yacc", srcFile, "c")
 			srcFiles[i] = cFile
-			deps = append(deps, genYacc(ctx, yaccRule(), srcFile, cFile, buildFlags.yacc)...)
+			deps = append(deps, genYacc(ctx, yaccRule(), srcFile, cFile, buildFlags.yacc, tools)...)
 		case ".yy":
 			cppFile := android.GenPathWithExt(ctx, "yacc", srcFile, "cpp")
 			srcFiles[i] = cppFile
-			deps = append(deps, genYacc(ctx, yaccRule(), srcFile, cppFile, buildFlags.yacc)...)
+			deps = append(deps, genYacc(ctx, yaccRule(), srcFile, cppFile, buildFlags.yacc, tools)...)
 		case ".l":
 			cFile := android.GenPathWithExt(ctx, "lex", srcFile, "c")
 			srcFiles[i] = cFile
-			genLex(ctx, srcFile, cFile)
+			genLex(ctx, lexRule(), srcFile, cFile, tools)
 		case ".ll":
 			cppFile := android.GenPathWithExt(ctx, "lex", srcFile, "cpp")
 			srcFiles[i] = cppFile
-			genLex(ctx, srcFile, cppFile)
+			genLex(ctx, lexRule(), srcFile, cppFile, tools)
 		case ".proto":
 			ccFile, headerFile := genProto(ctx, srcFile, buildFlags)
 			srcFiles[i] = ccFile
@@ -271,6 +289,10 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths,
 		aidlRule.Build(pctx, ctx, "aidl", "gen aidl")
 	}
 
+	if lexRule_ != nil {
+		lexRule_.Build(pctx, ctx, "lex", "gen lex")
+	}
+
 	if yaccRule_ != nil {
 		yaccRule_.Build(pctx, ctx, "yacc", "gen yacc")
 	}

cc/makevars.go

@@ -148,8 +148,6 @@ func makeVarsProvider(ctx android.MakeVarsContext) {
 	ctx.Strict("AIDL_CPP", "${aidlCmd}")
 	ctx.Strict("ALLOWED_MANUAL_INTERFACE_PATHS", strings.Join(allowedManualInterfacePaths, " "))
 
-	ctx.Strict("M4", "${m4Cmd}")
-
 	ctx.Strict("RS_GLOBAL_INCLUDES", "${config.RsGlobalIncludes}")
 
 	ctx.Strict("SOONG_STRIP_PATH", "${stripPath}")