Merge "Refactor mixed builds to only take one pass" am: 489128b8ef

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/2094705

Change-Id: I635d17c1de4c6e0d0ae3e093b01f6445fe214c1d
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Christopher Parsons
2022-05-20 15:52:31 +00:00
committed by Automerger Merge Worker
14 changed files with 383 additions and 285 deletions

View File

@@ -115,6 +115,27 @@ type Bazelable interface {
SetBaseModuleType(baseModuleType string)
}
// MixedBuildBuildable is an interface that module types should implement in order
// to be "handled by Bazel" in a mixed build.
type MixedBuildBuildable interface {
// IsMixedBuildSupported returns true if and only if this module should be
// "handled by Bazel" in a mixed build.
// This "escape hatch" allows modules with corner-case scenarios to opt out
// of being built with Bazel.
IsMixedBuildSupported(ctx BaseModuleContext) bool
// QueueBazelCall invokes request-queueing functions on the BazelContext
// so that these requests are handled when Bazel's cquery is invoked.
QueueBazelCall(ctx BaseModuleContext)
// ProcessBazelQueryResponse uses Bazel information (obtained from the BazelContext)
// to set module fields and providers to propagate this module's metadata upstream.
// This effectively "bridges the gap" between Bazel and Soong in a mixed build.
// Soong modules depending on this module should be oblivious to the fact that
// this module was handled by Bazel.
ProcessBazelQueryResponse(ctx ModuleContext)
}
// BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules.
type BazelModule interface {
Module
@@ -342,7 +363,7 @@ func shouldKeepExistingBuildFileForDir(allowlist bp2BuildConversionAllowlist, di
// converted or handcrafted Bazel target. As a side effect, calling this
// method will also log whether this module is mixed build enabled for
// metrics reporting.
func MixedBuildsEnabled(ctx ModuleContext) bool {
func MixedBuildsEnabled(ctx BaseModuleContext) bool {
mixedBuildEnabled := mixedBuildPossible(ctx)
ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled)
return mixedBuildEnabled
@@ -350,7 +371,7 @@ func MixedBuildsEnabled(ctx ModuleContext) bool {
// mixedBuildPossible returns true if a module is ready to be replaced by a
// converted or handcrafted Bazel target.
func mixedBuildPossible(ctx ModuleContext) bool {
func mixedBuildPossible(ctx BaseModuleContext) bool {
if ctx.Os() == Windows {
// Windows toolchains are not currently supported.
return false

View File

@@ -33,6 +33,26 @@ import (
"android/soong/bazel"
)
func init() {
RegisterMixedBuildsMutator(InitRegistrationContext)
}
func RegisterMixedBuildsMutator(ctx RegistrationContext) {
ctx.PostDepsMutators(func(ctx RegisterMutatorsContext) {
ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel()
})
}
func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) {
if m := ctx.Module(); m.Enabled() {
if mixedBuildMod, ok := m.(MixedBuildBuildable); ok {
if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) {
mixedBuildMod.QueueBazelCall(ctx)
}
}
}
}
type cqueryRequest interface {
// Name returns a string name for this request type. Such request type names must be unique,
// and must only consist of alphanumeric characters.
@@ -62,33 +82,32 @@ type cqueryKey struct {
configKey configKey
}
// bazelHandler is the interface for a helper object related to deferring to Bazel for
// processing a module (during Bazel mixed builds). Individual module types should define
// their own bazel handler if they support deferring to Bazel.
type BazelHandler interface {
// Issue query to Bazel to retrieve information about Bazel's view of the current module.
// If Bazel returns this information, set module properties on the current module to reflect
// the returned information.
// Returns true if information was available from Bazel, false if bazel invocation still needs to occur.
GenerateBazelBuildActions(ctx ModuleContext, label string) bool
}
// BazelContext is a context object useful for interacting with Bazel during
// the course of a build. Use of Bazel to evaluate part of the build graph
// is referred to as a "mixed build". (Some modules are managed by Soong,
// some are managed by Bazel). To facilitate interop between these build
// subgraphs, Soong may make requests to Bazel and evaluate their responses
// so that Soong modules may accurately depend on Bazel targets.
type BazelContext interface {
// The methods below involve queuing cquery requests to be later invoked
// by bazel. If any of these methods return (_, false), then the request
// has been queued to be run later.
// Add a cquery request to the bazel request queue. All queued requests
// will be sent to Bazel on a subsequent invocation of InvokeBazel.
QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey)
// ** Cquery Results Retrieval Functions
// The below functions pertain to retrieving cquery results from a prior
// InvokeBazel function call and parsing the results.
// Returns result files built by building the given bazel target label.
GetOutputFiles(label string, cfgKey configKey) ([]string, bool)
GetOutputFiles(label string, cfgKey configKey) ([]string, error)
// TODO(cparsons): Other cquery-related methods should be added here.
// Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order).
GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error)
GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error)
// Returns the executable binary resultant from building together the python sources
GetPythonBinary(label string, cfgKey configKey) (string, bool)
// TODO(b/232976601): Remove.
GetPythonBinary(label string, cfgKey configKey) (string, error)
// ** End cquery methods
// ** end Cquery Results Retrieval Functions
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext.
@@ -153,19 +172,23 @@ type MockBazelContext struct {
LabelToPythonBinary map[string]string
}
func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
result, ok := m.LabelToOutputFiles[label]
return result, ok
func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
panic("unimplemented")
}
func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
result, ok := m.LabelToCcInfo[label]
return result, ok, nil
func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
result, _ := m.LabelToOutputFiles[label]
return result, nil
}
func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
result, ok := m.LabelToPythonBinary[label]
return result, ok
func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
result, _ := m.LabelToCcInfo[label]
return result, nil
}
func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
result, _ := m.LabelToPythonBinary[label]
return result, nil
}
func (m MockBazelContext) InvokeBazel() error {
@@ -188,46 +211,53 @@ func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset {
var _ BazelContext = MockBazelContext{}
func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
rawString, ok := bazelCtx.cquery(label, cquery.GetOutputFiles, cfgKey)
var ret []string
if ok {
func (bazelCtx *bazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
key := cqueryKey{label, requestType, cfgKey}
bazelCtx.requestMutex.Lock()
defer bazelCtx.requestMutex.Unlock()
bazelCtx.requests[key] = true
}
func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
key := cqueryKey{label, cquery.GetOutputFiles, cfgKey}
if rawString, ok := bazelCtx.results[key]; ok {
bazelOutput := strings.TrimSpace(rawString)
ret = cquery.GetOutputFiles.ParseResult(bazelOutput)
return cquery.GetOutputFiles.ParseResult(bazelOutput), nil
}
return ret, ok
return nil, fmt.Errorf("no bazel response found for %v", key)
}
func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
result, ok := bazelCtx.cquery(label, cquery.GetCcInfo, cfgKey)
if !ok {
return cquery.CcInfo{}, ok, nil
}
bazelOutput := strings.TrimSpace(result)
ret, err := cquery.GetCcInfo.ParseResult(bazelOutput)
return ret, ok, err
}
func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
rawString, ok := bazelCtx.cquery(label, cquery.GetPythonBinary, cfgKey)
var ret string
if ok {
func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
key := cqueryKey{label, cquery.GetCcInfo, cfgKey}
if rawString, ok := bazelCtx.results[key]; ok {
bazelOutput := strings.TrimSpace(rawString)
ret = cquery.GetPythonBinary.ParseResult(bazelOutput)
return cquery.GetCcInfo.ParseResult(bazelOutput)
}
return ret, ok
return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key)
}
func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
key := cqueryKey{label, cquery.GetPythonBinary, cfgKey}
if rawString, ok := bazelCtx.results[key]; ok {
bazelOutput := strings.TrimSpace(rawString)
return cquery.GetPythonBinary.ParseResult(bazelOutput), nil
}
return "", fmt.Errorf("no bazel response found for %v", key)
}
func (n noopBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
panic("unimplemented")
}
func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
panic("unimplemented")
}
func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
panic("unimplemented")
}
func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
panic("unimplemented")
}
@@ -314,24 +344,6 @@ func (context *bazelContext) BazelEnabled() bool {
return true
}
// Adds a cquery request to the Bazel request queue, to be later invoked, or
// returns the result of the given request if the request was already made.
// If the given request was already made (and the results are available), then
// returns (result, true). If the request is queued but no results are available,
// then returns ("", false).
func (context *bazelContext) cquery(label string, requestType cqueryRequest,
cfgKey configKey) (string, bool) {
key := cqueryKey{label, requestType, cfgKey}
if result, ok := context.results[key]; ok {
return result, true
} else {
context.requestMutex.Lock()
defer context.requestMutex.Unlock()
context.requests[key] = true
return "", false
}
}
func pwdPrefix() string {
// Darwin doesn't have /proc
if runtime.GOOS != "darwin" {
@@ -916,7 +928,7 @@ func getConfigString(key cqueryKey) string {
return arch + "|" + os
}
func GetConfigKey(ctx ModuleContext) configKey {
func GetConfigKey(ctx BaseModuleContext) configKey {
return configKey{
// use string because Arch is not a valid key in go
arch: ctx.Arch().String(),

View File

@@ -5,6 +5,8 @@ import (
"path/filepath"
"reflect"
"testing"
"android/soong/bazel/cquery"
)
func TestRequestResultsAfterInvokeBazel(t *testing.T) {
@@ -13,17 +15,14 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) {
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{
bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot, 2)"}: `//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`,
})
g, ok := bazelContext.GetOutputFiles(label, cfg)
if ok {
t.Errorf("Did not expect cquery results prior to running InvokeBazel(), but got %s", g)
}
bazelContext.QueueBazelRequest(label, cquery.GetOutputFiles, cfg)
err := bazelContext.InvokeBazel()
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}
g, ok = bazelContext.GetOutputFiles(label, cfg)
if !ok {
t.Errorf("Expected cquery results after running InvokeBazel(), but got none")
g, err := bazelContext.GetOutputFiles(label, cfg)
if err != nil {
t.Errorf("Expected cquery results after running InvokeBazel(), but got err %v", err)
} else if w := []string{"out/foo/bar.txt"}; !reflect.DeepEqual(w, g) {
t.Errorf("Expected output %s, got %s", w, g)
}

View File

@@ -2047,7 +2047,7 @@ func (c *config) UseHostMusl() bool {
return Bool(c.productVariables.HostMusl)
}
func (c *config) LogMixedBuild(ctx ModuleContext, useBazel bool) {
func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) {
moduleName := ctx.Module().Name()
c.mixedBuildsLock.Lock()
defer c.mixedBuildsLock.Unlock()

View File

@@ -18,6 +18,7 @@ import (
"strings"
"android/soong/bazel"
"android/soong/bazel/cquery"
"github.com/google/blueprint"
)
@@ -101,6 +102,7 @@ type fileGroup struct {
srcs Paths
}
var _ MixedBuildBuildable = (*fileGroup)(nil)
var _ SourceFileProducer = (*fileGroup)(nil)
// filegroup contains a list of files that are referenced by other modules
@@ -114,42 +116,11 @@ func FileGroupFactory() Module {
return module
}
func (fg *fileGroup) maybeGenerateBazelBuildActions(ctx ModuleContext) {
if !MixedBuildsEnabled(ctx) {
return
}
archVariant := ctx.Arch().String()
osVariant := ctx.Os()
if len(fg.Srcs()) == 1 && fg.Srcs()[0].Base() == fg.Name() {
// This will be a regular file target, not filegroup, in Bazel.
// See FilegroupBp2Build for more information.
archVariant = Common.String()
osVariant = CommonOS
}
bazelCtx := ctx.Config().BazelContext
filePaths, ok := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{archVariant, osVariant})
if !ok {
return
}
bazelOuts := make(Paths, 0, len(filePaths))
for _, p := range filePaths {
src := PathForBazelOut(ctx, p)
bazelOuts = append(bazelOuts, src)
}
fg.srcs = bazelOuts
}
func (fg *fileGroup) GenerateAndroidBuildActions(ctx ModuleContext) {
fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs)
if fg.properties.Path != nil {
fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path))
}
fg.maybeGenerateBazelBuildActions(ctx)
}
func (fg *fileGroup) Srcs() Paths {
@@ -161,3 +132,38 @@ func (fg *fileGroup) MakeVars(ctx MakeVarsModuleContext) {
ctx.StrictRaw(makeVar, strings.Join(fg.srcs.Strings(), " "))
}
}
func (fg *fileGroup) QueueBazelCall(ctx BaseModuleContext) {
bazelCtx := ctx.Config().BazelContext
bazelCtx.QueueBazelRequest(
fg.GetBazelLabel(ctx, fg),
cquery.GetOutputFiles,
configKey{Common.String(), CommonOS})
}
func (fg *fileGroup) IsMixedBuildSupported(ctx BaseModuleContext) bool {
return true
}
func (fg *fileGroup) ProcessBazelQueryResponse(ctx ModuleContext) {
fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs)
if fg.properties.Path != nil {
fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path))
}
bazelCtx := ctx.Config().BazelContext
filePaths, err := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{Common.String(), CommonOS})
if err != nil {
ctx.ModuleErrorf(err.Error())
return
}
bazelOuts := make(Paths, 0, len(filePaths))
for _, p := range filePaths {
src := PathForBazelOut(ctx, p)
bazelOuts = append(bazelOuts, src)
}
fg.srcs = bazelOuts
}

View File

@@ -2275,7 +2275,11 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
return
}
m.module.GenerateAndroidBuildActions(ctx)
if mixedBuildMod, handled := m.isHandledByBazel(ctx); handled {
mixedBuildMod.ProcessBazelQueryResponse(ctx)
} else {
m.module.GenerateAndroidBuildActions(ctx)
}
if ctx.Failed() {
return
}
@@ -2331,6 +2335,18 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
m.variables = ctx.variables
}
func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) {
if !ctx.Config().BazelContext.BazelEnabled() {
return nil, false
}
if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok {
if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) {
return mixedBuildMod, true
}
}
return nil, false
}
// Check the supplied dist structure to make sure that it is valid.
//
// property - the base property, e.g. dist or dists[1], which is combined with the

View File

@@ -17,6 +17,7 @@ package cc
import (
"path/filepath"
"android/soong/bazel/cquery"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
@@ -562,25 +563,32 @@ func (binary *binaryDecorator) verifyHostBionicLinker(ctx ModuleContext, in, lin
}
type ccBinaryBazelHandler struct {
android.BazelHandler
BazelHandler
module *Module
}
func (handler *ccBinaryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
func (handler *ccBinaryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
if ok {
if len(filePaths) != 1 {
ctx.ModuleErrorf("expected exactly one output file for '%s', but got %s", label, filePaths)
return false
}
outputFilePath := android.PathForBazelOut(ctx, filePaths[0])
handler.module.outputFile = android.OptionalPathForPath(outputFilePath)
// TODO(b/220164721): We need to decide if we should return the stripped as the unstripped.
handler.module.linker.(*binaryDecorator).unstrippedOutputFile = outputFilePath
bazelCtx.QueueBazelRequest(label, cquery.GetOutputFiles, android.GetConfigKey(ctx))
}
func (handler *ccBinaryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
if err != nil {
ctx.ModuleErrorf(err.Error())
return
}
return ok
if len(filePaths) != 1 {
ctx.ModuleErrorf("expected exactly one output file for '%s', but got %s", label, filePaths)
return
}
outputFilePath := android.PathForBazelOut(ctx, filePaths[0])
handler.module.outputFile = android.OptionalPathForPath(outputFilePath)
// TODO(b/220164721): We need to decide if we should return the stripped as the unstripped.
handler.module.linker.(*binaryDecorator).unstrippedOutputFile = outputFilePath
}
func binaryBp2build(ctx android.TopDownMutatorContext, m *Module, typ string) {

View File

@@ -772,6 +772,19 @@ func IsTestPerSrcDepTag(depTag blueprint.DependencyTag) bool {
return ok && ccDepTag == testPerSrcDepTag
}
// bazelHandler is the interface for a helper object related to deferring to Bazel for
// processing a cc module (during Bazel mixed builds). Individual module types should define
// their own bazel handler if they support being handled by Bazel.
type BazelHandler interface {
// QueueBazelCall invokes request-queueing functions on the BazelContext
//so that these requests are handled when Bazel's cquery is invoked.
QueueBazelCall(ctx android.BaseModuleContext, label string)
// ProcessBazelQueryResponse uses information retrieved from Bazel to set properties
// on the current module with given label.
ProcessBazelQueryResponse(ctx android.ModuleContext, label string)
}
// Module contains the properties and members used by all C/C++ module types, and implements
// the blueprint.Module interface. It delegates to compiler, linker, and installer interfaces
// to construct the output file. Behavior can be customized with a Customizer, or "decorator",
@@ -811,7 +824,7 @@ type Module struct {
compiler compiler
linker linker
installer installer
bazelHandler android.BazelHandler
bazelHandler BazelHandler
features []feature
stl *stl
@@ -1773,31 +1786,51 @@ func GetSubnameProperty(actx android.ModuleContext, c LinkableInterface) string
return subName
}
// Returns true if Bazel was successfully used for the analysis of this module.
func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool {
var bazelModuleLabel string
var _ android.MixedBuildBuildable = (*Module)(nil)
func (c *Module) getBazelModuleLabel(ctx android.BaseModuleContext) string {
if c.typ() == fullLibrary && c.static() {
// cc_library is a special case in bp2build; two targets are generated -- one for each
// of the shared and static variants. The shared variant keeps the module name, but the
// static variant uses a different suffixed name.
bazelModuleLabel = bazelLabelForStaticModule(actx, c)
} else {
bazelModuleLabel = c.GetBazelLabel(actx, c)
return bazelLabelForStaticModule(ctx, c)
}
return c.GetBazelLabel(ctx, c)
}
func (c *Module) QueueBazelCall(ctx android.BaseModuleContext) {
c.bazelHandler.QueueBazelCall(ctx, c.getBazelModuleLabel(ctx))
}
func (c *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
return c.bazelHandler != nil
}
func (c *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) {
bazelModuleLabel := c.getBazelModuleLabel(ctx)
c.bazelHandler.ProcessBazelQueryResponse(ctx, bazelModuleLabel)
c.Properties.SubName = GetSubnameProperty(ctx, c)
apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo)
if !apexInfo.IsForPlatform() {
c.hideApexVariantFromMake = true
}
bazelActionsUsed := false
// Mixed builds mode is disabled for modules outside of device OS.
// TODO(b/200841190): Support non-device OS in mixed builds.
if android.MixedBuildsEnabled(actx) && c.bazelHandler != nil {
bazelActionsUsed = c.bazelHandler.GenerateBazelBuildActions(actx, bazelModuleLabel)
c.makeLinkType = GetMakeLinkType(ctx, c)
mctx := &moduleContext{
ModuleContext: ctx,
moduleContextImpl: moduleContextImpl{
mod: c,
},
}
return bazelActionsUsed
mctx.ctx = mctx
c.maybeInstall(mctx, apexInfo)
}
func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
// TODO(cparsons): Any logic in this method occurring prior to querying Bazel should be
// requested from Bazel instead.
// Handle the case of a test module split by `test_per_src` mutator.
//
// The `test_per_src` mutator adds an extra variation named "", depending on all the other
@@ -1824,11 +1857,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
}
ctx.ctx = ctx
if c.maybeGenerateBazelActions(actx) {
c.maybeInstall(ctx, apexInfo)
return
}
deps := c.depsToPaths(ctx)
if ctx.Failed() {
return

View File

@@ -642,18 +642,18 @@ type libraryDecorator struct {
}
type ccLibraryBazelHandler struct {
android.BazelHandler
BazelHandler
module *Module
}
// generateStaticBazelBuildActions constructs the StaticLibraryInfo Soong
// provider from a Bazel shared library's CcInfo provider.
func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool {
func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) {
rootStaticArchives := ccInfo.RootStaticArchives
if len(rootStaticArchives) != 1 {
ctx.ModuleErrorf("expected exactly one root archive file for '%s', but got %s", label, rootStaticArchives)
return false
return
}
outputFilePath := android.PathForBazelOut(ctx, rootStaticArchives[0])
handler.module.outputFile = android.OptionalPathForPath(outputFilePath)
@@ -679,17 +679,17 @@ func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx androi
Build(),
})
return true
return
}
// generateSharedBazelBuildActions constructs the SharedLibraryInfo Soong
// provider from a Bazel shared library's CcInfo provider.
func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool {
func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) {
rootDynamicLibraries := ccInfo.RootDynamicLibraries
if len(rootDynamicLibraries) != 1 {
ctx.ModuleErrorf("expected exactly one root dynamic library file for '%s', but got %s", label, rootDynamicLibraries)
return false
return
}
outputFilePath := android.PathForBazelOut(ctx, rootDynamicLibraries[0])
handler.module.outputFile = android.OptionalPathForPath(outputFilePath)
@@ -709,30 +709,27 @@ func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx androi
// TODO(b/190524881): Include transitive static libraries in this provider to support
// static libraries with deps. The provider key for this is TransitiveStaticLibrariesForOrdering.
})
return true
}
func (handler *ccLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
func (handler *ccLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx))
}
func (handler *ccLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
if err != nil {
ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err)
return false
}
if !ok {
return ok
return
}
if handler.module.static() {
if ok := handler.generateStaticBazelBuildActions(ctx, label, ccInfo); !ok {
return false
}
handler.generateStaticBazelBuildActions(ctx, label, ccInfo)
} else if handler.module.Shared() {
if ok := handler.generateSharedBazelBuildActions(ctx, label, ccInfo); !ok {
return false
}
handler.generateSharedBazelBuildActions(ctx, label, ccInfo)
} else {
return false
ctx.ModuleErrorf("Unhandled bazel case for %s (neither shared nor static!)", ctx.ModuleName())
}
handler.module.linker.(*libraryDecorator).setFlagExporterInfoFromCcInfo(ctx, ccInfo)
@@ -746,7 +743,6 @@ func (handler *ccLibraryBazelHandler) GenerateBazelBuildActions(ctx android.Modu
// implementation.
i.(*libraryDecorator).collectedSnapshotHeaders = android.Paths{}
}
return ok
}
func (library *libraryDecorator) setFlagExporterInfoFromCcInfo(ctx android.ModuleContext, ccInfo cquery.CcInfo) {

View File

@@ -17,6 +17,7 @@ package cc
import (
"android/soong/android"
"android/soong/bazel"
"android/soong/bazel/cquery"
)
func init() {
@@ -47,28 +48,30 @@ func RegisterLibraryHeadersBuildComponents(ctx android.RegistrationContext) {
ctx.RegisterModuleType("cc_prebuilt_library_headers", prebuiltLibraryHeaderFactory)
}
type libraryHeaderBazelHander struct {
android.BazelHandler
type libraryHeaderBazelHandler struct {
BazelHandler
module *Module
library *libraryDecorator
}
func (h *libraryHeaderBazelHander) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
func (handler *libraryHeaderBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx))
}
func (h *libraryHeaderBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
if err != nil {
ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err)
return false
}
if !ok {
return false
ctx.ModuleErrorf(err.Error())
return
}
outputPaths := ccInfo.OutputFiles
if len(outputPaths) != 1 {
ctx.ModuleErrorf("expected exactly one output file for %q, but got %q", label, outputPaths)
return false
return
}
outputPath := android.PathForBazelOut(ctx, outputPaths[0])
@@ -83,8 +86,6 @@ func (h *libraryHeaderBazelHander) GenerateBazelBuildActions(ctx android.ModuleC
// validation will fail. For now, set this to an empty list.
// TODO(cparsons): More closely mirror the collectHeadersForSnapshot implementation.
h.library.collectedSnapshotHeaders = android.Paths{}
return true
}
// cc_library_headers contains a set of c/c++ headers which are imported by
@@ -96,7 +97,7 @@ func LibraryHeaderFactory() android.Module {
library.HeaderOnly()
module.sdkMemberTypes = []android.SdkMemberType{headersLibrarySdkMemberType}
module.bazelable = true
module.bazelHandler = &libraryHeaderBazelHander{module: module, library: library}
module.bazelHandler = &libraryHeaderBazelHandler{module: module, library: library}
return module.Init()
}

View File

@@ -19,6 +19,7 @@ import (
"android/soong/android"
"android/soong/bazel"
"android/soong/bazel/cquery"
)
//
@@ -46,23 +47,30 @@ type objectLinker struct {
}
type objectBazelHandler struct {
android.BazelHandler
BazelHandler
module *Module
}
func (handler *objectBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
func (handler *objectBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
objPaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
if ok {
if len(objPaths) != 1 {
ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths)
return false
}
bazelCtx.QueueBazelRequest(label, cquery.GetOutputFiles, android.GetConfigKey(ctx))
}
handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0]))
func (handler *objectBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
objPaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
if err != nil {
ctx.ModuleErrorf(err.Error())
return
}
return ok
if len(objPaths) != 1 {
ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths)
return
}
handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0]))
}
type ObjectLinkerProperties struct {

View File

@@ -20,6 +20,7 @@ import (
"android/soong/android"
"android/soong/bazel"
"android/soong/bazel/cquery"
)
func init() {
@@ -406,25 +407,28 @@ type prebuiltObjectLinker struct {
}
type prebuiltStaticLibraryBazelHandler struct {
android.BazelHandler
BazelHandler
module *Module
library *libraryDecorator
}
func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
func (h *prebuiltStaticLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx))
}
func (h *prebuiltStaticLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
if err != nil {
ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err)
}
if !ok {
return false
ctx.ModuleErrorf(err.Error())
return
}
staticLibs := ccInfo.CcStaticLibraryFiles
if len(staticLibs) > 1 {
ctx.ModuleErrorf("expected 1 static library from bazel target %q, got %s", label, staticLibs)
return false
return
}
// TODO(b/184543518): cc_prebuilt_library_static may have properties for re-exporting flags
@@ -439,7 +443,7 @@ func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx androi
if len(staticLibs) == 0 {
h.module.outputFile = android.OptionalPath{}
return true
return
}
out := android.PathForBazelOut(ctx, staticLibs[0])
@@ -451,30 +455,31 @@ func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx androi
TransitiveStaticLibrariesForOrdering: depSet,
})
return true
}
type prebuiltSharedLibraryBazelHandler struct {
android.BazelHandler
BazelHandler
module *Module
library *libraryDecorator
}
func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
func (h *prebuiltSharedLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx))
}
func (h *prebuiltSharedLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
if err != nil {
ctx.ModuleErrorf("Error getting Bazel CcInfo for %s: %s", label, err)
}
if !ok {
return false
ctx.ModuleErrorf(err.Error())
return
}
sharedLibs := ccInfo.CcSharedLibraryFiles
if len(sharedLibs) != 1 {
ctx.ModuleErrorf("expected 1 shared library from bazel target %s, got %q", label, sharedLibs)
return false
return
}
// TODO(b/184543518): cc_prebuilt_library_shared may have properties for re-exporting flags
@@ -489,7 +494,7 @@ func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx androi
if len(sharedLibs) == 0 {
h.module.outputFile = android.OptionalPath{}
return true
return
}
out := android.PathForBazelOut(ctx, sharedLibs[0])
@@ -514,8 +519,6 @@ func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx androi
h.library.setFlagExporterInfoFromCcInfo(ctx, ccInfo)
h.module.maybeUnhideFromMake()
return true
}
func (p *prebuiltObjectLinker) prebuilt() *android.Prebuilt {

View File

@@ -129,44 +129,26 @@ func newConfig(availableEnv map[string]string) android.Config {
return configuration
}
// Bazel-enabled mode. Soong runs in two passes.
// First pass: Analyze the build tree, but only store all bazel commands
// needed to correctly evaluate the tree in the second pass.
// TODO(cparsons): Don't output any ninja file, as the second pass will overwrite
// the incorrect results from the first pass, and file I/O is expensive.
func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) {
firstCtx.EventHandler.Begin("mixed_build")
defer firstCtx.EventHandler.End("mixed_build")
// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a
// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata
// for modules that should be handled by Bazel.
func runMixedModeBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) {
ctx.EventHandler.Begin("mixed_build")
defer ctx.EventHandler.End("mixed_build")
firstCtx.EventHandler.Begin("prepare")
bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration)
firstCtx.EventHandler.End("prepare")
firstCtx.EventHandler.Begin("bazel")
// Invoke bazel commands and save results for second pass.
if err := configuration.BazelContext.InvokeBazel(); err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
bazelHook := func() error {
ctx.EventHandler.Begin("bazel")
defer ctx.EventHandler.End("bazel")
return configuration.BazelContext.InvokeBazel()
}
// Second pass: Full analysis, using the bazel command results. Output ninja file.
secondConfig, err := android.ConfigForAdditionalRun(configuration)
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
}
firstCtx.EventHandler.End("bazel")
ctx.SetBeforePrepareBuildActionsHook(bazelHook)
secondCtx := newContext(secondConfig)
secondCtx.EventHandler = firstCtx.EventHandler
secondCtx.EventHandler.Begin("analyze")
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig)
ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
secondCtx.EventHandler.End("analyze")
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, ctx.Context, configuration)
globListFiles := writeBuildGlobsNinjaFile(secondCtx, configuration.SoongOutDir(), configuration)
globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration)
ninjaDeps = append(ninjaDeps, globListFiles...)
writeDepFile(cmdlineArgs.OutFile, *secondCtx.EventHandler, ninjaDeps)
writeDepFile(cmdlineArgs.OutFile, *ctx.EventHandler, ninjaDeps)
}
// Run the code-generation phase to convert BazelTargetModules to BUILD files.

View File

@@ -25,6 +25,7 @@ import (
"strconv"
"strings"
"android/soong/bazel/cquery"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/proptools"
@@ -189,6 +190,8 @@ type Module struct {
modulePaths []string
}
var _ android.MixedBuildBuildable = (*Module)(nil)
type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask
type generateTask struct {
@@ -249,27 +252,36 @@ func toolDepsMutator(ctx android.BottomUpMutatorContext) {
}
}
// Returns true if information was available from Bazel, false if bazel invocation still needs to occur.
func (c *Module) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
func (g *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) {
g.generateCommonBuildActions(ctx)
label := g.GetBazelLabel(ctx, g)
bazelCtx := ctx.Config().BazelContext
filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
if ok {
var bazelOutputFiles android.Paths
exportIncludeDirs := map[string]bool{}
for _, bazelOutputFile := range filePaths {
bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOut(ctx, bazelOutputFile))
exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true
}
c.outputFiles = bazelOutputFiles
c.outputDeps = bazelOutputFiles
for includePath, _ := range exportIncludeDirs {
c.exportedIncludeDirs = append(c.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath))
}
filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
if err != nil {
ctx.ModuleErrorf(err.Error())
return
}
var bazelOutputFiles android.Paths
exportIncludeDirs := map[string]bool{}
for _, bazelOutputFile := range filePaths {
bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOut(ctx, bazelOutputFile))
exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true
}
g.outputFiles = bazelOutputFiles
g.outputDeps = bazelOutputFiles
for includePath, _ := range exportIncludeDirs {
g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath))
}
return ok
}
func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// generateCommonBuildActions contains build action generation logic
// common to both the mixed build case and the legacy case of genrule processing.
// To fully support genrule in mixed builds, the contents of this function should
// approach zero; there should be no genrule action registration done directly
// by Soong logic in the mixed-build case.
func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) {
g.subName = ctx.ModuleSubDir()
// Collect the module directory for IDE info in java/jdeps.go.
@@ -575,31 +587,37 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
}
g.outputFiles = outputFiles.Paths()
}
bazelModuleLabel := g.GetBazelLabel(ctx, g)
bazelActionsUsed := false
if android.MixedBuildsEnabled(ctx) {
bazelActionsUsed = g.GenerateBazelBuildActions(ctx, bazelModuleLabel)
}
if !bazelActionsUsed {
// For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of
// the genrules on AOSP. That will make things simpler to look at the graph in the common
// case. For larger sets of outputs, inject a phony target in between to limit ninja file
// growth.
if len(g.outputFiles) <= 6 {
g.outputDeps = g.outputFiles
} else {
phonyFile := android.PathForModuleGen(ctx, "genrule-phony")
ctx.Build(pctx, android.BuildParams{
Rule: blueprint.Phony,
Output: phonyFile,
Inputs: g.outputFiles,
})
g.outputDeps = android.Paths{phonyFile}
}
func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
g.generateCommonBuildActions(ctx)
// For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of
// the genrules on AOSP. That will make things simpler to look at the graph in the common
// case. For larger sets of outputs, inject a phony target in between to limit ninja file
// growth.
if len(g.outputFiles) <= 6 {
g.outputDeps = g.outputFiles
} else {
phonyFile := android.PathForModuleGen(ctx, "genrule-phony")
ctx.Build(pctx, android.BuildParams{
Rule: blueprint.Phony,
Output: phonyFile,
Inputs: g.outputFiles,
})
g.outputDeps = android.Paths{phonyFile}
}
}
func (g *Module) QueueBazelCall(ctx android.BaseModuleContext) {
bazelCtx := ctx.Config().BazelContext
bazelCtx.QueueBazelRequest(g.GetBazelLabel(ctx, g), cquery.GetOutputFiles, android.GetConfigKey(ctx))
}
func (g *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
return true
}
// Collect information for opening IDE project files in java/jdeps.go.
func (g *Module) IDEInfo(dpInfo *android.IdeInfo) {
dpInfo.Srcs = append(dpInfo.Srcs, g.Srcs().Strings()...)