diff --git a/android/bazel.go b/android/bazel.go index 67002ecf6..378eb6f29 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -115,6 +115,27 @@ type Bazelable interface { SetBaseModuleType(baseModuleType string) } +// MixedBuildBuildable is an interface that module types should implement in order +// to be "handled by Bazel" in a mixed build. +type MixedBuildBuildable interface { + // IsMixedBuildSupported returns true if and only if this module should be + // "handled by Bazel" in a mixed build. + // This "escape hatch" allows modules with corner-case scenarios to opt out + // of being built with Bazel. + IsMixedBuildSupported(ctx BaseModuleContext) bool + + // QueueBazelCall invokes request-queueing functions on the BazelContext + // so that these requests are handled when Bazel's cquery is invoked. + QueueBazelCall(ctx BaseModuleContext) + + // ProcessBazelQueryResponse uses Bazel information (obtained from the BazelContext) + // to set module fields and providers to propagate this module's metadata upstream. + // This effectively "bridges the gap" between Bazel and Soong in a mixed build. + // Soong modules depending on this module should be oblivious to the fact that + // this module was handled by Bazel. + ProcessBazelQueryResponse(ctx ModuleContext) +} + // BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules. type BazelModule interface { Module @@ -342,7 +363,7 @@ func shouldKeepExistingBuildFileForDir(allowlist bp2BuildConversionAllowlist, di // converted or handcrafted Bazel target. As a side effect, calling this // method will also log whether this module is mixed build enabled for // metrics reporting. -func MixedBuildsEnabled(ctx ModuleContext) bool { +func MixedBuildsEnabled(ctx BaseModuleContext) bool { mixedBuildEnabled := mixedBuildPossible(ctx) ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled) return mixedBuildEnabled @@ -350,7 +371,7 @@ func MixedBuildsEnabled(ctx ModuleContext) bool { // mixedBuildPossible returns true if a module is ready to be replaced by a // converted or handcrafted Bazel target. -func mixedBuildPossible(ctx ModuleContext) bool { +func mixedBuildPossible(ctx BaseModuleContext) bool { if ctx.Os() == Windows { // Windows toolchains are not currently supported. return false diff --git a/android/bazel_handler.go b/android/bazel_handler.go index c4eb0f318..4d9423a3c 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -33,6 +33,26 @@ import ( "android/soong/bazel" ) +func init() { + RegisterMixedBuildsMutator(InitRegistrationContext) +} + +func RegisterMixedBuildsMutator(ctx RegistrationContext) { + ctx.PostDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() + }) +} + +func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) { + if m := ctx.Module(); m.Enabled() { + if mixedBuildMod, ok := m.(MixedBuildBuildable); ok { + if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) { + mixedBuildMod.QueueBazelCall(ctx) + } + } + } +} + type cqueryRequest interface { // Name returns a string name for this request type. Such request type names must be unique, // and must only consist of alphanumeric characters. @@ -62,33 +82,32 @@ type cqueryKey struct { configKey configKey } -// bazelHandler is the interface for a helper object related to deferring to Bazel for -// processing a module (during Bazel mixed builds). Individual module types should define -// their own bazel handler if they support deferring to Bazel. -type BazelHandler interface { - // Issue query to Bazel to retrieve information about Bazel's view of the current module. - // If Bazel returns this information, set module properties on the current module to reflect - // the returned information. - // Returns true if information was available from Bazel, false if bazel invocation still needs to occur. - GenerateBazelBuildActions(ctx ModuleContext, label string) bool -} - +// BazelContext is a context object useful for interacting with Bazel during +// the course of a build. Use of Bazel to evaluate part of the build graph +// is referred to as a "mixed build". (Some modules are managed by Soong, +// some are managed by Bazel). To facilitate interop between these build +// subgraphs, Soong may make requests to Bazel and evaluate their responses +// so that Soong modules may accurately depend on Bazel targets. type BazelContext interface { - // The methods below involve queuing cquery requests to be later invoked - // by bazel. If any of these methods return (_, false), then the request - // has been queued to be run later. + // Add a cquery request to the bazel request queue. All queued requests + // will be sent to Bazel on a subsequent invocation of InvokeBazel. + QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) + + // ** Cquery Results Retrieval Functions + // The below functions pertain to retrieving cquery results from a prior + // InvokeBazel function call and parsing the results. // Returns result files built by building the given bazel target label. - GetOutputFiles(label string, cfgKey configKey) ([]string, bool) + GetOutputFiles(label string, cfgKey configKey) ([]string, error) - // TODO(cparsons): Other cquery-related methods should be added here. // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order). - GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) + GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) // Returns the executable binary resultant from building together the python sources - GetPythonBinary(label string, cfgKey configKey) (string, bool) + // TODO(b/232976601): Remove. + GetPythonBinary(label string, cfgKey configKey) (string, error) - // ** End cquery methods + // ** end Cquery Results Retrieval Functions // Issues commands to Bazel to receive results for all cquery requests // queued in the BazelContext. @@ -153,19 +172,23 @@ type MockBazelContext struct { LabelToPythonBinary map[string]string } -func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) { - result, ok := m.LabelToOutputFiles[label] - return result, ok +func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { + panic("unimplemented") } -func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) { - result, ok := m.LabelToCcInfo[label] - return result, ok, nil +func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { + result, _ := m.LabelToOutputFiles[label] + return result, nil } -func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) { - result, ok := m.LabelToPythonBinary[label] - return result, ok +func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { + result, _ := m.LabelToCcInfo[label] + return result, nil +} + +func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { + result, _ := m.LabelToPythonBinary[label] + return result, nil } func (m MockBazelContext) InvokeBazel() error { @@ -188,46 +211,53 @@ func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset { var _ BazelContext = MockBazelContext{} -func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) { - rawString, ok := bazelCtx.cquery(label, cquery.GetOutputFiles, cfgKey) - var ret []string - if ok { +func (bazelCtx *bazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { + key := cqueryKey{label, requestType, cfgKey} + bazelCtx.requestMutex.Lock() + defer bazelCtx.requestMutex.Unlock() + bazelCtx.requests[key] = true +} + +func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { + key := cqueryKey{label, cquery.GetOutputFiles, cfgKey} + if rawString, ok := bazelCtx.results[key]; ok { bazelOutput := strings.TrimSpace(rawString) - ret = cquery.GetOutputFiles.ParseResult(bazelOutput) + return cquery.GetOutputFiles.ParseResult(bazelOutput), nil } - return ret, ok + return nil, fmt.Errorf("no bazel response found for %v", key) } -func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) { - result, ok := bazelCtx.cquery(label, cquery.GetCcInfo, cfgKey) - if !ok { - return cquery.CcInfo{}, ok, nil - } - - bazelOutput := strings.TrimSpace(result) - ret, err := cquery.GetCcInfo.ParseResult(bazelOutput) - return ret, ok, err -} - -func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) { - rawString, ok := bazelCtx.cquery(label, cquery.GetPythonBinary, cfgKey) - var ret string - if ok { +func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { + key := cqueryKey{label, cquery.GetCcInfo, cfgKey} + if rawString, ok := bazelCtx.results[key]; ok { bazelOutput := strings.TrimSpace(rawString) - ret = cquery.GetPythonBinary.ParseResult(bazelOutput) + return cquery.GetCcInfo.ParseResult(bazelOutput) } - return ret, ok + return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key) } -func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) { +func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { + key := cqueryKey{label, cquery.GetPythonBinary, cfgKey} + if rawString, ok := bazelCtx.results[key]; ok { + bazelOutput := strings.TrimSpace(rawString) + return cquery.GetPythonBinary.ParseResult(bazelOutput), nil + } + return "", fmt.Errorf("no bazel response found for %v", key) +} + +func (n noopBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { panic("unimplemented") } -func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) { +func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { panic("unimplemented") } -func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) { +func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { + panic("unimplemented") +} + +func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { panic("unimplemented") } @@ -314,24 +344,6 @@ func (context *bazelContext) BazelEnabled() bool { return true } -// Adds a cquery request to the Bazel request queue, to be later invoked, or -// returns the result of the given request if the request was already made. -// If the given request was already made (and the results are available), then -// returns (result, true). If the request is queued but no results are available, -// then returns ("", false). -func (context *bazelContext) cquery(label string, requestType cqueryRequest, - cfgKey configKey) (string, bool) { - key := cqueryKey{label, requestType, cfgKey} - if result, ok := context.results[key]; ok { - return result, true - } else { - context.requestMutex.Lock() - defer context.requestMutex.Unlock() - context.requests[key] = true - return "", false - } -} - func pwdPrefix() string { // Darwin doesn't have /proc if runtime.GOOS != "darwin" { @@ -916,7 +928,7 @@ func getConfigString(key cqueryKey) string { return arch + "|" + os } -func GetConfigKey(ctx ModuleContext) configKey { +func GetConfigKey(ctx BaseModuleContext) configKey { return configKey{ // use string because Arch is not a valid key in go arch: ctx.Arch().String(), diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go index e5cff9031..cfdccd7c3 100644 --- a/android/bazel_handler_test.go +++ b/android/bazel_handler_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "reflect" "testing" + + "android/soong/bazel/cquery" ) func TestRequestResultsAfterInvokeBazel(t *testing.T) { @@ -13,17 +15,14 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) { bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot, 2)"}: `//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`, }) - g, ok := bazelContext.GetOutputFiles(label, cfg) - if ok { - t.Errorf("Did not expect cquery results prior to running InvokeBazel(), but got %s", g) - } + bazelContext.QueueBazelRequest(label, cquery.GetOutputFiles, cfg) err := bazelContext.InvokeBazel() if err != nil { t.Fatalf("Did not expect error invoking Bazel, but got %s", err) } - g, ok = bazelContext.GetOutputFiles(label, cfg) - if !ok { - t.Errorf("Expected cquery results after running InvokeBazel(), but got none") + g, err := bazelContext.GetOutputFiles(label, cfg) + if err != nil { + t.Errorf("Expected cquery results after running InvokeBazel(), but got err %v", err) } else if w := []string{"out/foo/bar.txt"}; !reflect.DeepEqual(w, g) { t.Errorf("Expected output %s, got %s", w, g) } diff --git a/android/config.go b/android/config.go index d69521747..9e96c6a50 100644 --- a/android/config.go +++ b/android/config.go @@ -2047,7 +2047,7 @@ func (c *config) UseHostMusl() bool { return Bool(c.productVariables.HostMusl) } -func (c *config) LogMixedBuild(ctx ModuleContext, useBazel bool) { +func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) { moduleName := ctx.Module().Name() c.mixedBuildsLock.Lock() defer c.mixedBuildsLock.Unlock() diff --git a/android/filegroup.go b/android/filegroup.go index 1bf5e07c2..485e0b955 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -18,6 +18,7 @@ import ( "strings" "android/soong/bazel" + "android/soong/bazel/cquery" "github.com/google/blueprint" ) @@ -101,6 +102,7 @@ type fileGroup struct { srcs Paths } +var _ MixedBuildBuildable = (*fileGroup)(nil) var _ SourceFileProducer = (*fileGroup)(nil) // filegroup contains a list of files that are referenced by other modules @@ -114,42 +116,11 @@ func FileGroupFactory() Module { return module } -func (fg *fileGroup) maybeGenerateBazelBuildActions(ctx ModuleContext) { - if !MixedBuildsEnabled(ctx) { - return - } - - archVariant := ctx.Arch().String() - osVariant := ctx.Os() - if len(fg.Srcs()) == 1 && fg.Srcs()[0].Base() == fg.Name() { - // This will be a regular file target, not filegroup, in Bazel. - // See FilegroupBp2Build for more information. - archVariant = Common.String() - osVariant = CommonOS - } - - bazelCtx := ctx.Config().BazelContext - filePaths, ok := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{archVariant, osVariant}) - if !ok { - return - } - - bazelOuts := make(Paths, 0, len(filePaths)) - for _, p := range filePaths { - src := PathForBazelOut(ctx, p) - bazelOuts = append(bazelOuts, src) - } - - fg.srcs = bazelOuts -} - func (fg *fileGroup) GenerateAndroidBuildActions(ctx ModuleContext) { fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs) if fg.properties.Path != nil { fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path)) } - - fg.maybeGenerateBazelBuildActions(ctx) } func (fg *fileGroup) Srcs() Paths { @@ -161,3 +132,38 @@ func (fg *fileGroup) MakeVars(ctx MakeVarsModuleContext) { ctx.StrictRaw(makeVar, strings.Join(fg.srcs.Strings(), " ")) } } + +func (fg *fileGroup) QueueBazelCall(ctx BaseModuleContext) { + bazelCtx := ctx.Config().BazelContext + + bazelCtx.QueueBazelRequest( + fg.GetBazelLabel(ctx, fg), + cquery.GetOutputFiles, + configKey{Common.String(), CommonOS}) +} + +func (fg *fileGroup) IsMixedBuildSupported(ctx BaseModuleContext) bool { + return true +} + +func (fg *fileGroup) ProcessBazelQueryResponse(ctx ModuleContext) { + fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs) + if fg.properties.Path != nil { + fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path)) + } + + bazelCtx := ctx.Config().BazelContext + filePaths, err := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{Common.String(), CommonOS}) + if err != nil { + ctx.ModuleErrorf(err.Error()) + return + } + + bazelOuts := make(Paths, 0, len(filePaths)) + for _, p := range filePaths { + src := PathForBazelOut(ctx, p) + bazelOuts = append(bazelOuts, src) + } + + fg.srcs = bazelOuts +} diff --git a/android/module.go b/android/module.go index ab68e24f1..97a46ddec 100644 --- a/android/module.go +++ b/android/module.go @@ -2275,7 +2275,11 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) return } - m.module.GenerateAndroidBuildActions(ctx) + if mixedBuildMod, handled := m.isHandledByBazel(ctx); handled { + mixedBuildMod.ProcessBazelQueryResponse(ctx) + } else { + m.module.GenerateAndroidBuildActions(ctx) + } if ctx.Failed() { return } @@ -2331,6 +2335,18 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) m.variables = ctx.variables } +func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) { + if !ctx.Config().BazelContext.BazelEnabled() { + return nil, false + } + if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok { + if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) { + return mixedBuildMod, true + } + } + return nil, false +} + // Check the supplied dist structure to make sure that it is valid. // // property - the base property, e.g. dist or dists[1], which is combined with the diff --git a/cc/binary.go b/cc/binary.go index c5017c1dd..7b5591aaf 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -17,6 +17,7 @@ package cc import ( "path/filepath" + "android/soong/bazel/cquery" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -562,25 +563,32 @@ func (binary *binaryDecorator) verifyHostBionicLinker(ctx ModuleContext, in, lin } type ccBinaryBazelHandler struct { - android.BazelHandler + BazelHandler module *Module } -func (handler *ccBinaryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (handler *ccBinaryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) - if ok { - if len(filePaths) != 1 { - ctx.ModuleErrorf("expected exactly one output file for '%s', but got %s", label, filePaths) - return false - } - outputFilePath := android.PathForBazelOut(ctx, filePaths[0]) - handler.module.outputFile = android.OptionalPathForPath(outputFilePath) - // TODO(b/220164721): We need to decide if we should return the stripped as the unstripped. - handler.module.linker.(*binaryDecorator).unstrippedOutputFile = outputFilePath + bazelCtx.QueueBazelRequest(label, cquery.GetOutputFiles, android.GetConfigKey(ctx)) +} + +func (handler *ccBinaryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) + if err != nil { + ctx.ModuleErrorf(err.Error()) + return } - return ok + + if len(filePaths) != 1 { + ctx.ModuleErrorf("expected exactly one output file for '%s', but got %s", label, filePaths) + return + } + outputFilePath := android.PathForBazelOut(ctx, filePaths[0]) + handler.module.outputFile = android.OptionalPathForPath(outputFilePath) + // TODO(b/220164721): We need to decide if we should return the stripped as the unstripped. + handler.module.linker.(*binaryDecorator).unstrippedOutputFile = outputFilePath } func binaryBp2build(ctx android.TopDownMutatorContext, m *Module, typ string) { diff --git a/cc/cc.go b/cc/cc.go index 86069208c..3d21f632e 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -772,6 +772,19 @@ func IsTestPerSrcDepTag(depTag blueprint.DependencyTag) bool { return ok && ccDepTag == testPerSrcDepTag } +// bazelHandler is the interface for a helper object related to deferring to Bazel for +// processing a cc module (during Bazel mixed builds). Individual module types should define +// their own bazel handler if they support being handled by Bazel. +type BazelHandler interface { + // QueueBazelCall invokes request-queueing functions on the BazelContext + //so that these requests are handled when Bazel's cquery is invoked. + QueueBazelCall(ctx android.BaseModuleContext, label string) + + // ProcessBazelQueryResponse uses information retrieved from Bazel to set properties + // on the current module with given label. + ProcessBazelQueryResponse(ctx android.ModuleContext, label string) +} + // Module contains the properties and members used by all C/C++ module types, and implements // the blueprint.Module interface. It delegates to compiler, linker, and installer interfaces // to construct the output file. Behavior can be customized with a Customizer, or "decorator", @@ -811,7 +824,7 @@ type Module struct { compiler compiler linker linker installer installer - bazelHandler android.BazelHandler + bazelHandler BazelHandler features []feature stl *stl @@ -1773,31 +1786,51 @@ func GetSubnameProperty(actx android.ModuleContext, c LinkableInterface) string return subName } -// Returns true if Bazel was successfully used for the analysis of this module. -func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool { - var bazelModuleLabel string +var _ android.MixedBuildBuildable = (*Module)(nil) + +func (c *Module) getBazelModuleLabel(ctx android.BaseModuleContext) string { if c.typ() == fullLibrary && c.static() { // cc_library is a special case in bp2build; two targets are generated -- one for each // of the shared and static variants. The shared variant keeps the module name, but the // static variant uses a different suffixed name. - bazelModuleLabel = bazelLabelForStaticModule(actx, c) - } else { - bazelModuleLabel = c.GetBazelLabel(actx, c) + return bazelLabelForStaticModule(ctx, c) + } + return c.GetBazelLabel(ctx, c) +} + +func (c *Module) QueueBazelCall(ctx android.BaseModuleContext) { + c.bazelHandler.QueueBazelCall(ctx, c.getBazelModuleLabel(ctx)) +} + +func (c *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool { + return c.bazelHandler != nil +} + +func (c *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) { + bazelModuleLabel := c.getBazelModuleLabel(ctx) + + c.bazelHandler.ProcessBazelQueryResponse(ctx, bazelModuleLabel) + + c.Properties.SubName = GetSubnameProperty(ctx, c) + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) + if !apexInfo.IsForPlatform() { + c.hideApexVariantFromMake = true } - bazelActionsUsed := false - // Mixed builds mode is disabled for modules outside of device OS. - // TODO(b/200841190): Support non-device OS in mixed builds. - if android.MixedBuildsEnabled(actx) && c.bazelHandler != nil { - bazelActionsUsed = c.bazelHandler.GenerateBazelBuildActions(actx, bazelModuleLabel) + c.makeLinkType = GetMakeLinkType(ctx, c) + + mctx := &moduleContext{ + ModuleContext: ctx, + moduleContextImpl: moduleContextImpl{ + mod: c, + }, } - return bazelActionsUsed + mctx.ctx = mctx + + c.maybeInstall(mctx, apexInfo) } func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { - // TODO(cparsons): Any logic in this method occurring prior to querying Bazel should be - // requested from Bazel instead. - // Handle the case of a test module split by `test_per_src` mutator. // // The `test_per_src` mutator adds an extra variation named "", depending on all the other @@ -1824,11 +1857,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { } ctx.ctx = ctx - if c.maybeGenerateBazelActions(actx) { - c.maybeInstall(ctx, apexInfo) - return - } - deps := c.depsToPaths(ctx) if ctx.Failed() { return diff --git a/cc/library.go b/cc/library.go index fdbbccbe7..57465292d 100644 --- a/cc/library.go +++ b/cc/library.go @@ -642,18 +642,18 @@ type libraryDecorator struct { } type ccLibraryBazelHandler struct { - android.BazelHandler + BazelHandler module *Module } // generateStaticBazelBuildActions constructs the StaticLibraryInfo Soong // provider from a Bazel shared library's CcInfo provider. -func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool { +func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) { rootStaticArchives := ccInfo.RootStaticArchives if len(rootStaticArchives) != 1 { ctx.ModuleErrorf("expected exactly one root archive file for '%s', but got %s", label, rootStaticArchives) - return false + return } outputFilePath := android.PathForBazelOut(ctx, rootStaticArchives[0]) handler.module.outputFile = android.OptionalPathForPath(outputFilePath) @@ -679,17 +679,17 @@ func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx androi Build(), }) - return true + return } // generateSharedBazelBuildActions constructs the SharedLibraryInfo Soong // provider from a Bazel shared library's CcInfo provider. -func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool { +func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) { rootDynamicLibraries := ccInfo.RootDynamicLibraries if len(rootDynamicLibraries) != 1 { ctx.ModuleErrorf("expected exactly one root dynamic library file for '%s', but got %s", label, rootDynamicLibraries) - return false + return } outputFilePath := android.PathForBazelOut(ctx, rootDynamicLibraries[0]) handler.module.outputFile = android.OptionalPathForPath(outputFilePath) @@ -709,30 +709,27 @@ func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx androi // TODO(b/190524881): Include transitive static libraries in this provider to support // static libraries with deps. The provider key for this is TransitiveStaticLibrariesForOrdering. }) - return true } -func (handler *ccLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (handler *ccLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) + bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx)) +} + +func (handler *ccLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) if err != nil { ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err) - return false - } - if !ok { - return ok + return } if handler.module.static() { - if ok := handler.generateStaticBazelBuildActions(ctx, label, ccInfo); !ok { - return false - } + handler.generateStaticBazelBuildActions(ctx, label, ccInfo) } else if handler.module.Shared() { - if ok := handler.generateSharedBazelBuildActions(ctx, label, ccInfo); !ok { - return false - } + handler.generateSharedBazelBuildActions(ctx, label, ccInfo) } else { - return false + ctx.ModuleErrorf("Unhandled bazel case for %s (neither shared nor static!)", ctx.ModuleName()) } handler.module.linker.(*libraryDecorator).setFlagExporterInfoFromCcInfo(ctx, ccInfo) @@ -746,7 +743,6 @@ func (handler *ccLibraryBazelHandler) GenerateBazelBuildActions(ctx android.Modu // implementation. i.(*libraryDecorator).collectedSnapshotHeaders = android.Paths{} } - return ok } func (library *libraryDecorator) setFlagExporterInfoFromCcInfo(ctx android.ModuleContext, ccInfo cquery.CcInfo) { diff --git a/cc/library_headers.go b/cc/library_headers.go index 41ebcc766..6fd956803 100644 --- a/cc/library_headers.go +++ b/cc/library_headers.go @@ -17,6 +17,7 @@ package cc import ( "android/soong/android" "android/soong/bazel" + "android/soong/bazel/cquery" ) func init() { @@ -47,28 +48,30 @@ func RegisterLibraryHeadersBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("cc_prebuilt_library_headers", prebuiltLibraryHeaderFactory) } -type libraryHeaderBazelHander struct { - android.BazelHandler +type libraryHeaderBazelHandler struct { + BazelHandler module *Module library *libraryDecorator } -func (h *libraryHeaderBazelHander) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (handler *libraryHeaderBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) + bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx)) +} + +func (h *libraryHeaderBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) if err != nil { - ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err) - return false - } - if !ok { - return false + ctx.ModuleErrorf(err.Error()) + return } outputPaths := ccInfo.OutputFiles if len(outputPaths) != 1 { ctx.ModuleErrorf("expected exactly one output file for %q, but got %q", label, outputPaths) - return false + return } outputPath := android.PathForBazelOut(ctx, outputPaths[0]) @@ -83,8 +86,6 @@ func (h *libraryHeaderBazelHander) GenerateBazelBuildActions(ctx android.ModuleC // validation will fail. For now, set this to an empty list. // TODO(cparsons): More closely mirror the collectHeadersForSnapshot implementation. h.library.collectedSnapshotHeaders = android.Paths{} - - return true } // cc_library_headers contains a set of c/c++ headers which are imported by @@ -96,7 +97,7 @@ func LibraryHeaderFactory() android.Module { library.HeaderOnly() module.sdkMemberTypes = []android.SdkMemberType{headersLibrarySdkMemberType} module.bazelable = true - module.bazelHandler = &libraryHeaderBazelHander{module: module, library: library} + module.bazelHandler = &libraryHeaderBazelHandler{module: module, library: library} return module.Init() } diff --git a/cc/object.go b/cc/object.go index bd5bd4517..a3be6b113 100644 --- a/cc/object.go +++ b/cc/object.go @@ -19,6 +19,7 @@ import ( "android/soong/android" "android/soong/bazel" + "android/soong/bazel/cquery" ) // @@ -46,23 +47,30 @@ type objectLinker struct { } type objectBazelHandler struct { - android.BazelHandler + BazelHandler module *Module } -func (handler *objectBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (handler *objectBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - objPaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) - if ok { - if len(objPaths) != 1 { - ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths) - return false - } + bazelCtx.QueueBazelRequest(label, cquery.GetOutputFiles, android.GetConfigKey(ctx)) +} - handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0])) +func (handler *objectBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + objPaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) + if err != nil { + ctx.ModuleErrorf(err.Error()) + return } - return ok + + if len(objPaths) != 1 { + ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths) + return + } + + handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0])) } type ObjectLinkerProperties struct { diff --git a/cc/prebuilt.go b/cc/prebuilt.go index f54c6f8d6..e8c60baea 100644 --- a/cc/prebuilt.go +++ b/cc/prebuilt.go @@ -20,6 +20,7 @@ import ( "android/soong/android" "android/soong/bazel" + "android/soong/bazel/cquery" ) func init() { @@ -406,25 +407,28 @@ type prebuiltObjectLinker struct { } type prebuiltStaticLibraryBazelHandler struct { - android.BazelHandler + BazelHandler module *Module library *libraryDecorator } -func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (h *prebuiltStaticLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) + bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx)) +} + +func (h *prebuiltStaticLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) if err != nil { - ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err) - } - if !ok { - return false + ctx.ModuleErrorf(err.Error()) + return } staticLibs := ccInfo.CcStaticLibraryFiles if len(staticLibs) > 1 { ctx.ModuleErrorf("expected 1 static library from bazel target %q, got %s", label, staticLibs) - return false + return } // TODO(b/184543518): cc_prebuilt_library_static may have properties for re-exporting flags @@ -439,7 +443,7 @@ func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx androi if len(staticLibs) == 0 { h.module.outputFile = android.OptionalPath{} - return true + return } out := android.PathForBazelOut(ctx, staticLibs[0]) @@ -451,30 +455,31 @@ func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx androi TransitiveStaticLibrariesForOrdering: depSet, }) - - return true } type prebuiltSharedLibraryBazelHandler struct { - android.BazelHandler + BazelHandler module *Module library *libraryDecorator } -func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (h *prebuiltSharedLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) + bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx)) +} + +func (h *prebuiltSharedLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) if err != nil { - ctx.ModuleErrorf("Error getting Bazel CcInfo for %s: %s", label, err) - } - if !ok { - return false + ctx.ModuleErrorf(err.Error()) + return } sharedLibs := ccInfo.CcSharedLibraryFiles if len(sharedLibs) != 1 { ctx.ModuleErrorf("expected 1 shared library from bazel target %s, got %q", label, sharedLibs) - return false + return } // TODO(b/184543518): cc_prebuilt_library_shared may have properties for re-exporting flags @@ -489,7 +494,7 @@ func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx androi if len(sharedLibs) == 0 { h.module.outputFile = android.OptionalPath{} - return true + return } out := android.PathForBazelOut(ctx, sharedLibs[0]) @@ -514,8 +519,6 @@ func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx androi h.library.setFlagExporterInfoFromCcInfo(ctx, ccInfo) h.module.maybeUnhideFromMake() - - return true } func (p *prebuiltObjectLinker) prebuilt() *android.Prebuilt { diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 4b3161b33..ad379d587 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -129,44 +129,26 @@ func newConfig(availableEnv map[string]string) android.Config { return configuration } -// Bazel-enabled mode. Soong runs in two passes. -// First pass: Analyze the build tree, but only store all bazel commands -// needed to correctly evaluate the tree in the second pass. -// TODO(cparsons): Don't output any ninja file, as the second pass will overwrite -// the incorrect results from the first pass, and file I/O is expensive. -func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) { - firstCtx.EventHandler.Begin("mixed_build") - defer firstCtx.EventHandler.End("mixed_build") +// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a +// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata +// for modules that should be handled by Bazel. +func runMixedModeBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) { + ctx.EventHandler.Begin("mixed_build") + defer ctx.EventHandler.End("mixed_build") - firstCtx.EventHandler.Begin("prepare") - bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration) - firstCtx.EventHandler.End("prepare") - - firstCtx.EventHandler.Begin("bazel") - // Invoke bazel commands and save results for second pass. - if err := configuration.BazelContext.InvokeBazel(); err != nil { - fmt.Fprintf(os.Stderr, "%s", err) - os.Exit(1) + bazelHook := func() error { + ctx.EventHandler.Begin("bazel") + defer ctx.EventHandler.End("bazel") + return configuration.BazelContext.InvokeBazel() } - // Second pass: Full analysis, using the bazel command results. Output ninja file. - secondConfig, err := android.ConfigForAdditionalRun(configuration) - if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) - os.Exit(1) - } - firstCtx.EventHandler.End("bazel") + ctx.SetBeforePrepareBuildActionsHook(bazelHook) - secondCtx := newContext(secondConfig) - secondCtx.EventHandler = firstCtx.EventHandler - secondCtx.EventHandler.Begin("analyze") - ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig) - ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - secondCtx.EventHandler.End("analyze") + ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, ctx.Context, configuration) - globListFiles := writeBuildGlobsNinjaFile(secondCtx, configuration.SoongOutDir(), configuration) + globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) ninjaDeps = append(ninjaDeps, globListFiles...) - writeDepFile(cmdlineArgs.OutFile, *secondCtx.EventHandler, ninjaDeps) + writeDepFile(cmdlineArgs.OutFile, *ctx.EventHandler, ninjaDeps) } // Run the code-generation phase to convert BazelTargetModules to BUILD files. diff --git a/genrule/genrule.go b/genrule/genrule.go index 3531ee6b8..8649b15b7 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" + "android/soong/bazel/cquery" "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/proptools" @@ -189,6 +190,8 @@ type Module struct { modulePaths []string } +var _ android.MixedBuildBuildable = (*Module)(nil) + type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask type generateTask struct { @@ -249,27 +252,36 @@ func toolDepsMutator(ctx android.BottomUpMutatorContext) { } } -// Returns true if information was available from Bazel, false if bazel invocation still needs to occur. -func (c *Module) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (g *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) { + g.generateCommonBuildActions(ctx) + + label := g.GetBazelLabel(ctx, g) bazelCtx := ctx.Config().BazelContext - filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) - if ok { - var bazelOutputFiles android.Paths - exportIncludeDirs := map[string]bool{} - for _, bazelOutputFile := range filePaths { - bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOut(ctx, bazelOutputFile)) - exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true - } - c.outputFiles = bazelOutputFiles - c.outputDeps = bazelOutputFiles - for includePath, _ := range exportIncludeDirs { - c.exportedIncludeDirs = append(c.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath)) - } + filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) + if err != nil { + ctx.ModuleErrorf(err.Error()) + return + } + + var bazelOutputFiles android.Paths + exportIncludeDirs := map[string]bool{} + for _, bazelOutputFile := range filePaths { + bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOut(ctx, bazelOutputFile)) + exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true + } + g.outputFiles = bazelOutputFiles + g.outputDeps = bazelOutputFiles + for includePath, _ := range exportIncludeDirs { + g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath)) } - return ok } -func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { +// generateCommonBuildActions contains build action generation logic +// common to both the mixed build case and the legacy case of genrule processing. +// To fully support genrule in mixed builds, the contents of this function should +// approach zero; there should be no genrule action registration done directly +// by Soong logic in the mixed-build case. +func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { g.subName = ctx.ModuleSubDir() // Collect the module directory for IDE info in java/jdeps.go. @@ -575,31 +587,37 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { } g.outputFiles = outputFiles.Paths() +} - bazelModuleLabel := g.GetBazelLabel(ctx, g) - bazelActionsUsed := false - if android.MixedBuildsEnabled(ctx) { - bazelActionsUsed = g.GenerateBazelBuildActions(ctx, bazelModuleLabel) - } - if !bazelActionsUsed { - // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of - // the genrules on AOSP. That will make things simpler to look at the graph in the common - // case. For larger sets of outputs, inject a phony target in between to limit ninja file - // growth. - if len(g.outputFiles) <= 6 { - g.outputDeps = g.outputFiles - } else { - phonyFile := android.PathForModuleGen(ctx, "genrule-phony") - ctx.Build(pctx, android.BuildParams{ - Rule: blueprint.Phony, - Output: phonyFile, - Inputs: g.outputFiles, - }) - g.outputDeps = android.Paths{phonyFile} - } +func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { + g.generateCommonBuildActions(ctx) + + // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of + // the genrules on AOSP. That will make things simpler to look at the graph in the common + // case. For larger sets of outputs, inject a phony target in between to limit ninja file + // growth. + if len(g.outputFiles) <= 6 { + g.outputDeps = g.outputFiles + } else { + phonyFile := android.PathForModuleGen(ctx, "genrule-phony") + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: phonyFile, + Inputs: g.outputFiles, + }) + g.outputDeps = android.Paths{phonyFile} } } +func (g *Module) QueueBazelCall(ctx android.BaseModuleContext) { + bazelCtx := ctx.Config().BazelContext + bazelCtx.QueueBazelRequest(g.GetBazelLabel(ctx, g), cquery.GetOutputFiles, android.GetConfigKey(ctx)) +} + +func (g *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool { + return true +} + // Collect information for opening IDE project files in java/jdeps.go. func (g *Module) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Srcs = append(dpInfo.Srcs, g.Srcs().Strings()...)