StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use aquery to declare bazel actions in the ninja file.

Browse Source 
This effectively moves execution of Bazel actions outside of soong_build
and alongside ninja execution of the actual ninja files, whether that be
by ninja or by Bazel itself.

This almost allows for mixed builds and Bazel-as-Ninja-executor to
coexist, but requires hacks explained in b/175307058.

Test: Treehugger
Test: lunch aosp_flame && USE_BAZEL_ANALYSIS=1 m libc
Test: lunch aosp_flame && USE_BAZEL=1 USE_BAZEL_ANALYSIS=1 m libc,
though this requires a hack of the main BUILD file. See b/175307058

Change-Id: Ia2f6b0f1057e8cea3809de66d8287f13d84b510c
This commit is contained in:
Chris Parsons
2020-12-10 17:19:18 -05:00
parent fde77e02f3
commit dbcb1ff469
6 changed files with 268 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
android/bazel_handler.go
View File

@@ -26,9 +26,10 @@ import (
"strings"
"sync"
"github.com/google/blueprint/bootstrap"
"android/soong/bazel"
"android/soong/shared"
"github.com/google/blueprint/bootstrap"
)
type CqueryRequestType int
@@ -60,6 +61,12 @@ type BazelContext interface {
// Returns true if bazel is enabled for the given configuration.
BazelEnabled() bool
// Returns the bazel output base (the root directory for all bazel intermediate outputs).
OutputBase() string
// Returns build statements which should get registered to reflect Bazel's outputs.
BuildStatementsToRegister() []bazel.BuildStatement
}
// A context object which tracks queued requests that need to be made to Bazel,
@@ -76,6 +83,9 @@ type bazelContext struct {
requestMutex sync.Mutex // requests can be written in parallel
results map[cqueryKey]string // Results of cquery requests after Bazel invocations
// Build statements which should get registered to reflect Bazel's outputs.
buildStatements []bazel.BuildStatement
}
var _ BazelContext = &bazelContext{}
@@ -103,6 +113,14 @@ func (m MockBazelContext) BazelEnabled() bool {
return true
}
func (m MockBazelContext) OutputBase() string {
return "outputbase"
}
func (m MockBazelContext) BuildStatementsToRegister() []bazel.BuildStatement {
return []bazel.BuildStatement{}
}
var _ BazelContext = MockBazelContext{}
func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
@@ -123,10 +141,18 @@ func (n noopBazelContext) InvokeBazel() error {
panic("unimplemented")
}
func (m noopBazelContext) OutputBase() string {
return ""
}
func (n noopBazelContext) BazelEnabled() bool {
return false
}
func (m noopBazelContext) BuildStatementsToRegister() []bazel.BuildStatement {
return []bazel.BuildStatement{}
}
func NewBazelContext(c *config) (BazelContext, error) {
// TODO(cparsons): Assess USE_BAZEL=1 instead once "mixed Soong/Bazel builds"
// are production ready.
@@ -241,14 +267,32 @@ local_repository(
func (context *bazelContext) mainBzlFileContents() []byte {
contents := `
#####################################################
# This file is generated by soong_build. Do not edit.
#####################################################
def _mixed_build_root_impl(ctx):
return [DefaultInfo(files = depset(ctx.files.deps))]
# Rule representing the root of the build, to depend on all Bazel targets that
# are required for the build. Building this target will build the entire Bazel
# build tree.
mixed_build_root = rule(
implementation = _mixed_build_root_impl,
attrs = {"deps" : attr.label_list()},
)
def _phony_root_impl(ctx):
return []
# Rule to depend on other targets but build nothing.
# This is useful as follows: building a target of this rule will generate
# symlink forests for all dependencies of the target, without executing any
# actions of the build.
phony_root = rule(
implementation = _phony_root_impl,
attrs = {"deps" : attr.label_list()},
)
`
return []byte(contents)
}
@@ -268,11 +312,15 @@ func canonicalizeLabel(label string) string {
func (context *bazelContext) mainBuildFileContents() []byte {
formatString := `
# This file is generated by soong_build. Do not edit.
load(":main.bzl", "mixed_build_root")
load(":main.bzl", "mixed_build_root", "phony_root")
mixed_build_root(name = "buildroot",
deps = [%s],
)
phony_root(name = "phonyroot",
deps = [":buildroot"],
)
`
var buildRootDeps []string = nil
for val, _ := range context.requests {
@@ -379,22 +427,46 @@ func (context *bazelContext) InvokeBazel() error {
}
}
// Issue a build command.
// TODO(cparsons): Invoking bazel execution during soong_build should be avoided;
// bazel actions should either be added to the Ninja file and executed later,
// or bazel should handle execution.
// Issue an aquery command to retrieve action information about the bazel build tree.
//
// TODO(cparsons): Use --target_pattern_file to avoid command line limits.
_, err = context.issueBazelCommand(bazel.BazelBuildPhonyRootRunName, "build", []string{buildroot_label})
var aqueryOutput string
aqueryOutput, err = context.issueBazelCommand(bazel.AqueryBuildRootRunName, "aquery",
[]string{fmt.Sprintf("deps(%s)", buildroot_label),
// Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's
// proto sources, which would add a number of unnecessary dependencies.
"--output=jsonproto"})
if err != nil {
return err
}
context.buildStatements = bazel.AqueryBuildStatements([]byte(aqueryOutput))
// Issue a build command of the phony root to generate symlink forests for dependencies of the
// Bazel build. This is necessary because aquery invocations do not generate this symlink forest,
// but some of symlinks may be required to resolve source dependencies of the build.
_, err = context.issueBazelCommand(bazel.BazelBuildPhonyRootRunName, "build",
[]string{"//:phonyroot"})
if err != nil {
return err
}
fmt.Printf("Build statements %s", context.buildStatements)
// Clear requests.
context.requests = map[cqueryKey]bool{}
return nil
}
func (context *bazelContext) BuildStatementsToRegister() []bazel.BuildStatement {
return context.buildStatements
}
func (context *bazelContext) OutputBase() string {
return context.outputBase
}
// Singleton used for registering BUILD file ninja dependencies (needed
// for correctness of builds which use Bazel.
func BazelSingleton() Singleton {
@@ -404,18 +476,45 @@ func BazelSingleton() Singleton {
type bazelSingleton struct{}
func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
if ctx.Config().BazelContext.BazelEnabled() {
bazelBuildList := absolutePath(filepath.Join(
filepath.Dir(bootstrap.ModuleListFile), "bazel.list"))
ctx.AddNinjaFileDeps(bazelBuildList)
// bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled.
if !ctx.Config().BazelContext.BazelEnabled() {
return
}
data, err := ioutil.ReadFile(bazelBuildList)
if err != nil {
ctx.Errorf(err.Error())
// Add ninja file dependencies for files which all bazel invocations require.
bazelBuildList := absolutePath(filepath.Join(
filepath.Dir(bootstrap.ModuleListFile), "bazel.list"))
ctx.AddNinjaFileDeps(bazelBuildList)
data, err := ioutil.ReadFile(bazelBuildList)
if err != nil {
ctx.Errorf(err.Error())
}
files := strings.Split(strings.TrimSpace(string(data)), "\n")
for _, file := range files {
ctx.AddNinjaFileDeps(file)
}
// Register bazel-owned build statements (obtained from the aquery invocation).
for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
rule := NewRuleBuilder(pctx, ctx)
cmd := rule.Command()
cmd.Text(fmt.Sprintf("cd %s/execroot/__main__ && %s",
ctx.Config().BazelContext.OutputBase(), buildStatement.Command))
for _, outputPath := range buildStatement.OutputPaths {
cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
}
files := strings.Split(strings.TrimSpace(string(data)), "\n")
for _, file := range files {
ctx.AddNinjaFileDeps(file)
for _, inputPath := range buildStatement.InputPaths {
cmd.Implicit(PathForBazelOut(ctx, inputPath))
}
// This is required to silence warnings pertaining to unexpected timestamps. Particularly,
// some Bazel builtins (such as files in the bazel_tools directory) have far-future
// timestamps. Without restat, Ninja would emit warnings that the input files of a
// build statement have later timestamps than the outputs.
rule.Restat()
rule.Build(fmt.Sprintf("bazel %s", index), buildStatement.Mnemonic)
}
}

29
android/paths.go
View File

@@ -1154,6 +1154,17 @@ func pathForModule(ctx ModuleContext) OutputPath {
return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir())
}
type BazelOutPath struct {
OutputPath
}
var _ Path = BazelOutPath{}
var _ objPathProvider = BazelOutPath{}
func (p BazelOutPath) objPathWithExt(ctx ModuleContext, subdir, ext string) ModuleObjPath {
return PathForModuleObj(ctx, subdir, pathtools.ReplaceExtension(p.path, ext))
}
// PathForVndkRefAbiDump returns an OptionalPath representing the path of the
// reference abi dump for the given module. This is not guaranteed to be valid.
func PathForVndkRefAbiDump(ctx ModuleContext, version, fileName string,
@@ -1192,6 +1203,24 @@ func PathForVndkRefAbiDump(ctx ModuleContext, version, fileName string,
fileName+ext)
}
// PathForBazelOut returns a Path representing the paths... under an output directory dedicated to
// bazel-owned outputs.
func PathForBazelOut(ctx PathContext, paths ...string) BazelOutPath {
execRootPathComponents := append([]string{"execroot", "__main__"}, paths...)
execRootPath := filepath.Join(execRootPathComponents...)
validatedExecRootPath, err := validatePath(execRootPath)
if err != nil {
reportPathError(ctx, err)
}
outputPath := OutputPath{basePath{"", ctx.Config(), ""},
ctx.Config().BazelContext.OutputBase()}
return BazelOutPath{
OutputPath: outputPath.withRel(validatedExecRootPath),
}
}
// PathForModuleOut returns a Path representing the paths... under the module's
// output directory.
func PathForModuleOut(ctx ModuleContext, paths ...string) ModuleOutPath {