Batch cquery requests for mixed builds am: b0f8ac4ff0 am: da4bb85447

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1471974

Change-Id: I75ec75d4f5e0f2b3b25de619ad197579b285020d
This commit is contained in:
Chris Parsons
2020-10-29 01:40:57 +00:00
committed by Automerger Merge Worker

View File

@@ -29,10 +29,16 @@ import (
"github.com/google/blueprint/bootstrap" "github.com/google/blueprint/bootstrap"
) )
type CqueryRequestType int
const (
getAllFiles CqueryRequestType = iota
)
// Map key to describe bazel cquery requests. // Map key to describe bazel cquery requests.
type cqueryKey struct { type cqueryKey struct {
label string label string
starlarkExpr string requestType CqueryRequestType
} }
type BazelContext interface { type BazelContext interface {
@@ -61,6 +67,7 @@ type bazelContext struct {
bazelPath string bazelPath string
outputBase string outputBase string
workspaceDir string workspaceDir string
buildDir string
requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel
requestMutex sync.Mutex // requests can be written in parallel requestMutex sync.Mutex // requests can be written in parallel
@@ -96,8 +103,7 @@ func (m MockBazelContext) BazelEnabled() bool {
var _ BazelContext = MockBazelContext{} var _ BazelContext = MockBazelContext{}
func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) { func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
starlarkExpr := "', '.join([f.path for f in target.files.to_list()])" result, ok := bazelCtx.cquery(label, getAllFiles)
result, ok := bazelCtx.cquery(label, starlarkExpr)
if ok { if ok {
bazelOutput := strings.TrimSpace(result) bazelOutput := strings.TrimSpace(result)
return strings.Split(bazelOutput, ", "), true return strings.Split(bazelOutput, ", "), true
@@ -125,7 +131,7 @@ func NewBazelContext(c *config) (BazelContext, error) {
return noopBazelContext{}, nil return noopBazelContext{}, nil
} }
bazelCtx := bazelContext{requests: make(map[cqueryKey]bool)} bazelCtx := bazelContext{buildDir: c.buildDir, requests: make(map[cqueryKey]bool)}
missingEnvVars := []string{} missingEnvVars := []string{}
if len(c.Getenv("BAZEL_HOME")) > 1 { if len(c.Getenv("BAZEL_HOME")) > 1 {
bazelCtx.homeDir = c.Getenv("BAZEL_HOME") bazelCtx.homeDir = c.Getenv("BAZEL_HOME")
@@ -163,8 +169,8 @@ func (context *bazelContext) BazelEnabled() bool {
// If the given request was already made (and the results are available), then // If the given request was already made (and the results are available), then
// returns (result, true). If the request is queued but no results are available, // returns (result, true). If the request is queued but no results are available,
// then returns ("", false). // then returns ("", false).
func (context *bazelContext) cquery(label string, starlarkExpr string) (string, bool) { func (context *bazelContext) cquery(label string, requestType CqueryRequestType) (string, bool) {
key := cqueryKey{label, starlarkExpr} key := cqueryKey{label, requestType}
if result, ok := context.results[key]; ok { if result, ok := context.results[key]; ok {
return result, true return result, true
} else { } else {
@@ -186,7 +192,8 @@ func pwdPrefix() string {
func (context *bazelContext) issueBazelCommand(command string, labels []string, func (context *bazelContext) issueBazelCommand(command string, labels []string,
extraFlags ...string) (string, error) { extraFlags ...string) (string, error) {
cmdFlags := []string{"--output_base=" + context.outputBase, command} cmdFlags := []string{"--bazelrc=build/bazel/common.bazelrc",
"--output_base=" + context.outputBase, command}
cmdFlags = append(cmdFlags, labels...) cmdFlags = append(cmdFlags, labels...)
cmdFlags = append(cmdFlags, extraFlags...) cmdFlags = append(cmdFlags, extraFlags...)
@@ -204,27 +211,113 @@ func (context *bazelContext) issueBazelCommand(command string, labels []string,
} }
} }
func (context *bazelContext) mainBzlFileContents() []byte {
contents := `
# This file is generated by soong_build. Do not edit.
def _mixed_build_root_impl(ctx):
return [DefaultInfo(files = depset(ctx.files.deps))]
mixed_build_root = rule(
implementation = _mixed_build_root_impl,
attrs = {"deps" : attr.label_list()},
)
`
return []byte(contents)
}
func (context *bazelContext) mainBuildFileContents() []byte {
formatString := `
# This file is generated by soong_build. Do not edit.
load(":main.bzl", "mixed_build_root")
mixed_build_root(name = "buildroot",
deps = [%s],
)
`
var buildRootDeps []string = nil
for val, _ := range context.requests {
buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", val.label))
}
buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
return []byte(fmt.Sprintf(formatString, buildRootDepsString))
}
func (context *bazelContext) cqueryStarlarkFileContents() []byte {
formatString := `
# This file is generated by soong_build. Do not edit.
getAllFilesLabels = {
%s
}
def format(target):
if str(target.label) in getAllFilesLabels:
return str(target.label) + ">>" + ', '.join([f.path for f in target.files.to_list()])
else:
# This target was not requested via cquery, and thus must be a dependency
# of a requested target.
return ""
`
var buildRootDeps []string = nil
// TODO(cparsons): Sort by request type instead of assuming all requests
// are of GetAllFiles type.
for val, _ := range context.requests {
buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", val.label))
}
buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
return []byte(fmt.Sprintf(formatString, buildRootDepsString))
}
// Issues commands to Bazel to receive results for all cquery requests // Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext. // queued in the BazelContext.
func (context *bazelContext) InvokeBazel() error { func (context *bazelContext) InvokeBazel() error {
context.results = make(map[cqueryKey]string) context.results = make(map[cqueryKey]string)
var labels []string
var cqueryOutput string var cqueryOutput string
var err error var err error
err = ioutil.WriteFile(
absolutePath(filepath.Join(context.buildDir, "main.bzl")),
context.mainBzlFileContents(), 0666)
if err != nil {
return err
}
err = ioutil.WriteFile(
absolutePath(filepath.Join(context.buildDir, "BUILD.bazel")),
context.mainBuildFileContents(), 0666)
if err != nil {
return err
}
cquery_file_relpath := filepath.Join(context.buildDir, "buildroot.cquery")
err = ioutil.WriteFile(
absolutePath(cquery_file_relpath),
context.cqueryStarlarkFileContents(), 0666)
if err != nil {
return err
}
buildroot_label := fmt.Sprintf("//%s:buildroot", context.buildDir)
cqueryOutput, err = context.issueBazelCommand("cquery",
[]string{fmt.Sprintf("deps(%s)", buildroot_label)},
"--output=starlark",
"--starlark:file="+cquery_file_relpath)
if err != nil {
return err
}
cqueryResults := map[string]string{}
for _, outputLine := range strings.Split(cqueryOutput, "\n") {
if strings.Contains(outputLine, ">>") {
splitLine := strings.SplitN(outputLine, ">>", 2)
cqueryResults[splitLine[0]] = splitLine[1]
}
}
for val, _ := range context.requests { for val, _ := range context.requests {
labels = append(labels, val.label) if cqueryResult, ok := cqueryResults[val.label]; ok {
context.results[val] = string(cqueryResult)
// TODO(cparsons): Combine requests into a batch cquery request.
// TODO(cparsons): Use --query_file to avoid command line limits.
cqueryOutput, err = context.issueBazelCommand("cquery", []string{val.label},
"--output=starlark",
"--starlark:expr="+val.starlarkExpr)
if err != nil {
return err
} else { } else {
context.results[val] = string(cqueryOutput) return fmt.Errorf("missing result for bazel target %s", val.label)
} }
} }
@@ -233,7 +326,7 @@ func (context *bazelContext) InvokeBazel() error {
// bazel actions should either be added to the Ninja file and executed later, // bazel actions should either be added to the Ninja file and executed later,
// or bazel should handle execution. // or bazel should handle execution.
// TODO(cparsons): Use --target_pattern_file to avoid command line limits. // TODO(cparsons): Use --target_pattern_file to avoid command line limits.
_, err = context.issueBazelCommand("build", labels) _, err = context.issueBazelCommand("build", []string{buildroot_label})
if err != nil { if err != nil {
return err return err