Batch cquery requests for mixed builds am: b0f8ac4ff0

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1471974

Change-Id: Ia8434f1f2621f8e1b43ee6da2ccc2578a6b2baf1

android/bazel_handler.go

@@ -29,10 +29,16 @@ import (
 	"github.com/google/blueprint/bootstrap"
 )
 
+type CqueryRequestType int
+
+const (
+	getAllFiles CqueryRequestType = iota
+)
+
 // Map key to describe bazel cquery requests.
 type cqueryKey struct {
-	label        string
+	label       string
-	starlarkExpr string
+	requestType CqueryRequestType
 }
 
 type BazelContext interface {
@@ -61,6 +67,7 @@ type bazelContext struct {
 	bazelPath    string
 	outputBase   string
 	workspaceDir string
+	buildDir     string
 
 	requests     map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel
 	requestMutex sync.Mutex         // requests can be written in parallel
@@ -96,8 +103,7 @@ func (m MockBazelContext) BazelEnabled() bool {
 var _ BazelContext = MockBazelContext{}
 
 func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
-	starlarkExpr := "', '.join([f.path for f in target.files.to_list()])"
+	result, ok := bazelCtx.cquery(label, getAllFiles)
-	result, ok := bazelCtx.cquery(label, starlarkExpr)
 	if ok {
 		bazelOutput := strings.TrimSpace(result)
 		return strings.Split(bazelOutput, ", "), true
@@ -125,7 +131,7 @@ func NewBazelContext(c *config) (BazelContext, error) {
 		return noopBazelContext{}, nil
 	}
 
-	bazelCtx := bazelContext{requests: make(map[cqueryKey]bool)}
+	bazelCtx := bazelContext{buildDir: c.buildDir, requests: make(map[cqueryKey]bool)}
 	missingEnvVars := []string{}
 	if len(c.Getenv("BAZEL_HOME")) > 1 {
 		bazelCtx.homeDir = c.Getenv("BAZEL_HOME")
@@ -163,8 +169,8 @@ func (context *bazelContext) BazelEnabled() bool {
 // If the given request was already made (and the results are available), then
 // returns (result, true). If the request is queued but no results are available,
 // then returns ("", false).
-func (context *bazelContext) cquery(label string, starlarkExpr string) (string, bool) {
+func (context *bazelContext) cquery(label string, requestType CqueryRequestType) (string, bool) {
-	key := cqueryKey{label, starlarkExpr}
+	key := cqueryKey{label, requestType}
 	if result, ok := context.results[key]; ok {
 		return result, true
 	} else {
@@ -186,7 +192,8 @@ func pwdPrefix() string {
 func (context *bazelContext) issueBazelCommand(command string, labels []string,
 	extraFlags ...string) (string, error) {
 
-	cmdFlags := []string{"--output_base=" + context.outputBase, command}
+	cmdFlags := []string{"--bazelrc=build/bazel/common.bazelrc",
+		"--output_base=" + context.outputBase, command}
 	cmdFlags = append(cmdFlags, labels...)
 	cmdFlags = append(cmdFlags, extraFlags...)
 
@@ -204,27 +211,113 @@ func (context *bazelContext) issueBazelCommand(command string, labels []string,
 	}
 }
 
+func (context *bazelContext) mainBzlFileContents() []byte {
+	contents := `
+# This file is generated by soong_build. Do not edit.
+def _mixed_build_root_impl(ctx):
+    return [DefaultInfo(files = depset(ctx.files.deps))]
+
+mixed_build_root = rule(
+    implementation = _mixed_build_root_impl,
+    attrs = {"deps" : attr.label_list()},
+)
+`
+	return []byte(contents)
+}
+
+func (context *bazelContext) mainBuildFileContents() []byte {
+	formatString := `
+# This file is generated by soong_build. Do not edit.
+load(":main.bzl", "mixed_build_root")
+
+mixed_build_root(name = "buildroot",
+    deps = [%s],
+)
+`
+	var buildRootDeps []string = nil
+	for val, _ := range context.requests {
+		buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", val.label))
+	}
+	buildRootDepsString := strings.Join(buildRootDeps, ",\n            ")
+
+	return []byte(fmt.Sprintf(formatString, buildRootDepsString))
+}
+
+func (context *bazelContext) cqueryStarlarkFileContents() []byte {
+	formatString := `
+# This file is generated by soong_build. Do not edit.
+getAllFilesLabels = {
+  %s
+}
+
+def format(target):
+  if str(target.label) in getAllFilesLabels:
+    return str(target.label) + ">>" + ', '.join([f.path for f in target.files.to_list()])
+  else:
+    # This target was not requested via cquery, and thus must be a dependency
+    # of a requested target.
+    return ""
+`
+	var buildRootDeps []string = nil
+	// TODO(cparsons): Sort by request type instead of assuming all requests
+	// are of GetAllFiles type.
+	for val, _ := range context.requests {
+		buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", val.label))
+	}
+	buildRootDepsString := strings.Join(buildRootDeps, ",\n  ")
+
+	return []byte(fmt.Sprintf(formatString, buildRootDepsString))
+}
+
 // Issues commands to Bazel to receive results for all cquery requests
 // queued in the BazelContext.
 func (context *bazelContext) InvokeBazel() error {
 	context.results = make(map[cqueryKey]string)
 
-	var labels []string
 	var cqueryOutput string
 	var err error
+	err = ioutil.WriteFile(
+		absolutePath(filepath.Join(context.buildDir, "main.bzl")),
+		context.mainBzlFileContents(), 0666)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(
+		absolutePath(filepath.Join(context.buildDir, "BUILD.bazel")),
+		context.mainBuildFileContents(), 0666)
+	if err != nil {
+		return err
+	}
+	cquery_file_relpath := filepath.Join(context.buildDir, "buildroot.cquery")
+	err = ioutil.WriteFile(
+		absolutePath(cquery_file_relpath),
+		context.cqueryStarlarkFileContents(), 0666)
+	if err != nil {
+		return err
+	}
+	buildroot_label := fmt.Sprintf("//%s:buildroot", context.buildDir)
+	cqueryOutput, err = context.issueBazelCommand("cquery",
+		[]string{fmt.Sprintf("deps(%s)", buildroot_label)},
+		"--output=starlark",
+		"--starlark:file="+cquery_file_relpath)
+
+	if err != nil {
+		return err
+	}
+
+	cqueryResults := map[string]string{}
+	for _, outputLine := range strings.Split(cqueryOutput, "\n") {
+		if strings.Contains(outputLine, ">>") {
+			splitLine := strings.SplitN(outputLine, ">>", 2)
+			cqueryResults[splitLine[0]] = splitLine[1]
+		}
+	}
+
 	for val, _ := range context.requests {
-		labels = append(labels, val.label)
+		if cqueryResult, ok := cqueryResults[val.label]; ok {
-
+			context.results[val] = string(cqueryResult)
-		// TODO(cparsons): Combine requests into a batch cquery request.
-		// TODO(cparsons): Use --query_file to avoid command line limits.
-		cqueryOutput, err = context.issueBazelCommand("cquery", []string{val.label},
-			"--output=starlark",
-			"--starlark:expr="+val.starlarkExpr)
-
-		if err != nil {
-			return err
 		} else {
-			context.results[val] = string(cqueryOutput)
+			return fmt.Errorf("missing result for bazel target %s", val.label)
 		}
 	}
 
@@ -233,7 +326,7 @@ func (context *bazelContext) InvokeBazel() error {
 	// bazel actions should either be added to the Ninja file and executed later,
 	// or bazel should handle execution.
 	// TODO(cparsons): Use --target_pattern_file to avoid command line limits.
-	_, err = context.issueBazelCommand("build", labels)
+	_, err = context.issueBazelCommand("build", []string{buildroot_label})
 
 	if err != nil {
 		return err