Fix the output handling of Bazel aquery for py_binary targets so soong's python_binary_host modules can be converted to py_binary and built with Bazel dependent targets in mixed build.

Test: USE_BAZEL_ANALYSIS=1 m genfunctosyscallnrs
Test: USE_BAZEL_ANALYSIS=1 m func_to_syscall_nrs
Test: USE_BAZEL_ANALYSIS=1 m libseccomp_policy
Bug: 201094425
Bug: 197135289
Bug: 204949078
Change-Id: If98630830d687ca814d6ae09c1e610f2c3a371b1

android/bazel.go

@@ -332,12 +332,8 @@ var (
 	// Per-module denylist to opt modules out of mixed builds. Such modules will
 	// still be generated via bp2build.
 	mixedBuildsDisabledList = []string{
-		"libbrotli",                            // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy
+		"libbrotli",               // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy
-		"func_to_syscall_nrs",                  // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
+		"minijail_constants_json", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
-		"libseccomp_policy_app_zygote_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
-		"libseccomp_policy_app_sources",        // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
-		"libseccomp_policy_system_sources",     // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
-		"minijail_constants_json",              // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
 	}
 
 	// Used for quicker lookups

bazel/aquery.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/google/blueprint/proptools"
@@ -59,6 +60,8 @@ type action struct {
 	InputDepSetIds       []int
 	Mnemonic             string
 	OutputIds            []int
+	TemplateContent      string
+	Substitutions        []KeyValuePair
 }
 
 // actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
@@ -100,6 +103,20 @@ type aqueryArtifactHandler struct {
 	artifactIdToPath map[int]string
 }
 
+// The tokens should be substituted with the value specified here, instead of the
+// one returned in 'substitutions' of TemplateExpand action.
+var TemplateActionOverriddenTokens = map[string]string{
+	// Uses "python3" for %python_binary% instead of the value returned by aquery
+	// which is "py3wrapper.sh". See removePy3wrapperScript.
+	"%python_binary%": "python3",
+}
+
+// This pattern matches the MANIFEST file created for a py_binary target.
+var manifestFilePattern = regexp.MustCompile(".*/.+\\.runfiles/MANIFEST$")
+
+// The file name of py3wrapper.sh, which is used by py_binary targets.
+var py3wrapperFileName = "/py3wrapper.sh"
+
 func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler, error) {
 	pathFragments := map[int]pathFragment{}
 	for _, pathFragment := range aqueryResult.PathFragments {
@@ -163,7 +180,31 @@ func (a *aqueryArtifactHandler) getInputPaths(depsetIds []int) ([]string, error)
 			}
 		}
 	}
-	return inputPaths, nil
+
+	// TODO(b/197135294): Clean up this custom runfiles handling logic when
+	// SourceSymlinkManifest and SymlinkTree actions are supported.
+	filteredInputPaths := filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths)
+
+	return filteredInputPaths, nil
+}
+
+// See go/python-binary-host-mixed-build for more details.
+// 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of
+// Bazel py_binary targets, so there is no Ninja build statements generated for creating it.
+// 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets,
+// but it doesn't contain sufficient information so no Ninja build statements are generated
+// for creating it.
+// So in mixed build mode, when these two are used as input of some Ninja build statement,
+// since there is no build statement to create them, they should be removed from input paths.
+func filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths []string) []string {
+	filteredInputPaths := []string{}
+	for _, path := range inputPaths {
+		if strings.HasSuffix(path, py3wrapperFileName) || manifestFilePattern.MatchString(path) {
+			continue
+		}
+		filteredInputPaths = append(filteredInputPaths, path)
+	}
+	return filteredInputPaths
 }
 
 func (a *aqueryArtifactHandler) artifactIdsFromDepsetId(depsetId int) ([]int, error) {
@@ -249,6 +290,45 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
 			// Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
 			buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in)
 			buildStatement.SymlinkPaths = outputPaths[:]
+		} else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 {
+			if len(outputPaths) != 1 {
+				return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths)
+			}
+			expandedTemplateContent := expandTemplateContent(actionEntry)
+			// The expandedTemplateContent is escaped for being used in double quotes and shell unescape,
+			// and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might
+			// change \n to space and mess up the format of Python programs.
+			// sed is used to convert \\n back to \n before saving to output file.
+			// See go/python-binary-host-mixed-build for more details.
+			command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`,
+				escapeCommandlineArgument(expandedTemplateContent), outputPaths[0])
+			buildStatement.Command = command
+		} else if isPythonZipperAction(actionEntry) {
+			if len(inputPaths) < 1 || len(outputPaths) != 1 {
+				return nil, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths)
+			}
+			buildStatement.InputPaths, buildStatement.Command = removePy3wrapperScript(buildStatement)
+			buildStatement.Command = addCommandForPyBinaryRunfilesDir(buildStatement, inputPaths[0], outputPaths[0])
+			// Add the python zip file as input of the corresponding python binary stub script in Ninja build statements.
+			// In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input,
+			// which is not sufficient without the python zip file from which runfiles directory is created for py_binary.
+			//
+			// The following logic relies on that Bazel aquery output returns actions in the order that
+			// PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions
+			// in that order, the following logic might not find the build statement generated for Python binary
+			// stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output.
+			// See go/python-binary-host-mixed-build for more details.
+			pythonZipFilePath := outputPaths[0]
+			pyBinaryFound := false
+			for i, _ := range buildStatements {
+				if len(buildStatements[i].OutputPaths) == 1 && buildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath {
+					buildStatements[i].InputPaths = append(buildStatements[i].InputPaths, pythonZipFilePath)
+					pyBinaryFound = true
+				}
+			}
+			if !pyBinaryFound {
+				return nil, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths)
+			}
 		} else if len(actionEntry.Arguments) < 1 {
 			return nil, fmt.Errorf("received action with no command: [%v]", buildStatement)
 		}
@@ -258,10 +338,85 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
 	return buildStatements, nil
 }
 
+// expandTemplateContent substitutes the tokens in a template.
+func expandTemplateContent(actionEntry action) string {
+	replacerString := []string{}
+	for _, pair := range actionEntry.Substitutions {
+		value := pair.Value
+		if val, ok := TemplateActionOverriddenTokens[pair.Key]; ok {
+			value = val
+		}
+		replacerString = append(replacerString, pair.Key, value)
+	}
+	replacer := strings.NewReplacer(replacerString...)
+	return replacer.Replace(actionEntry.TemplateContent)
+}
+
+func escapeCommandlineArgument(str string) string {
+	// \->\\, $->\$, `->\`, "->\", \n->\\n, '->'"'"'
+	replacer := strings.NewReplacer(
+		`\`, `\\`,
+		`$`, `\$`,
+		"`", "\\`",
+		`"`, `\"`,
+		"\n", "\\n",
+		`'`, `'"'"'`,
+	)
+	return replacer.Replace(str)
+}
+
+// removePy3wrapperScript removes py3wrapper.sh from the input paths and command of the action of
+// creating python zip file in mixed build mode. py3wrapper.sh is returned as input by aquery but
+// there is no action returned by aquery for creating it. So in mixed build "python3" is used
+// as the PYTHON_BINARY in python binary stub script, and py3wrapper.sh is not needed and should be
+// removed from input paths and command of creating python zip file.
+// See go/python-binary-host-mixed-build for more details.
+// TODO(b/205879240) remove this after py3wrapper.sh could be created in the mixed build mode.
+func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newCommand string) {
+	// Remove from inputs
+	filteredInputPaths := []string{}
+	for _, path := range bs.InputPaths {
+		if !strings.HasSuffix(path, py3wrapperFileName) {
+			filteredInputPaths = append(filteredInputPaths, path)
+		}
+	}
+	newInputPaths = filteredInputPaths
+
+	// Remove from command line
+	var re = regexp.MustCompile(`\S*` + py3wrapperFileName)
+	newCommand = re.ReplaceAllString(bs.Command, "")
+	return
+}
+
+// addCommandForPyBinaryRunfilesDir adds commands creating python binary runfiles directory.
+// runfiles directory is created by using MANIFEST file and MANIFEST file is the output of
+// SourceSymlinkManifest action is in aquery output of Bazel py_binary targets,
+// but since SourceSymlinkManifest doesn't contain sufficient information
+// so MANIFEST file could not be created, which also blocks the creation of runfiles directory.
+// See go/python-binary-host-mixed-build for more details.
+// TODO(b/197135294) create runfiles directory from MANIFEST file once it can be created from SourceSymlinkManifest action.
+func addCommandForPyBinaryRunfilesDir(bs BuildStatement, zipperCommandPath, zipFilePath string) string {
+	// Unzip the zip file, zipFilePath looks like <python_binary>.zip
+	runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles"
+	command := fmt.Sprintf("%s x %s -d %s", zipperCommandPath, zipFilePath, runfilesDirName)
+	// Create a symbolic link in <python_binary>.runfiles/, which is the expected structure
+	// when running the python binary stub script.
+	command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName)
+	return bs.Command + " && " + command
+}
+
 func isSymlinkAction(a action) bool {
 	return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink"
 }
 
+func isTemplateExpandAction(a action) bool {
+	return a.Mnemonic == "TemplateExpand"
+}
+
+func isPythonZipperAction(a action) bool {
+	return a.Mnemonic == "PythonZipper"
+}
+
 func shouldSkipAction(a action) bool {
 	// TODO(b/180945121): Handle complex symlink actions.
 	if a.Mnemonic == "SymlinkTree" || a.Mnemonic == "SourceSymlinkManifest" {

bazel/aquery_test.go

@@ -1015,6 +1015,355 @@ func TestSymlinkMultipleOutputs(t *testing.T) {
 	assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file"], output ["symlink" "other_symlink"]`)
 }
 
+func TestTemplateExpandActionSubstitutions(t *testing.T) {
+	const inputString = `
+{
+  "artifacts": [{
+    "id": 1,
+    "pathFragmentId": 1
+  }],
+  "actions": [{
+    "targetId": 1,
+    "actionKey": "x",
+    "mnemonic": "TemplateExpand",
+    "configurationId": 1,
+    "outputIds": [1],
+    "primaryOutputId": 1,
+    "executionPlatform": "//build/bazel/platforms:linux_x86_64",
+    "templateContent": "Test template substitutions: %token1%, %python_binary%",
+    "substitutions": [{
+      "key": "%token1%",
+      "value": "abcd"
+    },{
+      "key": "%python_binary%",
+      "value": "python3"
+    }]
+  }],
+  "pathFragments": [{
+    "id": 1,
+    "label": "template_file"
+  }]
+}`
+
+	actual, err := AqueryBuildStatements([]byte(inputString))
+
+	if err != nil {
+		t.Errorf("Unexpected error %q", err)
+	}
+
+	expectedBuildStatements := []BuildStatement{
+		BuildStatement{
+			Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > template_file && " +
+				"chmod a+x template_file'",
+			OutputPaths: []string{"template_file"},
+			Mnemonic:    "TemplateExpand",
+		},
+	}
+	assertBuildStatements(t, expectedBuildStatements, actual)
+}
+
+func TestTemplateExpandActionNoOutput(t *testing.T) {
+	const inputString = `
+{
+  "artifacts": [{
+    "id": 1,
+    "pathFragmentId": 1
+  }],
+  "actions": [{
+    "targetId": 1,
+    "actionKey": "x",
+    "mnemonic": "TemplateExpand",
+    "configurationId": 1,
+    "primaryOutputId": 1,
+    "executionPlatform": "//build/bazel/platforms:linux_x86_64",
+    "templateContent": "Test template substitutions: %token1%, %python_binary%",
+    "substitutions": [{
+      "key": "%token1%",
+      "value": "abcd"
+    },{
+      "key": "%python_binary%",
+      "value": "python3"
+    }]
+  }],
+  "pathFragments": [{
+    "id": 1,
+    "label": "template_file"
+  }]
+}`
+
+	_, err := AqueryBuildStatements([]byte(inputString))
+	assertError(t, err, `Expect 1 output to template expand action, got: output []`)
+}
+
+func TestPythonZipperActionSuccess(t *testing.T) {
+	const inputString = `
+{
+  "artifacts": [{
+    "id": 1,
+    "pathFragmentId": 1
+  },{
+    "id": 2,
+    "pathFragmentId": 2
+  },{
+    "id": 3,
+    "pathFragmentId": 3
+  },{
+    "id": 4,
+    "pathFragmentId": 4
+  },{
+    "id": 5,
+    "pathFragmentId": 10
+  },{
+    "id": 10,
+    "pathFragmentId": 20
+  }],
+  "actions": [{
+    "targetId": 1,
+    "actionKey": "x",
+    "mnemonic": "TemplateExpand",
+    "configurationId": 1,
+    "outputIds": [1],
+    "primaryOutputId": 1,
+    "executionPlatform": "//build/bazel/platforms:linux_x86_64",
+    "templateContent": "Test template substitutions: %token1%, %python_binary%",
+    "substitutions": [{
+      "key": "%token1%",
+      "value": "abcd"
+    },{
+      "key": "%python_binary%",
+      "value": "python3"
+    }]
+  },{
+    "targetId": 1,
+    "actionKey": "x",
+    "mnemonic": "PythonZipper",
+    "configurationId": 1,
+    "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
+    "outputIds": [2],
+    "inputDepSetIds": [1],
+    "primaryOutputId": 2
+  }],
+  "depSetOfFiles": [{
+    "id": 1,
+    "directArtifactIds": [4, 3, 5]
+  }],
+  "pathFragments": [{
+    "id": 1,
+    "label": "python_binary"
+  },{
+    "id": 2,
+    "label": "python_binary.zip"
+  },{
+    "id": 3,
+    "label": "python_binary.py"
+  },{
+    "id": 9,
+    "label": ".."
+  }, {
+    "id": 8,
+    "label": "bazel_tools",
+    "parentId": 9
+  }, {
+    "id": 7,
+    "label": "tools",
+    "parentId": 8
+  }, {
+    "id": 6,
+    "label": "zip",
+    "parentId": 7
+  }, {
+    "id": 5,
+    "label": "zipper",
+    "parentId": 6
+  }, {
+    "id": 4,
+    "label": "zipper",
+    "parentId": 5
+  },{
+    "id": 16,
+    "label": "bazel-out"
+  },{
+    "id": 15,
+    "label": "bazel_tools",
+    "parentId": 16
+  }, {
+    "id": 14,
+    "label": "k8-fastbuild",
+    "parentId": 15
+  }, {
+    "id": 13,
+    "label": "bin",
+    "parentId": 14
+  }, {
+    "id": 12,
+    "label": "tools",
+    "parentId": 13
+  }, {
+    "id": 11,
+    "label": "python",
+    "parentId": 12
+  }, {
+    "id": 10,
+    "label": "py3wrapper.sh",
+    "parentId": 11
+  },{
+    "id": 20,
+    "label": "python_binary"
+  }]
+}`
+	actual, err := AqueryBuildStatements([]byte(inputString))
+
+	if err != nil {
+		t.Errorf("Unexpected error %q", err)
+	}
+
+	expectedBuildStatements := []BuildStatement{
+		BuildStatement{
+			Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > python_binary && " +
+				"chmod a+x python_binary'",
+			InputPaths:  []string{"python_binary.zip"},
+			OutputPaths: []string{"python_binary"},
+			Mnemonic:    "TemplateExpand",
+		},
+		BuildStatement{
+			Command: "../bazel_tools/tools/zip/zipper/zipper cC python_binary.zip __main__.py=bazel-out/k8-fastbuild/bin/python_binary.temp " +
+				"__init__.py= runfiles/__main__/__init__.py= runfiles/__main__/python_binary.py=python_binary.py  && " +
+				"../bazel_tools/tools/zip/zipper/zipper x python_binary.zip -d python_binary.runfiles && ln -sf runfiles/__main__ python_binary.runfiles",
+			InputPaths:  []string{"../bazel_tools/tools/zip/zipper/zipper", "python_binary.py"},
+			OutputPaths: []string{"python_binary.zip"},
+			Mnemonic:    "PythonZipper",
+		},
+	}
+	assertBuildStatements(t, expectedBuildStatements, actual)
+}
+
+func TestPythonZipperActionNoInput(t *testing.T) {
+	const inputString = `
+{
+  "artifacts": [{
+    "id": 1,
+    "pathFragmentId": 1
+  },{
+    "id": 2,
+    "pathFragmentId": 2
+  }],
+  "actions": [{
+    "targetId": 1,
+    "actionKey": "x",
+    "mnemonic": "PythonZipper",
+    "configurationId": 1,
+    "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
+    "outputIds": [2],
+    "primaryOutputId": 2
+  }],
+  "pathFragments": [{
+    "id": 1,
+    "label": "python_binary"
+  },{
+    "id": 2,
+    "label": "python_binary.zip"
+  }]
+}`
+	_, err := AqueryBuildStatements([]byte(inputString))
+	assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input [], output ["python_binary.zip"]`)
+}
+
+func TestPythonZipperActionNoOutput(t *testing.T) {
+	const inputString = `
+{
+  "artifacts": [{
+    "id": 1,
+    "pathFragmentId": 1
+  },{
+    "id": 2,
+    "pathFragmentId": 2
+  },{
+    "id": 3,
+    "pathFragmentId": 3
+  },{
+    "id": 4,
+    "pathFragmentId": 4
+  },{
+    "id": 5,
+    "pathFragmentId": 10
+  }],
+  "actions": [{
+    "targetId": 1,
+    "actionKey": "x",
+    "mnemonic": "PythonZipper",
+    "configurationId": 1,
+    "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
+    "inputDepSetIds": [1]
+  }],
+  "depSetOfFiles": [{
+    "id": 1,
+    "directArtifactIds": [4, 3, 5]
+  }],
+  "pathFragments": [{
+    "id": 1,
+    "label": "python_binary"
+  },{
+    "id": 2,
+    "label": "python_binary.zip"
+  },{
+    "id": 3,
+    "label": "python_binary.py"
+  },{
+    "id": 9,
+    "label": ".."
+  }, {
+    "id": 8,
+    "label": "bazel_tools",
+    "parentId": 9
+  }, {
+    "id": 7,
+    "label": "tools",
+    "parentId": 8
+  }, {
+    "id": 6,
+    "label": "zip",
+    "parentId": 7
+  }, {
+    "id": 5,
+    "label": "zipper",
+    "parentId": 6
+  }, {
+    "id": 4,
+    "label": "zipper",
+    "parentId": 5
+  },{
+    "id": 16,
+    "label": "bazel-out"
+  },{
+    "id": 15,
+    "label": "bazel_tools",
+    "parentId": 16
+  }, {
+    "id": 14,
+    "label": "k8-fastbuild",
+    "parentId": 15
+  }, {
+    "id": 13,
+    "label": "bin",
+    "parentId": 14
+  }, {
+    "id": 12,
+    "label": "tools",
+    "parentId": 13
+  }, {
+    "id": 11,
+    "label": "python",
+    "parentId": 12
+  }, {
+    "id": 10,
+    "label": "py3wrapper.sh",
+    "parentId": 11
+  }]
+}`
+	_, err := AqueryBuildStatements([]byte(inputString))
+	assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["../bazel_tools/tools/zip/zipper/zipper" "python_binary.py"], output []`)
+}
+
 func assertError(t *testing.T, err error, expected string) {
 	t.Helper()
 	if err == nil {