diff --git a/android/bazel.go b/android/bazel.go index 63cc6c204..4f613df7e 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -332,12 +332,8 @@ var ( // Per-module denylist to opt modules out of mixed builds. Such modules will // still be generated via bp2build. mixedBuildsDisabledList = []string{ - "libbrotli", // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy - "func_to_syscall_nrs", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module. - "libseccomp_policy_app_zygote_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module. - "libseccomp_policy_app_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module. - "libseccomp_policy_system_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module. - "minijail_constants_json", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module. + "libbrotli", // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy + "minijail_constants_json", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module. } // Used for quicker lookups diff --git a/bazel/aquery.go b/bazel/aquery.go index 6d96b1ca1..4475b5e59 100644 --- a/bazel/aquery.go +++ b/bazel/aquery.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "path/filepath" + "regexp" "strings" "github.com/google/blueprint/proptools" @@ -59,6 +60,8 @@ type action struct { InputDepSetIds []int Mnemonic string OutputIds []int + TemplateContent string + Substitutions []KeyValuePair } // actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer. @@ -100,6 +103,20 @@ type aqueryArtifactHandler struct { artifactIdToPath map[int]string } +// The tokens should be substituted with the value specified here, instead of the +// one returned in 'substitutions' of TemplateExpand action. +var TemplateActionOverriddenTokens = map[string]string{ + // Uses "python3" for %python_binary% instead of the value returned by aquery + // which is "py3wrapper.sh". See removePy3wrapperScript. + "%python_binary%": "python3", +} + +// This pattern matches the MANIFEST file created for a py_binary target. +var manifestFilePattern = regexp.MustCompile(".*/.+\\.runfiles/MANIFEST$") + +// The file name of py3wrapper.sh, which is used by py_binary targets. +var py3wrapperFileName = "/py3wrapper.sh" + func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler, error) { pathFragments := map[int]pathFragment{} for _, pathFragment := range aqueryResult.PathFragments { @@ -163,7 +180,31 @@ func (a *aqueryArtifactHandler) getInputPaths(depsetIds []int) ([]string, error) } } } - return inputPaths, nil + + // TODO(b/197135294): Clean up this custom runfiles handling logic when + // SourceSymlinkManifest and SymlinkTree actions are supported. + filteredInputPaths := filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths) + + return filteredInputPaths, nil +} + +// See go/python-binary-host-mixed-build for more details. +// 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of +// Bazel py_binary targets, so there is no Ninja build statements generated for creating it. +// 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets, +// but it doesn't contain sufficient information so no Ninja build statements are generated +// for creating it. +// So in mixed build mode, when these two are used as input of some Ninja build statement, +// since there is no build statement to create them, they should be removed from input paths. +func filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths []string) []string { + filteredInputPaths := []string{} + for _, path := range inputPaths { + if strings.HasSuffix(path, py3wrapperFileName) || manifestFilePattern.MatchString(path) { + continue + } + filteredInputPaths = append(filteredInputPaths, path) + } + return filteredInputPaths } func (a *aqueryArtifactHandler) artifactIdsFromDepsetId(depsetId int) ([]int, error) { @@ -249,6 +290,45 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) { // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) buildStatement.SymlinkPaths = outputPaths[:] + } else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 { + if len(outputPaths) != 1 { + return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths) + } + expandedTemplateContent := expandTemplateContent(actionEntry) + // The expandedTemplateContent is escaped for being used in double quotes and shell unescape, + // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might + // change \n to space and mess up the format of Python programs. + // sed is used to convert \\n back to \n before saving to output file. + // See go/python-binary-host-mixed-build for more details. + command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`, + escapeCommandlineArgument(expandedTemplateContent), outputPaths[0]) + buildStatement.Command = command + } else if isPythonZipperAction(actionEntry) { + if len(inputPaths) < 1 || len(outputPaths) != 1 { + return nil, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths) + } + buildStatement.InputPaths, buildStatement.Command = removePy3wrapperScript(buildStatement) + buildStatement.Command = addCommandForPyBinaryRunfilesDir(buildStatement, inputPaths[0], outputPaths[0]) + // Add the python zip file as input of the corresponding python binary stub script in Ninja build statements. + // In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input, + // which is not sufficient without the python zip file from which runfiles directory is created for py_binary. + // + // The following logic relies on that Bazel aquery output returns actions in the order that + // PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions + // in that order, the following logic might not find the build statement generated for Python binary + // stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output. + // See go/python-binary-host-mixed-build for more details. + pythonZipFilePath := outputPaths[0] + pyBinaryFound := false + for i, _ := range buildStatements { + if len(buildStatements[i].OutputPaths) == 1 && buildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath { + buildStatements[i].InputPaths = append(buildStatements[i].InputPaths, pythonZipFilePath) + pyBinaryFound = true + } + } + if !pyBinaryFound { + return nil, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths) + } } else if len(actionEntry.Arguments) < 1 { return nil, fmt.Errorf("received action with no command: [%v]", buildStatement) } @@ -258,10 +338,85 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) { return buildStatements, nil } +// expandTemplateContent substitutes the tokens in a template. +func expandTemplateContent(actionEntry action) string { + replacerString := []string{} + for _, pair := range actionEntry.Substitutions { + value := pair.Value + if val, ok := TemplateActionOverriddenTokens[pair.Key]; ok { + value = val + } + replacerString = append(replacerString, pair.Key, value) + } + replacer := strings.NewReplacer(replacerString...) + return replacer.Replace(actionEntry.TemplateContent) +} + +func escapeCommandlineArgument(str string) string { + // \->\\, $->\$, `->\`, "->\", \n->\\n, '->'"'"' + replacer := strings.NewReplacer( + `\`, `\\`, + `$`, `\$`, + "`", "\\`", + `"`, `\"`, + "\n", "\\n", + `'`, `'"'"'`, + ) + return replacer.Replace(str) +} + +// removePy3wrapperScript removes py3wrapper.sh from the input paths and command of the action of +// creating python zip file in mixed build mode. py3wrapper.sh is returned as input by aquery but +// there is no action returned by aquery for creating it. So in mixed build "python3" is used +// as the PYTHON_BINARY in python binary stub script, and py3wrapper.sh is not needed and should be +// removed from input paths and command of creating python zip file. +// See go/python-binary-host-mixed-build for more details. +// TODO(b/205879240) remove this after py3wrapper.sh could be created in the mixed build mode. +func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newCommand string) { + // Remove from inputs + filteredInputPaths := []string{} + for _, path := range bs.InputPaths { + if !strings.HasSuffix(path, py3wrapperFileName) { + filteredInputPaths = append(filteredInputPaths, path) + } + } + newInputPaths = filteredInputPaths + + // Remove from command line + var re = regexp.MustCompile(`\S*` + py3wrapperFileName) + newCommand = re.ReplaceAllString(bs.Command, "") + return +} + +// addCommandForPyBinaryRunfilesDir adds commands creating python binary runfiles directory. +// runfiles directory is created by using MANIFEST file and MANIFEST file is the output of +// SourceSymlinkManifest action is in aquery output of Bazel py_binary targets, +// but since SourceSymlinkManifest doesn't contain sufficient information +// so MANIFEST file could not be created, which also blocks the creation of runfiles directory. +// See go/python-binary-host-mixed-build for more details. +// TODO(b/197135294) create runfiles directory from MANIFEST file once it can be created from SourceSymlinkManifest action. +func addCommandForPyBinaryRunfilesDir(bs BuildStatement, zipperCommandPath, zipFilePath string) string { + // Unzip the zip file, zipFilePath looks like .zip + runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles" + command := fmt.Sprintf("%s x %s -d %s", zipperCommandPath, zipFilePath, runfilesDirName) + // Create a symbolic link in .runfiles/, which is the expected structure + // when running the python binary stub script. + command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName) + return bs.Command + " && " + command +} + func isSymlinkAction(a action) bool { return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink" } +func isTemplateExpandAction(a action) bool { + return a.Mnemonic == "TemplateExpand" +} + +func isPythonZipperAction(a action) bool { + return a.Mnemonic == "PythonZipper" +} + func shouldSkipAction(a action) bool { // TODO(b/180945121): Handle complex symlink actions. if a.Mnemonic == "SymlinkTree" || a.Mnemonic == "SourceSymlinkManifest" { diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go index 69f11152e..ca58606d7 100644 --- a/bazel/aquery_test.go +++ b/bazel/aquery_test.go @@ -1015,6 +1015,355 @@ func TestSymlinkMultipleOutputs(t *testing.T) { assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file"], output ["symlink" "other_symlink"]`) } +func TestTemplateExpandActionSubstitutions(t *testing.T) { + const inputString = ` +{ + "artifacts": [{ + "id": 1, + "pathFragmentId": 1 + }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "TemplateExpand", + "configurationId": 1, + "outputIds": [1], + "primaryOutputId": 1, + "executionPlatform": "//build/bazel/platforms:linux_x86_64", + "templateContent": "Test template substitutions: %token1%, %python_binary%", + "substitutions": [{ + "key": "%token1%", + "value": "abcd" + },{ + "key": "%python_binary%", + "value": "python3" + }] + }], + "pathFragments": [{ + "id": 1, + "label": "template_file" + }] +}` + + actual, err := AqueryBuildStatements([]byte(inputString)) + + if err != nil { + t.Errorf("Unexpected error %q", err) + } + + expectedBuildStatements := []BuildStatement{ + BuildStatement{ + Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > template_file && " + + "chmod a+x template_file'", + OutputPaths: []string{"template_file"}, + Mnemonic: "TemplateExpand", + }, + } + assertBuildStatements(t, expectedBuildStatements, actual) +} + +func TestTemplateExpandActionNoOutput(t *testing.T) { + const inputString = ` +{ + "artifacts": [{ + "id": 1, + "pathFragmentId": 1 + }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "TemplateExpand", + "configurationId": 1, + "primaryOutputId": 1, + "executionPlatform": "//build/bazel/platforms:linux_x86_64", + "templateContent": "Test template substitutions: %token1%, %python_binary%", + "substitutions": [{ + "key": "%token1%", + "value": "abcd" + },{ + "key": "%python_binary%", + "value": "python3" + }] + }], + "pathFragments": [{ + "id": 1, + "label": "template_file" + }] +}` + + _, err := AqueryBuildStatements([]byte(inputString)) + assertError(t, err, `Expect 1 output to template expand action, got: output []`) +} + +func TestPythonZipperActionSuccess(t *testing.T) { + const inputString = ` +{ + "artifacts": [{ + "id": 1, + "pathFragmentId": 1 + },{ + "id": 2, + "pathFragmentId": 2 + },{ + "id": 3, + "pathFragmentId": 3 + },{ + "id": 4, + "pathFragmentId": 4 + },{ + "id": 5, + "pathFragmentId": 10 + },{ + "id": 10, + "pathFragmentId": 20 + }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "TemplateExpand", + "configurationId": 1, + "outputIds": [1], + "primaryOutputId": 1, + "executionPlatform": "//build/bazel/platforms:linux_x86_64", + "templateContent": "Test template substitutions: %token1%, %python_binary%", + "substitutions": [{ + "key": "%token1%", + "value": "abcd" + },{ + "key": "%python_binary%", + "value": "python3" + }] + },{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "PythonZipper", + "configurationId": 1, + "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"], + "outputIds": [2], + "inputDepSetIds": [1], + "primaryOutputId": 2 + }], + "depSetOfFiles": [{ + "id": 1, + "directArtifactIds": [4, 3, 5] + }], + "pathFragments": [{ + "id": 1, + "label": "python_binary" + },{ + "id": 2, + "label": "python_binary.zip" + },{ + "id": 3, + "label": "python_binary.py" + },{ + "id": 9, + "label": ".." + }, { + "id": 8, + "label": "bazel_tools", + "parentId": 9 + }, { + "id": 7, + "label": "tools", + "parentId": 8 + }, { + "id": 6, + "label": "zip", + "parentId": 7 + }, { + "id": 5, + "label": "zipper", + "parentId": 6 + }, { + "id": 4, + "label": "zipper", + "parentId": 5 + },{ + "id": 16, + "label": "bazel-out" + },{ + "id": 15, + "label": "bazel_tools", + "parentId": 16 + }, { + "id": 14, + "label": "k8-fastbuild", + "parentId": 15 + }, { + "id": 13, + "label": "bin", + "parentId": 14 + }, { + "id": 12, + "label": "tools", + "parentId": 13 + }, { + "id": 11, + "label": "python", + "parentId": 12 + }, { + "id": 10, + "label": "py3wrapper.sh", + "parentId": 11 + },{ + "id": 20, + "label": "python_binary" + }] +}` + actual, err := AqueryBuildStatements([]byte(inputString)) + + if err != nil { + t.Errorf("Unexpected error %q", err) + } + + expectedBuildStatements := []BuildStatement{ + BuildStatement{ + Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > python_binary && " + + "chmod a+x python_binary'", + InputPaths: []string{"python_binary.zip"}, + OutputPaths: []string{"python_binary"}, + Mnemonic: "TemplateExpand", + }, + BuildStatement{ + Command: "../bazel_tools/tools/zip/zipper/zipper cC python_binary.zip __main__.py=bazel-out/k8-fastbuild/bin/python_binary.temp " + + "__init__.py= runfiles/__main__/__init__.py= runfiles/__main__/python_binary.py=python_binary.py && " + + "../bazel_tools/tools/zip/zipper/zipper x python_binary.zip -d python_binary.runfiles && ln -sf runfiles/__main__ python_binary.runfiles", + InputPaths: []string{"../bazel_tools/tools/zip/zipper/zipper", "python_binary.py"}, + OutputPaths: []string{"python_binary.zip"}, + Mnemonic: "PythonZipper", + }, + } + assertBuildStatements(t, expectedBuildStatements, actual) +} + +func TestPythonZipperActionNoInput(t *testing.T) { + const inputString = ` +{ + "artifacts": [{ + "id": 1, + "pathFragmentId": 1 + },{ + "id": 2, + "pathFragmentId": 2 + }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "PythonZipper", + "configurationId": 1, + "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"], + "outputIds": [2], + "primaryOutputId": 2 + }], + "pathFragments": [{ + "id": 1, + "label": "python_binary" + },{ + "id": 2, + "label": "python_binary.zip" + }] +}` + _, err := AqueryBuildStatements([]byte(inputString)) + assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input [], output ["python_binary.zip"]`) +} + +func TestPythonZipperActionNoOutput(t *testing.T) { + const inputString = ` +{ + "artifacts": [{ + "id": 1, + "pathFragmentId": 1 + },{ + "id": 2, + "pathFragmentId": 2 + },{ + "id": 3, + "pathFragmentId": 3 + },{ + "id": 4, + "pathFragmentId": 4 + },{ + "id": 5, + "pathFragmentId": 10 + }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "PythonZipper", + "configurationId": 1, + "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"], + "inputDepSetIds": [1] + }], + "depSetOfFiles": [{ + "id": 1, + "directArtifactIds": [4, 3, 5] + }], + "pathFragments": [{ + "id": 1, + "label": "python_binary" + },{ + "id": 2, + "label": "python_binary.zip" + },{ + "id": 3, + "label": "python_binary.py" + },{ + "id": 9, + "label": ".." + }, { + "id": 8, + "label": "bazel_tools", + "parentId": 9 + }, { + "id": 7, + "label": "tools", + "parentId": 8 + }, { + "id": 6, + "label": "zip", + "parentId": 7 + }, { + "id": 5, + "label": "zipper", + "parentId": 6 + }, { + "id": 4, + "label": "zipper", + "parentId": 5 + },{ + "id": 16, + "label": "bazel-out" + },{ + "id": 15, + "label": "bazel_tools", + "parentId": 16 + }, { + "id": 14, + "label": "k8-fastbuild", + "parentId": 15 + }, { + "id": 13, + "label": "bin", + "parentId": 14 + }, { + "id": 12, + "label": "tools", + "parentId": 13 + }, { + "id": 11, + "label": "python", + "parentId": 12 + }, { + "id": 10, + "label": "py3wrapper.sh", + "parentId": 11 + }] +}` + _, err := AqueryBuildStatements([]byte(inputString)) + assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["../bazel_tools/tools/zip/zipper/zipper" "python_binary.py"], output []`) +} + func assertError(t *testing.T, err error, expected string) { t.Helper() if err == nil {