From 1da064c1e647f03fa4ef2742a5a01520093dbc63 Mon Sep 17 00:00:00 2001 From: Sasha Smundak Date: Wed, 8 Jun 2022 16:36:16 -0700 Subject: [PATCH] Handle WriteFile and SourceSymlinkManifest actions. Plus minor editorial changes. Bug: 232085015 Test: treehugger Change-Id: I966e9d6d306382dbb8eac6f8a495a2f152c7a22e --- android/bazel_handler.go | 44 ++++++++++++++++++++++++---- bazel/aquery.go | 46 +++++++++++++++++++++-------- bazel/aquery_test.go | 63 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 18 deletions(-) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index f1ec55eb9..eff03176d 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -34,6 +34,14 @@ import ( "android/soong/bazel" ) +var ( + writeBazelFile = pctx.AndroidStaticRule("bazelWriteFileRule", blueprint.RuleParams{ + Command: `sed "s/\\\\n/\n/g" ${out}.rsp >${out}`, + Rspfile: "${out}.rsp", + RspfileContent: "${content}", + }, "content") +) + func init() { RegisterMixedBuildsMutator(InitRegistrationContext) } @@ -778,7 +786,7 @@ func (context *bazelContext) InvokeBazel(config Config) error { } } - extraFlags := append([]string{"--output=jsonproto"}, coverageFlags...) + extraFlags := append([]string{"--output=jsonproto", "--include_file_write_contents"}, coverageFlags...) aqueryOutput, _, err = context.issueBazelCommand( context.paths, @@ -874,13 +882,37 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__") bazelOutDir := path.Join(executionRoot, "bazel-out") for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() { - if len(buildStatement.Command) < 1 { + if len(buildStatement.Command) > 0 { + rule := NewRuleBuilder(pctx, ctx) + createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx) + desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) + rule.Build(fmt.Sprintf("bazel %d", index), desc) + continue + } + // Certain actions returned by aquery (for instance FileWrite) do not contain a command + // and thus require special treatment. If BuildStatement were an interface implementing + // buildRule(ctx) function, the code here would just call it. + // Unfortunately, the BuildStatement is defined in + // the 'bazel' package, which cannot depend on 'android' package where ctx is defined, + // because this would cause circular dependency. So, until we move aquery processing + // to the 'android' package, we need to handle special cases here. + if buildStatement.Mnemonic == "FileWrite" || buildStatement.Mnemonic == "SourceSymlinkManifest" { + // Pass file contents as the value of the rule's "content" argument. + // Escape newlines and $ in the contents (the action "writeBazelFile" restores "\\n" + // back to the newline, and Ninja reads $$ as $. + escaped := strings.ReplaceAll(strings.ReplaceAll(buildStatement.FileContents, "\n", "\\n"), + "$", "$$") + ctx.Build(pctx, BuildParams{ + Rule: writeBazelFile, + Output: PathForBazelOut(ctx, buildStatement.OutputPaths[0]), + Description: fmt.Sprintf("%s %s", buildStatement.Mnemonic, buildStatement.OutputPaths[0]), + Args: map[string]string{ + "content": escaped, + }, + }) + } else { panic(fmt.Sprintf("unhandled build statement: %v", buildStatement)) } - rule := NewRuleBuilder(pctx, ctx) - createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx) - desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) - rule.Build(fmt.Sprintf("bazel %d", index), desc) } } diff --git a/bazel/aquery.go b/bazel/aquery.go index 1d1f49cdf..2853a70bc 100644 --- a/bazel/aquery.go +++ b/bazel/aquery.go @@ -83,6 +83,7 @@ type action struct { OutputIds []artifactId TemplateContent string Substitutions []KeyValuePair + FileContents string } // actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer. @@ -110,6 +111,7 @@ type BuildStatement struct { // input path string, but not both. InputDepsetHashes []string InputPaths []string + FileContents string } // A helper type for aquery processing which facilitates retrieval of path IDs from their @@ -346,12 +348,14 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDe } var buildStatement BuildStatement - if isSymlinkAction(actionEntry) { + if actionEntry.isSymlinkAction() { buildStatement, err = aqueryHandler.symlinkActionBuildStatement(actionEntry) - } else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 { + } else if actionEntry.isTemplateExpandAction() && len(actionEntry.Arguments) < 1 { buildStatement, err = aqueryHandler.templateExpandActionBuildStatement(actionEntry) - } else if isPythonZipperAction(actionEntry) { + } else if actionEntry.isPythonZipperAction() { buildStatement, err = aqueryHandler.pythonZipperActionBuildStatement(actionEntry, buildStatements) + } else if actionEntry.isFileWriteAction() { + buildStatement, err = aqueryHandler.fileWriteActionBuildStatement(actionEntry) } else if len(actionEntry.Arguments) < 1 { return nil, nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic) } else { @@ -532,6 +536,25 @@ func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry a return buildStatement, nil } +func (a *aqueryArtifactHandler) fileWriteActionBuildStatement(actionEntry action) (BuildStatement, error) { + outputPaths, _, err := a.getOutputPaths(actionEntry) + var depsetHashes []string + if err == nil { + depsetHashes, err = a.depsetContentHashes(actionEntry.InputDepSetIds) + } + if err != nil { + return BuildStatement{}, err + } + return BuildStatement{ + Depfile: nil, + OutputPaths: outputPaths, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + InputDepsetHashes: depsetHashes, + FileContents: actionEntry.FileContents, + }, nil +} + func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry action) (BuildStatement, error) { outputPaths, depfile, err := a.getOutputPaths(actionEntry) if err != nil { @@ -654,21 +677,25 @@ func addCommandForPyBinaryRunfilesDir(oldCommand string, zipFilePath string) str return oldCommand + " && " + command } -func isSymlinkAction(a action) bool { +func (a action) isSymlinkAction() bool { return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink" || a.Mnemonic == "ExecutableSymlink" } -func isTemplateExpandAction(a action) bool { +func (a action) isTemplateExpandAction() bool { return a.Mnemonic == "TemplateExpand" } -func isPythonZipperAction(a action) bool { +func (a action) isPythonZipperAction() bool { return a.Mnemonic == "PythonZipper" } +func (a action) isFileWriteAction() bool { + return a.Mnemonic == "FileWrite" || a.Mnemonic == "SourceSymlinkManifest" +} + func shouldSkipAction(a action) bool { // TODO(b/180945121): Handle complex symlink actions. - if a.Mnemonic == "SymlinkTree" || a.Mnemonic == "SourceSymlinkManifest" { + if a.Mnemonic == "SymlinkTree" { return true } // Middleman actions are not handled like other actions; they are handled separately as a @@ -681,11 +708,6 @@ func shouldSkipAction(a action) bool { if a.Mnemonic == "Fail" { return true } - // TODO(b/180946980): Handle FileWrite. The aquery proto currently contains no information - // about the contents that are written. - if a.Mnemonic == "FileWrite" { - return true - } if a.Mnemonic == "BaselineCoverage" { return true } diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go index c759d56ee..ba22f37a1 100644 --- a/bazel/aquery_test.go +++ b/bazel/aquery_test.go @@ -1008,6 +1008,69 @@ func TestPythonZipperActionNoOutput(t *testing.T) { assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["python_binary.py"], output []`) } +func TestFileWrite(t *testing.T) { + const inputString = ` +{ + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "FileWrite", + "configurationId": 1, + "outputIds": [1], + "primaryOutputId": 1, + "executionPlatform": "//build/bazel/platforms:linux_x86_64", + "fileContents": "file data\n" + }], + "pathFragments": [ + { "id": 1, "label": "foo.manifest" }] +} +` + actual, _, err := AqueryBuildStatements([]byte(inputString)) + if err != nil { + t.Errorf("Unexpected error %q", err) + } + assertBuildStatements(t, []BuildStatement{ + { + OutputPaths: []string{"foo.manifest"}, + Mnemonic: "FileWrite", + FileContents: "file data\n", + }, + }, actual) +} + +func TestSourceSymlinkManifest(t *testing.T) { + const inputString = ` +{ + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "SourceSymlinkManifest", + "configurationId": 1, + "outputIds": [1], + "primaryOutputId": 1, + "executionPlatform": "//build/bazel/platforms:linux_x86_64", + "fileContents": "symlink target\n" + }], + "pathFragments": [ + { "id": 1, "label": "foo.manifest" }] +} +` + actual, _, err := AqueryBuildStatements([]byte(inputString)) + if err != nil { + t.Errorf("Unexpected error %q", err) + } + assertBuildStatements(t, []BuildStatement{ + { + OutputPaths: []string{"foo.manifest"}, + Mnemonic: "SourceSymlinkManifest", + }, + }, actual) +} + func assertError(t *testing.T, err error, expected string) { t.Helper() if err == nil {