diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 69bcd2e4b..c935101ee 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -1643,4 +1643,14 @@ var ( "art_": DEFAULT_PRIORITIZED_WEIGHT, "ndk_library": DEFAULT_PRIORITIZED_WEIGHT, } + + BazelSandwichTargets = []struct { + Label string + Host bool + }{ + { + Label: "//build/bazel/examples/partitions:system_image", + Host: false, + }, + } ) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 94bc88b42..4c03ae6d1 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -186,6 +186,8 @@ type BazelContext interface { // Returns the depsets defined in Bazel's aquery response. AqueryDepsets() []bazel.AqueryDepset + + QueueBazelSandwichCqueryRequests(config Config) error } type bazelRunner interface { @@ -264,6 +266,10 @@ func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequ m.BazelRequests[key] = true } +func (m MockBazelContext) QueueBazelSandwichCqueryRequests(config Config) error { + panic("unimplemented") +} + func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) { result, ok := m.LabelToOutputFiles[label] if !ok { @@ -424,6 +430,10 @@ func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configK panic("unimplemented") } +func (n noopBazelContext) QueueBazelSandwichCqueryRequests(config Config) error { + panic("unimplemented") +} + func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) { panic("unimplemented") } @@ -1042,6 +1052,45 @@ var ( allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd} ) +func GetBazelSandwichCqueryRequests(config Config) ([]cqueryKey, error) { + result := make([]cqueryKey, 0, len(allowlists.BazelSandwichTargets)) + // Note that bazel "targets" are different from soong "targets", the bazel targets are + // synonymous with soong modules, and soong targets are a configuration a module is built in. + for _, target := range allowlists.BazelSandwichTargets { + var soongTarget Target + if target.Host { + soongTarget = config.BuildOSTarget + } else { + soongTarget = config.AndroidCommonTarget + } + + result = append(result, cqueryKey{ + label: target.Label, + requestType: cquery.GetOutputFiles, + configKey: configKey{ + arch: soongTarget.Arch.String(), + osType: soongTarget.Os, + }, + }) + } + return result, nil +} + +// QueueBazelSandwichCqueryRequests queues cquery requests for all the bazel labels in +// bazel_sandwich_targets. These will later be given phony targets so that they can be built on the +// command line. +func (context *mixedBuildBazelContext) QueueBazelSandwichCqueryRequests(config Config) error { + requests, err := GetBazelSandwichCqueryRequests(config) + if err != nil { + return err + } + for _, request := range requests { + context.QueueBazelRequest(request.label, request.requestType, request.configKey) + } + + return nil +} + // Issues commands to Bazel to receive results for all cquery requests // queued in the BazelContext. func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error { @@ -1255,6 +1304,11 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__") bazelOutDir := path.Join(executionRoot, "bazel-out") + rel, err := filepath.Rel(ctx.Config().OutDir(), executionRoot) + if err != nil { + ctx.Errorf("%s", err.Error()) + } + dotdotsToOutRoot := strings.Repeat("../", strings.Count(rel, "/")+1) for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() { // nil build statements are a valid case where we do not create an action because it is // unnecessary or handled by other processing @@ -1286,7 +1340,8 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { }) } } - createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset) + createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset, dotdotsToOutRoot) + desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) rule.Build(fmt.Sprintf("bazel %d", index), desc) continue @@ -1331,6 +1386,24 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { panic(fmt.Sprintf("unhandled build statement: %v", buildStatement)) } } + + // Create phony targets for all the bazel sandwich output files + requests, err := GetBazelSandwichCqueryRequests(ctx.Config()) + if err != nil { + ctx.Errorf(err.Error()) + } + for _, request := range requests { + files, err := ctx.Config().BazelContext.GetOutputFiles(request.label, request.configKey) + if err != nil { + ctx.Errorf(err.Error()) + } + filesAsPaths := make([]Path, 0, len(files)) + for _, file := range files { + filesAsPaths = append(filesAsPaths, PathForBazelOut(ctx, file)) + } + ctx.Phony("bazel_sandwich", filesAsPaths...) + } + ctx.Phony("checkbuild", PathForPhony(ctx, "bazel_sandwich")) } // Returns a out dir path for a sandboxed mixed build action @@ -1344,7 +1417,7 @@ func intermediatePathForSboxMixedBuildAction(ctx PathContext, statement *bazel.B } // Register bazel-owned build statements (obtained from the aquery invocation). -func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset) { +func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset, dotdotsToOutRoot string) { // executionRoot is the action cwd. if buildStatement.ShouldRunInSbox { // mkdir -p ensures that the directory exists when run via sbox @@ -1367,14 +1440,17 @@ func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement cmd.Flag(pair.Key + "=" + pair.Value) } + command := buildStatement.Command + command = strings.ReplaceAll(command, "{DOTDOTS_TO_OUTPUT_ROOT}", dotdotsToOutRoot) + // The actual Bazel action. - if len(buildStatement.Command) > 16*1024 { + if len(command) > 16*1024 { commandFile := PathForBazelOut(ctx, buildStatement.OutputPaths[0]+".sh") - WriteFileRule(ctx, commandFile, buildStatement.Command) + WriteFileRule(ctx, commandFile, command) cmd.Text("bash").Text(buildStatement.OutputPaths[0] + ".sh").Implicit(commandFile) } else { - cmd.Text(buildStatement.Command) + cmd.Text(command) } for _, outputPath := range buildStatement.OutputPaths { @@ -1403,6 +1479,9 @@ func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement cmd.Implicit(PathForPhony(ctx, otherDepsetName)) } } + for _, implicitPath := range buildStatement.ImplicitDeps { + cmd.Implicit(PathForArbitraryOutput(ctx, implicitPath)) + } if depfile := buildStatement.Depfile; depfile != nil { // The paths in depfile are relative to `executionRoot`. diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go index e08a4718a..9a3c8fc7c 100644 --- a/android/bazel_handler_test.go +++ b/android/bazel_handler_test.go @@ -181,7 +181,7 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) { cmd := RuleBuilderCommand{} ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))} - createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}) + createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "") if actual, expected := cmd.buf.String(), testCase.command; expected != actual { t.Errorf("expected: [%s], actual: [%s]", expected, actual) } @@ -224,7 +224,7 @@ func TestMixedBuildSandboxedAction(t *testing.T) { cmd := RuleBuilderCommand{} ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))} - createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}) + createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "") // Assert that the output is generated in an intermediate directory // fe05bcdcdc4928012781a5f1a2a77cbb5398e106 is the sha1 checksum of "one" if actual, expected := cmd.outputs[0].String(), "out/soong/mixed_build_sbox_intermediates/fe05bcdcdc4928012781a5f1a2a77cbb5398e106/test/exec_root/one"; expected != actual { diff --git a/android/paths.go b/android/paths.go index e16cb3781..325a953c4 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1029,16 +1029,16 @@ func (p basePath) withRel(rel string) basePath { return p } +func (p basePath) RelativeToTop() Path { + ensureTestOnly() + return p +} + // SourcePath is a Path representing a file path rooted from SrcDir type SourcePath struct { basePath } -func (p SourcePath) RelativeToTop() Path { - ensureTestOnly() - return p -} - var _ Path = SourcePath{} func (p SourcePath) withRel(rel string) SourcePath { @@ -1126,6 +1126,16 @@ func PathForSource(ctx PathContext, pathComponents ...string) SourcePath { return path } +// PathForArbitraryOutput creates a path for the given components. Unlike PathForOutput, +// the path is relative to the root of the output folder, not the out/soong folder. +func PathForArbitraryOutput(ctx PathContext, pathComponents ...string) Path { + p, err := validatePath(pathComponents...) + if err != nil { + reportPathError(ctx, err) + } + return basePath{path: filepath.Join(ctx.Config().OutDir(), p)} +} + // MaybeExistentPathForSource joins the provided path components and validates that the result // neither escapes the source dir nor is in the out dir. // It does not validate whether the path exists. diff --git a/bazel/aquery.go b/bazel/aquery.go index 2c080a11a..d77d59acf 100644 --- a/bazel/aquery.go +++ b/bazel/aquery.go @@ -17,15 +17,15 @@ package bazel import ( "crypto/sha256" "encoding/base64" + "encoding/json" "fmt" "path/filepath" + analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" "reflect" "sort" "strings" "sync" - analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" - "github.com/google/blueprint/metrics" "github.com/google/blueprint/proptools" "google.golang.org/protobuf/proto" @@ -119,6 +119,10 @@ type BuildStatement struct { // If ShouldRunInSbox is true, Soong will use sbox to created an isolated environment // and run the mixed build action there ShouldRunInSbox bool + // A list of files to add as implicit deps to the outputs of this BuildStatement. + // Unlike most properties in BuildStatement, these paths must be relative to the root of + // the whole out/ folder, instead of relative to ctx.Config().BazelContext.OutputBase() + ImplicitDeps []string } // A helper type for aquery processing which facilitates retrieval of path IDs from their @@ -581,6 +585,72 @@ func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry *ana }, nil } +type bazelSandwichJson struct { + Target string `json:"target"` + DependOnTarget *bool `json:"depend_on_target,omitempty"` + ImplicitDeps []string `json:"implicit_deps"` +} + +func (a *aqueryArtifactHandler) unresolvedSymlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { + outputPaths, depfile, err := a.getOutputPaths(actionEntry) + if err != nil { + return nil, err + } + if len(actionEntry.InputDepSetIds) != 0 || len(outputPaths) != 1 { + return nil, fmt.Errorf("expected 0 inputs and 1 output to symlink action, got: input %q, output %q", actionEntry.InputDepSetIds, outputPaths) + } + target := actionEntry.UnresolvedSymlinkTarget + if target == "" { + return nil, fmt.Errorf("expected an unresolved_symlink_target, but didn't get one") + } + if filepath.Clean(target) != target { + return nil, fmt.Errorf("expected %q, got %q", filepath.Clean(target), target) + } + if strings.HasPrefix(target, "/") { + return nil, fmt.Errorf("no absolute symlinks allowed: %s", target) + } + + out := outputPaths[0] + outDir := filepath.Dir(out) + var implicitDeps []string + if strings.HasPrefix(target, "bazel_sandwich:") { + j := bazelSandwichJson{} + err := json.Unmarshal([]byte(target[len("bazel_sandwich:"):]), &j) + if err != nil { + return nil, err + } + if proptools.BoolDefault(j.DependOnTarget, true) { + implicitDeps = append(implicitDeps, j.Target) + } + implicitDeps = append(implicitDeps, j.ImplicitDeps...) + dotDotsToReachCwd := "" + if outDir != "." { + dotDotsToReachCwd = strings.Repeat("../", strings.Count(outDir, "/")+1) + } + target = proptools.ShellEscapeIncludingSpaces(j.Target) + target = "{DOTDOTS_TO_OUTPUT_ROOT}" + dotDotsToReachCwd + target + } else { + target = proptools.ShellEscapeIncludingSpaces(target) + } + + outDir = proptools.ShellEscapeIncludingSpaces(outDir) + out = proptools.ShellEscapeIncludingSpaces(out) + // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). + command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, target) + symlinkPaths := outputPaths[:] + + buildStatement := &BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + SymlinkPaths: symlinkPaths, + ImplicitDeps: implicitDeps, + } + return buildStatement, nil +} + func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { outputPaths, depfile, err := a.getOutputPaths(actionEntry) if err != nil { @@ -690,6 +760,8 @@ func (a *aqueryArtifactHandler) actionToBuildStatement(actionEntry *analysis_v2_ return a.fileWriteActionBuildStatement(actionEntry) case "SymlinkTree": return a.symlinkTreeActionBuildStatement(actionEntry) + case "UnresolvedSymlink": + return a.unresolvedSymlinkActionBuildStatement(actionEntry) } if len(actionEntry.Arguments) < 1 { diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go index 19a584f23..32c87a0a6 100644 --- a/bazel/aquery_test.go +++ b/bazel/aquery_test.go @@ -357,9 +357,11 @@ func TestDepfiles(t *testing.T) { actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) if err != nil { t.Errorf("Unexpected error %q", err) + return } if expected := 1; len(actual) != expected { t.Fatalf("Expected %d build statements, got %d", expected, len(actual)) + return } bs := actual[0] @@ -544,6 +546,7 @@ func TestSymlinkTree(t *testing.T) { actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) if err != nil { t.Errorf("Unexpected error %q", err) + return } assertBuildStatements(t, []*BuildStatement{ &BuildStatement{ @@ -756,9 +759,11 @@ func TestMiddlemenAction(t *testing.T) { actualBuildStatements, actualDepsets, err := AqueryBuildStatements(data, &metrics.EventHandler{}) if err != nil { t.Errorf("Unexpected error %q", err) + return } if expected := 2; len(actualBuildStatements) != expected { t.Fatalf("Expected %d build statements, got %d %#v", expected, len(actualBuildStatements), actualBuildStatements) + return } expectedDepsetFiles := [][]string{ @@ -859,6 +864,7 @@ func TestSimpleSymlink(t *testing.T) { if err != nil { t.Errorf("Unexpected error %q", err) + return } expectedBuildStatements := []*BuildStatement{ @@ -907,6 +913,7 @@ func TestSymlinkQuotesPaths(t *testing.T) { actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) if err != nil { t.Errorf("Unexpected error %q", err) + return } expectedBuildStatements := []*BuildStatement{ @@ -1017,6 +1024,7 @@ func TestTemplateExpandActionSubstitutions(t *testing.T) { actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) if err != nil { t.Errorf("Unexpected error %q", err) + return } expectedBuildStatements := []*BuildStatement{ @@ -1088,6 +1096,7 @@ func TestFileWrite(t *testing.T) { actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) if err != nil { t.Errorf("Unexpected error %q", err) + return } assertBuildStatements(t, []*BuildStatement{ &BuildStatement{ @@ -1126,6 +1135,7 @@ func TestSourceSymlinkManifest(t *testing.T) { actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) if err != nil { t.Errorf("Unexpected error %q", err) + return } assertBuildStatements(t, []*BuildStatement{ &BuildStatement{ @@ -1136,6 +1146,126 @@ func TestSourceSymlinkManifest(t *testing.T) { }, actual) } +func TestUnresolvedSymlink(t *testing.T) { + const inputString = ` +{ + "artifacts": [ + { "id": 1, "path_fragment_id": 1 } + ], + "actions": [{ + "target_id": 1, + "action_key": "x", + "mnemonic": "UnresolvedSymlink", + "configuration_id": 1, + "output_ids": [1], + "primary_output_id": 1, + "execution_platform": "//build/bazel/platforms:linux_x86_64", + "unresolved_symlink_target": "symlink/target" + }], + "path_fragments": [ + { "id": 1, "label": "path/to/symlink" } + ] +} +` + data, err := JsonToActionGraphContainer(inputString) + if err != nil { + t.Error(err) + return + } + actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) + if err != nil { + t.Errorf("Unexpected error %q", err) + return + } + assertBuildStatements(t, []*BuildStatement{{ + Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf symlink/target path/to/symlink", + OutputPaths: []string{"path/to/symlink"}, + Mnemonic: "UnresolvedSymlink", + SymlinkPaths: []string{"path/to/symlink"}, + }}, actual) +} + +func TestUnresolvedSymlinkBazelSandwich(t *testing.T) { + const inputString = ` +{ + "artifacts": [ + { "id": 1, "path_fragment_id": 1 } + ], + "actions": [{ + "target_id": 1, + "action_key": "x", + "mnemonic": "UnresolvedSymlink", + "configuration_id": 1, + "output_ids": [1], + "primary_output_id": 1, + "execution_platform": "//build/bazel/platforms:linux_x86_64", + "unresolved_symlink_target": "bazel_sandwich:{\"target\":\"target/product/emulator_x86_64/system\"}" + }], + "path_fragments": [ + { "id": 1, "label": "path/to/symlink" } + ] +} +` + data, err := JsonToActionGraphContainer(inputString) + if err != nil { + t.Error(err) + return + } + actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) + if err != nil { + t.Errorf("Unexpected error %q", err) + return + } + assertBuildStatements(t, []*BuildStatement{{ + Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink", + OutputPaths: []string{"path/to/symlink"}, + Mnemonic: "UnresolvedSymlink", + SymlinkPaths: []string{"path/to/symlink"}, + ImplicitDeps: []string{"target/product/emulator_x86_64/system"}, + }}, actual) +} + +func TestUnresolvedSymlinkBazelSandwichWithAlternativeDeps(t *testing.T) { + const inputString = ` +{ + "artifacts": [ + { "id": 1, "path_fragment_id": 1 } + ], + "actions": [{ + "target_id": 1, + "action_key": "x", + "mnemonic": "UnresolvedSymlink", + "configuration_id": 1, + "output_ids": [1], + "primary_output_id": 1, + "execution_platform": "//build/bazel/platforms:linux_x86_64", + "unresolved_symlink_target": "bazel_sandwich:{\"depend_on_target\":false,\"implicit_deps\":[\"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp\"],\"target\":\"target/product/emulator_x86_64/system\"}" + }], + "path_fragments": [ + { "id": 1, "label": "path/to/symlink" } + ] +} +` + data, err := JsonToActionGraphContainer(inputString) + if err != nil { + t.Error(err) + return + } + actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) + if err != nil { + t.Errorf("Unexpected error %q", err) + return + } + assertBuildStatements(t, []*BuildStatement{{ + Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink", + OutputPaths: []string{"path/to/symlink"}, + Mnemonic: "UnresolvedSymlink", + SymlinkPaths: []string{"path/to/symlink"}, + // Note that the target of the symlink, target/product/emulator_x86_64/system, is not listed here + ImplicitDeps: []string{"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp"}, + }}, actual) +} + func assertError(t *testing.T, err error, expected string) { t.Helper() if err == nil { @@ -1201,6 +1331,9 @@ func buildStatementEquals(first *BuildStatement, second *BuildStatement) string if !reflect.DeepEqual(sortedStrings(first.SymlinkPaths), sortedStrings(second.SymlinkPaths)) { return "SymlinkPaths" } + if !reflect.DeepEqual(sortedStrings(first.ImplicitDeps), sortedStrings(second.ImplicitDeps)) { + return "ImplicitDeps" + } if first.Depfile != second.Depfile { return "Depfile" } diff --git a/bp2build/bp2build_product_config.go b/bp2build/bp2build_product_config.go index f56e6d86e..2513af898 100644 --- a/bp2build/bp2build_product_config.go +++ b/bp2build/bp2build_product_config.go @@ -97,6 +97,7 @@ load("@//build/bazel/product_config:android_product.bzl", "android_product") android_product( name = "mixed_builds_product-{VARIANT}", soong_variables = _soong_variables, + extra_constraints = ["@//build/bazel/platforms:mixed_builds"], ) `)), newFile( diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 5ea84bcba..62b3333b3 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -121,6 +121,10 @@ func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string { defer ctx.EventHandler.End("mixed_build") bazelHook := func() error { + err := ctx.Config().BazelContext.QueueBazelSandwichCqueryRequests(ctx.Config()) + if err != nil { + return err + } return ctx.Config().BazelContext.InvokeBazel(ctx.Config(), ctx) } ctx.SetBeforePrepareBuildActionsHook(bazelHook)