Merge changes from topic "build_go_source_mixed_builds" into main

* changes:
  Delete aliases to prebuilts
  Add functionality to sandbox mixed build actions

android/bazel_handler.go

@@ -16,6 +16,8 @@ package android
 
 import (
 	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
 	"fmt"
 	"os"
 	"path"
@@ -1222,7 +1224,11 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
 		ctx.AddNinjaFileDeps(file)
 	}
 
+	depsetHashToDepset := map[string]bazel.AqueryDepset{}
+
 	for _, depset := range ctx.Config().BazelContext.AqueryDepsets() {
+		depsetHashToDepset[depset.ContentHash] = depset
+
 		var outputs []Path
 		var orderOnlies []Path
 		for _, depsetDepHash := range depset.TransitiveDepSetHashes {
@@ -1257,7 +1263,30 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
 		}
 		if len(buildStatement.Command) > 0 {
 			rule := NewRuleBuilder(pctx, ctx)
-			createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx)
+			intermediateDir, intermediateDirHash := intermediatePathForSboxMixedBuildAction(ctx, buildStatement)
+			if buildStatement.ShouldRunInSbox {
+				// Create a rule to build the output inside a sandbox
+				// This will create two changes of working directory
+				// 1. From ANDROID_BUILD_TOP to sbox top
+				// 2. From sbox top to a a synthetic mixed build execution root relative to it
+				// Finally, the outputs will be copied to intermediateDir
+				rule.Sbox(intermediateDir,
+					PathForOutput(ctx, "mixed_build_sbox_intermediates", intermediateDirHash+".textproto")).
+					SandboxInputs().
+					// Since we will cd to mixed build execution root, set sbox's out subdir to empty
+					// Without this, we will try to copy from $SBOX_SANDBOX_DIR/out/out/bazel/output/execroot/__main__/...
+					SetSboxOutDirDirAsEmpty()
+
+				// Create another set of rules to copy files from the intermediate dir to mixed build execution root
+				for _, outputPath := range buildStatement.OutputPaths {
+					ctx.Build(pctx, BuildParams{
+						Rule:   CpIfChanged,
+						Input:  intermediateDir.Join(ctx, executionRoot, outputPath),
+						Output: PathForBazelOut(ctx, outputPath),
+					})
+				}
+			}
+			createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset)
 			desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
 			rule.Build(fmt.Sprintf("bazel %d", index), desc)
 			continue
@@ -1304,10 +1333,25 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
 	}
 }
 
+// Returns a out dir path for a sandboxed mixed build action
+func intermediatePathForSboxMixedBuildAction(ctx PathContext, statement *bazel.BuildStatement) (OutputPath, string) {
+	// An artifact can be generated by a single buildstatement.
+	// Use the hash of the first artifact to create a unique path
+	uniqueDir := sha1.New()
+	uniqueDir.Write([]byte(statement.OutputPaths[0]))
+	uniqueDirHashString := hex.EncodeToString(uniqueDir.Sum(nil))
+	return PathForOutput(ctx, "mixed_build_sbox_intermediates", uniqueDirHashString), uniqueDirHashString
+}
+
 // Register bazel-owned build statements (obtained from the aquery invocation).
-func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext) {
+func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset) {
 	// executionRoot is the action cwd.
-	cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot))
+	if buildStatement.ShouldRunInSbox {
+		// mkdir -p ensures that the directory exists when run via sbox
+		cmd.Text(fmt.Sprintf("mkdir -p '%s' && cd '%s' &&", executionRoot, executionRoot))
+	} else {
+		cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot))
+	}
 
 	// Remove old outputs, as some actions might not rerun if the outputs are detected.
 	if len(buildStatement.OutputPaths) > 0 {
@@ -1334,7 +1378,16 @@ func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement
 	}
 
 	for _, outputPath := range buildStatement.OutputPaths {
-		cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
+		if buildStatement.ShouldRunInSbox {
+			// The full path has three components that get joined together
+			// 1. intermediate output dir that `sbox` will place the artifacts at
+			// 2. mixed build execution root
+			// 3. artifact path returned by aquery
+			intermediateDir, _ := intermediatePathForSboxMixedBuildAction(ctx, buildStatement)
+			cmd.ImplicitOutput(intermediateDir.Join(ctx, executionRoot, outputPath))
+		} else {
+			cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
+		}
 	}
 	for _, inputPath := range buildStatement.OrderOnlyInputs {
 		cmd.OrderOnly(PathForBazelOut(ctx, inputPath))
@@ -1343,8 +1396,15 @@ func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement
 		cmd.Implicit(PathForBazelOut(ctx, inputPath))
 	}
 	for _, inputDepsetHash := range buildStatement.InputDepsetHashes {
-		otherDepsetName := bazelDepsetName(inputDepsetHash)
+		if buildStatement.ShouldRunInSbox {
-		cmd.Implicit(PathForPhony(ctx, otherDepsetName))
+			// Bazel depsets are phony targets that are used to group files.
+			// We need to copy the grouped files into the sandbox
+			ds, _ := depsetHashToDepset[inputDepsetHash]
+			cmd.Implicits(PathsForBazelOut(ctx, ds.DirectArtifacts))
+		} else {
+			otherDepsetName := bazelDepsetName(inputDepsetHash)
+			cmd.Implicit(PathForPhony(ctx, otherDepsetName))
+		}
 	}
 
 	if depfile := buildStatement.Depfile; depfile != nil {

android/bazel_handler_test.go

@@ -181,13 +181,62 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) {
 
 		cmd := RuleBuilderCommand{}
 		ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
-		createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx)
+		createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{})
 		if actual, expected := cmd.buf.String(), testCase.command; expected != actual {
 			t.Errorf("expected: [%s], actual: [%s]", expected, actual)
 		}
 	}
 }
 
+func TestMixedBuildSandboxedAction(t *testing.T) {
+	input := `{
+ "artifacts": [
+   { "id": 1, "path_fragment_id": 1 },
+   { "id": 2, "path_fragment_id": 2 }],
+ "actions": [{
+   "target_Id": 1,
+   "action_Key": "x",
+   "mnemonic": "x",
+   "arguments": ["touch", "foo"],
+   "input_dep_set_ids": [1],
+   "output_Ids": [1],
+   "primary_output_id": 1
+ }],
+ "dep_set_of_files": [
+   { "id": 1, "direct_artifact_ids": [1, 2] }],
+ "path_fragments": [
+   { "id": 1, "label": "one" },
+   { "id": 2, "label": "two" }]
+}`
+	data, err := JsonToActionGraphContainer(input)
+	if err != nil {
+		t.Error(err)
+	}
+	bazelContext, _ := testBazelContext(t, map[bazelCommand]string{aqueryCmd: string(data)})
+
+	err = bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{})
+	if err != nil {
+		t.Fatalf("TestMixedBuildSandboxedAction did not expect error invoking Bazel, but got %s", err)
+	}
+
+	statement := bazelContext.BuildStatementsToRegister()[0]
+	statement.ShouldRunInSbox = true
+
+	cmd := RuleBuilderCommand{}
+	ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
+	createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{})
+	// Assert that the output is generated in an intermediate directory
+	// fe05bcdcdc4928012781a5f1a2a77cbb5398e106 is the sha1 checksum of "one"
+	if actual, expected := cmd.outputs[0].String(), "out/soong/mixed_build_sbox_intermediates/fe05bcdcdc4928012781a5f1a2a77cbb5398e106/test/exec_root/one"; expected != actual {
+		t.Errorf("expected: [%s], actual: [%s]", expected, actual)
+	}
+
+	// Assert the actual command remains unchanged inside the sandbox
+	if actual, expected := cmd.buf.String(), "mkdir -p 'test/exec_root' && cd 'test/exec_root' && rm -rf 'one' && touch foo"; expected != actual {
+		t.Errorf("expected: [%s], actual: [%s]", expected, actual)
+	}
+}
+
 func TestCoverageFlagsAfterInvokeBazel(t *testing.T) {
 	testConfig.productVariables.ClangCoverage = boolPtr(true)
 

android/rule_builder.go

@@ -53,6 +53,7 @@ type RuleBuilder struct {
 	remoteable       RemoteRuleSupports
 	rbeParams        *remoteexec.REParams
 	outDir           WritablePath
+	sboxOutSubDir    string
 	sboxTools        bool
 	sboxInputs       bool
 	sboxManifestPath WritablePath
@@ -65,9 +66,18 @@ func NewRuleBuilder(pctx PackageContext, ctx BuilderContext) *RuleBuilder {
 		pctx:           pctx,
 		ctx:            ctx,
 		temporariesSet: make(map[WritablePath]bool),
+		sboxOutSubDir:  sboxOutSubDir,
 	}
 }
 
+// SetSboxOutDirDirAsEmpty sets the out subdirectory to an empty string
+// This is useful for sandboxing actions that change the execution root to a path in out/ (e.g mixed builds)
+// For such actions, SetSboxOutDirDirAsEmpty ensures that the path does not become $SBOX_SANDBOX_DIR/out/out/bazel/output/execroot/__main__/...
+func (rb *RuleBuilder) SetSboxOutDirDirAsEmpty() *RuleBuilder {
+	rb.sboxOutSubDir = ""
+	return rb
+}
+
 // RuleBuilderInstall is a tuple of install from and to locations.
 type RuleBuilderInstall struct {
 	From Path
@@ -585,7 +595,7 @@ func (r *RuleBuilder) Build(name string, desc string) {
 		for _, output := range outputs {
 			rel := Rel(r.ctx, r.outDir.String(), output.String())
 			command.CopyAfter = append(command.CopyAfter, &sbox_proto.Copy{
-				From: proto.String(filepath.Join(sboxOutSubDir, rel)),
+				From: proto.String(filepath.Join(r.sboxOutSubDir, rel)),
 				To:   proto.String(output.String()),
 			})
 		}

bazel/aquery.go

@@ -117,6 +117,9 @@ type BuildStatement struct {
 	InputPaths        []string
 	OrderOnlyInputs   []string
 	FileContents      string
+	// If ShouldRunInSbox is true, Soong will use sbox to created an isolated environment
+	// and run the mixed build action there
+	ShouldRunInSbox bool
 }
 
 // A helper type for aquery processing which facilitates retrieval of path IDs from their
@@ -517,6 +520,12 @@ func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry *analysis
 		Env:               actionEntry.EnvironmentVariables,
 		Mnemonic:          actionEntry.Mnemonic,
 	}
+	if buildStatement.Mnemonic == "GoToolchainBinaryBuild" {
+		// Unlike b's execution root, mixed build execution root contains a symlink to prebuilts/go
+		// This causes issues for `GOCACHE=$(mktemp -d) go build ...`
+		// To prevent this, sandbox this action in mixed builds as well
+		buildStatement.ShouldRunInSbox = true
+	}
 	return buildStatement, nil
 }
 

bp2build/build_conversion.go

@@ -471,17 +471,6 @@ func generateBazelTargetsGoBinary(ctx *android.Context, g *bootstrap.GoBinary, g
 	return []BazelTarget{binTarget}, nil
 }
 
-var (
-	// TODO - b/284483729: Remove this denyilst
-	// Temporary denylist of go binaries that are currently used in mixed builds
-	// This denylist allows us to rollout bp2build converters for go targets without affecting mixed builds
-	goBinaryDenylist = []string{
-		"soong_zip",
-		"zip2zip",
-		"bazel_notice_gen",
-	}
-)
-
 func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (conversionResults, []error) {
 	buildFileToTargets := make(map[string]BazelTargets)
 
@@ -574,7 +563,7 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
 				targets, targetErrs = generateBazelTargetsGoPackage(bpCtx, glib, nameToGoLibMap)
 				errs = append(errs, targetErrs...)
 				metrics.IncrementRuleClassCount("go_library")
-			} else if gbin, ok := m.(*bootstrap.GoBinary); ok && !android.InList(m.Name(), goBinaryDenylist) {
+			} else if gbin, ok := m.(*bootstrap.GoBinary); ok {
 				targets, targetErrs = generateBazelTargetsGoBinary(bpCtx, gbin, nameToGoLibMap)
 				errs = append(errs, targetErrs...)
 				metrics.IncrementRuleClassCount("go_binary")

cmd/zip2zip/BUILD.bazel

@@ -1,18 +0,0 @@
-# Copyright (C) 2022 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-alias(
-    name = "zip2zip",
-    actual = "//prebuilts/build-tools:linux-x86/bin/zip2zip",
-)

zip/cmd/BUILD.bazel

@@ -1,20 +0,0 @@
-# Copyright (C) 2022 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# TODO(b/194644518): Switch to the source version when Bazel can build go
-# binaries.
-alias(
-    name = "soong_zip",
-    actual = "//prebuilts/build-tools:linux-x86/bin/soong_zip",
-)