StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial implementation of the bazel sandwich

Browse Source 
The "bazel sandwich" is a mechanism for bazel to depend on make/soong
outputs. The name comes from the fact that bazel is now at the top
and bottom of the build graph. This is intended to allow us to work
on converting the partition builds to bazel while not all of the
dependencies of the partition have been converted.

It works by adding the bazel_sandwich_import_file rule, which emits a
dangling symlink that starts with bazel_sandwich:, and includes
information that the aquery handler in soong reads. The aquery handler
rewrites the symlink so that it points to a file generated by
make/soong, and adds a ninja dependency from the symlink to the file
it's targeting.

This allows us to depend on make-built files from bazel, but notably
it doesn't allow us to depend on analysis-time information from make.
This shouldn't be a problem for the partitions, but limits the use of
the bazel sandwich to similar, less complicated types of builds.

go/roboleaf-bazel-sandwich

Bug: 265127181
Test: m bazel_sandwich
Change-Id: Ic41bae7be0b55f251d04a6a95f846c50ce897adc
This commit is contained in:
Cole Faust
2023-06-28 17:18:20 -07:00
parent c796de83c8
commit a20d947329
8 changed files with 323 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
bazel/aquery.go
View File

@@ -17,15 +17,15 @@ package bazel
import (
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"path/filepath"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
"reflect"
"sort"
"strings"
"sync"
analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
"github.com/google/blueprint/metrics"
"github.com/google/blueprint/proptools"
"google.golang.org/protobuf/proto"
@@ -119,6 +119,10 @@ type BuildStatement struct {
// If ShouldRunInSbox is true, Soong will use sbox to created an isolated environment
// and run the mixed build action there
ShouldRunInSbox bool
// A list of files to add as implicit deps to the outputs of this BuildStatement.
// Unlike most properties in BuildStatement, these paths must be relative to the root of
// the whole out/ folder, instead of relative to ctx.Config().BazelContext.OutputBase()
ImplicitDeps []string
}
// A helper type for aquery processing which facilitates retrieval of path IDs from their
@@ -581,6 +585,72 @@ func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry *ana
}, nil
}
type bazelSandwichJson struct {
Target string `json:"target"`
DependOnTarget *bool `json:"depend_on_target,omitempty"`
ImplicitDeps []string `json:"implicit_deps"`
}
func (a *aqueryArtifactHandler) unresolvedSymlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
outputPaths, depfile, err := a.getOutputPaths(actionEntry)
if err != nil {
return nil, err
}
if len(actionEntry.InputDepSetIds) != 0 || len(outputPaths) != 1 {
return nil, fmt.Errorf("expected 0 inputs and 1 output to symlink action, got: input %q, output %q", actionEntry.InputDepSetIds, outputPaths)
}
target := actionEntry.UnresolvedSymlinkTarget
if target == "" {
return nil, fmt.Errorf("expected an unresolved_symlink_target, but didn't get one")
}
if filepath.Clean(target) != target {
return nil, fmt.Errorf("expected %q, got %q", filepath.Clean(target), target)
}
if strings.HasPrefix(target, "/") {
return nil, fmt.Errorf("no absolute symlinks allowed: %s", target)
}
out := outputPaths[0]
outDir := filepath.Dir(out)
var implicitDeps []string
if strings.HasPrefix(target, "bazel_sandwich:") {
j := bazelSandwichJson{}
err := json.Unmarshal([]byte(target[len("bazel_sandwich:"):]), &j)
if err != nil {
return nil, err
}
if proptools.BoolDefault(j.DependOnTarget, true) {
implicitDeps = append(implicitDeps, j.Target)
}
implicitDeps = append(implicitDeps, j.ImplicitDeps...)
dotDotsToReachCwd := ""
if outDir != "." {
dotDotsToReachCwd = strings.Repeat("../", strings.Count(outDir, "/")+1)
}
target = proptools.ShellEscapeIncludingSpaces(j.Target)
target = "{DOTDOTS_TO_OUTPUT_ROOT}" + dotDotsToReachCwd + target
} else {
target = proptools.ShellEscapeIncludingSpaces(target)
}
outDir = proptools.ShellEscapeIncludingSpaces(outDir)
out = proptools.ShellEscapeIncludingSpaces(out)
// Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, target)
symlinkPaths := outputPaths[:]
buildStatement := &BuildStatement{
Command: command,
Depfile: depfile,
OutputPaths: outputPaths,
Env: actionEntry.EnvironmentVariables,
Mnemonic: actionEntry.Mnemonic,
SymlinkPaths: symlinkPaths,
ImplicitDeps: implicitDeps,
}
return buildStatement, nil
}
func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
outputPaths, depfile, err := a.getOutputPaths(actionEntry)
if err != nil {
@@ -690,6 +760,8 @@ func (a *aqueryArtifactHandler) actionToBuildStatement(actionEntry *analysis_v2_
return a.fileWriteActionBuildStatement(actionEntry)
case "SymlinkTree":
return a.symlinkTreeActionBuildStatement(actionEntry)
case "UnresolvedSymlink":
return a.unresolvedSymlinkActionBuildStatement(actionEntry)
}
if len(actionEntry.Arguments) < 1 {

133
bazel/aquery_test.go
View File

@@ -357,9 +357,11 @@ func TestDepfiles(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
if expected := 1; len(actual) != expected {
t.Fatalf("Expected %d build statements, got %d", expected, len(actual))
return
}
bs := actual[0]
@@ -544,6 +546,7 @@ func TestSymlinkTree(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{
&BuildStatement{
@@ -756,9 +759,11 @@ func TestMiddlemenAction(t *testing.T) {
actualBuildStatements, actualDepsets, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
if expected := 2; len(actualBuildStatements) != expected {
t.Fatalf("Expected %d build statements, got %d %#v", expected, len(actualBuildStatements), actualBuildStatements)
return
}
expectedDepsetFiles := [][]string{
@@ -859,6 +864,7 @@ func TestSimpleSymlink(t *testing.T) {
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
expectedBuildStatements := []*BuildStatement{
@@ -907,6 +913,7 @@ func TestSymlinkQuotesPaths(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
expectedBuildStatements := []*BuildStatement{
@@ -1017,6 +1024,7 @@ func TestTemplateExpandActionSubstitutions(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
expectedBuildStatements := []*BuildStatement{
@@ -1088,6 +1096,7 @@ func TestFileWrite(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{
&BuildStatement{
@@ -1126,6 +1135,7 @@ func TestSourceSymlinkManifest(t *testing.T) {
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{
&BuildStatement{
@@ -1136,6 +1146,126 @@ func TestSourceSymlinkManifest(t *testing.T) {
}, actual)
}
func TestUnresolvedSymlink(t *testing.T) {
const inputString = `
{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 }
],
"actions": [{
"target_id": 1,
"action_key": "x",
"mnemonic": "UnresolvedSymlink",
"configuration_id": 1,
"output_ids": [1],
"primary_output_id": 1,
"execution_platform": "//build/bazel/platforms:linux_x86_64",
"unresolved_symlink_target": "symlink/target"
}],
"path_fragments": [
{ "id": 1, "label": "path/to/symlink" }
]
}
`
data, err := JsonToActionGraphContainer(inputString)
if err != nil {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{{
Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf symlink/target path/to/symlink",
OutputPaths: []string{"path/to/symlink"},
Mnemonic: "UnresolvedSymlink",
SymlinkPaths: []string{"path/to/symlink"},
}}, actual)
}
func TestUnresolvedSymlinkBazelSandwich(t *testing.T) {
const inputString = `
{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 }
],
"actions": [{
"target_id": 1,
"action_key": "x",
"mnemonic": "UnresolvedSymlink",
"configuration_id": 1,
"output_ids": [1],
"primary_output_id": 1,
"execution_platform": "//build/bazel/platforms:linux_x86_64",
"unresolved_symlink_target": "bazel_sandwich:{\"target\":\"target/product/emulator_x86_64/system\"}"
}],
"path_fragments": [
{ "id": 1, "label": "path/to/symlink" }
]
}
`
data, err := JsonToActionGraphContainer(inputString)
if err != nil {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{{
Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink",
OutputPaths: []string{"path/to/symlink"},
Mnemonic: "UnresolvedSymlink",
SymlinkPaths: []string{"path/to/symlink"},
ImplicitDeps: []string{"target/product/emulator_x86_64/system"},
}}, actual)
}
func TestUnresolvedSymlinkBazelSandwichWithAlternativeDeps(t *testing.T) {
const inputString = `
{
"artifacts": [
{ "id": 1, "path_fragment_id": 1 }
],
"actions": [{
"target_id": 1,
"action_key": "x",
"mnemonic": "UnresolvedSymlink",
"configuration_id": 1,
"output_ids": [1],
"primary_output_id": 1,
"execution_platform": "//build/bazel/platforms:linux_x86_64",
"unresolved_symlink_target": "bazel_sandwich:{\"depend_on_target\":false,\"implicit_deps\":[\"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp\"],\"target\":\"target/product/emulator_x86_64/system\"}"
}],
"path_fragments": [
{ "id": 1, "label": "path/to/symlink" }
]
}
`
data, err := JsonToActionGraphContainer(inputString)
if err != nil {
t.Error(err)
return
}
actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
if err != nil {
t.Errorf("Unexpected error %q", err)
return
}
assertBuildStatements(t, []*BuildStatement{{
Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink",
OutputPaths: []string{"path/to/symlink"},
Mnemonic: "UnresolvedSymlink",
SymlinkPaths: []string{"path/to/symlink"},
// Note that the target of the symlink, target/product/emulator_x86_64/system, is not listed here
ImplicitDeps: []string{"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp"},
}}, actual)
}
func assertError(t *testing.T, err error, expected string) {
t.Helper()
if err == nil {
@@ -1201,6 +1331,9 @@ func buildStatementEquals(first *BuildStatement, second *BuildStatement) string
if !reflect.DeepEqual(sortedStrings(first.SymlinkPaths), sortedStrings(second.SymlinkPaths)) {
return "SymlinkPaths"
}
if !reflect.DeepEqual(sortedStrings(first.ImplicitDeps), sortedStrings(second.ImplicitDeps)) {
return "ImplicitDeps"
}
if first.Depfile != second.Depfile {
return "Depfile"
}