StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reland: Rewrite sbox to use a textproto manifest

Browse Source 
This relands I3b918a6643cea77199fd39577ef71e34cdeacdb1 with a fix
to create the directory for the output depfile if doesn't exist.

In preparation for more complicated sandboxing that copies tools
and/or inputs into the sandbox directory, make sbox use a textproto
input that describes the commands to be run and the files to copy
in or out of the sandbox.

Bug: 124313442
Test: m checkbuild
Test: rule_builder_test.go
Test: genrule_test.go
Change-Id: I8af00c8c0b25a92f55a5032fcb525715ae8297c2
This commit is contained in:
Colin Cross
2020-11-12 08:29:30 -08:00
parent f18859626c
commit e16ce36818
12 changed files with 768 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
android/rule_builder.go
View File

@@ -20,27 +20,33 @@ import (
"path/filepath"
"sort"
"strings"
"testing"
"github.com/golang/protobuf/proto"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
"android/soong/cmd/sbox/sbox_proto"
"android/soong/shared"
)
const sboxOutDir = "__SBOX_OUT_DIR__"
const sboxSandboxBaseDir = "__SBOX_SANDBOX_DIR__"
const sboxOutSubDir = "out"
const sboxOutDir = sboxSandboxBaseDir + "/" + sboxOutSubDir
// RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
// graph.
type RuleBuilder struct {
commands []*RuleBuilderCommand
installs RuleBuilderInstalls
temporariesSet map[WritablePath]bool
restat bool
sbox bool
highmem bool
remoteable RemoteRuleSupports
sboxOutDir WritablePath
missingDeps []string
commands []*RuleBuilderCommand
installs RuleBuilderInstalls
temporariesSet map[WritablePath]bool
restat bool
sbox bool
highmem bool
remoteable RemoteRuleSupports
sboxOutDir WritablePath
sboxManifestPath WritablePath
missingDeps []string
}
// NewRuleBuilder returns a newly created RuleBuilder.
@@ -106,12 +112,14 @@ func (r *RuleBuilder) Remoteable(supports RemoteRuleSupports) *RuleBuilder {
return r
}
// Sbox marks the rule as needing to be wrapped by sbox. The WritablePath should point to the output
// directory that sbox will wipe. It should not be written to by any other rule. sbox will ensure
// that all outputs have been written, and will discard any output files that were not specified.
// Sbox marks the rule as needing to be wrapped by sbox. The outputDir should point to the output
// directory that sbox will wipe. It should not be written to by any other rule. manifestPath should
// point to a location where sbox's manifest will be written and must be outside outputDir. sbox
// will ensure that all outputs have been written, and will discard any output files that were not
// specified.
//
// Sbox is not compatible with Restat()
func (r *RuleBuilder) Sbox(outputDir WritablePath) *RuleBuilder {
func (r *RuleBuilder) Sbox(outputDir WritablePath, manifestPath WritablePath) *RuleBuilder {
if r.sbox {
panic("Sbox() may not be called more than once")
}
@@ -123,6 +131,7 @@ func (r *RuleBuilder) Sbox(outputDir WritablePath) *RuleBuilder {
}
r.sbox = true
r.sboxOutDir = outputDir
r.sboxManifestPath = manifestPath
return r
}
@@ -420,7 +429,8 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string
r.depFileMergerCmd(ctx, depFiles)
if r.sbox {
// Check for Rel() errors, as all depfiles should be in the output dir
// Check for Rel() errors, as all depfiles should be in the output dir. Errors
// will be reported to the ctx.
for _, path := range depFiles[1:] {
Rel(ctx, r.sboxOutDir.String(), path.String())
}
@@ -443,34 +453,60 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string
commandString := strings.Join(commands, " && ")
if r.sbox {
sboxOutputs := make([]string, len(outputs))
for i, output := range outputs {
sboxOutputs[i] = filepath.Join(sboxOutDir, Rel(ctx, r.sboxOutDir.String(), output.String()))
}
commandString = proptools.ShellEscape(commandString)
if !strings.HasPrefix(commandString, `'`) {
commandString = `'` + commandString + `'`
}
sboxCmd := &RuleBuilderCommand{}
sboxCmd.BuiltTool(ctx, "sbox").
Flag("-c").Text(commandString).
Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())).
Flag("--output-root").Text(r.sboxOutDir.String())
// If running the command inside sbox, write the rule data out to an sbox
// manifest.textproto.
manifest := sbox_proto.Manifest{}
command := sbox_proto.Command{}
manifest.Commands = append(manifest.Commands, &command)
command.Command = proto.String(commandString)
if depFile != nil {
sboxCmd.Flag("--depfile-out").Text(depFile.String())
manifest.OutputDepfile = proto.String(depFile.String())
}
// Add a hash of the list of input files to the xbox command line so that ninja reruns
// it when the list of input files changes.
sboxCmd.FlagWithArg("--input-hash ", hashSrcFiles(inputs))
// Add copy rules to the manifest to copy each output file from the sbox directory.
// to the output directory.
sboxOutputs := make([]string, len(outputs))
for i, output := range outputs {
rel := Rel(ctx, r.sboxOutDir.String(), output.String())
sboxOutputs[i] = filepath.Join(sboxOutDir, rel)
command.CopyAfter = append(command.CopyAfter, &sbox_proto.Copy{
From: proto.String(filepath.Join(sboxOutSubDir, rel)),
To: proto.String(output.String()),
})
}
sboxCmd.Flags(sboxOutputs)
// Add a hash of the list of input files to the manifest so that the textproto file
// changes when the list of input files changes and causes the sbox rule that
// depends on it to rerun.
command.InputHash = proto.String(hashSrcFiles(inputs))
// Verify that the manifest textproto is not inside the sbox output directory, otherwise
// it will get deleted when the sbox rule clears its output directory.
_, manifestInOutDir := MaybeRel(ctx, r.sboxOutDir.String(), r.sboxManifestPath.String())
if manifestInOutDir {
ReportPathErrorf(ctx, "sbox rule %q manifestPath %q must not be in outputDir %q",
name, r.sboxManifestPath.String(), r.sboxOutDir.String())
}
// Create a rule to write the manifest as a the textproto.
WriteFileRule(ctx, r.sboxManifestPath, proto.MarshalTextString(&manifest))
// Generate a new string to use as the command line of the sbox rule. This uses
// a RuleBuilderCommand as a convenience method of building the command line, then
// converts it to a string to replace commandString.
sboxCmd := &RuleBuilderCommand{}
sboxCmd.Text("rm -rf").Output(r.sboxOutDir)
sboxCmd.Text("&&")
sboxCmd.BuiltTool(ctx, "sbox").
Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())).
Flag("--manifest").Input(r.sboxManifestPath)
// Replace the command string, and add the sbox tool and manifest textproto to the
// dependencies of the final sbox rule.
commandString = sboxCmd.buf.String()
tools = append(tools, sboxCmd.tools...)
inputs = append(inputs, sboxCmd.inputs...)
} else {
// If not using sbox the rule will run the command directly, put the hash of the
// list of input files in a comment at the end of the command line to ensure ninja
@@ -890,6 +926,19 @@ func (c *RuleBuilderCommand) NinjaEscapedString() string {
return ninjaEscapeExceptForSpans(c.String(), c.unescapedSpans)
}
// RuleBuilderSboxProtoForTests takes the BuildParams for the manifest passed to RuleBuilder.Sbox()
// and returns sbox testproto generated by the RuleBuilder.
func RuleBuilderSboxProtoForTests(t *testing.T, params TestingBuildParams) *sbox_proto.Manifest {
t.Helper()
content := ContentFromFileRuleForTests(t, params)
manifest := sbox_proto.Manifest{}
err := proto.UnmarshalText(content, &manifest)
if err != nil {
t.Fatalf("failed to unmarshal manifest: %s", err.Error())
}
return &manifest
}
func ninjaEscapeExceptForSpans(s string, spans [][2]int) string {
if len(spans) == 0 {
return proptools.NinjaEscape(s)