StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pass rsp files into sbox and rewrapper

Browse Source 
The current implementation causes inputs listed in an rsp file used with
sbox to be duplicated 3 times in the build.ninja file; once as a
dependency of the rule, once in the rspfile_content field of the rule
with the paths rewritten to be relative to the sandbox, and once in the
rule to write the sbox manifest.  When RBE is enabled it also gets a
fourth copy in the list of files to be treated as inputs by rewrapper.

Reduce this to a single copy by using "$in" for the rspfile_content so
that the files only have to be listed in the input dependencies of the
rule, and then add support to sbox to rewrite the rsp file while copying
it into the sandbox, and pass it to rewrapper as well.

Test: m lint-check
Change-Id: I3f46f61119508d39a8bb231c99fc130153fb6f04
This commit is contained in:
Colin Cross
2021-03-23 13:44:30 -07:00
parent 045bfd9640
commit e55bd423df
9 changed files with 359 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/sbox/Android.bp
View File

@@ -21,6 +21,7 @@ blueprint_go_binary {
deps: [
"sbox_proto",
"soong-makedeps",
"soong-response",
],
srcs: [
"sbox.go",

90
cmd/sbox/sbox.go
View File

@@ -32,6 +32,7 @@ import (
"android/soong/cmd/sbox/sbox_proto"
"android/soong/makedeps"
"android/soong/response"
"github.com/golang/protobuf/proto"
)
@@ -218,6 +219,11 @@ func runCommand(command *sbox_proto.Command, tempDir string) (depFile string, er
return "", fmt.Errorf("command is required")
}
pathToTempDirInSbox := tempDir
if command.GetChdir() {
pathToTempDirInSbox = "."
}
err = os.MkdirAll(tempDir, 0777)
if err != nil {
return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
@@ -228,10 +234,9 @@ func runCommand(command *sbox_proto.Command, tempDir string) (depFile string, er
if err != nil {
return "", err
}
pathToTempDirInSbox := tempDir
if command.GetChdir() {
pathToTempDirInSbox = "."
err = copyRspFiles(command.RspFiles, tempDir, pathToTempDirInSbox)
if err != nil {
return "", err
}
if strings.Contains(rawCommand, depFilePlaceholder) {
@@ -409,6 +414,83 @@ func copyOneFile(from string, to string, executable bool) error {
return nil
}
// copyRspFiles copies rsp files into the sandbox with path mappings, and also copies the files
// listed into the sandbox.
func copyRspFiles(rspFiles []*sbox_proto.RspFile, toDir, toDirInSandbox string) error {
for _, rspFile := range rspFiles {
err := copyOneRspFile(rspFile, toDir, toDirInSandbox)
if err != nil {
return err
}
}
return nil
}
// copyOneRspFiles copies an rsp file into the sandbox with path mappings, and also copies the files
// listed into the sandbox.
func copyOneRspFile(rspFile *sbox_proto.RspFile, toDir, toDirInSandbox string) error {
in, err := os.Open(rspFile.GetFile())
if err != nil {
return err
}
defer in.Close()
files, err := response.ReadRspFile(in)
if err != nil {
return err
}
for i, from := range files {
// Convert the real path of the input file into the path inside the sandbox using the
// path mappings.
to := applyPathMappings(rspFile.PathMappings, from)
// Copy the file into the sandbox.
err := copyOneFile(from, joinPath(toDir, to), false)
if err != nil {
return err
}
// Rewrite the name in the list of files to be relative to the sandbox directory.
files[i] = joinPath(toDirInSandbox, to)
}
// Convert the real path of the rsp file into the path inside the sandbox using the path
// mappings.
outRspFile := joinPath(toDir, applyPathMappings(rspFile.PathMappings, rspFile.GetFile()))
err = os.MkdirAll(filepath.Dir(outRspFile), 0777)
if err != nil {
return err
}
out, err := os.Create(outRspFile)
if err != nil {
return err
}
defer out.Close()
// Write the rsp file with converted paths into the sandbox.
err = response.WriteRspFile(out, files)
if err != nil {
return err
}
return nil
}
// applyPathMappings takes a list of path mappings and a path, and returns the path with the first
// matching path mapping applied. If the path does not match any of the path mappings then it is
// returned unmodified.
func applyPathMappings(pathMappings []*sbox_proto.PathMapping, path string) string {
for _, mapping := range pathMappings {
if strings.HasPrefix(path, mapping.GetFrom()+"/") {
return joinPath(mapping.GetTo()+"/", strings.TrimPrefix(path, mapping.GetFrom()+"/"))
}
}
return path
}
// moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
// to moving files where the source and destination are in the same filesystem. This is OK for
// sbox because the temporary directory is inside the out directory. It updates the timestamp

159
cmd/sbox/sbox_proto/sbox.pb.go
View File

@@ -86,10 +86,13 @@ type Command struct {
CopyAfter []*Copy `protobuf:"bytes,4,rep,name=copy_after,json=copyAfter" json:"copy_after,omitempty"`
// An optional hash of the input files to ensure the textproto files and the sbox rule reruns
// when the lists of inputs changes, even if the inputs are not on the command line.
InputHash *string `protobuf:"bytes,5,opt,name=input_hash,json=inputHash" json:"input_hash,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
InputHash *string `protobuf:"bytes,5,opt,name=input_hash,json=inputHash" json:"input_hash,omitempty"`
// A list of files that will be copied before the sandboxed command, and whose contents should be
// copied as if they were listed in copy_before.
RspFiles []*RspFile `protobuf:"bytes,6,rep,name=rsp_files,json=rspFiles" json:"rsp_files,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Command) Reset() { *m = Command{} }
@@ -152,6 +155,13 @@ func (m *Command) GetInputHash() string {
return ""
}
func (m *Command) GetRspFiles() []*RspFile {
if m != nil {
return m.RspFiles
}
return nil
}
// Copy describes a from-to pair of files to copy. The paths may be relative, the root that they
// are relative to is specific to the context the Copy is used in and will be different for
// from and to.
@@ -211,10 +221,110 @@ func (m *Copy) GetExecutable() bool {
return false
}
// RspFile describes an rspfile that should be copied into the sandbox directory.
type RspFile struct {
// The path to the rsp file.
File *string `protobuf:"bytes,1,req,name=file" json:"file,omitempty"`
// A list of path mappings that should be applied to each file listed in the rsp file.
PathMappings []*PathMapping `protobuf:"bytes,2,rep,name=path_mappings,json=pathMappings" json:"path_mappings,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *RspFile) Reset() { *m = RspFile{} }
func (m *RspFile) String() string { return proto.CompactTextString(m) }
func (*RspFile) ProtoMessage() {}
func (*RspFile) Descriptor() ([]byte, []int) {
return fileDescriptor_9d0425bf0de86ed1, []int{3}
}
func (m *RspFile) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_RspFile.Unmarshal(m, b)
}
func (m *RspFile) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_RspFile.Marshal(b, m, deterministic)
}
func (m *RspFile) XXX_Merge(src proto.Message) {
xxx_messageInfo_RspFile.Merge(m, src)
}
func (m *RspFile) XXX_Size() int {
return xxx_messageInfo_RspFile.Size(m)
}
func (m *RspFile) XXX_DiscardUnknown() {
xxx_messageInfo_RspFile.DiscardUnknown(m)
}
var xxx_messageInfo_RspFile proto.InternalMessageInfo
func (m *RspFile) GetFile() string {
if m != nil && m.File != nil {
return *m.File
}
return ""
}
func (m *RspFile) GetPathMappings() []*PathMapping {
if m != nil {
return m.PathMappings
}
return nil
}
// PathMapping describes a mapping from a path outside the sandbox to the path inside the sandbox.
type PathMapping struct {
From *string `protobuf:"bytes,1,req,name=from" json:"from,omitempty"`
To *string `protobuf:"bytes,2,req,name=to" json:"to,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *PathMapping) Reset() { *m = PathMapping{} }
func (m *PathMapping) String() string { return proto.CompactTextString(m) }
func (*PathMapping) ProtoMessage() {}
func (*PathMapping) Descriptor() ([]byte, []int) {
return fileDescriptor_9d0425bf0de86ed1, []int{4}
}
func (m *PathMapping) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_PathMapping.Unmarshal(m, b)
}
func (m *PathMapping) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_PathMapping.Marshal(b, m, deterministic)
}
func (m *PathMapping) XXX_Merge(src proto.Message) {
xxx_messageInfo_PathMapping.Merge(m, src)
}
func (m *PathMapping) XXX_Size() int {
return xxx_messageInfo_PathMapping.Size(m)
}
func (m *PathMapping) XXX_DiscardUnknown() {
xxx_messageInfo_PathMapping.DiscardUnknown(m)
}
var xxx_messageInfo_PathMapping proto.InternalMessageInfo
func (m *PathMapping) GetFrom() string {
if m != nil && m.From != nil {
return *m.From
}
return ""
}
func (m *PathMapping) GetTo() string {
if m != nil && m.To != nil {
return *m.To
}
return ""
}
func init() {
proto.RegisterType((*Manifest)(nil), "sbox.Manifest")
proto.RegisterType((*Command)(nil), "sbox.Command")
proto.RegisterType((*Copy)(nil), "sbox.Copy")
proto.RegisterType((*RspFile)(nil), "sbox.RspFile")
proto.RegisterType((*PathMapping)(nil), "sbox.PathMapping")
}
func init() {
@@ -222,22 +332,27 @@ func init() {
}
var fileDescriptor_9d0425bf0de86ed1 = []byte{
// 268 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0x4f, 0x4b, 0xc3, 0x40,
0x10, 0xc5, 0xc9, 0x9f, 0xd2, 0x64, 0x6a, 0x7b, 0x18, 0x3c, 0xec, 0x45, 0x09, 0x01, 0x21, 0x45,
0xe8, 0xc1, 0x6f, 0x60, 0xf5, 0x20, 0x82, 0x97, 0x1c, 0x45, 0x08, 0x9b, 0x64, 0x43, 0x02, 0x4d,
0x26, 0xec, 0x6e, 0xa0, 0xfd, 0x56, 0x7e, 0x44, 0xd9, 0x49, 0x2a, 0x82, 0xb7, 0x99, 0xdf, 0xe3,
0xcd, 0x7b, 0x0c, 0x80, 0x29, 0xe9, 0x7c, 0x18, 0x35, 0x59, 0xc2, 0xd0, 0xcd, 0xe9, 0x17, 0x44,
0x1f, 0x72, 0xe8, 0x1a, 0x65, 0x2c, 0xee, 0x21, 0xaa, 0xa8, 0xef, 0xe5, 0x50, 0x1b, 0xe1, 0x25,
0x41, 0xb6, 0x79, 0xda, 0x1e, 0xd8, 0xf0, 0x32, 0xd3, 0xfc, 0x57, 0xc6, 0x07, 0xd8, 0xd1, 0x64,
0xc7, 0xc9, 0x16, 0xb5, 0x1a, 0x9b, 0xee, 0xa4, 0x84, 0x9f, 0x78, 0x59, 0x9c, 0x6f, 0x67, 0xfa,
0x3a, 0xc3, 0xf4, 0xdb, 0x83, 0xf5, 0x62, 0xc6, 0x47, 0xd8, 0x54, 0x34, 0x5e, 0x8a, 0x52, 0x35,
0xa4, 0xd5, 0x12, 0x00, 0xd7, 0x80, 0xf1, 0x92, 0x83, 0x93, 0x8f, 0xac, 0xe2, 0x2d, 0xac, 0xaa,
0xb6, 0xee, 0x34, 0x9f, 0x8d, 0xf2, 0x79, 0x41, 0x01, 0xeb, 0xa5, 0x81, 0x08, 0x12, 0x3f, 0x8b,
0xf3, 0xeb, 0x8a, 0x7b, 0x60, 0x77, 0x21, 0x1b, 0xab, 0xb4, 0x08, 0xff, 0xdd, 0x8e, 0x9d, 0xfa,
0xec, 0x44, 0xbc, 0x03, 0xe8, 0x06, 0xd7, 0xbc, 0x95, 0xa6, 0x15, 0x2b, 0xae, 0x1d, 0x33, 0x79,
0x93, 0xa6, 0x4d, 0xdf, 0x21, 0x74, 0x0e, 0x44, 0x08, 0x1b, 0x4d, 0xbd, 0xf0, 0x38, 0x88, 0x67,
0xdc, 0x81, 0x6f, 0x49, 0xf8, 0x4c, 0x7c, 0x4b, 0x78, 0x0f, 0xa0, 0xce, 0xaa, 0x9a, 0xac, 0x2c,
0x4f, 0x4a, 0x04, 0x5c, 0xf5, 0x0f, 0x39, 0xde, 0x7c, 0xf2, 0xc3, 0x0b, 0x7e, 0xf8, 0x4f, 0x00,
0x00, 0x00, 0xff, 0xff, 0x78, 0x37, 0x3e, 0x6a, 0x7d, 0x01, 0x00, 0x00,
// 342 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x5f, 0x4b, 0xc3, 0x30,
0x14, 0xc5, 0x69, 0xd7, 0xb9, 0xf6, 0xee, 0x0f, 0x18, 0x7c, 0xc8, 0x8b, 0x32, 0x0a, 0xc2, 0xa6,
0x30, 0xd0, 0x07, 0xdf, 0x9d, 0x22, 0x22, 0x0c, 0x24, 0xe0, 0x8b, 0x08, 0x25, 0xeb, 0x52, 0x5b,
0x58, 0x9b, 0x90, 0x64, 0xb0, 0x7d, 0x57, 0x3f, 0x8c, 0xe4, 0xa6, 0xd3, 0x82, 0x2f, 0xbe, 0xdd,
0x7b, 0x0e, 0xf7, 0xdc, 0x5f, 0xc2, 0x05, 0x30, 0x6b, 0xb9, 0x5f, 0x28, 0x2d, 0xad, 0x24, 0x91,
0xab, 0xd3, 0x0f, 0x88, 0x57, 0xbc, 0xa9, 0x0a, 0x61, 0x2c, 0x99, 0x43, 0x9c, 0xcb, 0xba, 0xe6,
0xcd, 0xc6, 0xd0, 0x60, 0xda, 0x9b, 0x0d, 0x6f, 0xc7, 0x0b, 0x1c, 0x78, 0xf0, 0x2a, 0xfb, 0xb1,
0xc9, 0x25, 0x4c, 0xe4, 0xce, 0xaa, 0x9d, 0xcd, 0x36, 0x42, 0x15, 0xd5, 0x56, 0xd0, 0x70, 0x1a,
0xcc, 0x12, 0x36, 0xf6, 0xea, 0xa3, 0x17, 0xd3, 0xaf, 0x00, 0x06, 0xed, 0x30, 0xb9, 0x86, 0x61,
0x2e, 0xd5, 0x21, 0x5b, 0x8b, 0x42, 0x6a, 0xd1, 0x2e, 0x80, 0xe3, 0x02, 0x75, 0x60, 0xe0, 0xec,
0x25, 0xba, 0xe4, 0x0c, 0xfa, 0x79, 0xb9, 0xa9, 0x34, 0xc6, 0xc6, 0xcc, 0x37, 0x84, 0xc2, 0xa0,
0x25, 0xa0, 0xbd, 0x69, 0x38, 0x4b, 0xd8, 0xb1, 0x25, 0x73, 0xc0, 0xe9, 0x8c, 0x17, 0x56, 0x68,
0x1a, 0xfd, 0xc9, 0x4e, 0x9c, 0x7b, 0xef, 0x4c, 0x72, 0x0e, 0x50, 0x35, 0x8e, 0xbc, 0xe4, 0xa6,
0xa4, 0x7d, 0xc4, 0x4e, 0x50, 0x79, 0xe6, 0xa6, 0x24, 0x57, 0x90, 0x68, 0xa3, 0x32, 0x87, 0x6f,
0xe8, 0x49, 0xf7, 0x17, 0x98, 0x51, 0x4f, 0xd5, 0x56, 0xb0, 0x58, 0xfb, 0xc2, 0xa4, 0x2f, 0x10,
0xb9, 0x74, 0x42, 0x20, 0x2a, 0xb4, 0xac, 0x69, 0x80, 0x50, 0x58, 0x93, 0x09, 0x84, 0x56, 0xd2,
0x10, 0x95, 0xd0, 0x4a, 0x72, 0x01, 0x20, 0xf6, 0x22, 0xdf, 0x59, 0xbe, 0xde, 0x0a, 0xda, 0xc3,
0x67, 0x75, 0x94, 0xf4, 0x0d, 0x06, 0xed, 0x02, 0x8c, 0x73, 0x5f, 0x7a, 0x8c, 0x73, 0xda, 0x1d,
0x8c, 0x15, 0xb7, 0x65, 0x56, 0x73, 0xa5, 0xaa, 0xe6, 0xd3, 0xd0, 0x10, 0xd1, 0x4e, 0x3d, 0xda,
0x2b, 0xb7, 0xe5, 0xca, 0x3b, 0x6c, 0xa4, 0x7e, 0x1b, 0x93, 0xde, 0xc0, 0xb0, 0x63, 0xfe, 0x87,
0x74, 0x39, 0x7a, 0xc7, 0x33, 0xc9, 0xf0, 0x4c, 0xbe, 0x03, 0x00, 0x00, 0xff, 0xff, 0x83, 0x82,
0xb0, 0xc3, 0x33, 0x02, 0x00, 0x00,
}

21
cmd/sbox/sbox_proto/sbox.proto
View File

@@ -47,6 +47,10 @@ message Command {
// An optional hash of the input files to ensure the textproto files and the sbox rule reruns
// when the lists of inputs changes, even if the inputs are not on the command line.
optional string input_hash = 5;
// A list of files that will be copied before the sandboxed command, and whose contents should be
// copied as if they were listed in copy_before.
repeated RspFile rsp_files = 6;
}
// Copy describes a from-to pair of files to copy. The paths may be relative, the root that they
@@ -58,4 +62,19 @@ message Copy {
// If true, make the file executable after copying it.
optional bool executable = 3;
}
}
// RspFile describes an rspfile that should be copied into the sandbox directory.
message RspFile {
// The path to the rsp file.
required string file = 1;
// A list of path mappings that should be applied to each file listed in the rsp file.
repeated PathMapping path_mappings = 2;
}
// PathMapping describes a mapping from a path outside the sandbox to the path inside the sandbox.
message PathMapping {
required string from = 1;
required string to = 2;
}