diff --git a/cmd/merge_zips/Android.bp b/cmd/merge_zips/Android.bp index 930d040f4..c516f99c1 100644 --- a/cmd/merge_zips/Android.bp +++ b/cmd/merge_zips/Android.bp @@ -22,7 +22,7 @@ blueprint_go_binary { "android-archive-zip", "blueprint-pathtools", "soong-jar", - "soong-zip", + "soong-response", ], srcs: [ "merge_zips.go", diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go index 274c8eee5..712c7fccd 100644 --- a/cmd/merge_zips/merge_zips.go +++ b/cmd/merge_zips/merge_zips.go @@ -25,12 +25,14 @@ import ( "os" "path/filepath" "sort" + "strings" + + "android/soong/response" "github.com/google/blueprint/pathtools" "android/soong/jar" "android/soong/third_party/zip" - soongZip "android/soong/zip" ) // Input zip: we can open it, close it, and obtain an array of entries @@ -690,15 +692,20 @@ func main() { inputs := make([]string, 0) for _, input := range args[1:] { if input[0] == '@' { - bytes, err := ioutil.ReadFile(input[1:]) + f, err := os.Open(strings.TrimPrefix(input[1:], "@")) if err != nil { log.Fatal(err) } - inputs = append(inputs, soongZip.ReadRespFile(bytes)...) - continue + + rspInputs, err := response.ReadRspFile(f) + f.Close() + if err != nil { + log.Fatal(err) + } + inputs = append(inputs, rspInputs...) + } else { + inputs = append(inputs, input) } - inputs = append(inputs, input) - continue } log.SetFlags(log.Lshortfile) diff --git a/response/Android.bp b/response/Android.bp new file mode 100644 index 000000000..e19981f8f --- /dev/null +++ b/response/Android.bp @@ -0,0 +1,16 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-response", + pkgPath: "android/soong/response", + deps: [ + ], + srcs: [ + "response.go", + ], + testSrcs: [ + "response_test.go", + ], +} diff --git a/response/response.go b/response/response.go new file mode 100644 index 000000000..e8ff4b2c9 --- /dev/null +++ b/response/response.go @@ -0,0 +1,70 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package response + +import ( + "io" + "io/ioutil" + "unicode" +) + +const noQuote = '\x00' + +// ReadRspFile reads a file in Ninja's response file format and returns its contents. +func ReadRspFile(r io.Reader) ([]string, error) { + var files []string + var file []byte + + buf, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + isEscaping := false + quotingStart := byte(noQuote) + for _, c := range buf { + switch { + case isEscaping: + if quotingStart == '"' { + if !(c == '"' || c == '\\') { + // '\"' or '\\' will be escaped under double quoting. + file = append(file, '\\') + } + } + file = append(file, c) + isEscaping = false + case c == '\\' && quotingStart != '\'': + isEscaping = true + case quotingStart == noQuote && (c == '\'' || c == '"'): + quotingStart = c + case quotingStart != noQuote && c == quotingStart: + quotingStart = noQuote + case quotingStart == noQuote && unicode.IsSpace(rune(c)): + // Current character is a space outside quotes + if len(file) != 0 { + files = append(files, string(file)) + } + file = file[:0] + default: + file = append(file, c) + } + } + + if len(file) != 0 { + files = append(files, string(file)) + } + + return files, nil +} diff --git a/response/response_test.go b/response/response_test.go new file mode 100644 index 000000000..99ce62359 --- /dev/null +++ b/response/response_test.go @@ -0,0 +1,96 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package response + +import ( + "bytes" + "reflect" + "testing" +) + +func TestReadRspFile(t *testing.T) { + testCases := []struct { + name, in string + out []string + }{ + { + name: "single quoting test case 1", + in: `./cmd '"'-C`, + out: []string{"./cmd", `"-C`}, + }, + { + name: "single quoting test case 2", + in: `./cmd '-C`, + out: []string{"./cmd", `-C`}, + }, + { + name: "single quoting test case 3", + in: `./cmd '\"'-C`, + out: []string{"./cmd", `\"-C`}, + }, + { + name: "single quoting test case 4", + in: `./cmd '\\'-C`, + out: []string{"./cmd", `\\-C`}, + }, + { + name: "none quoting test case 1", + in: `./cmd \'-C`, + out: []string{"./cmd", `'-C`}, + }, + { + name: "none quoting test case 2", + in: `./cmd \\-C`, + out: []string{"./cmd", `\-C`}, + }, + { + name: "none quoting test case 3", + in: `./cmd \"-C`, + out: []string{"./cmd", `"-C`}, + }, + { + name: "double quoting test case 1", + in: `./cmd "'"-C`, + out: []string{"./cmd", `'-C`}, + }, + { + name: "double quoting test case 2", + in: `./cmd "\\"-C`, + out: []string{"./cmd", `\-C`}, + }, + { + name: "double quoting test case 3", + in: `./cmd "\""-C`, + out: []string{"./cmd", `"-C`}, + }, + { + name: "ninja rsp file", + in: "'a'\nb\n'@'\n'foo'\\''bar'\n'foo\"bar'", + out: []string{"a", "b", "@", "foo'bar", `foo"bar`}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + got, err := ReadRspFile(bytes.NewBuffer([]byte(testCase.in))) + if err != nil { + t.Errorf("unexpected error: %q", err) + } + if !reflect.DeepEqual(got, testCase.out) { + t.Errorf("expected %q got %q", testCase.out, got) + } + }) + } +} diff --git a/zip/Android.bp b/zip/Android.bp index b28adbd51..14541eb1c 100644 --- a/zip/Android.bp +++ b/zip/Android.bp @@ -25,6 +25,7 @@ bootstrap_go_package { "android-archive-zip", "blueprint-pathtools", "soong-jar", + "soong-response", ], srcs: [ "zip.go", diff --git a/zip/cmd/main.go b/zip/cmd/main.go index fc976f689..cbc73eda6 100644 --- a/zip/cmd/main.go +++ b/zip/cmd/main.go @@ -24,7 +24,6 @@ package main import ( "flag" "fmt" - "io/ioutil" "os" "runtime" "runtime/pprof" @@ -32,6 +31,7 @@ import ( "strconv" "strings" + "android/soong/response" "android/soong/zip" ) @@ -125,12 +125,18 @@ func main() { var expandedArgs []string for _, arg := range os.Args { if strings.HasPrefix(arg, "@") { - bytes, err := ioutil.ReadFile(strings.TrimPrefix(arg, "@")) + f, err := os.Open(strings.TrimPrefix(arg, "@")) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + + respArgs, err := response.ReadRspFile(f) + f.Close() if err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } - respArgs := zip.ReadRespFile(bytes) expandedArgs = append(expandedArgs, respArgs...) } else { expandedArgs = append(expandedArgs, arg) diff --git a/zip/zip.go b/zip/zip.go index 088ed0d04..a6490d452 100644 --- a/zip/zip.go +++ b/zip/zip.go @@ -29,7 +29,8 @@ import ( "sync" "syscall" "time" - "unicode" + + "android/soong/response" "github.com/google/blueprint/pathtools" @@ -164,14 +165,12 @@ func (b *FileArgsBuilder) RspFile(name string) *FileArgsBuilder { } defer f.Close() - list, err := ioutil.ReadAll(f) + arg := b.state + arg.SourceFiles, err = response.ReadRspFile(f) if err != nil { b.err = err return b } - - arg := b.state - arg.SourceFiles = ReadRespFile(list) for i := range arg.SourceFiles { arg.SourceFiles[i] = pathtools.MatchEscape(arg.SourceFiles[i]) } @@ -253,49 +252,6 @@ type ZipArgs struct { Filesystem pathtools.FileSystem } -const NOQUOTE = '\x00' - -func ReadRespFile(bytes []byte) []string { - var args []string - var arg []rune - - isEscaping := false - quotingStart := NOQUOTE - for _, c := range string(bytes) { - switch { - case isEscaping: - if quotingStart == '"' { - if !(c == '"' || c == '\\') { - // '\"' or '\\' will be escaped under double quoting. - arg = append(arg, '\\') - } - } - arg = append(arg, c) - isEscaping = false - case c == '\\' && quotingStart != '\'': - isEscaping = true - case quotingStart == NOQUOTE && (c == '\'' || c == '"'): - quotingStart = c - case quotingStart != NOQUOTE && c == quotingStart: - quotingStart = NOQUOTE - case quotingStart == NOQUOTE && unicode.IsSpace(c): - // Current character is a space outside quotes - if len(arg) != 0 { - args = append(args, string(arg)) - } - arg = arg[:0] - default: - arg = append(arg, c) - } - } - - if len(arg) != 0 { - args = append(args, string(arg)) - } - - return args -} - func zipTo(args ZipArgs, w io.Writer) error { if args.EmulateJar { args.AddDirectoryEntriesToZip = true diff --git a/zip/zip_test.go b/zip/zip_test.go index b456ef8f2..a37ae41e4 100644 --- a/zip/zip_test.go +++ b/zip/zip_test.go @@ -535,78 +535,6 @@ func TestZip(t *testing.T) { } } -func TestReadRespFile(t *testing.T) { - testCases := []struct { - name, in string - out []string - }{ - { - name: "single quoting test case 1", - in: `./cmd '"'-C`, - out: []string{"./cmd", `"-C`}, - }, - { - name: "single quoting test case 2", - in: `./cmd '-C`, - out: []string{"./cmd", `-C`}, - }, - { - name: "single quoting test case 3", - in: `./cmd '\"'-C`, - out: []string{"./cmd", `\"-C`}, - }, - { - name: "single quoting test case 4", - in: `./cmd '\\'-C`, - out: []string{"./cmd", `\\-C`}, - }, - { - name: "none quoting test case 1", - in: `./cmd \'-C`, - out: []string{"./cmd", `'-C`}, - }, - { - name: "none quoting test case 2", - in: `./cmd \\-C`, - out: []string{"./cmd", `\-C`}, - }, - { - name: "none quoting test case 3", - in: `./cmd \"-C`, - out: []string{"./cmd", `"-C`}, - }, - { - name: "double quoting test case 1", - in: `./cmd "'"-C`, - out: []string{"./cmd", `'-C`}, - }, - { - name: "double quoting test case 2", - in: `./cmd "\\"-C`, - out: []string{"./cmd", `\-C`}, - }, - { - name: "double quoting test case 3", - in: `./cmd "\""-C`, - out: []string{"./cmd", `"-C`}, - }, - { - name: "ninja rsp file", - in: "'a'\nb\n'@'\n'foo'\\''bar'\n'foo\"bar'", - out: []string{"a", "b", "@", "foo'bar", `foo"bar`}, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - got := ReadRespFile([]byte(testCase.in)) - if !reflect.DeepEqual(got, testCase.out) { - t.Errorf("expected %q got %q", testCase.out, got) - } - }) - } -} - func TestSrcJar(t *testing.T) { mockFs := pathtools.MockFs(map[string][]byte{ "wrong_package.java": []byte("package foo;"),