From 674dd932d8a60877b71147bd426d7b88a613f4ad Mon Sep 17 00:00:00 2001 From: Nan Zhang Date: Fri, 26 Jan 2018 18:30:36 -0800 Subject: [PATCH] Add Respfile support for soong_zip. Sometime the size of our command line passed to soong_zip go program exceeds the cmdline size limit. So add an RespFile support with "@" special character prefix. The args in the cmdline will be considered together with the args in RespFile during soong_zip running. Test: real tests in my local machine, and compare the res/libphonenumber.jar before and after changes. ./cmd -o test.zip '""'-C -> [./cmd,-o,test.zip,""-C] ./cmd -o test.zip '-C -f -> [./cmd,-o,test.zip,-C -f] ./cmd -o test.zip '\"'-C -f -> [./cmd,-o,test.zip,\"-C -f] ./cmd -o test.zip '\\'-C -f -> [./cmd,-o,test.zip,\\-C -f] ./cmd -o test.zip '\a'-C -f -> [./cmd,-o,test.zip,\a-C -f] ./cmd -o test.zip \'-C -> [./cmd,-o,test.zip,'-C] ./cmd -o test.zip \\-C -> [./cmd,-o,test.zip,\-C] ./cmd -o test.zip \"-C -> [./cmd,-o,test.zip,"-C] ./cmd -o test.zip "'"-C -> [./cmd,-o,test.zip,'-C] ./cmd -o test.zip "\\"-C -f -> [./cmd,-o,test.zip,\a-C -f] ./cmd -o test.zip "\""-C -f -> [./cmd,-o,test.zip,"a-C -f] Bug: b/72484223 Change-Id: I83c3630b70c8396c8e8a3f266244d868d754c4e8 --- java/builder.go | 6 ++-- zip/Android.bp | 3 ++ zip/cmd/main.go | 57 ++++++++++++++++++++------------ zip/zip.go | 44 +++++++++++++++++++++++++ zip/zip_test.go | 87 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 175 insertions(+), 22 deletions(-) create mode 100644 zip/zip_test.go diff --git a/java/builder.go b/java/builder.go index 2fd4ac047..72574f197 100644 --- a/java/builder.go +++ b/java/builder.go @@ -123,8 +123,10 @@ var ( jar = pctx.AndroidStaticRule("jar", blueprint.RuleParams{ - Command: `${config.SoongZipCmd} -jar -o $out $jarArgs`, - CommandDeps: []string{"${config.SoongZipCmd}"}, + Command: `${config.SoongZipCmd} -jar -o $out @$out.rsp`, + CommandDeps: []string{"${config.SoongZipCmd}"}, + Rspfile: "$out.rsp", + RspfileContent: "$jarArgs", }, "jarArgs") diff --git a/zip/Android.bp b/zip/Android.bp index 3bb4f25d6..259e010aa 100644 --- a/zip/Android.bp +++ b/zip/Android.bp @@ -26,5 +26,8 @@ bootstrap_go_package { "zip.go", "rate_limit.go", ], + testSrcs: [ + "zip_test.go", + ], } diff --git a/zip/cmd/main.go b/zip/cmd/main.go index c0418f7b5..60017aa94 100644 --- a/zip/cmd/main.go +++ b/zip/cmd/main.go @@ -120,30 +120,12 @@ func (d *dir) Set(s string) error { } var ( - out = flag.String("o", "", "file to write zip file to") - manifest = flag.String("m", "", "input jar manifest file name") - directories = flag.Bool("d", false, "include directories in zip") - rootPrefix = flag.String("P", "", "path prefix within the zip at which to place files") - relativeRoot = flag.String("C", "", "path to use as relative root of files in following -f, -l, or -D arguments") - parallelJobs = flag.Int("j", runtime.NumCPU(), "number of parallel threads to use") - compLevel = flag.Int("L", 5, "deflate compression level (0-9)") - emulateJar = flag.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'") - writeIfChanged = flag.Bool("write_if_changed", false, "only update resultant .zip if it has changed") + rootPrefix, relativeRoot *string fArgs zip.FileArgs nonDeflatedFiles = make(uniqueSet) - - cpuProfile = flag.String("cpuprofile", "", "write cpu profile to file") - traceFile = flag.String("trace", "", "write trace to file") ) -func init() { - flag.Var(&listFiles{}, "l", "file containing list of .class files") - flag.Var(&dir{}, "D", "directory to include in zip") - flag.Var(&file{}, "f", "file to include in zip") - flag.Var(&nonDeflatedFiles, "s", "file path to be stored within the zip without compression") -} - func usage() { fmt.Fprintf(os.Stderr, "usage: zip -o zipfile [-m manifest] -C dir [-f|-l file]...\n") flag.PrintDefaults() @@ -151,7 +133,42 @@ func usage() { } func main() { - flag.Parse() + var expandedArgs []string + for _, arg := range os.Args { + if strings.HasPrefix(arg, "@") { + bytes, err := ioutil.ReadFile(strings.TrimPrefix(arg, "@")) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + respArgs := zip.ReadRespFile(bytes) + expandedArgs = append(expandedArgs, respArgs...) + } else { + expandedArgs = append(expandedArgs, arg) + } + } + + flags := flag.NewFlagSet("flags", flag.ExitOnError) + + out := flags.String("o", "", "file to write zip file to") + manifest := flags.String("m", "", "input jar manifest file name") + directories := flags.Bool("d", false, "include directories in zip") + rootPrefix = flags.String("P", "", "path prefix within the zip at which to place files") + relativeRoot = flags.String("C", "", "path to use as relative root of files in following -f, -l, or -D arguments") + parallelJobs := flags.Int("j", runtime.NumCPU(), "number of parallel threads to use") + compLevel := flags.Int("L", 5, "deflate compression level (0-9)") + emulateJar := flags.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'") + writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed") + + cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file") + traceFile := flags.String("trace", "", "write trace to file") + + flags.Var(&listFiles{}, "l", "file containing list of .class files") + flags.Var(&dir{}, "D", "directory to include in zip") + flags.Var(&file{}, "f", "file to include in zip") + flags.Var(&nonDeflatedFiles, "s", "file path to be stored within the zip without compression") + + flags.Parse(expandedArgs[1:]) err := zip.Run(zip.ZipArgs{ FileArgs: fArgs, diff --git a/zip/zip.go b/zip/zip.go index c878a0cce..b7e37646e 100644 --- a/zip/zip.go +++ b/zip/zip.go @@ -31,6 +31,7 @@ import ( "strings" "sync" "time" + "unicode" "github.com/google/blueprint/pathtools" @@ -132,6 +133,49 @@ type ZipArgs struct { WriteIfChanged bool } +const NOQUOTE = '\x00' + +func ReadRespFile(bytes []byte) []string { + var args []string + var arg []rune + + isEscaping := false + quotingStart := NOQUOTE + for _, c := range string(bytes) { + switch { + case isEscaping: + if quotingStart == '"' { + if !(c == '"' || c == '\\') { + // '\"' or '\\' will be escaped under double quoting. + arg = append(arg, '\\') + } + } + arg = append(arg, c) + isEscaping = false + case c == '\\' && quotingStart != '\'': + isEscaping = true + case quotingStart == NOQUOTE && (c == '\'' || c == '"'): + quotingStart = c + case quotingStart != NOQUOTE && c == quotingStart: + quotingStart = NOQUOTE + case quotingStart == NOQUOTE && unicode.IsSpace(c): + // Current character is a space outside quotes + if len(arg) != 0 { + args = append(args, string(arg)) + } + arg = arg[:0] + default: + arg = append(arg, c) + } + } + + if len(arg) != 0 { + args = append(args, string(arg)) + } + + return args +} + func Run(args ZipArgs) (err error) { if args.CpuProfileFilePath != "" { f, err := os.Create(args.CpuProfileFilePath) diff --git a/zip/zip_test.go b/zip/zip_test.go new file mode 100644 index 000000000..03e7958fb --- /dev/null +++ b/zip/zip_test.go @@ -0,0 +1,87 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zip + +import ( + "reflect" + "testing" +) + +func TestReadRespFile(t *testing.T) { + testCases := []struct { + name, in string + out []string + }{ + { + name: "single quoting test case 1", + in: `./cmd '"'-C`, + out: []string{"./cmd", `"-C`}, + }, + { + name: "single quoting test case 2", + in: `./cmd '-C`, + out: []string{"./cmd", `-C`}, + }, + { + name: "single quoting test case 3", + in: `./cmd '\"'-C`, + out: []string{"./cmd", `\"-C`}, + }, + { + name: "single quoting test case 4", + in: `./cmd '\\'-C`, + out: []string{"./cmd", `\\-C`}, + }, + { + name: "none quoting test case 1", + in: `./cmd \'-C`, + out: []string{"./cmd", `'-C`}, + }, + { + name: "none quoting test case 2", + in: `./cmd \\-C`, + out: []string{"./cmd", `\-C`}, + }, + { + name: "none quoting test case 3", + in: `./cmd \"-C`, + out: []string{"./cmd", `"-C`}, + }, + { + name: "double quoting test case 1", + in: `./cmd "'"-C`, + out: []string{"./cmd", `'-C`}, + }, + { + name: "double quoting test case 2", + in: `./cmd "\\"-C`, + out: []string{"./cmd", `\-C`}, + }, + { + name: "double quoting test case 3", + in: `./cmd "\""-C`, + out: []string{"./cmd", `"-C`}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + got := ReadRespFile([]byte(testCase.in)) + if !reflect.DeepEqual(got, testCase.out) { + t.Errorf("expected %q got %q", testCase.out, got) + } + }) + } +}