From 11b5c51d4e03369420b587dfdba2ee09be3afa33 Mon Sep 17 00:00:00 2001 From: Jeff Gaston Date: Thu, 12 Oct 2017 12:19:14 -0700 Subject: [PATCH] split soong_zip into a library and a binary to make it faster/easier to invoke from other Go programs (such as multiproduct_kati) Bug: 67478260 Test: m -j Change-Id: Idd2671a44290550197c88f53dd11a6dd39c85cc5 --- Android.bp | 1 + zip/Android.bp | 29 ++++ {cmd/soong_zip => zip/cmd}/Android.bp | 6 +- zip/cmd/main.go | 171 +++++++++++++++++++++++ {cmd/soong_zip => zip}/rate_limit.go | 2 +- cmd/soong_zip/soong_zip.go => zip/zip.go | 120 +--------------- 6 files changed, 205 insertions(+), 124 deletions(-) create mode 100644 zip/Android.bp rename {cmd/soong_zip => zip/cmd}/Android.bp (87%) create mode 100644 zip/cmd/main.go rename {cmd/soong_zip => zip}/rate_limit.go (99%) rename cmd/soong_zip/soong_zip.go => zip/zip.go (84%) diff --git a/Android.bp b/Android.bp index 32b89d1a3..1f6ebe2ad 100644 --- a/Android.bp +++ b/Android.bp @@ -5,6 +5,7 @@ subdirs = [ "fs", "finder", "jar", + "zip", "third_party/zip", "ui/*", ] diff --git a/zip/Android.bp b/zip/Android.bp new file mode 100644 index 000000000..d7080899b --- /dev/null +++ b/zip/Android.bp @@ -0,0 +1,29 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +subdirs = ["cmd"] + +bootstrap_go_package { + name: "soong-zip", + pkgPath: "android/soong/zip", + deps: [ + "android-archive-zip", + "soong-jar", + ], + srcs: [ + "zip.go", + "rate_limit.go", + ], +} + diff --git a/cmd/soong_zip/Android.bp b/zip/cmd/Android.bp similarity index 87% rename from cmd/soong_zip/Android.bp rename to zip/cmd/Android.bp index 3e6933627..6029a694f 100644 --- a/cmd/soong_zip/Android.bp +++ b/zip/cmd/Android.bp @@ -15,11 +15,9 @@ blueprint_go_binary { name: "soong_zip", deps: [ - "android-archive-zip", - "soong-jar", + "soong-zip", ], srcs: [ - "soong_zip.go", - "rate_limit.go", + "main.go", ], } diff --git a/zip/cmd/main.go b/zip/cmd/main.go new file mode 100644 index 000000000..348728c25 --- /dev/null +++ b/zip/cmd/main.go @@ -0,0 +1,171 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + + "android/soong/zip" +) + +type byteReaderCloser struct { + *bytes.Reader + io.Closer +} + +type pathMapping struct { + dest, src string + zipMethod uint16 +} + +type uniqueSet map[string]bool + +func (u *uniqueSet) String() string { + return `""` +} + +func (u *uniqueSet) Set(s string) error { + if _, found := (*u)[s]; found { + return fmt.Errorf("File %q was specified twice as a file to not deflate", s) + } else { + (*u)[s] = true + } + + return nil +} + +type file struct{} + +type listFiles struct{} + +type dir struct{} + +func (f *file) String() string { + return `""` +} + +func (f *file) Set(s string) error { + if *relativeRoot == "" { + return fmt.Errorf("must pass -C before -f") + } + + fArgs = append(fArgs, zip.FileArg{ + PathPrefixInZip: filepath.Clean(*rootPrefix), + SourcePrefixToStrip: filepath.Clean(*relativeRoot), + SourceFiles: []string{s}, + }) + + return nil +} + +func (l *listFiles) String() string { + return `""` +} + +func (l *listFiles) Set(s string) error { + if *relativeRoot == "" { + return fmt.Errorf("must pass -C before -l") + } + + list, err := ioutil.ReadFile(s) + if err != nil { + return err + } + + fArgs = append(fArgs, zip.FileArg{ + PathPrefixInZip: filepath.Clean(*rootPrefix), + SourcePrefixToStrip: filepath.Clean(*relativeRoot), + SourceFiles: strings.Split(string(list), "\n"), + }) + + return nil +} + +func (d *dir) String() string { + return `""` +} + +func (d *dir) Set(s string) error { + if *relativeRoot == "" { + return fmt.Errorf("must pass -C before -D") + } + + fArgs = append(fArgs, zip.FileArg{ + PathPrefixInZip: filepath.Clean(*rootPrefix), + SourcePrefixToStrip: filepath.Clean(*relativeRoot), + GlobDir: filepath.Clean(s), + }) + + return nil +} + +var ( + out = flag.String("o", "", "file to write zip file to") + manifest = flag.String("m", "", "input jar manifest file name") + directories = flag.Bool("d", false, "include directories in zip") + rootPrefix = flag.String("P", "", "path prefix within the zip at which to place files") + relativeRoot = flag.String("C", "", "path to use as relative root of files in following -f, -l, or -D arguments") + parallelJobs = flag.Int("j", runtime.NumCPU(), "number of parallel threads to use") + compLevel = flag.Int("L", 5, "deflate compression level (0-9)") + emulateJar = flag.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'") + + fArgs zip.FileArgs + nonDeflatedFiles = make(uniqueSet) + + cpuProfile = flag.String("cpuprofile", "", "write cpu profile to file") + traceFile = flag.String("trace", "", "write trace to file") +) + +func init() { + flag.Var(&listFiles{}, "l", "file containing list of .class files") + flag.Var(&dir{}, "D", "directory to include in zip") + flag.Var(&file{}, "f", "file to include in zip") + flag.Var(&nonDeflatedFiles, "s", "file path to be stored within the zip without compression") +} + +func usage() { + fmt.Fprintf(os.Stderr, "usage: zip -o zipfile [-m manifest] -C dir [-f|-l file]...\n") + flag.PrintDefaults() + os.Exit(2) +} + +func main() { + flag.Parse() + + err := zip.Run(zip.ZipArgs{ + FileArgs: fArgs, + OutputFilePath: *out, + CpuProfileFilePath: *cpuProfile, + TraceFilePath: *traceFile, + EmulateJar: *emulateJar, + AddDirectoryEntriesToZip: *directories, + CompressionLevel: *compLevel, + ManifestSourcePath: *manifest, + NumParallelJobs: *parallelJobs, + NonDeflatedFiles: nonDeflatedFiles, + }) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } +} diff --git a/cmd/soong_zip/rate_limit.go b/zip/rate_limit.go similarity index 99% rename from cmd/soong_zip/rate_limit.go rename to zip/rate_limit.go index 9cb5fdd51..0ea2ef084 100644 --- a/cmd/soong_zip/rate_limit.go +++ b/zip/rate_limit.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package zip import ( "fmt" diff --git a/cmd/soong_zip/soong_zip.go b/zip/zip.go similarity index 84% rename from cmd/soong_zip/soong_zip.go rename to zip/zip.go index 2bcc9a5f4..95520fe31 100644 --- a/cmd/soong_zip/soong_zip.go +++ b/zip/zip.go @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package zip import ( "bytes" "compress/flate" "errors" - "flag" "fmt" "hash/crc32" "io" @@ -26,7 +25,6 @@ import ( "log" "os" "path/filepath" - "runtime" "runtime/pprof" "runtime/trace" "sort" @@ -83,122 +81,6 @@ func (u *uniqueSet) Set(s string) error { return nil } -type file struct{} - -type listFiles struct{} - -type dir struct{} - -func (f *file) String() string { - return `""` -} - -func (f *file) Set(s string) error { - if *relativeRoot == "" { - return fmt.Errorf("must pass -C before -f") - } - - fArgs = append(fArgs, FileArg{ - PathPrefixInZip: filepath.Clean(*rootPrefix), - SourcePrefixToStrip: filepath.Clean(*relativeRoot), - SourceFiles: []string{s}, - }) - - return nil -} - -func (l *listFiles) String() string { - return `""` -} - -func (l *listFiles) Set(s string) error { - if *relativeRoot == "" { - return fmt.Errorf("must pass -C before -l") - } - - list, err := ioutil.ReadFile(s) - if err != nil { - return err - } - - fArgs = append(fArgs, FileArg{ - PathPrefixInZip: filepath.Clean(*rootPrefix), - SourcePrefixToStrip: filepath.Clean(*relativeRoot), - SourceFiles: strings.Split(string(list), "\n"), - }) - - return nil -} - -func (d *dir) String() string { - return `""` -} - -func (d *dir) Set(s string) error { - if *relativeRoot == "" { - return fmt.Errorf("must pass -C before -D") - } - - fArgs = append(fArgs, FileArg{ - PathPrefixInZip: filepath.Clean(*rootPrefix), - SourcePrefixToStrip: filepath.Clean(*relativeRoot), - GlobDir: filepath.Clean(s), - }) - - return nil -} - -var ( - out = flag.String("o", "", "file to write zip file to") - manifest = flag.String("m", "", "input jar manifest file name") - directories = flag.Bool("d", false, "include directories in zip") - rootPrefix = flag.String("P", "", "path prefix within the zip at which to place files") - relativeRoot = flag.String("C", "", "path to use as relative root of files in following -f, -l, or -D arguments") - parallelJobs = flag.Int("j", runtime.NumCPU(), "number of parallel threads to use") - compLevel = flag.Int("L", 5, "deflate compression level (0-9)") - emulateJar = flag.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'") - - fArgs FileArgs - nonDeflatedFiles = make(uniqueSet) - - cpuProfile = flag.String("cpuprofile", "", "write cpu profile to file") - traceFile = flag.String("trace", "", "write trace to file") -) - -func init() { - flag.Var(&listFiles{}, "l", "file containing list of .class files") - flag.Var(&dir{}, "D", "directory to include in zip") - flag.Var(&file{}, "f", "file to include in zip") - flag.Var(&nonDeflatedFiles, "s", "file path to be stored within the zip without compression") -} - -func usage() { - fmt.Fprintf(os.Stderr, "usage: soong_zip -o zipfile [-m manifest] -C dir [-f|-l file]...\n") - flag.PrintDefaults() - os.Exit(2) -} - -func main() { - flag.Parse() - - err := Run(ZipArgs{ - FileArgs: fArgs, - OutputFilePath: *out, - CpuProfileFilePath: *cpuProfile, - TraceFilePath: *traceFile, - EmulateJar: *emulateJar, - AddDirectoryEntriesToZip: *directories, - CompressionLevel: *compLevel, - ManifestSourcePath: *manifest, - NumParallelJobs: *parallelJobs, - NonDeflatedFiles: nonDeflatedFiles, - }) - if err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - os.Exit(1) - } -} - type FileArg struct { PathPrefixInZip, SourcePrefixToStrip string SourceFiles []string