split soong_zip into a library and a binary
to make it faster/easier to invoke from other Go programs (such as multiproduct_kati) Bug: 67478260 Test: m -j Change-Id: Idd2671a44290550197c88f53dd11a6dd39c85cc5
This commit is contained in:
Jeff Gaston
@@ -1,25 +0,0 @@
|
||||
// Copyright 2016 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
blueprint_go_binary {
|
||||
name: "soong_zip",
|
||||
deps: [
|
||||
"android-archive-zip",
|
||||
"soong-jar",
|
||||
],
|
||||
srcs: [
|
||||
"soong_zip.go",
|
||||
"rate_limit.go",
|
||||
],
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
// Copyright 2016 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
type RateLimit struct {
|
||||
requests chan request
|
||||
completions chan int64
|
||||
|
||||
stop chan struct{}
|
||||
}
|
||||
|
||||
type request struct {
|
||||
size int64
|
||||
serviced chan struct{}
|
||||
}
|
||||
|
||||
// NewRateLimit starts a new rate limiter that permits the usage of up to <capacity> at once,
|
||||
// except when no capacity is in use, in which case the first caller is always permitted
|
||||
func NewRateLimit(capacity int64) *RateLimit {
|
||||
ret := &RateLimit{
|
||||
requests: make(chan request),
|
||||
completions: make(chan int64),
|
||||
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
|
||||
go ret.monitorChannels(capacity)
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// RequestExecution blocks until another execution of size <size> can be allowed to run.
|
||||
func (r *RateLimit) Request(size int64) {
|
||||
request := request{
|
||||
size: size,
|
||||
serviced: make(chan struct{}, 1),
|
||||
}
|
||||
|
||||
// wait for the request to be received
|
||||
r.requests <- request
|
||||
|
||||
// wait for the request to be accepted
|
||||
<-request.serviced
|
||||
}
|
||||
|
||||
// Finish declares the completion of an execution of size <size>
|
||||
func (r *RateLimit) Finish(size int64) {
|
||||
r.completions <- size
|
||||
}
|
||||
|
||||
// Stop the background goroutine
|
||||
func (r *RateLimit) Stop() {
|
||||
close(r.stop)
|
||||
}
|
||||
|
||||
// monitorChannels processes incoming requests from channels
|
||||
func (r *RateLimit) monitorChannels(capacity int64) {
|
||||
var usedCapacity int64
|
||||
var currentRequest *request
|
||||
|
||||
for {
|
||||
var requests chan request
|
||||
if currentRequest == nil {
|
||||
// If we don't already have a queued request, then we should check for a new request
|
||||
requests = r.requests
|
||||
}
|
||||
|
||||
select {
|
||||
case newRequest := <-requests:
|
||||
currentRequest = &newRequest
|
||||
case amountCompleted := <-r.completions:
|
||||
usedCapacity -= amountCompleted
|
||||
|
||||
if usedCapacity < 0 {
|
||||
panic(fmt.Sprintf("usedCapacity < 0: %v (decreased by %v)", usedCapacity, amountCompleted))
|
||||
}
|
||||
case <-r.stop:
|
||||
return
|
||||
}
|
||||
|
||||
if currentRequest != nil {
|
||||
accepted := false
|
||||
if usedCapacity == 0 {
|
||||
accepted = true
|
||||
} else {
|
||||
if capacity >= usedCapacity+currentRequest.size {
|
||||
accepted = true
|
||||
}
|
||||
}
|
||||
if accepted {
|
||||
usedCapacity += currentRequest.size
|
||||
currentRequest.serviced <- struct{}{}
|
||||
currentRequest = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A CPURateLimiter limits the number of active calls based on CPU requirements
|
||||
type CPURateLimiter struct {
|
||||
impl *RateLimit
|
||||
}
|
||||
|
||||
func NewCPURateLimiter(capacity int64) *CPURateLimiter {
|
||||
if capacity <= 0 {
|
||||
capacity = int64(runtime.NumCPU())
|
||||
}
|
||||
impl := NewRateLimit(capacity)
|
||||
return &CPURateLimiter{impl: impl}
|
||||
}
|
||||
|
||||
func (e CPURateLimiter) Request() {
|
||||
e.impl.Request(1)
|
||||
}
|
||||
|
||||
func (e CPURateLimiter) Finish() {
|
||||
e.impl.Finish(1)
|
||||
}
|
||||
|
||||
func (e CPURateLimiter) Stop() {
|
||||
e.impl.Stop()
|
||||
}
|
||||
|
||||
// A MemoryRateLimiter limits the number of active calls based on Memory requirements
|
||||
type MemoryRateLimiter struct {
|
||||
*RateLimit
|
||||
}
|
||||
|
||||
func NewMemoryRateLimiter(capacity int64) *MemoryRateLimiter {
|
||||
if capacity <= 0 {
|
||||
capacity = 512 * 1024 * 1024 // 512MB
|
||||
}
|
||||
impl := NewRateLimit(capacity)
|
||||
return &MemoryRateLimiter{RateLimit: impl}
|
||||
}
|
||||
@@ -1,878 +0,0 @@
|
||||
// Copyright 2015 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/flate"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"runtime/trace"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"android/soong/jar"
|
||||
"android/soong/third_party/zip"
|
||||
)
|
||||
|
||||
// Block size used during parallel compression of a single file.
|
||||
const parallelBlockSize = 1 * 1024 * 1024 // 1MB
|
||||
|
||||
// Minimum file size to use parallel compression. It requires more
|
||||
// flate.Writer allocations, since we can't change the dictionary
|
||||
// during Reset
|
||||
const minParallelFileSize = parallelBlockSize * 6
|
||||
|
||||
// Size of the ZIP compression window (32KB)
|
||||
const windowSize = 32 * 1024
|
||||
|
||||
type nopCloser struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
func (nopCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type byteReaderCloser struct {
|
||||
*bytes.Reader
|
||||
io.Closer
|
||||
}
|
||||
|
||||
type pathMapping struct {
|
||||
dest, src string
|
||||
zipMethod uint16
|
||||
}
|
||||
|
||||
type uniqueSet map[string]bool
|
||||
|
||||
func (u *uniqueSet) String() string {
|
||||
return `""`
|
||||
}
|
||||
|
||||
func (u *uniqueSet) Set(s string) error {
|
||||
if _, found := (*u)[s]; found {
|
||||
return fmt.Errorf("File %q was specified twice as a file to not deflate", s)
|
||||
} else {
|
||||
(*u)[s] = true
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type file struct{}
|
||||
|
||||
type listFiles struct{}
|
||||
|
||||
type dir struct{}
|
||||
|
||||
func (f *file) String() string {
|
||||
return `""`
|
||||
}
|
||||
|
||||
func (f *file) Set(s string) error {
|
||||
if *relativeRoot == "" {
|
||||
return fmt.Errorf("must pass -C before -f")
|
||||
}
|
||||
|
||||
fArgs = append(fArgs, FileArg{
|
||||
PathPrefixInZip: filepath.Clean(*rootPrefix),
|
||||
SourcePrefixToStrip: filepath.Clean(*relativeRoot),
|
||||
SourceFiles: []string{s},
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *listFiles) String() string {
|
||||
return `""`
|
||||
}
|
||||
|
||||
func (l *listFiles) Set(s string) error {
|
||||
if *relativeRoot == "" {
|
||||
return fmt.Errorf("must pass -C before -l")
|
||||
}
|
||||
|
||||
list, err := ioutil.ReadFile(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fArgs = append(fArgs, FileArg{
|
||||
PathPrefixInZip: filepath.Clean(*rootPrefix),
|
||||
SourcePrefixToStrip: filepath.Clean(*relativeRoot),
|
||||
SourceFiles: strings.Split(string(list), "\n"),
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *dir) String() string {
|
||||
return `""`
|
||||
}
|
||||
|
||||
func (d *dir) Set(s string) error {
|
||||
if *relativeRoot == "" {
|
||||
return fmt.Errorf("must pass -C before -D")
|
||||
}
|
||||
|
||||
fArgs = append(fArgs, FileArg{
|
||||
PathPrefixInZip: filepath.Clean(*rootPrefix),
|
||||
SourcePrefixToStrip: filepath.Clean(*relativeRoot),
|
||||
GlobDir: filepath.Clean(s),
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
out = flag.String("o", "", "file to write zip file to")
|
||||
manifest = flag.String("m", "", "input jar manifest file name")
|
||||
directories = flag.Bool("d", false, "include directories in zip")
|
||||
rootPrefix = flag.String("P", "", "path prefix within the zip at which to place files")
|
||||
relativeRoot = flag.String("C", "", "path to use as relative root of files in following -f, -l, or -D arguments")
|
||||
parallelJobs = flag.Int("j", runtime.NumCPU(), "number of parallel threads to use")
|
||||
compLevel = flag.Int("L", 5, "deflate compression level (0-9)")
|
||||
emulateJar = flag.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'")
|
||||
|
||||
fArgs FileArgs
|
||||
nonDeflatedFiles = make(uniqueSet)
|
||||
|
||||
cpuProfile = flag.String("cpuprofile", "", "write cpu profile to file")
|
||||
traceFile = flag.String("trace", "", "write trace to file")
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.Var(&listFiles{}, "l", "file containing list of .class files")
|
||||
flag.Var(&dir{}, "D", "directory to include in zip")
|
||||
flag.Var(&file{}, "f", "file to include in zip")
|
||||
flag.Var(&nonDeflatedFiles, "s", "file path to be stored within the zip without compression")
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintf(os.Stderr, "usage: soong_zip -o zipfile [-m manifest] -C dir [-f|-l file]...\n")
|
||||
flag.PrintDefaults()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
err := Run(ZipArgs{
|
||||
FileArgs: fArgs,
|
||||
OutputFilePath: *out,
|
||||
CpuProfileFilePath: *cpuProfile,
|
||||
TraceFilePath: *traceFile,
|
||||
EmulateJar: *emulateJar,
|
||||
AddDirectoryEntriesToZip: *directories,
|
||||
CompressionLevel: *compLevel,
|
||||
ManifestSourcePath: *manifest,
|
||||
NumParallelJobs: *parallelJobs,
|
||||
NonDeflatedFiles: nonDeflatedFiles,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
type FileArg struct {
|
||||
PathPrefixInZip, SourcePrefixToStrip string
|
||||
SourceFiles []string
|
||||
GlobDir string
|
||||
}
|
||||
|
||||
type FileArgs []FileArg
|
||||
|
||||
type ZipWriter struct {
|
||||
time time.Time
|
||||
createdFiles map[string]string
|
||||
createdDirs map[string]string
|
||||
directories bool
|
||||
|
||||
errors chan error
|
||||
writeOps chan chan *zipEntry
|
||||
|
||||
cpuRateLimiter *CPURateLimiter
|
||||
memoryRateLimiter *MemoryRateLimiter
|
||||
|
||||
compressorPool sync.Pool
|
||||
compLevel int
|
||||
}
|
||||
|
||||
type zipEntry struct {
|
||||
fh *zip.FileHeader
|
||||
|
||||
// List of delayed io.Reader
|
||||
futureReaders chan chan io.Reader
|
||||
|
||||
// Only used for passing into the MemoryRateLimiter to ensure we
|
||||
// release as much memory as much as we request
|
||||
allocatedSize int64
|
||||
}
|
||||
|
||||
type ZipArgs struct {
|
||||
FileArgs FileArgs
|
||||
OutputFilePath string
|
||||
CpuProfileFilePath string
|
||||
TraceFilePath string
|
||||
EmulateJar bool
|
||||
AddDirectoryEntriesToZip bool
|
||||
CompressionLevel int
|
||||
ManifestSourcePath string
|
||||
NumParallelJobs int
|
||||
NonDeflatedFiles map[string]bool
|
||||
}
|
||||
|
||||
func Run(args ZipArgs) (err error) {
|
||||
if args.CpuProfileFilePath != "" {
|
||||
f, err := os.Create(args.CpuProfileFilePath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
defer f.Close()
|
||||
pprof.StartCPUProfile(f)
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
if args.TraceFilePath != "" {
|
||||
f, err := os.Create(args.TraceFilePath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
defer f.Close()
|
||||
err = trace.Start(f)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
defer trace.Stop()
|
||||
}
|
||||
|
||||
if args.OutputFilePath == "" {
|
||||
return fmt.Errorf("output file path must be nonempty")
|
||||
}
|
||||
|
||||
if args.EmulateJar {
|
||||
args.AddDirectoryEntriesToZip = true
|
||||
}
|
||||
|
||||
w := &ZipWriter{
|
||||
time: jar.DefaultTime,
|
||||
createdDirs: make(map[string]string),
|
||||
createdFiles: make(map[string]string),
|
||||
directories: args.AddDirectoryEntriesToZip,
|
||||
compLevel: args.CompressionLevel,
|
||||
}
|
||||
pathMappings := []pathMapping{}
|
||||
|
||||
for _, fa := range args.FileArgs {
|
||||
srcs := fa.SourceFiles
|
||||
if fa.GlobDir != "" {
|
||||
srcs = append(srcs, recursiveGlobFiles(fa.GlobDir)...)
|
||||
}
|
||||
for _, src := range srcs {
|
||||
if err := fillPathPairs(fa.PathPrefixInZip,
|
||||
fa.SourcePrefixToStrip, src, &pathMappings, args.NonDeflatedFiles); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return w.write(args.OutputFilePath, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
|
||||
|
||||
}
|
||||
|
||||
func fillPathPairs(prefix, rel, src string, pathMappings *[]pathMapping, nonDeflatedFiles map[string]bool) error {
|
||||
src = strings.TrimSpace(src)
|
||||
if src == "" {
|
||||
return nil
|
||||
}
|
||||
src = filepath.Clean(src)
|
||||
dest, err := filepath.Rel(rel, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dest = filepath.Join(prefix, dest)
|
||||
|
||||
zipMethod := zip.Deflate
|
||||
if _, found := nonDeflatedFiles[dest]; found {
|
||||
zipMethod = zip.Store
|
||||
}
|
||||
*pathMappings = append(*pathMappings,
|
||||
pathMapping{dest: dest, src: src, zipMethod: zipMethod})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func jarSort(mappings []pathMapping) {
|
||||
less := func(i int, j int) (smaller bool) {
|
||||
return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest)
|
||||
}
|
||||
sort.SliceStable(mappings, less)
|
||||
}
|
||||
|
||||
type readerSeekerCloser interface {
|
||||
io.Reader
|
||||
io.ReaderAt
|
||||
io.Closer
|
||||
io.Seeker
|
||||
}
|
||||
|
||||
func (z *ZipWriter) write(out string, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error {
|
||||
f, err := os.Create(out)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
os.Remove(out)
|
||||
}
|
||||
}()
|
||||
|
||||
z.errors = make(chan error)
|
||||
defer close(z.errors)
|
||||
|
||||
// This channel size can be essentially unlimited -- it's used as a fifo
|
||||
// queue decouple the CPU and IO loads. Directories don't require any
|
||||
// compression time, but still cost some IO. Similar with small files that
|
||||
// can be very fast to compress. Some files that are more difficult to
|
||||
// compress won't take a corresponding longer time writing out.
|
||||
//
|
||||
// The optimum size here depends on your CPU and IO characteristics, and
|
||||
// the the layout of your zip file. 1000 was chosen mostly at random as
|
||||
// something that worked reasonably well for a test file.
|
||||
//
|
||||
// The RateLimit object will put the upper bounds on the number of
|
||||
// parallel compressions and outstanding buffers.
|
||||
z.writeOps = make(chan chan *zipEntry, 1000)
|
||||
z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs))
|
||||
z.memoryRateLimiter = NewMemoryRateLimiter(0)
|
||||
defer func() {
|
||||
z.cpuRateLimiter.Stop()
|
||||
z.memoryRateLimiter.Stop()
|
||||
}()
|
||||
|
||||
if manifest != "" && !emulateJar {
|
||||
return errors.New("must specify --jar when specifying a manifest via -m")
|
||||
}
|
||||
|
||||
if emulateJar {
|
||||
// manifest may be empty, in which case addManifest will fill in a default
|
||||
pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
|
||||
|
||||
jarSort(pathMappings)
|
||||
}
|
||||
|
||||
go func() {
|
||||
var err error
|
||||
defer close(z.writeOps)
|
||||
|
||||
for _, ele := range pathMappings {
|
||||
if emulateJar && ele.dest == jar.ManifestFile {
|
||||
err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
|
||||
} else {
|
||||
err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar)
|
||||
}
|
||||
if err != nil {
|
||||
z.errors <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
zipw := zip.NewWriter(f)
|
||||
|
||||
var currentWriteOpChan chan *zipEntry
|
||||
var currentWriter io.WriteCloser
|
||||
var currentReaders chan chan io.Reader
|
||||
var currentReader chan io.Reader
|
||||
var done bool
|
||||
|
||||
for !done {
|
||||
var writeOpsChan chan chan *zipEntry
|
||||
var writeOpChan chan *zipEntry
|
||||
var readersChan chan chan io.Reader
|
||||
|
||||
if currentReader != nil {
|
||||
// Only read and process errors
|
||||
} else if currentReaders != nil {
|
||||
readersChan = currentReaders
|
||||
} else if currentWriteOpChan != nil {
|
||||
writeOpChan = currentWriteOpChan
|
||||
} else {
|
||||
writeOpsChan = z.writeOps
|
||||
}
|
||||
|
||||
select {
|
||||
case writeOp, ok := <-writeOpsChan:
|
||||
if !ok {
|
||||
done = true
|
||||
}
|
||||
|
||||
currentWriteOpChan = writeOp
|
||||
|
||||
case op := <-writeOpChan:
|
||||
currentWriteOpChan = nil
|
||||
|
||||
if op.fh.Method == zip.Deflate {
|
||||
currentWriter, err = zipw.CreateCompressedHeader(op.fh)
|
||||
} else {
|
||||
var zw io.Writer
|
||||
|
||||
op.fh.CompressedSize64 = op.fh.UncompressedSize64
|
||||
|
||||
zw, err = zipw.CreateHeaderAndroid(op.fh)
|
||||
currentWriter = nopCloser{zw}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentReaders = op.futureReaders
|
||||
if op.futureReaders == nil {
|
||||
currentWriter.Close()
|
||||
currentWriter = nil
|
||||
}
|
||||
z.memoryRateLimiter.Finish(op.allocatedSize)
|
||||
|
||||
case futureReader, ok := <-readersChan:
|
||||
if !ok {
|
||||
// Done with reading
|
||||
currentWriter.Close()
|
||||
currentWriter = nil
|
||||
currentReaders = nil
|
||||
}
|
||||
|
||||
currentReader = futureReader
|
||||
|
||||
case reader := <-currentReader:
|
||||
_, err = io.Copy(currentWriter, reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentReader = nil
|
||||
|
||||
case err = <-z.errors:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// One last chance to catch an error
|
||||
select {
|
||||
case err = <-z.errors:
|
||||
return err
|
||||
default:
|
||||
zipw.Close()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// imports (possibly with compression) <src> into the zip at sub-path <dest>
|
||||
func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error {
|
||||
var fileSize int64
|
||||
var executable bool
|
||||
|
||||
if s, err := os.Lstat(src); err != nil {
|
||||
return err
|
||||
} else if s.IsDir() {
|
||||
if z.directories {
|
||||
return z.writeDirectory(dest, src, emulateJar)
|
||||
}
|
||||
return nil
|
||||
} else {
|
||||
if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if prev, exists := z.createdDirs[dest]; exists {
|
||||
return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
|
||||
}
|
||||
if prev, exists := z.createdFiles[dest]; exists {
|
||||
return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
|
||||
}
|
||||
|
||||
z.createdFiles[dest] = src
|
||||
|
||||
if s.Mode()&os.ModeSymlink != 0 {
|
||||
return z.writeSymlink(dest, src)
|
||||
} else if !s.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s is not a file, directory, or symlink", src)
|
||||
}
|
||||
|
||||
fileSize = s.Size()
|
||||
executable = s.Mode()&0100 != 0
|
||||
}
|
||||
|
||||
r, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header := &zip.FileHeader{
|
||||
Name: dest,
|
||||
Method: method,
|
||||
UncompressedSize64: uint64(fileSize),
|
||||
}
|
||||
|
||||
if executable {
|
||||
header.SetMode(0700)
|
||||
}
|
||||
|
||||
return z.writeFileContents(header, r)
|
||||
}
|
||||
|
||||
func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
|
||||
if prev, exists := z.createdDirs[dest]; exists {
|
||||
return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
|
||||
}
|
||||
if prev, exists := z.createdFiles[dest]; exists {
|
||||
return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
|
||||
}
|
||||
|
||||
if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fh, buf, err := jar.ManifestFileContents(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
|
||||
|
||||
return z.writeFileContents(fh, reader)
|
||||
}
|
||||
|
||||
func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r readerSeekerCloser) (err error) {
|
||||
|
||||
header.SetModTime(z.time)
|
||||
|
||||
compressChan := make(chan *zipEntry, 1)
|
||||
z.writeOps <- compressChan
|
||||
|
||||
// Pre-fill a zipEntry, it will be sent in the compressChan once
|
||||
// we're sure about the Method and CRC.
|
||||
ze := &zipEntry{
|
||||
fh: header,
|
||||
}
|
||||
|
||||
ze.allocatedSize = int64(header.UncompressedSize64)
|
||||
z.cpuRateLimiter.Request()
|
||||
z.memoryRateLimiter.Request(ze.allocatedSize)
|
||||
|
||||
fileSize := int64(header.UncompressedSize64)
|
||||
if fileSize == 0 {
|
||||
fileSize = int64(header.UncompressedSize)
|
||||
}
|
||||
|
||||
if header.Method == zip.Deflate && fileSize >= minParallelFileSize {
|
||||
wg := new(sync.WaitGroup)
|
||||
|
||||
// Allocate enough buffer to hold all readers. We'll limit
|
||||
// this based on actual buffer sizes in RateLimit.
|
||||
ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
|
||||
|
||||
// Calculate the CRC in the background, since reading the entire
|
||||
// file could take a while.
|
||||
//
|
||||
// We could split this up into chunks as well, but it's faster
|
||||
// than the compression. Due to the Go Zip API, we also need to
|
||||
// know the result before we can begin writing the compressed
|
||||
// data out to the zipfile.
|
||||
wg.Add(1)
|
||||
go z.crcFile(r, ze, compressChan, wg)
|
||||
|
||||
for start := int64(0); start < fileSize; start += parallelBlockSize {
|
||||
sr := io.NewSectionReader(r, start, parallelBlockSize)
|
||||
resultChan := make(chan io.Reader, 1)
|
||||
ze.futureReaders <- resultChan
|
||||
|
||||
z.cpuRateLimiter.Request()
|
||||
|
||||
last := !(start+parallelBlockSize < fileSize)
|
||||
var dict []byte
|
||||
if start >= windowSize {
|
||||
dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go z.compressPartialFile(sr, dict, last, resultChan, wg)
|
||||
}
|
||||
|
||||
close(ze.futureReaders)
|
||||
|
||||
// Close the file handle after all readers are done
|
||||
go func(wg *sync.WaitGroup, closer io.Closer) {
|
||||
wg.Wait()
|
||||
closer.Close()
|
||||
}(wg, r)
|
||||
} else {
|
||||
go func() {
|
||||
z.compressWholeFile(ze, r, compressChan)
|
||||
r.Close()
|
||||
}()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
defer z.cpuRateLimiter.Finish()
|
||||
|
||||
crc := crc32.NewIEEE()
|
||||
_, err := io.Copy(crc, r)
|
||||
if err != nil {
|
||||
z.errors <- err
|
||||
return
|
||||
}
|
||||
|
||||
ze.fh.CRC32 = crc.Sum32()
|
||||
resultChan <- ze
|
||||
close(resultChan)
|
||||
}
|
||||
|
||||
func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
|
||||
result, err := z.compressBlock(r, dict, last)
|
||||
if err != nil {
|
||||
z.errors <- err
|
||||
return
|
||||
}
|
||||
|
||||
z.cpuRateLimiter.Finish()
|
||||
|
||||
resultChan <- result
|
||||
}
|
||||
|
||||
func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
var fw *flate.Writer
|
||||
var err error
|
||||
if len(dict) > 0 {
|
||||
// There's no way to Reset a Writer with a new dictionary, so
|
||||
// don't use the Pool
|
||||
fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
|
||||
} else {
|
||||
var ok bool
|
||||
if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
|
||||
fw.Reset(buf)
|
||||
} else {
|
||||
fw, err = flate.NewWriter(buf, z.compLevel)
|
||||
}
|
||||
defer z.compressorPool.Put(fw)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = io.Copy(fw, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if last {
|
||||
fw.Close()
|
||||
} else {
|
||||
fw.Flush()
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
|
||||
|
||||
crc := crc32.NewIEEE()
|
||||
_, err := io.Copy(crc, r)
|
||||
if err != nil {
|
||||
z.errors <- err
|
||||
return
|
||||
}
|
||||
|
||||
ze.fh.CRC32 = crc.Sum32()
|
||||
|
||||
_, err = r.Seek(0, 0)
|
||||
if err != nil {
|
||||
z.errors <- err
|
||||
return
|
||||
}
|
||||
|
||||
readFile := func(reader io.ReadSeeker) ([]byte, error) {
|
||||
_, err := reader.Seek(0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
ze.futureReaders = make(chan chan io.Reader, 1)
|
||||
futureReader := make(chan io.Reader, 1)
|
||||
ze.futureReaders <- futureReader
|
||||
close(ze.futureReaders)
|
||||
|
||||
if ze.fh.Method == zip.Deflate {
|
||||
compressed, err := z.compressBlock(r, nil, true)
|
||||
if err != nil {
|
||||
z.errors <- err
|
||||
return
|
||||
}
|
||||
if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
|
||||
futureReader <- compressed
|
||||
} else {
|
||||
buf, err := readFile(r)
|
||||
if err != nil {
|
||||
z.errors <- err
|
||||
return
|
||||
}
|
||||
ze.fh.Method = zip.Store
|
||||
futureReader <- bytes.NewReader(buf)
|
||||
}
|
||||
} else {
|
||||
buf, err := readFile(r)
|
||||
if err != nil {
|
||||
z.errors <- err
|
||||
return
|
||||
}
|
||||
ze.fh.Method = zip.Store
|
||||
futureReader <- bytes.NewReader(buf)
|
||||
}
|
||||
|
||||
z.cpuRateLimiter.Finish()
|
||||
|
||||
close(futureReader)
|
||||
|
||||
compressChan <- ze
|
||||
close(compressChan)
|
||||
}
|
||||
|
||||
// writeDirectory annotates that dir is a directory created for the src file or directory, and adds
|
||||
// the directory entry to the zip file if directories are enabled.
|
||||
func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error {
|
||||
// clean the input
|
||||
dir = filepath.Clean(dir)
|
||||
|
||||
// discover any uncreated directories in the path
|
||||
zipDirs := []string{}
|
||||
for dir != "" && dir != "." {
|
||||
if _, exists := z.createdDirs[dir]; exists {
|
||||
break
|
||||
}
|
||||
|
||||
if prev, exists := z.createdFiles[dir]; exists {
|
||||
return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev)
|
||||
}
|
||||
|
||||
z.createdDirs[dir] = src
|
||||
// parent directories precede their children
|
||||
zipDirs = append([]string{dir}, zipDirs...)
|
||||
|
||||
dir = filepath.Dir(dir)
|
||||
}
|
||||
|
||||
if z.directories {
|
||||
// make a directory entry for each uncreated directory
|
||||
for _, cleanDir := range zipDirs {
|
||||
var dirHeader *zip.FileHeader
|
||||
|
||||
if emulateJar && cleanDir+"/" == jar.MetaDir {
|
||||
dirHeader = jar.MetaDirFileHeader()
|
||||
} else {
|
||||
dirHeader = &zip.FileHeader{
|
||||
Name: cleanDir + "/",
|
||||
}
|
||||
dirHeader.SetMode(0700 | os.ModeDir)
|
||||
}
|
||||
|
||||
dirHeader.SetModTime(z.time)
|
||||
|
||||
ze := make(chan *zipEntry, 1)
|
||||
ze <- &zipEntry{
|
||||
fh: dirHeader,
|
||||
}
|
||||
close(ze)
|
||||
z.writeOps <- ze
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *ZipWriter) writeSymlink(rel, file string) error {
|
||||
fileHeader := &zip.FileHeader{
|
||||
Name: rel,
|
||||
}
|
||||
fileHeader.SetModTime(z.time)
|
||||
fileHeader.SetMode(0700 | os.ModeSymlink)
|
||||
|
||||
dest, err := os.Readlink(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ze := make(chan *zipEntry, 1)
|
||||
futureReaders := make(chan chan io.Reader, 1)
|
||||
futureReader := make(chan io.Reader, 1)
|
||||
futureReaders <- futureReader
|
||||
close(futureReaders)
|
||||
futureReader <- bytes.NewBufferString(dest)
|
||||
close(futureReader)
|
||||
|
||||
ze <- &zipEntry{
|
||||
fh: fileHeader,
|
||||
futureReaders: futureReaders,
|
||||
}
|
||||
close(ze)
|
||||
z.writeOps <- ze
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func recursiveGlobFiles(path string) []string {
|
||||
var files []string
|
||||
filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
|
||||
if !info.IsDir() {
|
||||
files = append(files, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return files
|
||||
}
|
||||
Reference in New Issue
Block a user