diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go index 95ff70b06..f383de90a 100644 --- a/cmd/merge_zips/merge_zips.go +++ b/cmd/merge_zips/merge_zips.go @@ -250,7 +250,12 @@ func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoin addMapping(jar.MetaDir, dirSource) } - fh, buf, err := jar.ManifestFileContents(manifest) + contents, err := ioutil.ReadFile(manifest) + if err != nil { + return err + } + + fh, buf, err := jar.ManifestFileContents(contents) if err != nil { return err } diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 374868c1d..4933f3745 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -328,7 +328,7 @@ func main() { NumParallelJobs: runtime.NumCPU(), CompressionLevel: 5, } - if err := zip.Run(args); err != nil { + if err := zip.Zip(args); err != nil { log.Fatalf("Error zipping logs: %v", err) } } @@ -409,7 +409,7 @@ func buildProduct(mpctx *mpContext, product string) { NumParallelJobs: runtime.NumCPU(), CompressionLevel: 5, } - if err := zip.Run(args); err != nil { + if err := zip.Zip(args); err != nil { log.Fatalf("Error zipping artifacts: %v", err) } } diff --git a/jar/jar.go b/jar/jar.go index 653e5eedb..fa0e69350 100644 --- a/jar/jar.go +++ b/jar/jar.go @@ -17,7 +17,6 @@ package jar import ( "bytes" "fmt" - "io/ioutil" "os" "strings" "time" @@ -81,10 +80,9 @@ func MetaDirFileHeader() *zip.FileHeader { return dirHeader } -// Convert manifest source path to zip header and contents. If path is empty uses a default -// manifest. -func ManifestFileContents(src string) (*zip.FileHeader, []byte, error) { - b, err := manifestContents(src) +// Create a manifest zip header and contents using the provided contents if any. +func ManifestFileContents(contents []byte) (*zip.FileHeader, []byte, error) { + b, err := manifestContents(contents) if err != nil { return nil, nil, err } @@ -100,26 +98,16 @@ func ManifestFileContents(src string) (*zip.FileHeader, []byte, error) { return fh, b, nil } -// Convert manifest source path to contents. If path is empty uses a default manifest. -func manifestContents(src string) ([]byte, error) { - var givenBytes []byte - var err error - - if src != "" { - givenBytes, err = ioutil.ReadFile(src) - if err != nil { - return nil, err - } - } - +// Create manifest contents, using the provided contents if any. +func manifestContents(contents []byte) ([]byte, error) { manifestMarker := []byte("Manifest-Version:") header := append(manifestMarker, []byte(" 1.0\nCreated-By: soong_zip\n")...) var finalBytes []byte - if !bytes.Contains(givenBytes, manifestMarker) { - finalBytes = append(append(header, givenBytes...), byte('\n')) + if !bytes.Contains(contents, manifestMarker) { + finalBytes = append(append(header, contents...), byte('\n')) } else { - finalBytes = givenBytes + finalBytes = contents } return finalBytes, nil diff --git a/java/java.go b/java/java.go index 0bd7857a0..7fd534446 100644 --- a/java/java.go +++ b/java/java.go @@ -95,6 +95,9 @@ type CompilerProperties struct { // list of java libraries that will be compiled into the resulting jar Static_libs []string `android:"arch_variant"` + // list of native libraries that will be provided in or alongside the resulting jar + Jni_libs []string `android:"arch_variant"` + // manifest file to be included in resulting jar Manifest *string diff --git a/zip/cmd/main.go b/zip/cmd/main.go index a2fbf4113..1125602d0 100644 --- a/zip/cmd/main.go +++ b/zip/cmd/main.go @@ -187,7 +187,7 @@ func main() { os.Exit(1) } - err := zip.Run(zip.ZipArgs{ + err := zip.Zip(zip.ZipArgs{ FileArgs: fileArgsBuilder.FileArgs(), OutputFilePath: *out, EmulateJar: *emulateJar, diff --git a/zip/zip.go b/zip/zip.go index 96f4535ae..e7de6f8cc 100644 --- a/zip/zip.go +++ b/zip/zip.go @@ -22,7 +22,6 @@ import ( "hash/crc32" "io" "io/ioutil" - "log" "os" "path/filepath" "sort" @@ -178,6 +177,8 @@ type ZipWriter struct { compressorPool sync.Pool compLevel int + + fs pathtools.FileSystem } type zipEntry struct { @@ -201,6 +202,7 @@ type ZipArgs struct { NumParallelJobs int NonDeflatedFiles map[string]bool WriteIfChanged bool + Filesystem pathtools.FileSystem } const NOQUOTE = '\x00' @@ -246,22 +248,24 @@ func ReadRespFile(bytes []byte) []string { return args } -func Run(args ZipArgs) (err error) { - if args.OutputFilePath == "" { - return fmt.Errorf("output file path must be nonempty") - } - +func ZipTo(args ZipArgs, w io.Writer) error { if args.EmulateJar { args.AddDirectoryEntriesToZip = true } - w := &ZipWriter{ + z := &ZipWriter{ time: jar.DefaultTime, createdDirs: make(map[string]string), createdFiles: make(map[string]string), directories: args.AddDirectoryEntriesToZip, compLevel: args.CompressionLevel, + fs: args.Filesystem, } + + if z.fs == nil { + z.fs = pathtools.OsFs + } + pathMappings := []pathMapping{} noCompression := args.CompressionLevel == 0 @@ -274,11 +278,19 @@ func Run(args ZipArgs) (err error) { for _, src := range srcs { err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression) if err != nil { - log.Fatal(err) + return err } } } + return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs) +} + +func Zip(args ZipArgs) error { + if args.OutputFilePath == "" { + return fmt.Errorf("output file path must be nonempty") + } + buf := &bytes.Buffer{} var out io.Writer = buf @@ -298,7 +310,7 @@ func Run(args ZipArgs) (err error) { out = f } - err = w.write(out, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs) + err := ZipTo(args, out) if err != nil { return err } @@ -351,13 +363,6 @@ func jarSort(mappings []pathMapping) { sort.SliceStable(mappings, less) } -type readerSeekerCloser interface { - io.Reader - io.ReaderAt - io.Closer - io.Seeker -} - func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error { z.errors = make(chan error) defer close(z.errors) @@ -504,7 +509,7 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er var fileSize int64 var executable bool - if s, err := os.Lstat(src); err != nil { + if s, err := z.fs.Lstat(src); err != nil { return err } else if s.IsDir() { if z.directories { @@ -535,7 +540,7 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er executable = s.Mode()&0100 != 0 } - r, err := os.Open(src) + r, err := z.fs.Open(src) if err != nil { return err } @@ -565,7 +570,21 @@ func (z *ZipWriter) addManifest(dest string, src string, method uint16) error { return err } - fh, buf, err := jar.ManifestFileContents(src) + var contents []byte + if src != "" { + f, err := z.fs.Open(src) + if err != nil { + return err + } + + contents, err = ioutil.ReadAll(f) + f.Close() + if err != nil { + return err + } + } + + fh, buf, err := jar.ManifestFileContents(contents) if err != nil { return err } @@ -575,7 +594,7 @@ func (z *ZipWriter) addManifest(dest string, src string, method uint16) error { return z.writeFileContents(fh, reader) } -func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r readerSeekerCloser) (err error) { +func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) { header.SetModTime(z.time) @@ -845,7 +864,7 @@ func (z *ZipWriter) writeSymlink(rel, file string) error { fileHeader.SetModTime(z.time) fileHeader.SetMode(0777 | os.ModeSymlink) - dest, err := os.Readlink(file) + dest, err := z.fs.Readlink(file) if err != nil { return err } diff --git a/zip/zip_test.go b/zip/zip_test.go index 03e7958fb..0c2105c67 100644 --- a/zip/zip_test.go +++ b/zip/zip_test.go @@ -15,10 +15,395 @@ package zip import ( + "bytes" + "hash/crc32" + "io" + "os" "reflect" + "syscall" "testing" + + "android/soong/third_party/zip" + + "github.com/google/blueprint/pathtools" ) +var ( + fileA = []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + fileB = []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB") + fileC = []byte("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC") + fileEmpty = []byte("") + fileManifest = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\n\n") + + fileCustomManifest = []byte("Custom manifest: true\n") + customManifestAfter = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\nCustom manifest: true\n\n") +) + +var mockFs = pathtools.MockFs(map[string][]byte{ + "a/a/a": fileA, + "a/a/b": fileB, + "a/a/c -> ../../c": nil, + "a/a/d -> b": nil, + "c": fileC, + "l": []byte("a/a/a\na/a/b\nc\n"), + "l2": []byte("missing\n"), + "manifest.txt": fileCustomManifest, +}) + +func fh(name string, contents []byte, method uint16) zip.FileHeader { + return zip.FileHeader{ + Name: name, + Method: method, + CRC32: crc32.ChecksumIEEE(contents), + UncompressedSize64: uint64(len(contents)), + ExternalAttrs: 0, + } +} + +func fhManifest(contents []byte) zip.FileHeader { + return zip.FileHeader{ + Name: "META-INF/MANIFEST.MF", + Method: zip.Store, + CRC32: crc32.ChecksumIEEE(contents), + UncompressedSize64: uint64(len(contents)), + ExternalAttrs: (syscall.S_IFREG | 0700) << 16, + } +} + +func fhLink(name string, to string) zip.FileHeader { + return zip.FileHeader{ + Name: name, + Method: zip.Store, + CRC32: crc32.ChecksumIEEE([]byte(to)), + UncompressedSize64: uint64(len(to)), + ExternalAttrs: (syscall.S_IFLNK | 0777) << 16, + } +} + +func fhDir(name string) zip.FileHeader { + return zip.FileHeader{ + Name: name, + Method: zip.Store, + CRC32: crc32.ChecksumIEEE(nil), + UncompressedSize64: 0, + ExternalAttrs: (syscall.S_IFDIR|0700)<<16 | 0x10, + } +} + +func fileArgsBuilder() *FileArgsBuilder { + return &FileArgsBuilder{ + fs: mockFs, + } +} + +func TestZip(t *testing.T) { + testCases := []struct { + name string + args *FileArgsBuilder + compressionLevel int + emulateJar bool + nonDeflatedFiles map[string]bool + dirEntries bool + manifest string + + files []zip.FileHeader + err error + }{ + { + name: "empty args", + args: fileArgsBuilder(), + + files: []zip.FileHeader{}, + }, + { + name: "files", + args: fileArgsBuilder(). + File("a/a/a"). + File("a/a/b"). + File("c"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("a/a/a", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + fh("c", fileC, zip.Deflate), + }, + }, + { + name: "stored files", + args: fileArgsBuilder(). + File("a/a/a"). + File("a/a/b"). + File("c"), + compressionLevel: 0, + + files: []zip.FileHeader{ + fh("a/a/a", fileA, zip.Store), + fh("a/a/b", fileB, zip.Store), + fh("c", fileC, zip.Store), + }, + }, + { + name: "symlinks in zip", + args: fileArgsBuilder(). + File("a/a/a"). + File("a/a/b"). + File("a/a/c"). + File("a/a/d"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("a/a/a", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + fhLink("a/a/c", "../../c"), + fhLink("a/a/d", "b"), + }, + }, + { + name: "list", + args: fileArgsBuilder(). + List("l"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("a/a/a", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + fh("c", fileC, zip.Deflate), + }, + }, + { + name: "prefix in zip", + args: fileArgsBuilder(). + PathPrefixInZip("foo"). + File("a/a/a"). + File("a/a/b"). + File("c"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("foo/a/a/a", fileA, zip.Deflate), + fh("foo/a/a/b", fileB, zip.Deflate), + fh("foo/c", fileC, zip.Deflate), + }, + }, + { + name: "relative root", + args: fileArgsBuilder(). + SourcePrefixToStrip("a"). + File("a/a/a"). + File("a/a/b"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("a/a", fileA, zip.Deflate), + fh("a/b", fileB, zip.Deflate), + }, + }, + { + name: "multiple relative root", + args: fileArgsBuilder(). + SourcePrefixToStrip("a"). + File("a/a/a"). + SourcePrefixToStrip("a/a"). + File("a/a/b"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("a/a", fileA, zip.Deflate), + fh("b", fileB, zip.Deflate), + }, + }, + { + name: "emulate jar", + args: fileArgsBuilder(). + File("a/a/a"). + File("a/a/b"), + compressionLevel: 9, + emulateJar: true, + + files: []zip.FileHeader{ + fhDir("META-INF/"), + fhManifest(fileManifest), + fhDir("a/"), + fhDir("a/a/"), + fh("a/a/a", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + }, + }, + { + name: "emulate jar with manifest", + args: fileArgsBuilder(). + File("a/a/a"). + File("a/a/b"), + compressionLevel: 9, + emulateJar: true, + manifest: "manifest.txt", + + files: []zip.FileHeader{ + fhDir("META-INF/"), + fhManifest(customManifestAfter), + fhDir("a/"), + fhDir("a/a/"), + fh("a/a/a", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + }, + }, + { + name: "dir entries", + args: fileArgsBuilder(). + File("a/a/a"). + File("a/a/b"), + compressionLevel: 9, + dirEntries: true, + + files: []zip.FileHeader{ + fhDir("a/"), + fhDir("a/a/"), + fh("a/a/a", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + }, + }, + { + name: "junk paths", + args: fileArgsBuilder(). + JunkPaths(true). + File("a/a/a"). + File("a/a/b"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("a", fileA, zip.Deflate), + fh("b", fileB, zip.Deflate), + }, + }, + { + name: "non deflated files", + args: fileArgsBuilder(). + File("a/a/a"). + File("a/a/b"), + compressionLevel: 9, + nonDeflatedFiles: map[string]bool{"a/a/a": true}, + + files: []zip.FileHeader{ + fh("a/a/a", fileA, zip.Store), + fh("a/a/b", fileB, zip.Deflate), + }, + }, + + // errors + { + name: "error missing file", + args: fileArgsBuilder(). + File("missing"), + err: os.ErrNotExist, + }, + { + name: "error missing file in list", + args: fileArgsBuilder(). + List("l2"), + err: os.ErrNotExist, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + if test.args.Error() != nil { + t.Fatal(test.args.Error()) + } + + args := ZipArgs{} + args.FileArgs = test.args.FileArgs() + args.CompressionLevel = test.compressionLevel + args.EmulateJar = test.emulateJar + args.AddDirectoryEntriesToZip = test.dirEntries + args.NonDeflatedFiles = test.nonDeflatedFiles + args.ManifestSourcePath = test.manifest + args.Filesystem = mockFs + + buf := &bytes.Buffer{} + err := ZipTo(args, buf) + + if (err != nil) != (test.err != nil) { + t.Fatalf("want error %v, got %v", test.err, err) + } else if test.err != nil { + if os.IsNotExist(test.err) { + if !os.IsNotExist(test.err) { + t.Fatalf("want error %v, got %v", test.err, err) + } + } else { + t.Fatalf("want error %v, got %v", test.err, err) + } + return + } + + br := bytes.NewReader(buf.Bytes()) + zr, err := zip.NewReader(br, int64(br.Len())) + if err != nil { + t.Fatal(err) + } + + var files []zip.FileHeader + for _, f := range zr.File { + r, err := f.Open() + if err != nil { + t.Fatalf("error when opening %s: %s", f.Name, err) + } + + crc := crc32.NewIEEE() + len, err := io.Copy(crc, r) + r.Close() + if err != nil { + t.Fatalf("error when reading %s: %s", f.Name, err) + } + + if uint64(len) != f.UncompressedSize64 { + t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len) + } + + if crc.Sum32() != f.CRC32 { + t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc) + } + + files = append(files, f.FileHeader) + } + + if len(files) != len(test.files) { + t.Fatalf("want %d files, got %d", len(test.files), len(files)) + } + + for i := range files { + want := test.files[i] + got := files[i] + + if want.Name != got.Name { + t.Errorf("incorrect file %d want %q got %q", i, want.Name, got.Name) + continue + } + + if want.UncompressedSize64 != got.UncompressedSize64 { + t.Errorf("incorrect file %s length want %v got %v", want.Name, + want.UncompressedSize64, got.UncompressedSize64) + } + + if want.ExternalAttrs != got.ExternalAttrs { + t.Errorf("incorrect file %s attrs want %x got %x", want.Name, + want.ExternalAttrs, got.ExternalAttrs) + } + + if want.CRC32 != got.CRC32 { + t.Errorf("incorrect file %s crc want %v got %v", want.Name, + want.CRC32, got.CRC32) + } + + if want.Method != got.Method { + t.Errorf("incorrect file %s method want %v got %v", want.Name, + want.Method, got.Method) + } + } + }) + } +} + func TestReadRespFile(t *testing.T) { testCases := []struct { name, in string