diff --git a/jar/Android.bp b/jar/Android.bp index 6c2e60e47..2563474af 100644 --- a/jar/Android.bp +++ b/jar/Android.bp @@ -18,8 +18,10 @@ bootstrap_go_package { srcs: [ "jar.go", ], + testSrcs: [ + "jar_test.go", + ], deps: [ "android-archive-zip", ], } - diff --git a/jar/jar.go b/jar/jar.go index fa0e69350..a8f06a4f3 100644 --- a/jar/jar.go +++ b/jar/jar.go @@ -17,9 +17,12 @@ package jar import ( "bytes" "fmt" + "io" "os" "strings" + "text/scanner" "time" + "unicode" "android/soong/third_party/zip" ) @@ -112,3 +115,111 @@ func manifestContents(contents []byte) ([]byte, error) { return finalBytes, nil } + +var javaIgnorableIdentifier = &unicode.RangeTable{ + R16: []unicode.Range16{ + {0x00, 0x08, 1}, + {0x0e, 0x1b, 1}, + {0x7f, 0x9f, 1}, + }, + LatinOffset: 3, +} + +func javaIdentRune(ch rune, i int) bool { + if unicode.IsLetter(ch) { + return true + } + if unicode.IsDigit(ch) && i > 0 { + return true + } + + if unicode.In(ch, + unicode.Nl, // letter number + unicode.Sc, // currency symbol + unicode.Pc, // connecting punctuation + ) { + return true + } + + if unicode.In(ch, + unicode.Cf, // format + unicode.Mc, // combining mark + unicode.Mn, // non-spacing mark + javaIgnorableIdentifier, + ) && i > 0 { + return true + } + + return false +} + +// JavaPackage parses the package out of a java source file by looking for the package statement, or the first valid +// non-package statement, in which case it returns an empty string for the package. +func JavaPackage(r io.Reader, src string) (string, error) { + var s scanner.Scanner + var sErr error + + s.Init(r) + s.Filename = src + s.Error = func(s *scanner.Scanner, msg string) { + sErr = fmt.Errorf("error parsing %q: %s", src, msg) + } + s.IsIdentRune = javaIdentRune + + tok := s.Scan() + if sErr != nil { + return "", sErr + } + if tok == scanner.Ident { + switch s.TokenText() { + case "package": + // Nothing + case "import": + // File has no package statement, first keyword is an import + return "", nil + case "class", "enum", "interface": + // File has no package statement, first keyword is a type declaration + return "", nil + case "public", "protected", "private", "abstract", "static", "final", "strictfp": + // File has no package statement, first keyword is a modifier + return "", nil + case "module", "open": + // File has no package statement, first keyword is a module declaration + return "", nil + default: + return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText()) + } + } else if tok == '@' { + // File has no package statement, first token is an annotation + return "", nil + } else if tok == scanner.EOF { + // File no package statement, it has no non-whitespace non-comment tokens + return "", nil + } else { + return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText()) + } + + var pkg string + for { + tok = s.Scan() + if sErr != nil { + return "", sErr + } + if tok != scanner.Ident { + return "", fmt.Errorf(`expected "package ;", got "package %s%s"`, pkg, s.TokenText()) + } + pkg += s.TokenText() + + tok = s.Scan() + if sErr != nil { + return "", sErr + } + if tok == ';' { + return pkg, nil + } else if tok == '.' { + pkg += "." + } else { + return "", fmt.Errorf(`expected "package ;", got "package %s%s"`, pkg, s.TokenText()) + } + } +} diff --git a/jar/jar_test.go b/jar/jar_test.go new file mode 100644 index 000000000..c92011e12 --- /dev/null +++ b/jar/jar_test.go @@ -0,0 +1,182 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jar + +import ( + "bytes" + "io" + "testing" +) + +func TestGetJavaPackage(t *testing.T) { + type args struct { + r io.Reader + src string + } + tests := []struct { + name string + in string + want string + wantErr bool + }{ + { + name: "simple", + in: "package foo.bar;", + want: "foo.bar", + }, + { + name: "comment", + in: "/* test */\npackage foo.bar;", + want: "foo.bar", + }, + { + name: "no package", + in: "import foo.bar;", + want: "", + }, + { + name: "missing semicolon error", + in: "package foo.bar", + wantErr: true, + }, + { + name: "parser error", + in: "/*", + wantErr: true, + }, + { + name: "parser ident error", + in: "package 0foo.bar;", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := bytes.NewBufferString(tt.in) + got, err := JavaPackage(buf, "") + if (err != nil) != tt.wantErr { + t.Errorf("JavaPackage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("JavaPackage() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_javaIdentRune(t *testing.T) { + // runes that should be valid anywhere in an identifier + validAnywhere := []rune{ + // letters, $, _ + 'a', + 'A', + '$', + '_', + + // assorted unicode + '𐐀', + '𐐨', + 'Dž', + 'ῼ', + 'ʰ', + '゚', + 'ƻ', + '㡢', + '₩', + '_', + 'Ⅰ', + '𐍊', + } + + // runes that should be invalid as the first rune in an identifier, but valid anywhere else + validAfterFirst := []rune{ + // digits + '0', + + // assorted unicode + '᥍', + '𝟎', + 'ྂ', + '𝆀', + + // control characters + '\x00', + '\b', + '\u000e', + '\u001b', + '\u007f', + '\u009f', + '\u00ad', + 0xE007F, + + // zero width space + '\u200b', + } + + // runes that should never be valid in an identifier + invalid := []rune{ + ';', + 0x110000, + } + + validFirst := validAnywhere + invalidFirst := append(validAfterFirst, invalid...) + validPart := append(validAnywhere, validAfterFirst...) + invalidPart := invalid + + check := func(t *testing.T, ch rune, i int, want bool) { + t.Helper() + if got := javaIdentRune(ch, i); got != want { + t.Errorf("javaIdentRune() = %v, want %v", got, want) + } + } + + t.Run("first", func(t *testing.T) { + t.Run("valid", func(t *testing.T) { + for _, ch := range validFirst { + t.Run(string(ch), func(t *testing.T) { + check(t, ch, 0, true) + }) + } + }) + + t.Run("invalid", func(t *testing.T) { + for _, ch := range invalidFirst { + t.Run(string(ch), func(t *testing.T) { + check(t, ch, 0, false) + }) + } + }) + }) + + t.Run("part", func(t *testing.T) { + t.Run("valid", func(t *testing.T) { + for _, ch := range validPart { + t.Run(string(ch), func(t *testing.T) { + check(t, ch, 1, true) + }) + } + }) + + t.Run("invalid", func(t *testing.T) { + for _, ch := range invalidPart { + t.Run(string(ch), func(t *testing.T) { + check(t, ch, 1, false) + }) + } + }) + }) +} diff --git a/zip/cmd/main.go b/zip/cmd/main.go index 6f40a3e0a..fba2e4b1e 100644 --- a/zip/cmd/main.go +++ b/zip/cmd/main.go @@ -136,6 +136,7 @@ func main() { writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed") ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist") symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them") + srcJar := flags.Bool("srcjar", false, "move .java files to locations that match their package statement") parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use") cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file") @@ -191,6 +192,7 @@ func main() { FileArgs: fileArgsBuilder.FileArgs(), OutputFilePath: *out, EmulateJar: *emulateJar, + SrcJar: *srcJar, AddDirectoryEntriesToZip: *directories, CompressionLevel: *compLevel, ManifestSourcePath: *manifest, diff --git a/zip/zip.go b/zip/zip.go index 1f5fe4335..707c4ef16 100644 --- a/zip/zip.go +++ b/zip/zip.go @@ -210,6 +210,7 @@ type ZipArgs struct { FileArgs []FileArg OutputFilePath string EmulateJar bool + SrcJar bool AddDirectoryEntriesToZip bool CompressionLevel int ManifestSourcePath string @@ -364,7 +365,7 @@ func ZipTo(args ZipArgs, w io.Writer) error { } } - return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs) + return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs) } func Zip(args ZipArgs) error { @@ -446,7 +447,9 @@ func jarSort(mappings []pathMapping) { sort.SliceStable(mappings, less) } -func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error { +func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool, + parallelJobs int) error { + z.errors = make(chan error) defer close(z.errors) @@ -489,7 +492,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri if emulateJar && ele.dest == jar.ManifestFile { err = z.addManifest(ele.dest, ele.src, ele.zipMethod) } else { - err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar) + err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar) } if err != nil { z.errors <- err @@ -588,7 +591,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri } // imports (possibly with compression) into the zip at sub-path -func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error { +func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error { var fileSize int64 var executable bool @@ -606,12 +609,9 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er return nil } return err - } else if s.IsDir() { - if z.directories { - return z.writeDirectory(dest, src, emulateJar) - } - return nil - } else { + } + + createParentDirs := func(dest, src string) error { if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil { return err } @@ -625,32 +625,64 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er z.createdFiles[dest] = src - if s.Mode()&os.ModeSymlink != 0 { - return z.writeSymlink(dest, src) - } else if !s.Mode().IsRegular() { - return fmt.Errorf("%s is not a file, directory, or symlink", src) + return nil + } + + if s.IsDir() { + if z.directories { + return z.writeDirectory(dest, src, emulateJar) + } + return nil + } else if s.Mode()&os.ModeSymlink != 0 { + err = createParentDirs(dest, src) + if err != nil { + return err + } + + return z.writeSymlink(dest, src) + } else if s.Mode().IsRegular() { + r, err := z.fs.Open(src) + if err != nil { + return err + } + + if srcJar && filepath.Ext(src) == ".java" { + // rewrite the destination using the package path if it can be determined + pkg, err := jar.JavaPackage(r, src) + if err != nil { + // ignore errors for now, leaving the file at in its original location in the zip + } else { + dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src)) + } + + _, err = r.Seek(0, io.SeekStart) + if err != nil { + return err + } } fileSize = s.Size() executable = s.Mode()&0100 != 0 - } - r, err := z.fs.Open(src) - if err != nil { - return err - } + header := &zip.FileHeader{ + Name: dest, + Method: method, + UncompressedSize64: uint64(fileSize), + } - header := &zip.FileHeader{ - Name: dest, - Method: method, - UncompressedSize64: uint64(fileSize), - } + if executable { + header.SetMode(0700) + } - if executable { - header.SetMode(0700) - } + err = createParentDirs(dest, src) + if err != nil { + return err + } - return z.writeFileContents(header, r) + return z.writeFileContents(header, r) + } else { + return fmt.Errorf("%s is not a file, directory, or symlink", src) + } } func (z *ZipWriter) addManifest(dest string, src string, method uint16) error { diff --git a/zip/zip_test.go b/zip/zip_test.go index 93c5f3dee..84317d1b6 100644 --- a/zip/zip_test.go +++ b/zip/zip_test.go @@ -40,14 +40,15 @@ var ( ) var mockFs = pathtools.MockFs(map[string][]byte{ - "a/a/a": fileA, - "a/a/b": fileB, - "a/a/c -> ../../c": nil, - "a/a/d -> b": nil, - "c": fileC, - "l": []byte("a/a/a\na/a/b\nc\n"), - "l2": []byte("missing\n"), - "manifest.txt": fileCustomManifest, + "a/a/a": fileA, + "a/a/b": fileB, + "a/a/c -> ../../c": nil, + "dangling -> missing": nil, + "a/a/d -> b": nil, + "c": fileC, + "l": []byte("a/a/a\na/a/b\nc\n"), + "l2": []byte("missing\n"), + "manifest.txt": fileCustomManifest, }) func fh(name string, contents []byte, method uint16) zip.FileHeader { @@ -209,6 +210,17 @@ func TestZip(t *testing.T) { fh("a/a/d", fileB, zip.Deflate), }, }, + { + name: "dangling symlinks", + args: fileArgsBuilder(). + File("dangling"), + compressionLevel: 9, + storeSymlinks: true, + + files: []zip.FileHeader{ + fhLink("dangling", "missing"), + }, + }, { name: "list", args: fileArgsBuilder(). @@ -554,3 +566,70 @@ func TestReadRespFile(t *testing.T) { }) } } + +func TestSrcJar(t *testing.T) { + mockFs := pathtools.MockFs(map[string][]byte{ + "wrong_package.java": []byte("package foo;"), + "foo/correct_package.java": []byte("package foo;"), + "src/no_package.java": nil, + "src2/parse_error.java": []byte("error"), + }) + + want := []string{ + "foo/", + "foo/wrong_package.java", + "foo/correct_package.java", + "no_package.java", + "src2/", + "src2/parse_error.java", + } + + args := ZipArgs{} + args.FileArgs = NewFileArgsBuilder().File("**/*.java").FileArgs() + + args.SrcJar = true + args.AddDirectoryEntriesToZip = true + args.Filesystem = mockFs + args.Stderr = &bytes.Buffer{} + + buf := &bytes.Buffer{} + err := ZipTo(args, buf) + if err != nil { + t.Fatalf("got error %v", err) + } + + br := bytes.NewReader(buf.Bytes()) + zr, err := zip.NewReader(br, int64(br.Len())) + if err != nil { + t.Fatal(err) + } + + var got []string + for _, f := range zr.File { + r, err := f.Open() + if err != nil { + t.Fatalf("error when opening %s: %s", f.Name, err) + } + + crc := crc32.NewIEEE() + len, err := io.Copy(crc, r) + r.Close() + if err != nil { + t.Fatalf("error when reading %s: %s", f.Name, err) + } + + if uint64(len) != f.UncompressedSize64 { + t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len) + } + + if crc.Sum32() != f.CRC32 { + t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc) + } + + got = append(got, f.Name) + } + + if !reflect.DeepEqual(want, got) { + t.Errorf("want files %q, got %q", want, got) + } +}