Support moving sources in srcjars in soong_zip

Add a -srcjar argument to soong_zip that causes it to read the
package statement of each .java file and use that to place the
source file at a path that matches the package.

Test: jar_test.go, zip_test.go
Change-Id: I36017e42445ba3b0a82a10a8d81e8ac0cca096f2

jar/Android.bp

@@ -18,8 +18,10 @@ bootstrap_go_package {
     srcs: [
         "jar.go",
     ],
+    testSrcs: [
+        "jar_test.go",
+    ],
     deps: [
         "android-archive-zip",
     ],
 }
-

jar/jar.go

@@ -17,9 +17,12 @@ package jar
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"strings"
+	"text/scanner"
 	"time"
+	"unicode"
 
 	"android/soong/third_party/zip"
 )
@@ -112,3 +115,111 @@ func manifestContents(contents []byte) ([]byte, error) {
 
 	return finalBytes, nil
 }
+
+var javaIgnorableIdentifier = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x00, 0x08, 1},
+		{0x0e, 0x1b, 1},
+		{0x7f, 0x9f, 1},
+	},
+	LatinOffset: 3,
+}
+
+func javaIdentRune(ch rune, i int) bool {
+	if unicode.IsLetter(ch) {
+		return true
+	}
+	if unicode.IsDigit(ch) && i > 0 {
+		return true
+	}
+
+	if unicode.In(ch,
+		unicode.Nl, // letter number
+		unicode.Sc, // currency symbol
+		unicode.Pc, // connecting punctuation
+	) {
+		return true
+	}
+
+	if unicode.In(ch,
+		unicode.Cf, // format
+		unicode.Mc, // combining mark
+		unicode.Mn, // non-spacing mark
+		javaIgnorableIdentifier,
+	) && i > 0 {
+		return true
+	}
+
+	return false
+}
+
+// JavaPackage parses the package out of a java source file by looking for the package statement, or the first valid
+// non-package statement, in which case it returns an empty string for the package.
+func JavaPackage(r io.Reader, src string) (string, error) {
+	var s scanner.Scanner
+	var sErr error
+
+	s.Init(r)
+	s.Filename = src
+	s.Error = func(s *scanner.Scanner, msg string) {
+		sErr = fmt.Errorf("error parsing %q: %s", src, msg)
+	}
+	s.IsIdentRune = javaIdentRune
+
+	tok := s.Scan()
+	if sErr != nil {
+		return "", sErr
+	}
+	if tok == scanner.Ident {
+		switch s.TokenText() {
+		case "package":
+		// Nothing
+		case "import":
+			// File has no package statement, first keyword is an import
+			return "", nil
+		case "class", "enum", "interface":
+			// File has no package statement, first keyword is a type declaration
+			return "", nil
+		case "public", "protected", "private", "abstract", "static", "final", "strictfp":
+			// File has no package statement, first keyword is a modifier
+			return "", nil
+		case "module", "open":
+			// File has no package statement, first keyword is a module declaration
+			return "", nil
+		default:
+			return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText())
+		}
+	} else if tok == '@' {
+		// File has no package statement, first token is an annotation
+		return "", nil
+	} else if tok == scanner.EOF {
+		// File no package statement, it has no non-whitespace non-comment tokens
+		return "", nil
+	} else {
+		return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText())
+	}
+
+	var pkg string
+	for {
+		tok = s.Scan()
+		if sErr != nil {
+			return "", sErr
+		}
+		if tok != scanner.Ident {
+			return "", fmt.Errorf(`expected "package <package>;", got "package %s%s"`, pkg, s.TokenText())
+		}
+		pkg += s.TokenText()
+
+		tok = s.Scan()
+		if sErr != nil {
+			return "", sErr
+		}
+		if tok == ';' {
+			return pkg, nil
+		} else if tok == '.' {
+			pkg += "."
+		} else {
+			return "", fmt.Errorf(`expected "package <package>;", got "package %s%s"`, pkg, s.TokenText())
+		}
+	}
+}

jar/jar_test.go

@@ -0,0 +1,182 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jar
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+func TestGetJavaPackage(t *testing.T) {
+	type args struct {
+		r   io.Reader
+		src string
+	}
+	tests := []struct {
+		name    string
+		in      string
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "simple",
+			in:   "package foo.bar;",
+			want: "foo.bar",
+		},
+		{
+			name: "comment",
+			in:   "/* test */\npackage foo.bar;",
+			want: "foo.bar",
+		},
+		{
+			name: "no package",
+			in:   "import foo.bar;",
+			want: "",
+		},
+		{
+			name:    "missing semicolon error",
+			in:      "package foo.bar",
+			wantErr: true,
+		},
+		{
+			name:    "parser error",
+			in:      "/*",
+			wantErr: true,
+		},
+		{
+			name:    "parser ident error",
+			in:      "package 0foo.bar;",
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			buf := bytes.NewBufferString(tt.in)
+			got, err := JavaPackage(buf, "<test>")
+			if (err != nil) != tt.wantErr {
+				t.Errorf("JavaPackage() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("JavaPackage() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_javaIdentRune(t *testing.T) {
+	// runes that should be valid anywhere in an identifier
+	validAnywhere := []rune{
+		// letters, $, _
+		'a',
+		'A',
+		'$',
+		'_',
+
+		// assorted unicode
+		'𐐀',
+		'𐐨',
+		'Dž',
+		'ῼ',
+		'ʰ',
+		'ﾟ',
+		'ƻ',
+		'㡢',
+		'￦',
+		'＿',
+		'Ⅰ',
+		'𐍊',
+	}
+
+	// runes that should be invalid as the first rune in an identifier, but valid anywhere else
+	validAfterFirst := []rune{
+		// digits
+		'0',
+
+		// assorted unicode
+		'᥍',
+		'𝟎',
+		'ྂ',
+		'𝆀',
+
+		// control characters
+		'\x00',
+		'\b',
+		'\u000e',
+		'\u001b',
+		'\u007f',
+		'\u009f',
+		'\u00ad',
+		0xE007F,
+
+		// zero width space
+		'\u200b',
+	}
+
+	// runes that should never be valid in an identifier
+	invalid := []rune{
+		';',
+		0x110000,
+	}
+
+	validFirst := validAnywhere
+	invalidFirst := append(validAfterFirst, invalid...)
+	validPart := append(validAnywhere, validAfterFirst...)
+	invalidPart := invalid
+
+	check := func(t *testing.T, ch rune, i int, want bool) {
+		t.Helper()
+		if got := javaIdentRune(ch, i); got != want {
+			t.Errorf("javaIdentRune() = %v, want %v", got, want)
+		}
+	}
+
+	t.Run("first", func(t *testing.T) {
+		t.Run("valid", func(t *testing.T) {
+			for _, ch := range validFirst {
+				t.Run(string(ch), func(t *testing.T) {
+					check(t, ch, 0, true)
+				})
+			}
+		})
+
+		t.Run("invalid", func(t *testing.T) {
+			for _, ch := range invalidFirst {
+				t.Run(string(ch), func(t *testing.T) {
+					check(t, ch, 0, false)
+				})
+			}
+		})
+	})
+
+	t.Run("part", func(t *testing.T) {
+		t.Run("valid", func(t *testing.T) {
+			for _, ch := range validPart {
+				t.Run(string(ch), func(t *testing.T) {
+					check(t, ch, 1, true)
+				})
+			}
+		})
+
+		t.Run("invalid", func(t *testing.T) {
+			for _, ch := range invalidPart {
+				t.Run(string(ch), func(t *testing.T) {
+					check(t, ch, 1, false)
+				})
+			}
+		})
+	})
+}

zip/cmd/main.go

@@ -136,6 +136,7 @@ func main() {
 	writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed")
 	ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist")
 	symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them")
+	srcJar := flags.Bool("srcjar", false, "move .java files to locations that match their package statement")
 
 	parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use")
 	cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file")
@@ -191,6 +192,7 @@ func main() {
 		FileArgs:                 fileArgsBuilder.FileArgs(),
 		OutputFilePath:           *out,
 		EmulateJar:               *emulateJar,
+		SrcJar:                   *srcJar,
 		AddDirectoryEntriesToZip: *directories,
 		CompressionLevel:         *compLevel,
 		ManifestSourcePath:       *manifest,

zip/zip.go

@@ -210,6 +210,7 @@ type ZipArgs struct {
 	FileArgs                 []FileArg
 	OutputFilePath           string
 	EmulateJar               bool
+	SrcJar                   bool
 	AddDirectoryEntriesToZip bool
 	CompressionLevel         int
 	ManifestSourcePath       string
@@ -364,7 +365,7 @@ func ZipTo(args ZipArgs, w io.Writer) error {
 		}
 	}
 
-	return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
+	return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs)
 }
 
 func Zip(args ZipArgs) error {
@@ -446,7 +447,9 @@ func jarSort(mappings []pathMapping) {
 	sort.SliceStable(mappings, less)
 }
 
-func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error {
+func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool,
+	parallelJobs int) error {
+
 	z.errors = make(chan error)
 	defer close(z.errors)
 
@@ -489,7 +492,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri
 			if emulateJar && ele.dest == jar.ManifestFile {
 				err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
 			} else {
-				err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar)
+				err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar)
 			}
 			if err != nil {
 				z.errors <- err
@@ -588,7 +591,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri
 }
 
 // imports (possibly with compression) <src> into the zip at sub-path <dest>
-func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error {
+func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error {
 	var fileSize int64
 	var executable bool
 
@@ -606,12 +609,9 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
 			return nil
 		}
 		return err
-	} else if s.IsDir() {
+	}
-		if z.directories {
+
-			return z.writeDirectory(dest, src, emulateJar)
+	createParentDirs := func(dest, src string) error {
-		}
-		return nil
-	} else {
 		if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
 			return err
 		}
@@ -625,32 +625,64 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
 
 		z.createdFiles[dest] = src
 
-		if s.Mode()&os.ModeSymlink != 0 {
+		return nil
-			return z.writeSymlink(dest, src)
+	}
-		} else if !s.Mode().IsRegular() {
+
-			return fmt.Errorf("%s is not a file, directory, or symlink", src)
+	if s.IsDir() {
+		if z.directories {
+			return z.writeDirectory(dest, src, emulateJar)
+		}
+		return nil
+	} else if s.Mode()&os.ModeSymlink != 0 {
+		err = createParentDirs(dest, src)
+		if err != nil {
+			return err
+		}
+
+		return z.writeSymlink(dest, src)
+	} else if s.Mode().IsRegular() {
+		r, err := z.fs.Open(src)
+		if err != nil {
+			return err
+		}
+
+		if srcJar && filepath.Ext(src) == ".java" {
+			// rewrite the destination using the package path if it can be determined
+			pkg, err := jar.JavaPackage(r, src)
+			if err != nil {
+				// ignore errors for now, leaving the file at in its original location in the zip
+			} else {
+				dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src))
+			}
+
+			_, err = r.Seek(0, io.SeekStart)
+			if err != nil {
+				return err
+			}
 		}
 
 		fileSize = s.Size()
 		executable = s.Mode()&0100 != 0
-	}
 
-	r, err := z.fs.Open(src)
+		header := &zip.FileHeader{
-	if err != nil {
+			Name:               dest,
-		return err
+			Method:             method,
-	}
+			UncompressedSize64: uint64(fileSize),
+		}
 
-	header := &zip.FileHeader{
+		if executable {
-		Name:               dest,
+			header.SetMode(0700)
-		Method:             method,
+		}
-		UncompressedSize64: uint64(fileSize),
-	}
 
-	if executable {
+		err = createParentDirs(dest, src)
-		header.SetMode(0700)
+		if err != nil {
-	}
+			return err
+		}
 
-	return z.writeFileContents(header, r)
+		return z.writeFileContents(header, r)
+	} else {
+		return fmt.Errorf("%s is not a file, directory, or symlink", src)
+	}
 }
 
 func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {

zip/zip_test.go

@@ -40,14 +40,15 @@ var (
 )
 
 var mockFs = pathtools.MockFs(map[string][]byte{
-	"a/a/a":            fileA,
+	"a/a/a":               fileA,
-	"a/a/b":            fileB,
+	"a/a/b":               fileB,
-	"a/a/c -> ../../c": nil,
+	"a/a/c -> ../../c":    nil,
-	"a/a/d -> b":       nil,
+	"dangling -> missing": nil,
-	"c":                fileC,
+	"a/a/d -> b":          nil,
-	"l":                []byte("a/a/a\na/a/b\nc\n"),
+	"c":                   fileC,
-	"l2":               []byte("missing\n"),
+	"l":                   []byte("a/a/a\na/a/b\nc\n"),
-	"manifest.txt":     fileCustomManifest,
+	"l2":                  []byte("missing\n"),
+	"manifest.txt":        fileCustomManifest,
 })
 
 func fh(name string, contents []byte, method uint16) zip.FileHeader {
@@ -209,6 +210,17 @@ func TestZip(t *testing.T) {
 				fh("a/a/d", fileB, zip.Deflate),
 			},
 		},
+		{
+			name: "dangling symlinks",
+			args: fileArgsBuilder().
+				File("dangling"),
+			compressionLevel: 9,
+			storeSymlinks:    true,
+
+			files: []zip.FileHeader{
+				fhLink("dangling", "missing"),
+			},
+		},
 		{
 			name: "list",
 			args: fileArgsBuilder().
@@ -554,3 +566,70 @@ func TestReadRespFile(t *testing.T) {
 		})
 	}
 }
+
+func TestSrcJar(t *testing.T) {
+	mockFs := pathtools.MockFs(map[string][]byte{
+		"wrong_package.java":       []byte("package foo;"),
+		"foo/correct_package.java": []byte("package foo;"),
+		"src/no_package.java":      nil,
+		"src2/parse_error.java":    []byte("error"),
+	})
+
+	want := []string{
+		"foo/",
+		"foo/wrong_package.java",
+		"foo/correct_package.java",
+		"no_package.java",
+		"src2/",
+		"src2/parse_error.java",
+	}
+
+	args := ZipArgs{}
+	args.FileArgs = NewFileArgsBuilder().File("**/*.java").FileArgs()
+
+	args.SrcJar = true
+	args.AddDirectoryEntriesToZip = true
+	args.Filesystem = mockFs
+	args.Stderr = &bytes.Buffer{}
+
+	buf := &bytes.Buffer{}
+	err := ZipTo(args, buf)
+	if err != nil {
+		t.Fatalf("got error %v", err)
+	}
+
+	br := bytes.NewReader(buf.Bytes())
+	zr, err := zip.NewReader(br, int64(br.Len()))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var got []string
+	for _, f := range zr.File {
+		r, err := f.Open()
+		if err != nil {
+			t.Fatalf("error when opening %s: %s", f.Name, err)
+		}
+
+		crc := crc32.NewIEEE()
+		len, err := io.Copy(crc, r)
+		r.Close()
+		if err != nil {
+			t.Fatalf("error when reading %s: %s", f.Name, err)
+		}
+
+		if uint64(len) != f.UncompressedSize64 {
+			t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len)
+		}
+
+		if crc.Sum32() != f.CRC32 {
+			t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc)
+		}
+
+		got = append(got, f.Name)
+	}
+
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("want files %q, got %q", want, got)
+	}
+}