Merge changes I69f80d12,I40d48644 into main am: ed89d33fb1 am: 22283ef0d4 am: 03a60cd0da

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/2663338

Change-Id: If1bf38ec6f89c4a8085505b696854226c25acf08
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

cmd/merge_zips/merge_zips.go

@@ -122,7 +122,7 @@ func (be ZipEntryFromBuffer) Size() uint64 {
 }
 
 func (be ZipEntryFromBuffer) WriteToZip(dest string, zw *zip.Writer) error {
-	w, err := zw.CreateHeader(be.fh)
+	w, err := zw.CreateHeaderAndroid(be.fh)
 	if err != nil {
 		return err
 	}
@@ -562,6 +562,8 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
 		}
 	}
 
+	var jarServices jar.Services
+
 	// Finally, add entries from all the input zips.
 	for _, inputZip := range inputZips {
 		_, copyFully := zipsToNotStrip[inputZip.Name()]
@@ -570,6 +572,14 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
 		}
 
 		for i, entry := range inputZip.Entries() {
+			if emulateJar && jarServices.IsServiceFile(entry) {
+				// If this is a jar, collect service files to combine  instead of adding them to the zip.
+				err := jarServices.AddServiceFile(entry)
+				if err != nil {
+					return err
+				}
+				continue
+			}
 			if copyFully || !out.isEntryExcluded(entry.Name) {
 				if err := out.copyEntry(inputZip, i); err != nil {
 					return err
@@ -585,6 +595,16 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
 	}
 
 	if emulateJar {
+		// Combine all the service files into a single list of combined service files and add them to the zip.
+		for _, serviceFile := range jarServices.ServiceFiles() {
+			_, err := out.addZipEntry(serviceFile.Name, ZipEntryFromBuffer{
+				fh:      serviceFile.FileHeader,
+				content: serviceFile.Contents,
+			})
+			if err != nil {
+				return err
+			}
+		}
 		return out.writeEntries(out.jarSorted())
 	} else if sortEntries {
 		return out.writeEntries(out.alphanumericSorted())

cmd/merge_zips/merge_zips_test.go

@@ -17,6 +17,7 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"hash/crc32"
 	"os"
 	"strconv"
 	"strings"
@@ -27,28 +28,34 @@ import (
 )
 
 type testZipEntry struct {
-	name string
-	mode os.FileMode
-	data []byte
+	name   string
+	mode   os.FileMode
+	data   []byte
+	method uint16
 }
 
 var (
-	A     = testZipEntry{"A", 0755, []byte("foo")}
-	a     = testZipEntry{"a", 0755, []byte("foo")}
-	a2    = testZipEntry{"a", 0755, []byte("FOO2")}
-	a3    = testZipEntry{"a", 0755, []byte("Foo3")}
-	bDir  = testZipEntry{"b/", os.ModeDir | 0755, nil}
-	bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil}
-	bbb   = testZipEntry{"b/b/b", 0755, nil}
-	ba    = testZipEntry{"b/a", 0755, []byte("foob")}
-	bc    = testZipEntry{"b/c", 0755, []byte("bar")}
-	bd    = testZipEntry{"b/d", 0700, []byte("baz")}
-	be    = testZipEntry{"b/e", 0700, []byte("")}
+	A     = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate}
+	a     = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate}
+	a2    = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate}
+	a3    = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate}
+	bDir  = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate}
+	bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate}
+	bbb   = testZipEntry{"b/b/b", 0755, nil, zip.Deflate}
+	ba    = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate}
+	bc    = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate}
+	bd    = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate}
+	be    = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate}
 
-	metainfDir     = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil}
-	manifestFile   = testZipEntry{jar.ManifestFile, 0755, []byte("manifest")}
-	manifestFile2  = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2")}
-	moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")}
+	service1a        = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store}
+	service1b        = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate}
+	service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store}
+	service2         = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate}
+
+	metainfDir     = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate}
+	manifestFile   = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate}
+	manifestFile2  = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate}
+	moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate}
 )
 
 type testInputZip struct {
@@ -236,6 +243,15 @@ func TestMergeZips(t *testing.T) {
 				"in1": true,
 			},
 		},
+		{
+			name: "services",
+			in: [][]testZipEntry{
+				{service1a, service2},
+				{service1b},
+			},
+			jar: true,
+			out: []testZipEntry{service1combined, service2},
+		},
 	}
 
 	for _, test := range testCases {
@@ -256,7 +272,7 @@ func TestMergeZips(t *testing.T) {
 
 			closeErr := writer.Close()
 			if closeErr != nil {
-				t.Fatal(err)
+				t.Fatal(closeErr)
 			}
 
 			if test.err != "" {
@@ -266,12 +282,16 @@ func TestMergeZips(t *testing.T) {
 					t.Fatal("incorrect err, want:", test.err, "got:", err)
 				}
 				return
+			} else if err != nil {
+				t.Fatal("unexpected err: ", err)
 			}
 
 			if !bytes.Equal(want, out.Bytes()) {
 				t.Error("incorrect zip output")
 				t.Errorf("want:\n%s", dumpZip(want))
 				t.Errorf("got:\n%s", dumpZip(out.Bytes()))
+				os.WriteFile("/tmp/got.zip", out.Bytes(), 0755)
+				os.WriteFile("/tmp/want.zip", want, 0755)
 			}
 		})
 	}
@@ -286,8 +306,14 @@ func testZipEntriesToBuf(entries []testZipEntry) []byte {
 			Name: e.name,
 		}
 		fh.SetMode(e.mode)
+		fh.Method = e.method
+		fh.UncompressedSize64 = uint64(len(e.data))
+		fh.CRC32 = crc32.ChecksumIEEE(e.data)
+		if fh.Method == zip.Store {
+			fh.CompressedSize64 = fh.UncompressedSize64
+		}
 
-		w, err := zw.CreateHeader(&fh)
+		w, err := zw.CreateHeaderAndroid(&fh)
 		if err != nil {
 			panic(err)
 		}

jar/Android.bp

@@ -21,6 +21,7 @@ bootstrap_go_package {
     pkgPath: "android/soong/jar",
     srcs: [
         "jar.go",
+        "services.go",
     ],
     testSrcs: [
         "jar_test.go",

jar/services.go

@@ -0,0 +1,128 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jar
+
+import (
+	"android/soong/third_party/zip"
+	"bufio"
+	"hash/crc32"
+	"sort"
+	"strings"
+)
+
+const servicesPrefix = "META-INF/services/"
+
+// Services is used to collect service files from multiple zip files and produce a list of ServiceFiles containing
+// the unique lines from all the input zip entries with the same name.
+type Services struct {
+	services map[string]*ServiceFile
+}
+
+// ServiceFile contains the combined contents of all input zip entries with a single name.
+type ServiceFile struct {
+	Name       string
+	FileHeader *zip.FileHeader
+	Contents   []byte
+	Lines      []string
+}
+
+// IsServiceFile returns true if the zip entry is in the META-INF/services/ directory.
+func (Services) IsServiceFile(entry *zip.File) bool {
+	return strings.HasPrefix(entry.Name, servicesPrefix)
+}
+
+// AddServiceFile adds a zip entry in the META-INF/services/ directory to the list of service files that need
+// to be combined.
+func (j *Services) AddServiceFile(entry *zip.File) error {
+	if j.services == nil {
+		j.services = map[string]*ServiceFile{}
+	}
+
+	service := entry.Name
+	serviceFile := j.services[service]
+	fh := entry.FileHeader
+	if serviceFile == nil {
+		serviceFile = &ServiceFile{
+			Name:       service,
+			FileHeader: &fh,
+		}
+		j.services[service] = serviceFile
+	}
+
+	f, err := entry.Open()
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line != "" {
+			serviceFile.Lines = append(serviceFile.Lines, line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ServiceFiles returns the list of combined service files, each containing all the unique lines from the
+// corresponding service files in the input zip entries.
+func (j *Services) ServiceFiles() []ServiceFile {
+	services := make([]ServiceFile, 0, len(j.services))
+
+	for _, serviceFile := range j.services {
+		serviceFile.Lines = dedupServicesLines(serviceFile.Lines)
+		serviceFile.Lines = append(serviceFile.Lines, "")
+		serviceFile.Contents = []byte(strings.Join(serviceFile.Lines, "\n"))
+
+		serviceFile.FileHeader.UncompressedSize64 = uint64(len(serviceFile.Contents))
+		serviceFile.FileHeader.CRC32 = crc32.ChecksumIEEE(serviceFile.Contents)
+		if serviceFile.FileHeader.Method == zip.Store {
+			serviceFile.FileHeader.CompressedSize64 = serviceFile.FileHeader.UncompressedSize64
+		}
+
+		services = append(services, *serviceFile)
+	}
+
+	sort.Slice(services, func(i, j int) bool {
+		return services[i].Name < services[j].Name
+	})
+
+	return services
+}
+
+func dedupServicesLines(in []string) []string {
+	writeIndex := 0
+outer:
+	for readIndex := 0; readIndex < len(in); readIndex++ {
+		for compareIndex := 0; compareIndex < writeIndex; compareIndex++ {
+			if interface{}(in[readIndex]) == interface{}(in[compareIndex]) {
+				// The value at readIndex already exists somewhere in the output region
+				// of the slice before writeIndex, skip it.
+				continue outer
+			}
+		}
+		if readIndex != writeIndex {
+			in[writeIndex] = in[readIndex]
+		}
+		writeIndex++
+	}
+	return in[0:writeIndex]
+}

java/androidmk.go

@@ -346,6 +346,9 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries {
 				// App module names can be overridden.
 				entries.SetString("LOCAL_MODULE", app.installApkName)
 				entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", app.appProperties.PreventInstall)
+				if app.headerJarFile != nil {
+					entries.SetPath("LOCAL_SOONG_HEADER_JAR", app.headerJarFile)
+				}
 				entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", app.exportPackage)
 				if app.dexJarFile.IsSet() {
 					entries.SetPath("LOCAL_SOONG_DEX_JAR", app.dexJarFile.Path())

java/base.go

@@ -1485,7 +1485,13 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) {
 
 	j.implementationJarFile = outputFile
 	if j.headerJarFile == nil {
-		j.headerJarFile = j.implementationJarFile
+		// If this module couldn't generate a header jar (for example due to api generating annotation processors)
+		// then use the implementation jar.  Run it through zip2zip first to remove any files in META-INF/services
+		// so that javac on modules that depend on this module don't pick up annotation processors (which may be
+		// missing their implementations) from META-INF/services/javax.annotation.processing.Processor.
+		headerJarFile := android.PathForModuleOut(ctx, "javac-header", jarName)
+		convertImplementationJarToHeaderJar(ctx, j.implementationJarFile, headerJarFile)
+		j.headerJarFile = headerJarFile
 	}
 
 	// enforce syntax check to jacoco filters for any build (http://b/183622051)

java/builder.go

@@ -271,6 +271,12 @@ var (
 			Description: "Check zip alignment",
 		},
 	)
+
+	convertImplementationJarToHeaderJarRule = pctx.AndroidStaticRule("convertImplementationJarToHeaderJar",
+		blueprint.RuleParams{
+			Command:     `${config.Zip2ZipCmd} -i ${in} -o ${out} -x 'META-INF/services/**/*'`,
+			CommandDeps: []string{"${config.Zip2ZipCmd}"},
+		})
 )
 
 func init() {
@@ -633,6 +639,15 @@ func TransformJarsToJar(ctx android.ModuleContext, outputFile android.WritablePa
 	})
 }
 
+func convertImplementationJarToHeaderJar(ctx android.ModuleContext, implementationJarFile android.Path,
+	headerJarFile android.WritablePath) {
+	ctx.Build(pctx, android.BuildParams{
+		Rule:   convertImplementationJarToHeaderJarRule,
+		Input:  implementationJarFile,
+		Output: headerJarFile,
+	})
+}
+
 func TransformJarJar(ctx android.ModuleContext, outputFile android.WritablePath,
 	classesJar android.Path, rulesFile android.Path) {
 	ctx.Build(pctx, android.BuildParams{

java/device_host_converter_test.go

@@ -135,6 +135,7 @@ func TestHostForDevice(t *testing.T) {
 
 	hostModule := ctx.ModuleForTests("host_module", config.BuildOSCommonTarget.String())
 	hostJavac := hostModule.Output("javac/host_module.jar")
+	hostJavacHeader := hostModule.Output("javac-header/host_module.jar")
 	hostRes := hostModule.Output("res/host_module.jar")
 
 	hostImportModule := ctx.ModuleForTests("host_import_module", config.BuildOSCommonTarget.String())
@@ -148,7 +149,7 @@ func TestHostForDevice(t *testing.T) {
 
 	// check classpath of device module with dependency on host_for_device_module
 	expectedClasspath := "-classpath " + strings.Join(android.Paths{
-		hostJavac.Output,
+		hostJavacHeader.Output,
 		hostImportCombined.Output,
 	}.Strings(), ":")
 

java/fuzz_test.go

@@ -71,8 +71,8 @@ func TestJavaFuzz(t *testing.T) {
 	}
 
 	baz := result.ModuleForTests("baz", osCommonTarget).Rule("javac").Output.String()
-	barOut := filepath.Join("out", "soong", ".intermediates", "bar", osCommonTarget, "javac", "bar.jar")
-	bazOut := filepath.Join("out", "soong", ".intermediates", "baz", osCommonTarget, "javac", "baz.jar")
+	barOut := filepath.Join("out", "soong", ".intermediates", "bar", osCommonTarget, "javac-header", "bar.jar")
+	bazOut := filepath.Join("out", "soong", ".intermediates", "baz", osCommonTarget, "javac-header", "baz.jar")
 
 	android.AssertStringDoesContain(t, "foo classpath", javac.Args["classpath"], barOut)
 	android.AssertStringDoesContain(t, "foo classpath", javac.Args["classpath"], bazOut)

sdk/java_sdk_test.go

@@ -160,7 +160,7 @@ java_import {
 }
 `),
 		checkAllCopyRules(`
-.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/myjavalib.jar
+.intermediates/myjavalib/linux_glibc_common/javac-header/myjavalib.jar -> java/myjavalib.jar
 aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl
 `),
 	)
@@ -206,7 +206,7 @@ java_import {
 `),
 		checkAllCopyRules(`
 .intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/android/myjavalib.jar
-.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar
+.intermediates/myjavalib/linux_glibc_common/javac-header/myjavalib.jar -> java/linux_glibc/myjavalib.jar
 `),
 	)
 }
@@ -799,7 +799,7 @@ java_system_modules_import {
     libs: ["mysdk_system-module"],
 }
 `),
-		checkAllCopyRules(".intermediates/system-module/linux_glibc_common/javac/system-module.jar -> java/system-module.jar"),
+		checkAllCopyRules(".intermediates/system-module/linux_glibc_common/javac-header/system-module.jar -> java/system-module.jar"),
 	)
 }
 
@@ -879,7 +879,7 @@ java_import {
 }
 `),
 		checkAllCopyRules(`
-.intermediates/hostjavalib/linux_glibc_common/javac/hostjavalib.jar -> java/hostjavalib.jar
+.intermediates/hostjavalib/linux_glibc_common/javac-header/hostjavalib.jar -> java/hostjavalib.jar
 .intermediates/androidjavalib/android_common/turbine-combined/androidjavalib.jar -> java/androidjavalib.jar
 .intermediates/myjavalib/android_common/javac/myjavalib.jar -> java/android/myjavalib.jar
 .intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar

third_party/zip/android.go

@@ -170,7 +170,7 @@ func (w *Writer) CreateCompressedHeader(fh *FileHeader) (io.WriteCloser, error)
 func (w *Writer) CreateHeaderAndroid(fh *FileHeader) (io.Writer, error) {
 	writeDataDescriptor := fh.Method != Store
 	if writeDataDescriptor {
-		fh.Flags &= DataDescriptorFlag
+		fh.Flags |= DataDescriptorFlag
 	} else {
 		fh.Flags &= ^uint16(DataDescriptorFlag)
 	}