Replace some uses of zip with merge_zips, soong_zip, and zip2zip

zip maintains permissions when adding files to zips, which causes
the final jars and apks to change based on the umask of the build.
Move some of the rules that were using zip to soong_zip instead.

Since soong_zip doesn't support adding files to existing zips,
create a new zip file for each part of the package creation
process and then merge them together at the end with merge_zips.

Bug: 69500920
Test: m checkbuild
Change-Id: Id253df776ce19ec4cac3a36fa470a50461ea3cad

core/definitions.mk

@@ -2301,28 +2301,6 @@ $(hide) $(DX_COMMAND) \
 $(hide) rm -f $(dir $@)d8_input.jar
 endef
 
-# Create a mostly-empty .jar file that we'll add to later.
-# The MacOS jar tool doesn't like creating empty jar files,
-# so we need to give it something.
-# $(1) package to create
-define create-empty-package
-@mkdir -p $(dir $(1))
-$(hide) touch $(dir $(1))zipdummy
-$(hide) $(JAR) cf $(1) -C $(dir $(1)) zipdummy
-$(hide) zip -qd $(1) zipdummy
-$(hide) rm $(dir $(1))zipdummy
-endef
-
-# Copy an arhchive file and delete any class files and empty folders inside.
-# $(1): the source archive file.
-# $(2): the destination archive file.
-define initialize-package-file
-@mkdir -p $(dir $(2))
-$(hide) cp -f $(1) $(2)
-$(hide) zip -qd $(2) "*.class" "*/" \
-    || true # Ignore the error when nothing to delete.
-endef
-
 #TODO: we kinda want to build different asset packages for
 #      different configurations, then combine them later (or something).
 #      Per-locale, etc.
@@ -2333,8 +2311,8 @@ endef
 #values; applications can override these by explicitly stating
 #them in their manifest.
 # $(1) the package file
-define add-assets-to-package
+define create-assets-package
-$(hide) $(AAPT_ASAN_OPTIONS) $(AAPT) package -u $(PRIVATE_AAPT_FLAGS) \
+$(hide) $(AAPT_ASAN_OPTIONS) $(AAPT) package $(PRIVATE_AAPT_FLAGS) \
     $(addprefix -c , $(PRIVATE_PRODUCT_AAPT_CONFIG)) \
     $(addprefix --preferred-density , $(PRIVATE_PRODUCT_AAPT_PREF_CONFIG)) \
     $(addprefix -M , $(PRIVATE_ANDROID_MANIFEST)) \
@@ -2369,32 +2347,43 @@ ifdef TARGET_BUILD_APPS
 JNI_COMPRESS_FLAGS :=
 ZIPALIGN_PAGE_ALIGN_FLAGS :=
 else
-JNI_COMPRESS_FLAGS := -0
+JNI_COMPRESS_FLAGS := -L 0
 ZIPALIGN_PAGE_ALIGN_FLAGS := -p
 endif
 
 # $(1): the package file
-define add-jni-shared-libs-to-package
+define create-jni-shared-libs-package
-$(hide) rm -rf $(dir $(1))lib
+rm -rf $(dir $(1))lib
-$(hide) mkdir -p $(addprefix $(dir $(1))lib/,$(PRIVATE_JNI_SHARED_LIBRARIES_ABI))
+mkdir -p $(addprefix $(dir $(1))lib/,$(PRIVATE_JNI_SHARED_LIBRARIES_ABI))
 $(foreach abi,$(PRIVATE_JNI_SHARED_LIBRARIES_ABI),\
   $(call _add-jni-shared-libs-to-package-per-abi,$(1),$(abi),\
     $(patsubst $(abi):%,%,$(filter $(abi):%,$(PRIVATE_JNI_SHARED_LIBRARIES)))))
-$(hide) (cd $(dir $(1)) && zip -qrX $(JNI_COMPRESS_FLAGS) $(notdir $(1)) lib)
+$(SOONG_ZIP) $(JNI_COMPRESS_FLAGS) -o $(1) -C $(dir $(1)) -D $(dir $(1))lib
-$(hide) rm -rf $(dir $(1))lib
+rm -rf $(dir $(1))lib
 endef
 
-# $(1): the package file.
+# $(1): the jar file.
-define add-dex-to-package
+# $(2): the classes.dex file.
-$(hide) find $(dir $(PRIVATE_DEX_FILE)) -maxdepth 1 -name "classes*.dex" | sort | xargs zip -qjX $(1)
+define create-dex-jar
+find $(dir $(2)) -maxdepth 1 -name "classes*.dex" | sort > $(1).lst
+$(SOONG_ZIP) -o $(1) -C $(dir $(2)) -l $(1).lst
 endef
 
-# Add java resources added by the current module.
+# Add java resources added by the current module to an existing package.
-# $(1) destination package
+# $(1) destination package.
-#
 define add-java-resources-to
+  $(call _java-resources,$(1),u)
+endef
+
+# Add java resources added by the current module to a new jar.
+# $(1) destination jar.
+define create-java-resources-jar
+  $(call _java-resources,$(1),c)
+endef
+
+define _java-resources
 $(call dump-words-to-file, $(PRIVATE_EXTRA_JAR_ARGS), $(1).jar-arg-list)
-$(hide) $(JAR) uf $(1) @$(1).jar-arg-list
+$(hide) $(JAR) $(2)f $(1) @$(1).jar-arg-list
 @rm -f $(1).jar-arg-list
 endef
 
@@ -2409,6 +2398,12 @@ define add-jar-resources-to-package
   $(JAR) uf $(1) $(call jar-args-sorted-files-in-directory,$(3))
 endef
 
+# $(1): the output resources jar.
+# $(2): the input jar
+define extract-resources-jar
+  $(ZIP2ZIP) -i $(2) -o $(1) -x '**/*.class' -x '**/*/'
+endef
+
 # Sign a package using the specified key/cert.
 #
 define sign-package
@@ -3510,3 +3505,9 @@ $(if $(call _invalid-name-chars,$($(1))), \
   $(call pretty-error,Invalid characters in module stem \($(1)\): $(call _invalid-name-chars,$($(1)))))
 endef
 .KATI_READONLY := verify-module-stem
+
+$(KATI_obsolete_var \
+  create-empty-package \
+  initialize-package-file \
+  add-jni-shared-libs-to-package,\
+  These functions have been removed)

core/dpi_specific_apk.mk

@@ -47,21 +47,24 @@ $(built_dpi_apk) : $(R_file_stamp)
 $(built_dpi_apk) : $(all_library_res_package_export_deps)
 $(built_dpi_apk) : $(private_key) $(certificate) $(SIGNAPK_JAR)
 $(built_dpi_apk) : $(AAPT)
+$(built_dpi_apk) : $(MERGE_ZIPS) $(SOONG_ZIP) $(ZIP2ZIP)
 $(built_dpi_apk) : $(all_res_assets) $(jni_shared_libraries) $(full_android_manifest)
 	@echo "target Package: $(PRIVATE_MODULE) ($@)"
-	$(if $(PRIVATE_SOURCE_ARCHIVE),\
+	rm -rf $@.parts
-	  $(call initialize-package-file,$(PRIVATE_SOURCE_ARCHIVE),$@),\
+	mkdir -p $@.parts
-	  $(call create-empty-package,$@))
+	$(call create-assets-package,$@.parts/apk.zip)
-	$(call add-assets-to-package,$@)
 ifneq ($(jni_shared_libraries),)
-	$(call add-jni-shared-libs-to-package,$@)
+	$(call create-jni-shared-libs-package,$@.parts/jni.zip)
 endif
 ifeq ($(full_classes_jar),)
 # We don't build jar, need to add the Java resources here.
-	$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call add-java-resources-to,$@))
+	$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call create-java-resources-jar,$@.parts/res.zip))
 else
-	$(call add-dex-to-package,$@)
+	$(call create-dex-jar,$@.parts/dex.zip,$(PRIVATE_DEX_FILE))
+	$(call extract-resources-jar,$@.parts/res.zip,$(PRIVATE_SOURCE_ARCHIVE))
 endif
+	$(MERGE_ZIPS) $@ $@.parts/*.zip
+	rm -rf $@.parts
 	$(sign-package)
 
 # Set up global variables to register this apk to the higher-level dependency graph.

core/host_dalvik_java_library.mk

@@ -171,10 +171,15 @@ $(built_dex): $(full_classes_jar) $(DX) $(ZIP2ZIP)
 
 $(LOCAL_BUILT_MODULE): PRIVATE_DEX_FILE := $(built_dex)
 $(LOCAL_BUILT_MODULE): PRIVATE_SOURCE_ARCHIVE := $(full_classes_jarjar_jar)
+$(LOCAL_BUILT_MODULE): $(MERGE_ZIPS) $(SOONG_ZIP) $(ZIP2ZIP)
 $(LOCAL_BUILT_MODULE): $(built_dex) $(java_resource_sources)
 	@echo "Host Jar: $(PRIVATE_MODULE) ($@)"
-	$(call initialize-package-file,$(PRIVATE_SOURCE_ARCHIVE),$@)
+	rm -rf $@.parts
-	$(call add-dex-to-package,$@)
+	mkdir -p $@.parts
+	$(call create-dex-jar,$@.parts/dex.zip,$(PRIVATE_DEX_FILE))
+	$(call extract-resources-jar,$@.parts/res.zip,$(PRIVATE_SOURCE_ARCHIVE))
+	$(MERGE_ZIPS) -j $@ $@.parts/dex.zip $@.parts/res.zip
+	rm -rf $@.parts
 
 endif # !LOCAL_IS_STATIC_JAVA_LIBRARY
 

core/java_library.mk

@@ -65,10 +65,14 @@ else # !LOCAL_IS_STATIC_JAVA_LIBRARY
 
 $(common_javalib.jar): PRIVATE_DEX_FILE := $(built_dex)
 $(common_javalib.jar): PRIVATE_SOURCE_ARCHIVE := $(full_classes_pre_proguard_jar)
+$(common_javalib.jar): $(MERGE_ZIPS) $(SOONG_ZIP) $(ZIP2ZIP)
 $(common_javalib.jar) : $(built_dex) $(java_resource_sources) | $(ZIPTIME) $(ZIPALIGN)
 	@echo "target Jar: $(PRIVATE_MODULE) ($@)"
-	$(call initialize-package-file,$(PRIVATE_SOURCE_ARCHIVE),$@.tmp)
+	rm -rf $@.parts && mkdir -p $@.parts
-	$(call add-dex-to-package,$@.tmp)
+	$(call create-dex-jar,$@.parts/dex.zip,$(PRIVATE_DEX_FILE))
+	$(call extract-resources-jar,$@.parts/res.zip,$(PRIVATE_SOURCE_ARCHIVE))
+	$(MERGE_ZIPS) -j $@.tmp $@.parts/dex.zip $@.parts/res.zip
+	rm -rf $@.parts
 	$(hide) $(ZIPTIME) $@.tmp
 	$(call commit-change-for-toc,$@)
 ifeq (true, $(LOCAL_UNCOMPRESS_DEX))

core/package_internal.mk

@@ -413,15 +413,14 @@ else  # LOCAL_USE_AAPT2
     resource_export_package := $(intermediates.COMMON)/package-export.apk
     $(R_file_stamp): $(resource_export_package)
 
-    # add-assets-to-package looks at PRODUCT_AAPT_CONFIG, but this target
+    # create-assets-package looks at PRODUCT_AAPT_CONFIG, but this target
     # can't know anything about PRODUCT.  Clear it out just for this target.
     $(resource_export_package): PRIVATE_PRODUCT_AAPT_CONFIG :=
     $(resource_export_package): PRIVATE_PRODUCT_AAPT_PREF_CONFIG :=
     $(resource_export_package): PRIVATE_RESOURCE_LIST := $(all_res_assets)
     $(resource_export_package): $(all_res_assets) $(full_android_manifest) $(rs_generated_res_zip) $(AAPT)
 	@echo "target Export Resources: $(PRIVATE_MODULE) ($@)"
-	$(call create-empty-package,$@)
+	$(call create-assets-package,$@)
-	$(call add-assets-to-package,$@)
   endif
 
 endif  # LOCAL_USE_AAPT2
@@ -609,9 +608,8 @@ $(LOCAL_BUILT_MODULE): PRIVATE_INSTALLED_MODULE := $(LOCAL_INSTALLED_MODULE)
 endif
 
 $(LOCAL_BUILT_MODULE): PRIVATE_RESOURCE_INTERMEDIATES_DIR := $(intermediates.COMMON)/resources
-$(LOCAL_BUILT_MODULE): PRIVATE_FULL_CLASSES_JAR := $(full_classes_jar)
 $(LOCAL_BUILT_MODULE) : $(jni_shared_libraries)
-$(LOCAL_BUILT_MODULE) : $(JAR_ARGS)
+$(LOCAL_BUILT_MODULE) : $(JAR_ARGS) $(SOONG_ZIP) $(MERGE_ZIPS) $(ZIP2ZIP)
 ifeq ($(LOCAL_USE_AAPT2),true)
 $(LOCAL_BUILT_MODULE): PRIVATE_RES_PACKAGE := $(my_res_package)
 $(LOCAL_BUILT_MODULE) : $(my_res_package) $(AAPT2) | $(ACP)
@@ -623,26 +621,25 @@ ifdef LOCAL_COMPRESSED_MODULE
 $(LOCAL_BUILT_MODULE) : $(MINIGZIP)
 endif
 	@echo "target Package: $(PRIVATE_MODULE) ($@)"
+	rm -rf $@.parts
+	mkdir -p $@.parts
 ifeq ($(LOCAL_USE_AAPT2),true)
-	$(call copy-file-to-new-target)
+	cp -f $< $@.parts/apk.zip
 else  # ! LOCAL_USE_AAPT2
-	$(if $(PRIVATE_SOURCE_ARCHIVE),\
+	$(call create-assets-package,$@.parts/apk.zip)
-	  $(call initialize-package-file,$(PRIVATE_SOURCE_ARCHIVE),$@),\
-	  $(call create-empty-package,$@))
-	$(call add-assets-to-package,$@)
 endif  # LOCAL_USE_AAPT2
 ifneq ($(jni_shared_libraries),)
-	$(call add-jni-shared-libs-to-package,$@)
+	$(call create-jni-shared-libs-package,$@.parts/jni.zip)
 endif
 ifeq ($(full_classes_jar),)
 # We don't build jar, need to add the Java resources here.
-	$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call add-java-resources-to,$@))
+	$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call create-java-resources-jar,$@.parts/res.zip))
 else  # full_classes_jar
-	$(call add-dex-to-package,$@)
+	$(call create-dex-jar,$@.parts/dex.zip,$(PRIVATE_DEX_FILE))
-ifeq ($(LOCAL_USE_AAPT2),true)
+	$(call extract-resources-jar,$@.parts/res.zip,$(PRIVATE_SOURCE_ARCHIVE))
-	$(call add-jar-resources-to-package,$@,$(PRIVATE_FULL_CLASSES_JAR),$(PRIVATE_RESOURCE_INTERMEDIATES_DIR))
-endif
 endif  # full_classes_jar
+	$(MERGE_ZIPS) $@ $@.parts/*.zip
+	rm -rf $@.parts
 ifeq (true, $(LOCAL_UNCOMPRESS_DEX))
 	@# No need to align, sign-package below will do it.
 	$(uncompress-dexs)
@@ -687,7 +684,7 @@ $(built_odex): PRIVATE_DEX_FILE := $(built_dex)
 # Use pattern rule - we may have multiple built odex files.
 $(built_odex) : $(dir $(LOCAL_BUILT_MODULE))% : $(built_dex)
 	$(hide) mkdir -p $(dir $@) && rm -f $@
-	$(call add-dex-to-package,$@)
+	$(call create-dex-jar,$@,$(PRIVATE_DEX_FILE))
 ifeq (true, $(LOCAL_UNCOMPRESS_DEX))
 	$(uncompress-dexs)
 	$(align-package)