Merge "Revert "Remove all ZIP64LIMIT hack"" am: 7b640806da

Original change: https://android-review.googlesource.com/c/platform/build/+/2535093

Change-Id: I315fd1f9274cfda7b45c84b229ca6d6f040aa3a4
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

tools/releasetools/add_img_to_target_files.py

@@ -842,13 +842,14 @@ def ReplaceUpdatedFiles(zip_filename, files_list):
   SYSTEM/ after rebuilding recovery.
   """
   common.ZipDelete(zip_filename, files_list)
-  with zipfile.ZipFile(zip_filename, "a",
+  output_zip = zipfile.ZipFile(zip_filename, "a",
                                compression=zipfile.ZIP_DEFLATED,
-                               allowZip64=True) as output_zip:
+                               allowZip64=True)
   for item in files_list:
     file_path = os.path.join(OPTIONS.input_tmp, item)
     assert os.path.exists(file_path)
     common.ZipWrite(output_zip, file_path, arcname=item)
+  common.ZipClose(output_zip)
 
 
 def HasPartition(partition_name):
@@ -1191,7 +1192,7 @@ def AddImagesToTargetFiles(filename):
   AddVbmetaDigest(output_zip)
 
   if output_zip:
-    output_zip.close()
+    common.ZipClose(output_zip)
     if OPTIONS.replace_updated_files_list:
       ReplaceUpdatedFiles(output_zip.filename,
                           OPTIONS.replace_updated_files_list)

tools/releasetools/apex_utils.py

@@ -431,7 +431,7 @@ def SignUncompressedApex(avbtool, apex_file, payload_key, container_key,
   apex_zip = zipfile.ZipFile(apex_file, 'a', allowZip64=True)
   common.ZipWrite(apex_zip, payload_file, arcname=APEX_PAYLOAD_IMAGE)
   common.ZipWrite(apex_zip, payload_public_key, arcname=APEX_PUBKEY)
-  apex_zip.close()
+  common.ZipClose(apex_zip)
 
   # 3. Sign the APEX container with container_key.
   signed_apex = common.MakeTempFile(prefix='apex-container-', suffix='.apex')

tools/releasetools/check_ota_package_signature.py

@@ -142,7 +142,7 @@ def VerifyAbOtaPayload(cert, package):
   """Verifies the payload and metadata signatures in an A/B OTA payload."""
   package_zip = zipfile.ZipFile(package, 'r', allowZip64=True)
   if 'payload.bin' not in package_zip.namelist():
-    package_zip.close()
+    common.ZipClose(package_zip)
     return
 
   print('Verifying A/B OTA payload signatures...')
@@ -160,7 +160,7 @@ def VerifyAbOtaPayload(cert, package):
          '--in_file=' + payload_file,
          '--public_key=' + pubkey]
   common.RunAndCheckOutput(cmd)
-  package_zip.close()
+  common.ZipClose(package_zip)
 
   # Verified successfully upon reaching here.
   print('\nPayload signatures VERIFIED\n\n')

tools/releasetools/common.py

@@ -2809,6 +2809,18 @@ class PasswordManager(object):
 def ZipWrite(zip_file, filename, arcname=None, perms=0o644,
              compress_type=None):
 
+  # http://b/18015246
+  # Python 2.7's zipfile implementation wrongly thinks that zip64 is required
+  # for files larger than 2GiB. We can work around this by adjusting their
+  # limit. Note that `zipfile.writestr()` will not work for strings larger than
+  # 2GiB. The Python interpreter sometimes rejects strings that large (though
+  # it isn't clear to me exactly what circumstances cause this).
+  # `zipfile.write()` must be used directly to work around this.
+  #
+  # This mess can be avoided if we port to python3.
+  saved_zip64_limit = zipfile.ZIP64_LIMIT
+  zipfile.ZIP64_LIMIT = (1 << 32) - 1
+
   if compress_type is None:
     compress_type = zip_file.compression
   if arcname is None:
@@ -2834,13 +2846,14 @@ def ZipWrite(zip_file, filename, arcname=None, perms=0o644,
   finally:
     os.chmod(filename, saved_stat.st_mode)
     os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime))
+    zipfile.ZIP64_LIMIT = saved_zip64_limit
 
 
 def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
                 compress_type=None):
   """Wrap zipfile.writestr() function to work around the zip64 limit.
 
-  Python's zip implementation won't allow writing a string
+  Even with the ZIP64_LIMIT workaround, it won't allow writing a string
   longer than 2GiB. It gives 'OverflowError: size does not fit in an int'
   when calling crc32(bytes).
 
@@ -2849,6 +2862,9 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
   when we know the string won't be too long.
   """
 
+  saved_zip64_limit = zipfile.ZIP64_LIMIT
+  zipfile.ZIP64_LIMIT = (1 << 32) - 1
+
   if not isinstance(zinfo_or_arcname, zipfile.ZipInfo):
     zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname)
     zinfo.compress_type = zip_file.compression
@@ -2881,6 +2897,7 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
   zinfo.date_time = (2009, 1, 1, 0, 0, 0)
 
   zip_file.writestr(zinfo, data)
+  zipfile.ZIP64_LIMIT = saved_zip64_limit
 
 
 def ZipDelete(zip_filename, entries, force=False):
@@ -2913,6 +2930,18 @@ def ZipDelete(zip_filename, entries, force=False):
   os.replace(new_zipfile, zip_filename)
 
 
+def ZipClose(zip_file):
+  # http://b/18015246
+  # zipfile also refers to ZIP64_LIMIT during close() when it writes out the
+  # central directory.
+  saved_zip64_limit = zipfile.ZIP64_LIMIT
+  zipfile.ZIP64_LIMIT = (1 << 32) - 1
+
+  zip_file.close()
+
+  zipfile.ZIP64_LIMIT = saved_zip64_limit
+
+
 class DeviceSpecificParams(object):
   module = None
 

tools/releasetools/non_ab_ota.py

@@ -272,7 +272,7 @@ endif;
 
   # We haven't written the metadata entry, which will be done in
   # FinalizeMetadata.
-  output_zip.close()
+  common.ZipClose(output_zip)
 
   needed_property_files = (
       NonAbOtaPropertyFiles(),
@@ -526,7 +526,7 @@ endif;
 
   # We haven't written the metadata entry yet, which will be handled in
   # FinalizeMetadata().
-  output_zip.close()
+  common.ZipClose(output_zip)
 
   # Sign the generated zip package unless no_signing is specified.
   needed_property_files = (

tools/releasetools/ota_from_target_files.py

@@ -495,7 +495,7 @@ def GetTargetFilesZipForSecondaryImages(input_file, skip_postinstall=False):
       else:
         common.ZipWrite(target_zip, unzipped_file, arcname=info.filename)
 
-  target_zip.close()
+  common.ZipClose(target_zip)
 
   return target_file
 
@@ -632,7 +632,7 @@ def GetTargetFilesZipForPartialUpdates(input_file, ab_partitions):
 
     # TODO(xunchang) handle META/postinstall_config.txt'
 
-  partial_target_zip.close()
+  common.ZipClose(partial_target_zip)
 
   return partial_target_file
 
@@ -717,7 +717,7 @@ def GetTargetFilesZipForRetrofitDynamicPartitions(input_file,
   # Write new ab_partitions.txt file
   common.ZipWrite(target_zip, new_ab_partitions, arcname=AB_PARTITIONS)
 
-  target_zip.close()
+  common.ZipClose(target_zip)
 
   return target_file
 
@@ -1052,11 +1052,11 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None):
     common.ZipWriteStr(output_zip, "apex_info.pb", ota_apex_info,
                        compress_type=zipfile.ZIP_STORED)
 
-  target_zip.close()
+  common.ZipClose(target_zip)
 
   # We haven't written the metadata entry yet, which will be handled in
   # FinalizeMetadata().
-  output_zip.close()
+  common.ZipClose(output_zip)
 
   FinalizeMetadata(metadata, staging_file, output_file,
                    package_key=OPTIONS.package_key)

tools/releasetools/ota_utils.py

@@ -22,7 +22,7 @@ import zipfile
 
 import ota_metadata_pb2
 import common
-from common import (ZipDelete, OPTIONS, MakeTempFile,
+from common import (ZipDelete, ZipClose, OPTIONS, MakeTempFile,
                     ZipWriteStr, BuildInfo, LoadDictionaryFromFile,
                     SignFile, PARTITIONS_WITH_BUILD_PROP, PartitionBuildProps,
                     GetRamdiskFormat, ParseUpdateEngineConfig)

tools/releasetools/sign_target_files_apks.py

@@ -908,7 +908,7 @@ def WriteOtacerts(output_zip, filename, keys):
   certs_zip = zipfile.ZipFile(temp_file, "w", allowZip64=True)
   for k in keys:
     common.ZipWrite(certs_zip, k)
-  certs_zip.close()
+  common.ZipClose(certs_zip)
   common.ZipWriteStr(output_zip, filename, temp_file.getvalue())
 
 
@@ -1545,8 +1545,8 @@ def main(argv):
                      platform_api_level, codename_to_api_level_map,
                      compressed_extension)
 
-  input_zip.close()
+  common.ZipClose(input_zip)
-  output_zip.close()
+  common.ZipClose(output_zip)
 
   if OPTIONS.vendor_partitions and OPTIONS.vendor_otatools:
     BuildVendorPartitions(args[1])

tools/releasetools/test_common.py

@@ -461,7 +461,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
       os.utime(test_file_name, (1234567, 1234567))
       expected_stat = os.stat(test_file_name)
       common.ZipWrite(zip_file, test_file_name, **extra_zipwrite_args)
-      zip_file.close()
+      common.ZipClose(zip_file)
 
       self._verify(zip_file, zip_file_name, arcname, sha1_hash.hexdigest(),
                    test_file_name, expected_stat, expected_mode,
@@ -494,7 +494,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
         expected_mode = extra_args.get("perms", zinfo_perms)
 
       common.ZipWriteStr(zip_file, zinfo_or_arcname, contents, **extra_args)
-      zip_file.close()
+      common.ZipClose(zip_file)
 
       self._verify(zip_file, zip_file_name, arcname, sha1(contents).hexdigest(),
                    expected_mode=expected_mode,
@@ -538,7 +538,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
 
       common.ZipWrite(zip_file, test_file_name, **extra_args)
       common.ZipWriteStr(zip_file, arcname_small, small, **extra_args)
-      zip_file.close()
+      common.ZipClose(zip_file)
 
       # Verify the contents written by ZipWrite().
       self._verify(zip_file, zip_file_name, arcname_large,
@@ -553,6 +553,12 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
       os.remove(zip_file_name)
       os.remove(test_file_name)
 
+  def _test_reset_ZIP64_LIMIT(self, func, *args):
+    default_limit = (1 << 31) - 1
+    self.assertEqual(default_limit, zipfile.ZIP64_LIMIT)
+    func(*args)
+    self.assertEqual(default_limit, zipfile.ZIP64_LIMIT)
+
   def test_ZipWrite(self):
     file_contents = os.urandom(1024)
     self._test_ZipWrite(file_contents)
@@ -577,7 +583,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
     })
 
   def test_ZipWrite_resets_ZIP64_LIMIT(self):
-    self._test_ZipWrite("")
+    self._test_reset_ZIP64_LIMIT(self._test_ZipWrite, "")
 
   def test_ZipWriteStr(self):
     random_string = os.urandom(1024)
@@ -628,9 +634,9 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
     })
 
   def test_ZipWriteStr_resets_ZIP64_LIMIT(self):
-    self._test_ZipWriteStr('foo', b'')
+    self._test_reset_ZIP64_LIMIT(self._test_ZipWriteStr, 'foo', b'')
     zinfo = zipfile.ZipInfo(filename="foo")
-    self._test_ZipWriteStr(zinfo, b'')
+    self._test_reset_ZIP64_LIMIT(self._test_ZipWriteStr, zinfo, b'')
 
   def test_bug21309935(self):
     zip_file = tempfile.NamedTemporaryFile(delete=False)
@@ -652,7 +658,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
       zinfo = zipfile.ZipInfo(filename="qux")
       zinfo.external_attr = 0o700 << 16
       common.ZipWriteStr(zip_file, zinfo, random_string, perms=0o400)
-      zip_file.close()
+      common.ZipClose(zip_file)
 
       self._verify(zip_file, zip_file_name, "foo",
                    sha1(random_string).hexdigest(),
@@ -679,7 +685,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
       common.ZipWrite(output_zip, entry_file.name, arcname='Test1')
       common.ZipWrite(output_zip, entry_file.name, arcname='Test2')
       common.ZipWrite(output_zip, entry_file.name, arcname='Test3')
-      output_zip.close()
+      common.ZipClose(output_zip)
     zip_file.close()
 
     try:
@@ -727,8 +733,8 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase):
       common.ZipWrite(output_zip, entry_file.name, arcname='Foo3')
       common.ZipWrite(output_zip, entry_file.name, arcname='Bar4')
       common.ZipWrite(output_zip, entry_file.name, arcname='Dir5/Baz5')
-      output_zip.close()
+      common.ZipClose(output_zip)
-    output_zip.close()
+    common.ZipClose(output_zip)
     return zip_file
 
   @test_utils.SkipIfExternalToolsUnavailable()
@@ -1231,8 +1237,7 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase):
     self.assertEqual(
         '1-5 9-10',
         sparse_image.file_map['//system/file1'].extra['text_str'])
-    self.assertTrue(
+    self.assertTrue(sparse_image.file_map['//system/file2'].extra['incomplete'])
-        sparse_image.file_map['//system/file2'].extra['incomplete'])
     self.assertTrue(
         sparse_image.file_map['/system/app/file3'].extra['incomplete'])
 
@@ -1661,7 +1666,6 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase):
     self.assertRaises(common.ExternalError, common._GenerateGkiCertificate,
                       test_file.name, 'generic_kernel')
 
-
 class InstallRecoveryScriptFormatTest(test_utils.ReleaseToolsTestCase):
   """Checks the format of install-recovery.sh.