From 0ff15de32a41a00c0ef1a5dbcf2826ef9c1c87db Mon Sep 17 00:00:00 2001 From: Tao Bao Date: Wed, 20 Mar 2019 11:26:06 -0700 Subject: [PATCH 1/2] releasetools: common.UnzipTemp() filters out non-matching patterns. common.UnzipTemp() calls `unzip` to do the unzipping, which will complain if there's non-existent names in the given list. Prior to this CL, callers had to do the work to remove non-existent entries. This CL filters out the given patterns in common.UnzipTemp()/common.UnzipToDir() to make callers' works easier. Bug: 128848294 Test: `m dist` with aosp_taimen-userdebug (which calls ota_from_target_files.py on a target_files.zip that doesn't contain RADIO/*). Test: `python -m unittest test_common.CommonZipTest` Change-Id: I5e741c27ea8d0b8126c398a7e1b56a8deb4a3d7f --- tools/releasetools/common.py | 25 ++++-- tools/releasetools/ota_from_target_files.py | 9 +-- tools/releasetools/test_common.py | 84 +++++++++++++++++++++ 3 files changed, 103 insertions(+), 15 deletions(-) diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 780b9c1550..34c13596c7 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -17,6 +17,7 @@ from __future__ import print_function import collections import copy import errno +import fnmatch import getopt import getpass import gzip @@ -771,21 +772,29 @@ def Gunzip(in_filename, out_filename): shutil.copyfileobj(in_file, out_file) -def UnzipToDir(filename, dirname, pattern=None): +def UnzipToDir(filename, dirname, patterns=None): """Unzips the archive to the given directory. Args: filename: The name of the zip file to unzip. - dirname: Where the unziped files will land. - - pattern: Files to unzip from the archive. If omitted, will unzip the entire - archvie. + patterns: Files to unzip from the archive. If omitted, will unzip the entire + archvie. Non-matching patterns will be filtered out. If there's no match + after the filtering, no file will be unzipped. """ - cmd = ["unzip", "-o", "-q", filename, "-d", dirname] - if pattern is not None: - cmd.extend(pattern) + if patterns is not None: + # Filter out non-matching patterns. unzip will complain otherwise. + with zipfile.ZipFile(filename) as input_zip: + names = input_zip.namelist() + filtered = [ + pattern for pattern in patterns if fnmatch.filter(names, pattern)] + + # There isn't any matching files. Don't unzip anything. + if not filtered: + return + cmd.extend(filtered) + RunAndCheckOutput(cmd) diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index fe40936463..37f4e38d12 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -236,7 +236,7 @@ METADATA_NAME = 'META-INF/com/android/metadata' POSTINSTALL_CONFIG = 'META/postinstall_config.txt' DYNAMIC_PARTITION_INFO = 'META/dynamic_partitions_info.txt' AB_PARTITIONS = 'META/ab_partitions.txt' -UNZIP_PATTERN = ['IMAGES/*', 'META/*'] +UNZIP_PATTERN = ['IMAGES/*', 'META/*', 'RADIO/*'] RETROFIT_DAP_UNZIP_PATTERN = ['OTA/super_*.img', AB_PARTITIONS] @@ -1802,12 +1802,7 @@ def GetTargetFilesZipForSecondaryImages(input_file, skip_postinstall=False): infolist = input_zip.infolist() namelist = input_zip.namelist() - # Additionally unzip 'RADIO/*' if exists. - unzip_pattern = UNZIP_PATTERN[:] - if any([entry.startswith('RADIO/') for entry in namelist]): - unzip_pattern.append('RADIO/*') - input_tmp = common.UnzipTemp(input_file, unzip_pattern) - + input_tmp = common.UnzipTemp(input_file, UNZIP_PATTERN) for info in infolist: unzipped_file = os.path.join(input_tmp, *info.filename.split('/')) if info.filename == 'IMAGES/system_other.img': diff --git a/tools/releasetools/test_common.py b/tools/releasetools/test_common.py index 8709124193..d4fa5f3a59 100644 --- a/tools/releasetools/test_common.py +++ b/tools/releasetools/test_common.py @@ -359,6 +359,90 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): finally: os.remove(zip_file.name) + @staticmethod + def _test_UnzipTemp_createZipFile(): + zip_file = common.MakeTempFile(suffix='.zip') + output_zip = zipfile.ZipFile( + zip_file, 'w', compression=zipfile.ZIP_DEFLATED) + contents = os.urandom(1024) + with tempfile.NamedTemporaryFile() as entry_file: + entry_file.write(contents) + common.ZipWrite(output_zip, entry_file.name, arcname='Test1') + common.ZipWrite(output_zip, entry_file.name, arcname='Test2') + common.ZipWrite(output_zip, entry_file.name, arcname='Foo3') + common.ZipWrite(output_zip, entry_file.name, arcname='Bar4') + common.ZipWrite(output_zip, entry_file.name, arcname='Dir5/Baz5') + common.ZipClose(output_zip) + common.ZipClose(output_zip) + return zip_file + + def test_UnzipTemp(self): + zip_file = self._test_UnzipTemp_createZipFile() + unzipped_dir = common.UnzipTemp(zip_file) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test1'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test2'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Foo3'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Bar4'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Dir5/Baz5'))) + + def test_UnzipTemp_withPatterns(self): + zip_file = self._test_UnzipTemp_createZipFile() + + unzipped_dir = common.UnzipTemp(zip_file, ['Test1']) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test1'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Test2'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Foo3'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Bar4'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Dir5/Baz5'))) + + unzipped_dir = common.UnzipTemp(zip_file, ['Test1', 'Foo3']) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test1'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Test2'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Foo3'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Bar4'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Dir5/Baz5'))) + + unzipped_dir = common.UnzipTemp(zip_file, ['Test*', 'Foo3*']) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test1'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test2'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Foo3'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Bar4'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Dir5/Baz5'))) + + unzipped_dir = common.UnzipTemp(zip_file, ['*Test1', '*Baz*']) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test1'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Test2'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Foo3'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Bar4'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Dir5/Baz5'))) + + def test_UnzipTemp_withEmptyPatterns(self): + zip_file = self._test_UnzipTemp_createZipFile() + unzipped_dir = common.UnzipTemp(zip_file, []) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Test1'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Test2'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Foo3'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Bar4'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Dir5/Baz5'))) + + def test_UnzipTemp_withPartiallyMatchingPatterns(self): + zip_file = self._test_UnzipTemp_createZipFile() + unzipped_dir = common.UnzipTemp(zip_file, ['Test*', 'Nonexistent*']) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test1'))) + self.assertTrue(os.path.exists(os.path.join(unzipped_dir, 'Test2'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Foo3'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Bar4'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Dir5/Baz5'))) + + def test_UnzipTemp_withNoMatchingPatterns(self): + zip_file = self._test_UnzipTemp_createZipFile() + unzipped_dir = common.UnzipTemp(zip_file, ['Foo4', 'Nonexistent*']) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Test1'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Test2'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Foo3'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Bar4'))) + self.assertFalse(os.path.exists(os.path.join(unzipped_dir, 'Dir5/Baz5'))) + class CommonApkUtilsTest(test_utils.ReleaseToolsTestCase): """Tests the APK utils related functions.""" From 359862db12c6c09c003a16efd617ada26fab833b Mon Sep 17 00:00:00 2001 From: Tao Bao Date: Wed, 20 Mar 2019 12:24:58 -0700 Subject: [PATCH 2/2] Revert^2 "releasetools: check_target_files_signatures.py checks APEXes." This reverts commit 5516d37f4197f31f714d28bb307c469a15bae5da. The previous issue in unzipping non-matching files has been addressed with commit a49054ca2f2959f50f3188914ec0faebc90ebcbe. This CL rolls forward to allow dumping container certifcates for APEXes. Bug: 128848294 Test: Run check_target_files_signatures.py on target_files.zips w/ and w/o APEX files. Change-Id: I662aab3d96fc40ac8e5e206e32b73ac763220b70 --- tools/releasetools/check_target_files_signatures.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/releasetools/check_target_files_signatures.py b/tools/releasetools/check_target_files_signatures.py index 9b7695479b..4b0d4c752d 100755 --- a/tools/releasetools/check_target_files_signatures.py +++ b/tools/releasetools/check_target_files_signatures.py @@ -168,6 +168,7 @@ def CertFromPKCS7(data, filename): class APK(object): + def __init__(self, full_filename, filename): self.filename = filename self.certs = None @@ -244,12 +245,12 @@ class TargetFiles(object): # must decompress them individually before we perform any analysis. # This is the list of wildcards of files we extract from |filename|. - apk_extensions = ['*.apk'] + apk_extensions = ['*.apk', '*.apex'] self.certmap, compressed_extension = common.ReadApkCerts( - zipfile.ZipFile(filename, "r")) + zipfile.ZipFile(filename)) if compressed_extension: - apk_extensions.append("*.apk" + compressed_extension) + apk_extensions.append('*.apk' + compressed_extension) d = common.UnzipTemp(filename, apk_extensions) self.apks = {} @@ -272,7 +273,7 @@ class TargetFiles(object): os.remove(os.path.join(dirpath, fn)) fn = uncompressed_fn - if fn.endswith(".apk"): + if fn.endswith(('.apk', '.apex')): fullname = os.path.join(dirpath, fn) displayname = fullname[len(d)+1:] apk = APK(fullname, displayname)