diff --git a/tools/releasetools/sparse_img.py b/tools/releasetools/sparse_img.py index 33678966e8..a2490813e5 100644 --- a/tools/releasetools/sparse_img.py +++ b/tools/releasetools/sparse_img.py @@ -249,8 +249,9 @@ class SparseImage(object): with open(fn) as f: for line in f: - fn, ranges = line.split(None, 1) - ranges = rangelib.RangeSet.parse(ranges) + fn, ranges_text = line.rstrip().split(None, 1) + ranges = rangelib.RangeSet.parse(ranges_text) + ranges.extra['text_str'] = ranges_text if allow_shared_blocks: # Find the shared blocks that have been claimed by others. If so, tag @@ -261,9 +262,6 @@ class SparseImage(object): if not non_shared: continue - # There shouldn't anything in the extra dict yet. - assert not ranges.extra, "Non-empty RangeSet.extra" - # Put the non-shared RangeSet as the value in the block map, which # has a copy of the original RangeSet. non_shared.extra['uses_shared_blocks'] = ranges diff --git a/tools/releasetools/test_validate_target_files.py b/tools/releasetools/test_validate_target_files.py index 9c816eb416..65045150a3 100644 --- a/tools/releasetools/test_validate_target_files.py +++ b/tools/releasetools/test_validate_target_files.py @@ -238,14 +238,14 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase): system_root = os.path.join(input_tmp, "SYSTEM") os.mkdir(system_root) - # Write the test file that contain multiple blocks of zeros, and these - # zero blocks will be omitted by kernel. And the test files will occupy one - # block range each in the final system image. + # Write test files that contain multiple blocks of zeros, and these zero + # blocks will be omitted by kernel. Each test file will occupy one block in + # the final system image. with open(os.path.join(system_root, 'a'), 'w') as f: - f.write("aaa") + f.write('aaa') f.write('\0' * 4096 * 3) with open(os.path.join(system_root, 'b'), 'w') as f: - f.write("bbb") + f.write('bbb') f.write('\0' * 4096 * 3) raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map') @@ -254,7 +254,7 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase): # Parse the generated file map and update the block ranges for each file. file_map_list = {} image_ranges = RangeSet() - with open(raw_file_map, 'r') as f: + with open(raw_file_map) as f: for line in f.readlines(): info = line.split() self.assertEqual(2, len(info)) @@ -265,7 +265,7 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase): mock_shared_block = RangeSet("10-20").subtract(image_ranges).first(1) with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f: for key in sorted(file_map_list.keys()): - line = "{} {}\n".format( + line = '{} {}\n'.format( key, file_map_list[key].union(mock_shared_block)) f.write(line) @@ -277,9 +277,55 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase): for name in all_entries: input_zip.write(os.path.join(input_tmp, name), arcname=name) - input_zip = zipfile.ZipFile(input_file, 'r') - info_dict = {'extfs_sparse_flag': '-s'} - # Expect the validation to pass and both files are skipped due to # 'incomplete' block range. - ValidateFileConsistency(input_zip, input_tmp, info_dict) + with zipfile.ZipFile(input_file) as input_zip: + info_dict = {'extfs_sparse_flag': '-s'} + ValidateFileConsistency(input_zip, input_tmp, info_dict) + + @test_utils.SkipIfExternalToolsUnavailable() + def test_ValidateFileConsistency_nonMonotonicRanges(self): + input_tmp = common.MakeTempDir() + os.mkdir(os.path.join(input_tmp, 'IMAGES')) + system_image = os.path.join(input_tmp, 'IMAGES', 'system.img') + system_root = os.path.join(input_tmp, "SYSTEM") + os.mkdir(system_root) + + # Write the test file that contain three blocks of 'a', 'b', 'c'. + with open(os.path.join(system_root, 'abc'), 'w') as f: + f.write('a' * 4096 + 'b' * 4096 + 'c' * 4096) + raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map') + self._generate_system_image(system_image, system_root, raw_file_map) + + # Parse the generated file map and manipulate the block ranges of 'abc' to + # be 'cba'. + file_map_list = {} + with open(raw_file_map) as f: + for line in f.readlines(): + info = line.split() + self.assertEqual(2, len(info)) + ranges = RangeSet(info[1]) + self.assertTrue(ranges.monotonic) + blocks = reversed(list(ranges.next_item())) + file_map_list[info[0]] = ' '.join([str(block) for block in blocks]) + + # Update the contents of 'abc' to be 'cba'. + with open(os.path.join(system_root, 'abc'), 'w') as f: + f.write('c' * 4096 + 'b' * 4096 + 'a' * 4096) + + # Update the system.map. + with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f: + for key in sorted(file_map_list.keys()): + f.write('{} {}\n'.format(key, file_map_list[key])) + + # Get the target zip file. + input_file = common.MakeTempFile() + all_entries = ['SYSTEM/', 'SYSTEM/abc', 'IMAGES/', + 'IMAGES/system.map', 'IMAGES/system.img'] + with zipfile.ZipFile(input_file, 'w') as input_zip: + for name in all_entries: + input_zip.write(os.path.join(input_tmp, name), arcname=name) + + with zipfile.ZipFile(input_file) as input_zip: + info_dict = {'extfs_sparse_flag': '-s'} + ValidateFileConsistency(input_zip, input_tmp, info_dict) diff --git a/tools/releasetools/validate_target_files.py b/tools/releasetools/validate_target_files.py index a5e1930b62..383ef7bc07 100755 --- a/tools/releasetools/validate_target_files.py +++ b/tools/releasetools/validate_target_files.py @@ -36,20 +36,21 @@ import logging import os.path import re import zipfile +from hashlib import sha1 import common +import rangelib def _ReadFile(file_name, unpacked_name, round_up=False): """Constructs and returns a File object. Rounds up its size if needed.""" - assert os.path.exists(unpacked_name) with open(unpacked_name, 'rb') as f: file_data = f.read() file_size = len(file_data) if round_up: file_size_rounded_up = common.RoundUpTo4K(file_size) - file_data += '\0' * (file_size_rounded_up - file_size) + file_data += b'\0' * (file_size_rounded_up - file_size) return common.File(file_name, file_data) @@ -96,13 +97,15 @@ def ValidateFileConsistency(input_zip, input_tmp, info_dict): logging.warning('Skipping %s that has incomplete block list', entry) continue - # TODO(b/79951650): Handle files with non-monotonic ranges. + # If the file has non-monotonic ranges, read each range in order. if not file_ranges.monotonic: - logging.warning( - 'Skipping %s that has non-monotonic ranges: %s', entry, file_ranges) - continue - - blocks_sha1 = image.RangeSha1(file_ranges) + h = sha1() + for file_range in file_ranges.extra['text_str'].split(' '): + for data in image.ReadRangeSet(rangelib.RangeSet(file_range)): + h.update(data) + blocks_sha1 = h.hexdigest() + else: + blocks_sha1 = image.RangeSha1(file_ranges) # The filename under unpacked directory, such as SYSTEM/bin/sh. unpacked_name = os.path.join(