From c0f77ee489552782f76526d97374a875de2a796e Mon Sep 17 00:00:00 2001 From: xunchang Date: Wed, 20 Feb 2019 15:03:43 -0800 Subject: [PATCH] Check the completeness of original blocks during target files validation The validate_target_files.py checks the 'incomplete' field of the range in file_map. And range has already considered the shared blocks and could be smaller in size than the original file range. Therefore, the 'incomplete' flag was set on the original range in common.py; and we should switch to use the original range also during validation. I also checked another flag usage in CanUseImgdiff(), and it has explicitly rejected cases of shared blocks. Bug: 124868891 Test: unit tests pass Change-Id: I03959625d7b81fd83420db98f01d23f54064bcd2 --- .../test_validate_target_files.py | 67 +++++++++++++++++-- tools/releasetools/validate_target_files.py | 10 +-- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/tools/releasetools/test_validate_target_files.py b/tools/releasetools/test_validate_target_files.py index a6a8876c25..5f619ec79a 100644 --- a/tools/releasetools/test_validate_target_files.py +++ b/tools/releasetools/test_validate_target_files.py @@ -19,11 +19,13 @@ import os import os.path import shutil +import zipfile import common import test_utils -import verity_utils -from validate_target_files import ValidateVerifiedBootImages +from rangelib import RangeSet +from validate_target_files import (ValidateVerifiedBootImages, + ValidateFileConsistency) from verity_utils import CreateVerityImageBuilder @@ -107,7 +109,8 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase): AssertionError, ValidateVerifiedBootImages, input_tmp, info_dict, options) - def _generate_system_image(self, output_file): + def _generate_system_image(self, output_file, system_root=None, + file_map=None): prop_dict = { 'partition_size': str(1024 * 1024), 'verity': 'true', @@ -120,9 +123,12 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase): image_size = verity_image_builder.CalculateMaxImageSize() # Use an empty root directory. - system_root = common.MakeTempDir() + if not system_root: + system_root = common.MakeTempDir() cmd = ['mkuserimg_mke2fs', '-s', system_root, output_file, 'ext4', '/system', str(image_size), '-j', '0'] + if file_map: + cmd.extend(['-B', file_map]) proc = common.Run(cmd) stdoutdata, _ = proc.communicate() self.assertEqual( @@ -155,3 +161,56 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase): 'verity_key_mincrypt' : verity_key_mincrypt, } ValidateVerifiedBootImages(input_tmp, info_dict, options) + + def test_ValidateFileConsistency_incompleteRange(self): + input_tmp = common.MakeTempDir() + os.mkdir(os.path.join(input_tmp, 'IMAGES')) + system_image = os.path.join(input_tmp, 'IMAGES', 'system.img') + system_root = os.path.join(input_tmp, "SYSTEM") + os.mkdir(system_root) + + # Write the test file that contain multiple blocks of zeros, and these + # zero blocks will be omitted by kernel. And the test files will occupy one + # block range each in the final system image. + with open(os.path.join(system_root, 'a'), 'w') as f: + f.write("aaa") + f.write('\0' * 4096 * 3) + with open(os.path.join(system_root, 'b'), 'w') as f: + f.write("bbb") + f.write('\0' * 4096 * 3) + + raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map') + self._generate_system_image(system_image, system_root, raw_file_map) + + # Parse the generated file map and update the block ranges for each file. + file_map_list = {} + image_ranges = RangeSet() + with open(raw_file_map, 'r') as f: + for line in f.readlines(): + info = line.split() + self.assertEqual(2, len(info)) + image_ranges = image_ranges.union(RangeSet(info[1])) + file_map_list[info[0]] = RangeSet(info[1]) + + # Add one unoccupied block as the shared block for all test files. + mock_shared_block = RangeSet("10-20").subtract(image_ranges).first(1) + with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f: + for key in sorted(file_map_list.keys()): + line = "{} {}\n".format( + key, file_map_list[key].union(mock_shared_block)) + f.write(line) + + # Prepare for the target zip file + input_file = common.MakeTempFile() + all_entries = ['SYSTEM/', 'SYSTEM/b', 'SYSTEM/a', 'IMAGES/', + 'IMAGES/system.map', 'IMAGES/system.img'] + with zipfile.ZipFile(input_file, 'w') as input_zip: + for name in all_entries: + input_zip.write(os.path.join(input_tmp, name), arcname=name) + + input_zip = zipfile.ZipFile(input_file, 'r') + info_dict = {'extfs_sparse_flag': '-s'} + + # Expect the validation to pass and both files are skipped due to + # 'incomplete' block range. + ValidateFileConsistency(input_zip, input_tmp, info_dict) diff --git a/tools/releasetools/validate_target_files.py b/tools/releasetools/validate_target_files.py index eeb802bc40..275939c94d 100755 --- a/tools/releasetools/validate_target_files.py +++ b/tools/releasetools/validate_target_files.py @@ -84,11 +84,6 @@ def ValidateFileConsistency(input_zip, input_tmp, info_dict): # bytes past the file length, which is expected to be padded with '\0's. ranges = image.file_map[entry] - incomplete = ranges.extra.get('incomplete', False) - if incomplete: - logging.warning('Skipping %s that has incomplete block list', entry) - continue - # Use the original RangeSet if applicable, which includes the shared # blocks. And this needs to happen before checking the monotonicity flag. if ranges.extra.get('uses_shared_blocks'): @@ -96,6 +91,11 @@ def ValidateFileConsistency(input_zip, input_tmp, info_dict): else: file_ranges = ranges + incomplete = file_ranges.extra.get('incomplete', False) + if incomplete: + logging.warning('Skipping %s that has incomplete block list', entry) + continue + # TODO(b/79951650): Handle files with non-monotonic ranges. if not file_ranges.monotonic: logging.warning(