From 1caead09f94c2fefa2eea6df5168f338d217709e Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Fri, 23 Sep 2022 10:06:03 -0700 Subject: [PATCH] Use AVB footer to determine caremap Care maps need to store the original image size, which excludes bytes used by hash tree or FEC code. We used to propagate original image size using the global OPTIONS dictionary. This is bad coding practice, and also fragile because we have to make sure what's stored in dictionary and what's on disk are consistent. Instead, let's read the content of images on disk, and parse the AVB footer. The AVB footer contains the ground truth original image size. Test: build OTA, make sure the care maps have valid range Bug: 246504616 Change-Id: I9250b478ab34dda60578a6b8c23ae6d7a9385788 --- tools/releasetools/Android.bp | 1 + tools/releasetools/add_img_to_target_files.py | 50 ++++++++++++---- tools/releasetools/common.py | 15 ++++- tools/releasetools/sparse_img.py | 7 ++- .../test_add_img_to_target_files.py | 60 +++++-------------- tools/releasetools/test_utils.py | 28 ++++++++- 6 files changed, 98 insertions(+), 63 deletions(-) diff --git a/tools/releasetools/Android.bp b/tools/releasetools/Android.bp index 8c91470bbd..4fdc7074bb 100644 --- a/tools/releasetools/Android.bp +++ b/tools/releasetools/Android.bp @@ -37,6 +37,7 @@ python_defaults { "releasetools_build_image", "releasetools_build_super_image", "releasetools_common", + "libavbtool", ], required: [ "care_map_generator", diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py index b1772663a9..465e1a8f9e 100644 --- a/tools/releasetools/add_img_to_target_files.py +++ b/tools/releasetools/add_img_to_target_files.py @@ -46,6 +46,7 @@ Usage: add_img_to_target_files [flag] target_files from __future__ import print_function +import avbtool import datetime import logging import os @@ -62,12 +63,12 @@ import build_super_image import common import verity_utils import ota_metadata_pb2 - -from apex_utils import GetApexInfoFromTargetFiles -from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, MakeTempFile, ZipWrite import rangelib import sparse_img +from apex_utils import GetApexInfoFromTargetFiles +from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, IsSparseImage, MakeTempFile, ZipWrite + if sys.hexversion < 0x02070000: print("Python 2.7 or newer is required.", file=sys.stderr) sys.exit(1) @@ -87,6 +88,13 @@ FIXED_FILE_TIMESTAMP = int(( datetime.datetime.utcfromtimestamp(0)).total_seconds()) +def ParseAvbFooter(img_path) -> avbtool.AvbFooter: + with open(img_path, 'rb') as fp: + fp.seek(-avbtool.AvbFooter.SIZE, os.SEEK_END) + data = fp.read(avbtool.AvbFooter.SIZE) + return avbtool.AvbFooter(data) + + def GetCareMap(which, imgname): """Returns the care_map string for the given partition. @@ -100,15 +108,35 @@ def GetCareMap(which, imgname): """ assert which in PARTITIONS_WITH_CARE_MAP - # which + "_image_size" contains the size that the actual filesystem image - # resides in, which is all that needs to be verified. The additional blocks in - # the image file contain verity metadata, by reading which would trigger - # invalid reads. - image_size = OPTIONS.info_dict.get(which + "_image_size") - if not image_size: + is_sparse_img = IsSparseImage(imgname) + unsparsed_image_size = os.path.getsize(imgname) + + # A verified image contains original image + hash tree data + FEC data + # + AVB footer, all concatenated together. The caremap specifies a range + # of blocks that update_verifier should read on top of dm-verity device + # to verify correctness of OTA updates. When reading off of dm-verity device, + # the hashtree and FEC part of image isn't available. So caremap should + # only contain the original image blocks. + try: + avbfooter = None + if is_sparse_img: + with tempfile.NamedTemporaryFile() as tmpfile: + img = sparse_img.SparseImage(imgname) + unsparsed_image_size = img.total_blocks * img.blocksize + for data in img.ReadBlocks(img.total_blocks - 1, 1): + tmpfile.write(data) + tmpfile.flush() + avbfooter = ParseAvbFooter(tmpfile.name) + else: + avbfooter = ParseAvbFooter(imgname) + except LookupError as e: + logger.warning( + "Failed to parse avbfooter for partition %s image %s, %s", which, imgname, e) return None - disable_sparse = OPTIONS.info_dict.get(which + "_disable_sparse") + image_size = avbfooter.original_image_size + assert image_size < unsparsed_image_size, f"AVB footer's original image size {image_size} is larger than or equal to image size on disk {unsparsed_image_size}, this can't happen because a verified image = original image + hash tree data + FEC data + avbfooter." + assert image_size > 0 image_blocks = int(image_size) // 4096 - 1 # It's OK for image_blocks to be 0, because care map ranges are inclusive. @@ -118,7 +146,7 @@ def GetCareMap(which, imgname): # For sparse images, we will only check the blocks that are listed in the care # map, i.e. the ones with meaningful data. - if "extfs_sparse_flag" in OPTIONS.info_dict and not disable_sparse: + if is_sparse_img: simg = sparse_img.SparseImage(imgname) care_map_ranges = simg.care_map.intersect( rangelib.RangeSet("0-{}".format(image_blocks))) diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index cc75d2c9cd..bbdff6e50b 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -2865,7 +2865,7 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None, zipfile.ZIP64_LIMIT = saved_zip64_limit -def ZipDelete(zip_filename, entries): +def ZipDelete(zip_filename, entries, force=False): """Deletes entries from a ZIP file. Since deleting entries from a ZIP file is not supported, it shells out to @@ -2883,8 +2883,15 @@ def ZipDelete(zip_filename, entries): # If list is empty, nothing to do if not entries: return - cmd = ["zip", "-d", zip_filename] + entries - RunAndCheckOutput(cmd) + if force: + cmd = ["zip", "-q", "-d", zip_filename] + entries + else: + cmd = ["zip", "-d", zip_filename] + entries + if force: + p = Run(cmd) + p.wait() + else: + RunAndCheckOutput(cmd) def ZipClose(zip_file): @@ -3979,6 +3986,8 @@ def GetBootImageTimestamp(boot_img): def IsSparseImage(filepath): + if not os.path.exists(filepath): + return False with open(filepath, 'rb') as fp: # Magic for android sparse image format # https://source.android.com/devices/bootloader/images diff --git a/tools/releasetools/sparse_img.py b/tools/releasetools/sparse_img.py index e824a6410c..a2f7e9ed7d 100644 --- a/tools/releasetools/sparse_img.py +++ b/tools/releasetools/sparse_img.py @@ -80,7 +80,7 @@ class SparseImage(object): self.offset_map = offset_map = [] self.clobbered_blocks = rangelib.RangeSet(data=clobbered_blocks) - for i in range(total_chunks): + for _ in range(total_chunks): header_bin = f.read(12) header = struct.unpack("<2H2I", header_bin) chunk_type = header[0] @@ -166,6 +166,11 @@ class SparseImage(object): def ReadRangeSet(self, ranges): return [d for d in self._GetRangeData(ranges)] + def ReadBlocks(self, start=0, num_blocks=None): + if num_blocks is None: + num_blocks = self.total_blocks + return self._GetRangeData([(start, start + num_blocks)]) + def TotalSha1(self, include_clobbered_blocks=False): """Return the SHA-1 hash of all data in the 'care' regions. diff --git a/tools/releasetools/test_add_img_to_target_files.py b/tools/releasetools/test_add_img_to_target_files.py index 5077b38150..7b5476db3e 100644 --- a/tools/releasetools/test_add_img_to_target_files.py +++ b/tools/releasetools/test_add_img_to_target_files.py @@ -16,6 +16,7 @@ import os import os.path +import tempfile import zipfile import common @@ -124,9 +125,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): def _test_AddCareMapForAbOta(): """Helper function to set up the test for test_AddCareMapForAbOta().""" OPTIONS.info_dict = { - 'extfs_sparse_flag': '-s', - 'system_image_size': 65536, - 'vendor_image_size': 40960, 'system_verity_block_device': '/dev/block/system', 'vendor_verity_block_device': '/dev/block/vendor', 'system.build.prop': common.PartitionBuildProps.FromDictionary( @@ -149,9 +147,9 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): system_image = test_utils.construct_sparse_image([ (0xCAC1, 6), (0xCAC3, 4), - (0xCAC1, 8)]) + (0xCAC1, 6)], "system") vendor_image = test_utils.construct_sparse_image([ - (0xCAC2, 12)]) + (0xCAC2, 10)], "vendor") image_paths = { 'system': system_image, @@ -210,9 +208,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): """Tests the case for device using AVB.""" image_paths = self._test_AddCareMapForAbOta() OPTIONS.info_dict = { - 'extfs_sparse_flag': '-s', - 'system_image_size': 65536, - 'vendor_image_size': 40960, 'avb_system_hashtree_enable': 'true', 'avb_vendor_hashtree_enable': 'true', 'system.build.prop': common.PartitionBuildProps.FromDictionary( @@ -244,9 +239,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): """Tests the case for partitions without fingerprint.""" image_paths = self._test_AddCareMapForAbOta() OPTIONS.info_dict = { - 'extfs_sparse_flag': '-s', - 'system_image_size': 65536, - 'vendor_image_size': 40960, 'system_verity_block_device': '/dev/block/system', 'vendor_verity_block_device': '/dev/block/vendor', } @@ -266,9 +258,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): """Tests the case for partitions with thumbprint.""" image_paths = self._test_AddCareMapForAbOta() OPTIONS.info_dict = { - 'extfs_sparse_flag': '-s', - 'system_image_size': 65536, - 'vendor_image_size': 40960, 'system_verity_block_device': '/dev/block/system', 'vendor_verity_block_device': '/dev/block/vendor', 'system.build.prop': common.PartitionBuildProps.FromDictionary( @@ -298,9 +287,7 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): @test_utils.SkipIfExternalToolsUnavailable() def test_AddCareMapForAbOta_skipPartition(self): image_paths = self._test_AddCareMapForAbOta() - - # Remove vendor_image_size to invalidate the care_map for vendor.img. - del OPTIONS.info_dict['vendor_image_size'] + test_utils.erase_avb_footer(image_paths["vendor"]) care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb') AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths) @@ -314,10 +301,8 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): @test_utils.SkipIfExternalToolsUnavailable() def test_AddCareMapForAbOta_skipAllPartitions(self): image_paths = self._test_AddCareMapForAbOta() - - # Remove the image_size properties for all the partitions. - del OPTIONS.info_dict['system_image_size'] - del OPTIONS.info_dict['vendor_image_size'] + test_utils.erase_avb_footer(image_paths["system"]) + test_utils.erase_avb_footer(image_paths["vendor"]) care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb') AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths) @@ -396,35 +381,18 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): sparse_image = test_utils.construct_sparse_image([ (0xCAC1, 6), (0xCAC3, 4), - (0xCAC1, 6)]) - OPTIONS.info_dict = { - 'extfs_sparse_flag': '-s', - 'system_image_size': 53248, - } + (0xCAC1, 6)], "system") name, care_map = GetCareMap('system', sparse_image) self.assertEqual('system', name) - self.assertEqual(RangeSet("0-5 10-12").to_string_raw(), care_map) + self.assertEqual(RangeSet("0-5 10-15").to_string_raw(), care_map) def test_GetCareMap_invalidPartition(self): self.assertRaises(AssertionError, GetCareMap, 'oem', None) - def test_GetCareMap_invalidAdjustedPartitionSize(self): - sparse_image = test_utils.construct_sparse_image([ - (0xCAC1, 6), - (0xCAC3, 4), - (0xCAC1, 6)]) - OPTIONS.info_dict = { - 'extfs_sparse_flag': '-s', - 'system_image_size': -45056, - } - self.assertRaises(AssertionError, GetCareMap, 'system', sparse_image) - def test_GetCareMap_nonSparseImage(self): - OPTIONS.info_dict = { - 'system_image_size': 53248, - } - # 'foo' is the image filename, which is expected to be not used by - # GetCareMap(). - name, care_map = GetCareMap('system', 'foo') - self.assertEqual('system', name) - self.assertEqual(RangeSet("0-12").to_string_raw(), care_map) + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.truncate(4096 * 13) + test_utils.append_avb_footer(tmpfile.name, "system") + name, care_map = GetCareMap('system', tmpfile.name) + self.assertEqual('system', name) + self.assertEqual(RangeSet("0-12").to_string_raw(), care_map) diff --git a/tools/releasetools/test_utils.py b/tools/releasetools/test_utils.py index e30d2b98e3..5bbcf7f355 100755 --- a/tools/releasetools/test_utils.py +++ b/tools/releasetools/test_utils.py @@ -19,6 +19,7 @@ Utils for running unittests. """ +import avbtool import logging import os import os.path @@ -57,12 +58,14 @@ def get_testdata_dir(): current_dir = os.path.dirname(os.path.realpath(__file__)) return os.path.join(current_dir, 'testdata') + def get_current_dir(): """Returns the current dir, relative to the script dir.""" # The script dir is the one we want, which could be different from pwd. current_dir = os.path.dirname(os.path.realpath(__file__)) return current_dir + def get_search_path(): """Returns the search path that has 'framework/signapk.jar' under.""" @@ -83,14 +86,33 @@ def get_search_path(): # In relative to 'build/make/tools/releasetools' in the Android source. ['..'] * 4 + ['out', 'host', 'linux-x86'], # Or running the script unpacked from otatools.zip. - ['..']): + ['..']): full_path = os.path.realpath(os.path.join(current_dir, *path)) if signapk_exists(full_path): return full_path return None -def construct_sparse_image(chunks): +def append_avb_footer(file_path: str, partition_name: str = ""): + avb = avbtool.AvbTool() + try: + args = ["avbtool", "add_hashtree_footer", "--image", file_path, + "--partition_name", partition_name, "--do_not_generate_fec"] + avb.run(args) + except SystemExit: + raise ValueError(f"Failed to append hashtree footer {args}") + + +def erase_avb_footer(file_path: str): + avb = avbtool.AvbTool() + try: + args = ["avbtool", "erase_footer", "--image", file_path] + avb.run(args) + except SystemExit: + raise ValueError(f"Failed to erase hashtree footer {args}") + + +def construct_sparse_image(chunks, partition_name: str = ""): """Returns a sparse image file constructed from the given chunks. From system/core/libsparse/sparse_format.h. @@ -151,6 +173,7 @@ def construct_sparse_image(chunks): if data_size != 0: fp.write(os.urandom(data_size)) + append_avb_footer(sparse_image, partition_name) return sparse_image @@ -201,6 +224,7 @@ class ReleaseToolsTestCase(unittest.TestCase): def tearDown(self): common.Cleanup() + class PropertyFilesTestCase(ReleaseToolsTestCase): @staticmethod