StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use AVB footer to determine caremap

Browse Source 
Care maps need to store the original image size, which excludes bytes
used by hash tree or FEC code.

We used to propagate original image size using the global OPTIONS
dictionary. This is bad coding practice, and also fragile because we
have to make sure what's stored in dictionary and what's on disk are
consistent. Instead, let's read the content of images on disk, and parse
the AVB footer. The AVB footer contains the ground truth original image
size.

Test: build OTA, make sure the care maps have valid range
Bug: 246504616
Change-Id: I9250b478ab34dda60578a6b8c23ae6d7a9385788
This commit is contained in:
Kelvin Zhang
2022-09-23 10:06:03 -07:00
parent c6ea3aa6e0
commit 1caead09f9
6 changed files with 98 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tools/releasetools/Android.bp
View File

@@ -37,6 +37,7 @@ python_defaults {
"releasetools_build_image",
"releasetools_build_super_image",
"releasetools_common",
"libavbtool",
],
required: [
"care_map_generator",

50
tools/releasetools/add_img_to_target_files.py
View File

@@ -46,6 +46,7 @@ Usage: add_img_to_target_files [flag] target_files
from __future__ import print_function
import avbtool
import datetime
import logging
import os
@@ -62,12 +63,12 @@ import build_super_image
import common
import verity_utils
import ota_metadata_pb2
from apex_utils import GetApexInfoFromTargetFiles
from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, MakeTempFile, ZipWrite
import rangelib
import sparse_img
from apex_utils import GetApexInfoFromTargetFiles
from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, IsSparseImage, MakeTempFile, ZipWrite
if sys.hexversion < 0x02070000:
print("Python 2.7 or newer is required.", file=sys.stderr)
sys.exit(1)
@@ -87,6 +88,13 @@ FIXED_FILE_TIMESTAMP = int((
datetime.datetime.utcfromtimestamp(0)).total_seconds())
def ParseAvbFooter(img_path) -> avbtool.AvbFooter:
with open(img_path, 'rb') as fp:
fp.seek(-avbtool.AvbFooter.SIZE, os.SEEK_END)
data = fp.read(avbtool.AvbFooter.SIZE)
return avbtool.AvbFooter(data)
def GetCareMap(which, imgname):
"""Returns the care_map string for the given partition.
@@ -100,15 +108,35 @@ def GetCareMap(which, imgname):
"""
assert which in PARTITIONS_WITH_CARE_MAP
# which + "_image_size" contains the size that the actual filesystem image
# resides in, which is all that needs to be verified. The additional blocks in
# the image file contain verity metadata, by reading which would trigger
# invalid reads.
image_size = OPTIONS.info_dict.get(which + "_image_size")
if not image_size:
is_sparse_img = IsSparseImage(imgname)
unsparsed_image_size = os.path.getsize(imgname)
# A verified image contains original image + hash tree data + FEC data
# + AVB footer, all concatenated together. The caremap specifies a range
# of blocks that update_verifier should read on top of dm-verity device
# to verify correctness of OTA updates. When reading off of dm-verity device,
# the hashtree and FEC part of image isn't available. So caremap should
# only contain the original image blocks.
try:
avbfooter = None
if is_sparse_img:
with tempfile.NamedTemporaryFile() as tmpfile:
img = sparse_img.SparseImage(imgname)
unsparsed_image_size = img.total_blocks * img.blocksize
for data in img.ReadBlocks(img.total_blocks - 1, 1):
tmpfile.write(data)
tmpfile.flush()
avbfooter = ParseAvbFooter(tmpfile.name)
else:
avbfooter = ParseAvbFooter(imgname)
except LookupError as e:
logger.warning(
"Failed to parse avbfooter for partition %s image %s, %s", which, imgname, e)
return None
disable_sparse = OPTIONS.info_dict.get(which + "_disable_sparse")
image_size = avbfooter.original_image_size
assert image_size < unsparsed_image_size, f"AVB footer's original image size {image_size} is larger than or equal to image size on disk {unsparsed_image_size}, this can't happen because a verified image = original image + hash tree data + FEC data + avbfooter."
assert image_size > 0
image_blocks = int(image_size) // 4096 - 1
# It's OK for image_blocks to be 0, because care map ranges are inclusive.
@@ -118,7 +146,7 @@ def GetCareMap(which, imgname):
# For sparse images, we will only check the blocks that are listed in the care
# map, i.e. the ones with meaningful data.
if "extfs_sparse_flag" in OPTIONS.info_dict and not disable_sparse:
if is_sparse_img:
simg = sparse_img.SparseImage(imgname)
care_map_ranges = simg.care_map.intersect(
rangelib.RangeSet("0-{}".format(image_blocks)))

15
tools/releasetools/common.py
View File

@@ -2865,7 +2865,7 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
zipfile.ZIP64_LIMIT = saved_zip64_limit
def ZipDelete(zip_filename, entries):
def ZipDelete(zip_filename, entries, force=False):
"""Deletes entries from a ZIP file.
Since deleting entries from a ZIP file is not supported, it shells out to
@@ -2883,8 +2883,15 @@ def ZipDelete(zip_filename, entries):
# If list is empty, nothing to do
if not entries:
return
cmd = ["zip", "-d", zip_filename] + entries
RunAndCheckOutput(cmd)
if force:
cmd = ["zip", "-q", "-d", zip_filename] + entries
else:
cmd = ["zip", "-d", zip_filename] + entries
if force:
p = Run(cmd)
p.wait()
else:
RunAndCheckOutput(cmd)
def ZipClose(zip_file):
@@ -3979,6 +3986,8 @@ def GetBootImageTimestamp(boot_img):
def IsSparseImage(filepath):
if not os.path.exists(filepath):
return False
with open(filepath, 'rb') as fp:
# Magic for android sparse image format
# https://source.android.com/devices/bootloader/images

7
tools/releasetools/sparse_img.py
View File

@@ -80,7 +80,7 @@ class SparseImage(object):
self.offset_map = offset_map = []
self.clobbered_blocks = rangelib.RangeSet(data=clobbered_blocks)
for i in range(total_chunks):
for _ in range(total_chunks):
header_bin = f.read(12)
header = struct.unpack("<2H2I", header_bin)
chunk_type = header[0]
@@ -166,6 +166,11 @@ class SparseImage(object):
def ReadRangeSet(self, ranges):
return [d for d in self._GetRangeData(ranges)]
def ReadBlocks(self, start=0, num_blocks=None):
if num_blocks is None:
num_blocks = self.total_blocks
return self._GetRangeData([(start, start + num_blocks)])
def TotalSha1(self, include_clobbered_blocks=False):
"""Return the SHA-1 hash of all data in the 'care' regions.

60
tools/releasetools/test_add_img_to_target_files.py
View File

@@ -16,6 +16,7 @@
import os
import os.path
import tempfile
import zipfile
import common
@@ -124,9 +125,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
def _test_AddCareMapForAbOta():
"""Helper function to set up the test for test_AddCareMapForAbOta()."""
OPTIONS.info_dict = {
'extfs_sparse_flag': '-s',
'system_image_size': 65536,
'vendor_image_size': 40960,
'system_verity_block_device': '/dev/block/system',
'vendor_verity_block_device': '/dev/block/vendor',
'system.build.prop': common.PartitionBuildProps.FromDictionary(
@@ -149,9 +147,9 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
system_image = test_utils.construct_sparse_image([
(0xCAC1, 6),
(0xCAC3, 4),
(0xCAC1, 8)])
(0xCAC1, 6)], "system")
vendor_image = test_utils.construct_sparse_image([
(0xCAC2, 12)])
(0xCAC2, 10)], "vendor")
image_paths = {
'system': system_image,
@@ -210,9 +208,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
"""Tests the case for device using AVB."""
image_paths = self._test_AddCareMapForAbOta()
OPTIONS.info_dict = {
'extfs_sparse_flag': '-s',
'system_image_size': 65536,
'vendor_image_size': 40960,
'avb_system_hashtree_enable': 'true',
'avb_vendor_hashtree_enable': 'true',
'system.build.prop': common.PartitionBuildProps.FromDictionary(
@@ -244,9 +239,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
"""Tests the case for partitions without fingerprint."""
image_paths = self._test_AddCareMapForAbOta()
OPTIONS.info_dict = {
'extfs_sparse_flag': '-s',
'system_image_size': 65536,
'vendor_image_size': 40960,
'system_verity_block_device': '/dev/block/system',
'vendor_verity_block_device': '/dev/block/vendor',
}
@@ -266,9 +258,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
"""Tests the case for partitions with thumbprint."""
image_paths = self._test_AddCareMapForAbOta()
OPTIONS.info_dict = {
'extfs_sparse_flag': '-s',
'system_image_size': 65536,
'vendor_image_size': 40960,
'system_verity_block_device': '/dev/block/system',
'vendor_verity_block_device': '/dev/block/vendor',
'system.build.prop': common.PartitionBuildProps.FromDictionary(
@@ -298,9 +287,7 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
@test_utils.SkipIfExternalToolsUnavailable()
def test_AddCareMapForAbOta_skipPartition(self):
image_paths = self._test_AddCareMapForAbOta()
# Remove vendor_image_size to invalidate the care_map for vendor.img.
del OPTIONS.info_dict['vendor_image_size']
test_utils.erase_avb_footer(image_paths["vendor"])
care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')
AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)
@@ -314,10 +301,8 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
@test_utils.SkipIfExternalToolsUnavailable()
def test_AddCareMapForAbOta_skipAllPartitions(self):
image_paths = self._test_AddCareMapForAbOta()
# Remove the image_size properties for all the partitions.
del OPTIONS.info_dict['system_image_size']
del OPTIONS.info_dict['vendor_image_size']
test_utils.erase_avb_footer(image_paths["system"])
test_utils.erase_avb_footer(image_paths["vendor"])
care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')
AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)
@@ -396,35 +381,18 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
sparse_image = test_utils.construct_sparse_image([
(0xCAC1, 6),
(0xCAC3, 4),
(0xCAC1, 6)])
OPTIONS.info_dict = {
'extfs_sparse_flag': '-s',
'system_image_size': 53248,
}
(0xCAC1, 6)], "system")
name, care_map = GetCareMap('system', sparse_image)
self.assertEqual('system', name)
self.assertEqual(RangeSet("0-5 10-12").to_string_raw(), care_map)
self.assertEqual(RangeSet("0-5 10-15").to_string_raw(), care_map)
def test_GetCareMap_invalidPartition(self):
self.assertRaises(AssertionError, GetCareMap, 'oem', None)
def test_GetCareMap_invalidAdjustedPartitionSize(self):
sparse_image = test_utils.construct_sparse_image([
(0xCAC1, 6),
(0xCAC3, 4),
(0xCAC1, 6)])
OPTIONS.info_dict = {
'extfs_sparse_flag': '-s',
'system_image_size': -45056,
}
self.assertRaises(AssertionError, GetCareMap, 'system', sparse_image)
def test_GetCareMap_nonSparseImage(self):
OPTIONS.info_dict = {
'system_image_size': 53248,
}
# 'foo' is the image filename, which is expected to be not used by
# GetCareMap().
name, care_map = GetCareMap('system', 'foo')
self.assertEqual('system', name)
self.assertEqual(RangeSet("0-12").to_string_raw(), care_map)
with tempfile.NamedTemporaryFile() as tmpfile:
tmpfile.truncate(4096 * 13)
test_utils.append_avb_footer(tmpfile.name, "system")
name, care_map = GetCareMap('system', tmpfile.name)
self.assertEqual('system', name)
self.assertEqual(RangeSet("0-12").to_string_raw(), care_map)

28
tools/releasetools/test_utils.py
View File

@@ -19,6 +19,7 @@
Utils for running unittests.
"""
import avbtool
import logging
import os
import os.path
@@ -57,12 +58,14 @@ def get_testdata_dir():
current_dir = os.path.dirname(os.path.realpath(__file__))
return os.path.join(current_dir, 'testdata')
def get_current_dir():
"""Returns the current dir, relative to the script dir."""
# The script dir is the one we want, which could be different from pwd.
current_dir = os.path.dirname(os.path.realpath(__file__))
return current_dir
def get_search_path():
"""Returns the search path that has 'framework/signapk.jar' under."""
@@ -83,14 +86,33 @@ def get_search_path():
# In relative to 'build/make/tools/releasetools' in the Android source.
['..'] * 4 + ['out', 'host', 'linux-x86'],
# Or running the script unpacked from otatools.zip.
['..']):
['..']):
full_path = os.path.realpath(os.path.join(current_dir, *path))
if signapk_exists(full_path):
return full_path
return None
def construct_sparse_image(chunks):
def append_avb_footer(file_path: str, partition_name: str = ""):
avb = avbtool.AvbTool()
try:
args = ["avbtool", "add_hashtree_footer", "--image", file_path,
"--partition_name", partition_name, "--do_not_generate_fec"]
avb.run(args)
except SystemExit:
raise ValueError(f"Failed to append hashtree footer {args}")
def erase_avb_footer(file_path: str):
avb = avbtool.AvbTool()
try:
args = ["avbtool", "erase_footer", "--image", file_path]
avb.run(args)
except SystemExit:
raise ValueError(f"Failed to erase hashtree footer {args}")
def construct_sparse_image(chunks, partition_name: str = ""):
"""Returns a sparse image file constructed from the given chunks.
From system/core/libsparse/sparse_format.h.
@@ -151,6 +173,7 @@ def construct_sparse_image(chunks):
if data_size != 0:
fp.write(os.urandom(data_size))
append_avb_footer(sparse_image, partition_name)
return sparse_image
@@ -201,6 +224,7 @@ class ReleaseToolsTestCase(unittest.TestCase):
def tearDown(self):
common.Cleanup()
class PropertyFilesTestCase(ReleaseToolsTestCase):
@staticmethod