diff --git a/tools/releasetools/blockimgdiff.py b/tools/releasetools/blockimgdiff.py index be5b32a591..24c5b2de7f 100644 --- a/tools/releasetools/blockimgdiff.py +++ b/tools/releasetools/blockimgdiff.py @@ -272,6 +272,7 @@ class ImgdiffStats(object): # Reasons for not applying imgdiff on APKs. SKIPPED_TRIMMED = "Not used imgdiff due to trimmed RangeSet" SKIPPED_NONMONOTONIC = "Not used imgdiff due to having non-monotonic ranges" + SKIPPED_SHARED_BLOCKS = "Not used imgdiff due to using shared blocks" SKIPPED_INCOMPLETE = "Not used imgdiff due to incomplete RangeSet" # The list of valid reasons, which will also be the dumped order in a report. @@ -280,6 +281,7 @@ class ImgdiffStats(object): USED_IMGDIFF_LARGE_APK, SKIPPED_TRIMMED, SKIPPED_NONMONOTONIC, + SKIPPED_SHARED_BLOCKS, SKIPPED_INCOMPLETE, ) @@ -414,6 +416,7 @@ class BlockImageDiff(object): - The file type is supported by imgdiff; - The source and target blocks are monotonic (i.e. the data is stored with blocks in increasing order); + - Both files don't contain shared blocks; - Both files have complete lists of blocks; - We haven't removed any blocks from the source set. @@ -437,6 +440,11 @@ class BlockImageDiff(object): self.imgdiff_stats.Log(name, ImgdiffStats.SKIPPED_NONMONOTONIC) return False + if (tgt_ranges.extra.get('uses_shared_blocks') or + src_ranges.extra.get('uses_shared_blocks')): + self.imgdiff_stats.Log(name, ImgdiffStats.SKIPPED_SHARED_BLOCKS) + return False + if tgt_ranges.extra.get('incomplete') or src_ranges.extra.get('incomplete'): self.imgdiff_stats.Log(name, ImgdiffStats.SKIPPED_INCOMPLETE) return False diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 16600ed1a6..370710ecaf 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -625,7 +625,7 @@ def UnzipTemp(filename, pattern=None): return tmp, zipfile.ZipFile(filename, "r") -def GetSparseImage(which, tmpdir, input_zip): +def GetSparseImage(which, tmpdir, input_zip, allow_shared_blocks): """Returns a SparseImage object suitable for passing to BlockImageDiff. This function loads the specified sparse image from the given path, and @@ -637,6 +637,7 @@ def GetSparseImage(which, tmpdir, input_zip): which: The partition name, which must be "system" or "vendor". tmpdir: The directory that contains the prebuilt image and block map file. input_zip: The target-files ZIP archive. + allow_shared_blocks: Whether having shared blocks is allowed. Returns: A SparseImage object, with file_map info loaded. @@ -655,7 +656,8 @@ def GetSparseImage(which, tmpdir, input_zip): # unconditionally. Note that they are still part of care_map. (Bug: 20939131) clobbered_blocks = "0" - image = sparse_img.SparseImage(path, mappath, clobbered_blocks) + image = sparse_img.SparseImage(path, mappath, clobbered_blocks, + allow_shared_blocks=allow_shared_blocks) # block.map may contain less blocks, because mke2fs may skip allocating blocks # if they contain all zeros. We can't reconstruct such a file from its block @@ -669,6 +671,13 @@ def GetSparseImage(which, tmpdir, input_zip): info = input_zip.getinfo(arcname) ranges = image.file_map[entry] + + # If a RangeSet has been tagged as using shared blocks while loading the + # image, its block list must be already incomplete due to that reason. Don't + # give it 'incomplete' tag to avoid messing up the imgdiff stats. + if ranges.extra.get('uses_shared_blocks'): + continue + if RoundUpTo4K(info.file_size) > ranges.size() * 4096: ranges.extra['incomplete'] = True diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index b5c8392bad..a22145a566 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -786,11 +786,15 @@ else if get_stage("%(bcb_dev)s") == "3/3" then script.ShowProgress(system_progress, 0) + # See the notes in WriteBlockIncrementalOTAPackage(). + allow_shared_blocks = target_info.get('ext4_share_dup_blocks') == "true" + # Full OTA is done as an "incremental" against an empty source image. This # has the effect of writing new data from the package to the entire # partition, but lets us reuse the updater code that writes incrementals to # do it. - system_tgt = common.GetSparseImage("system", OPTIONS.input_tmp, input_zip) + system_tgt = common.GetSparseImage("system", OPTIONS.input_tmp, input_zip, + allow_shared_blocks) system_tgt.ResetFileMap() system_diff = common.BlockDifference("system", system_tgt, src=None) system_diff.WriteScript(script, output_zip) @@ -801,7 +805,8 @@ else if get_stage("%(bcb_dev)s") == "3/3" then if HasVendorPartition(input_zip): script.ShowProgress(0.1, 0) - vendor_tgt = common.GetSparseImage("vendor", OPTIONS.input_tmp, input_zip) + vendor_tgt = common.GetSparseImage("vendor", OPTIONS.input_tmp, input_zip, + allow_shared_blocks) vendor_tgt.ResetFileMap() vendor_diff = common.BlockDifference("vendor", vendor_tgt) vendor_diff.WriteScript(script, output_zip) @@ -978,8 +983,16 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): target_recovery = common.GetBootableImage( "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") - system_src = common.GetSparseImage("system", OPTIONS.source_tmp, source_zip) - system_tgt = common.GetSparseImage("system", OPTIONS.target_tmp, target_zip) + # When target uses 'BOARD_EXT4_SHARE_DUP_BLOCKS := true', images may contain + # shared blocks (i.e. some blocks will show up in multiple files' block + # list). We can only allocate such shared blocks to the first "owner", and + # disable imgdiff for all later occurrences. + allow_shared_blocks = (source_info.get('ext4_share_dup_blocks') == "true" or + target_info.get('ext4_share_dup_blocks') == "true") + system_src = common.GetSparseImage("system", OPTIONS.source_tmp, source_zip, + allow_shared_blocks) + system_tgt = common.GetSparseImage("system", OPTIONS.target_tmp, target_zip, + allow_shared_blocks) blockimgdiff_version = max( int(i) for i in target_info.get("blockimgdiff_versions", "1").split(",")) @@ -1004,8 +1017,10 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): if HasVendorPartition(target_zip): if not HasVendorPartition(source_zip): raise RuntimeError("can't generate incremental that adds /vendor") - vendor_src = common.GetSparseImage("vendor", OPTIONS.source_tmp, source_zip) - vendor_tgt = common.GetSparseImage("vendor", OPTIONS.target_tmp, target_zip) + vendor_src = common.GetSparseImage("vendor", OPTIONS.source_tmp, source_zip, + allow_shared_blocks) + vendor_tgt = common.GetSparseImage("vendor", OPTIONS.target_tmp, target_zip, + allow_shared_blocks) # Check first block of vendor partition for remount R/W only if # disk type is ext4 diff --git a/tools/releasetools/sparse_img.py b/tools/releasetools/sparse_img.py index c978be8f53..083da7a490 100644 --- a/tools/releasetools/sparse_img.py +++ b/tools/releasetools/sparse_img.py @@ -33,7 +33,7 @@ class SparseImage(object): """ def __init__(self, simg_fn, file_map_fn=None, clobbered_blocks=None, - mode="rb", build_map=True): + mode="rb", build_map=True, allow_shared_blocks=False): self.simg_f = f = open(simg_fn, mode) header_bin = f.read(28) @@ -129,7 +129,8 @@ class SparseImage(object): self.extended = extended if file_map_fn: - self.LoadFileBlockMap(file_map_fn, self.clobbered_blocks) + self.LoadFileBlockMap(file_map_fn, self.clobbered_blocks, + allow_shared_blocks) else: self.file_map = {"__DATA": self.care_map} @@ -209,7 +210,14 @@ class SparseImage(object): yield fill_data * (this_read * (self.blocksize >> 2)) to_read -= this_read - def LoadFileBlockMap(self, fn, clobbered_blocks): + def LoadFileBlockMap(self, fn, clobbered_blocks, allow_shared_blocks): + """Loads the given block map file. + + Args: + fn: The filename of the block map file. + clobbered_blocks: A RangeSet instance for the clobbered blocks. + allow_shared_blocks: Whether having shared blocks is allowed. + """ remaining = self.care_map self.file_map = out = {} @@ -217,6 +225,18 @@ class SparseImage(object): for line in f: fn, ranges = line.split(None, 1) ranges = rangelib.RangeSet.parse(ranges) + + if allow_shared_blocks: + # Find the shared blocks that have been claimed by others. + shared_blocks = ranges.subtract(remaining) + if shared_blocks: + ranges = ranges.subtract(shared_blocks) + if not ranges: + continue + + # Tag the entry so that we can skip applying imgdiff on this file. + ranges.extra['uses_shared_blocks'] = True + out[fn] = ranges assert ranges.size() == ranges.intersect(remaining).size()