From bd4301e19025814ffbf97e1bdc5cd7df0ddafba8 Mon Sep 17 00:00:00 2001 From: Yifan Hong Date: Wed, 3 Apr 2019 17:03:40 -0700 Subject: [PATCH 1/3] Revert "Fix dynamic partition size check for devices with recovery" Reason for revert: size check is removed for devices with dynamic partitions because it doesn't make sense. Bug: 122377935 Bug: 120043292 Bug: 124489494 Test: build cuttlefish This reverts commit accf09b2e0bf39c8ab8b006d11e341b4dc982787. Change-Id: I289faf11a08acbcef36924eb747a15f55124ce79 --- core/Makefile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/core/Makefile b/core/Makefile index 52aa0e80d8..2120c95808 100644 --- a/core/Makefile +++ b/core/Makefile @@ -2286,19 +2286,6 @@ endif # INSTALLED_BOOTIMAGE_TARGET $(INSTALLED_SYSTEMIMAGE_TARGET): $(BUILT_SYSTEMIMAGE) $(RECOVERY_FROM_BOOT_PATCH) @echo "Install system fs image: $@" $(copy-file-to-target) -ifdef RECOVERY_FROM_BOOT_PATCH -ifeq ($(PRODUCT_USE_DYNAMIC_PARTITION_SIZE),true) -ifeq ($(BOARD_SYSTEMIMAGE_PARTITION_SIZE),) - # system image size is dynamic, hence system_size in generated_system_image_info.txt does not - # have room for recovery from boot patch. Increase system_size so that check-all-partition-sizes - # accounts for the size of the patch. - sed -i'.bak' -e 's/^system_size=.*$$/system_size='"$$(( \ - $(call read-image-prop-dictionary,$(systemimage_intermediates)/generated_system_image_info.txt,system_size) + \ - $$($(call get-file-size,$(RECOVERY_FROM_BOOT_PATCH))) ))"'/' \ - $(systemimage_intermediates)/generated_system_image_info.txt -endif -endif -endif $(hide) $(call assert-max-image-size,$@ $(RECOVERY_FROM_BOOT_PATCH),\ $(call read-image-prop-dictionary,\ $(systemimage_intermediates)/generated_system_image_info.txt,system_size)) From 7ad83b634e8c70ed87b1c850683804a1128a4d6c Mon Sep 17 00:00:00 2001 From: Yifan Hong Date: Thu, 4 Apr 2019 10:57:39 -0700 Subject: [PATCH 2/3] sparse_img.py --get_partition_size return size of partition Also, move code from build_super_image.py to sparse_img.py. Test: sparse_img.py on sparse and non-sparse images Bug: 122377935 Change-Id: Ie91fdfdbb54298ea27eb20d1b5363aeb1470356e --- tools/releasetools/build_super_image.py | 10 +--------- tools/releasetools/sparse_img.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) mode change 100644 => 100755 tools/releasetools/sparse_img.py diff --git a/tools/releasetools/build_super_image.py b/tools/releasetools/build_super_image.py index 38ea3d68e4..0150ba15c3 100755 --- a/tools/releasetools/build_super_image.py +++ b/tools/releasetools/build_super_image.py @@ -58,16 +58,8 @@ logger = logging.getLogger(__name__) UNZIP_PATTERN = ["IMAGES/*", "META/*"] -def GetPartitionSizeFromImage(img): - try: - simg = sparse_img.SparseImage(img) - return simg.blocksize * simg.total_blocks - except ValueError: - return os.path.getsize(img) - - def GetArgumentsForImage(partition, group, image=None): - image_size = GetPartitionSizeFromImage(image) if image else 0 + image_size = sparse_img.GetImagePartitionSize(image) if image else 0 cmd = ["--partition", "{}:readonly:{}:{}".format(partition, image_size, group)] diff --git a/tools/releasetools/sparse_img.py b/tools/releasetools/sparse_img.py old mode 100644 new mode 100755 index 7919ba4a17..33678966e8 --- a/tools/releasetools/sparse_img.py +++ b/tools/releasetools/sparse_img.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python +# # Copyright (C) 2014 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import argparse import bisect import logging import os @@ -344,3 +349,21 @@ class SparseImage(object): """Throw away the file map and treat the entire image as undifferentiated data.""" self.file_map = {"__DATA": self.care_map} + + +def GetImagePartitionSize(img): + try: + simg = SparseImage(img, build_map=False) + return simg.blocksize * simg.total_blocks + except ValueError: + return os.path.getsize(img) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('image') + parser.add_argument('--get_partition_size', action='store_true', + help='Return partition size of the image') + args = parser.parse_args() + if args.get_partition_size: + print(GetImagePartitionSize(args.image)) From 67e4336912f145710819d413bf82823ca02c1ef8 Mon Sep 17 00:00:00 2001 From: Yifan Hong Date: Wed, 3 Apr 2019 17:18:08 -0700 Subject: [PATCH 3/3] Only assert-max-image-size for static partitions. assert-max-image-size doesn't make sense for dynamic partitions, as build_image.py always find the right size for the output image. Hence: - build_image.py no longer need to write generated_*_info.txt (which contains the size of the image). - assert-max-image-size on the static BOARD_*IMAGE_PARTITION_SIZE. If a partition is dynamic, that variable isn't set, and assert-max-image-size becomes a no-op. If the partition is static, assert-max-image-size checks the static partition size as it used to be. - Fix read-size-of-partitions to use the size of the partition by reading the image directly (instead of using generated_*_info.txt). For devices without AVB, with DAP enabled, and does not have RESERVED_SIZE for partitions, because of right sizing, the original code always warns about approaching size limits. Since such checks doesn't make sense for dynamic partitions, remove them. Test: builds on device with dynamic partitions Test: builds on cuttlefish with DAP enabled (without AVB), no more size limit warnings: WARNING: out/target/product/vsoc_x86/vendor.img approaching size limit (X now; limit X) Fixes: 122377935 Change-Id: I75e1b8322197cb18cf397d02aefd49d777bb6405 --- CleanSpec.mk | 2 ++ core/Makefile | 51 +++++++++++-------------------- tools/releasetools/build_image.py | 16 ++-------- 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/CleanSpec.mk b/CleanSpec.mk index 76bc0c5103..1c02da69f6 100644 --- a/CleanSpec.mk +++ b/CleanSpec.mk @@ -615,6 +615,8 @@ $(call add-clean-step, rm -rf $(HOST_OUT)/framework/vts-tradefed.jar) $(call add-clean-step, rm -rf $(PRODUCT_OUT)/system/etc/security/avb/) $(call add-clean-step, rm -rf $(PRODUCT_OUT)/super.img) + +$(call add-clean-step, find $(PRODUCT_OUT) -type f -name "generated_*_image_info.txt" -print0 | xargs -0 rm -f) # ************************************************ # NEWER CLEAN STEPS MUST BE AT THE END OF THE LIST # ************************************************ diff --git a/core/Makefile b/core/Makefile index 2120c95808..0a004dddb5 100644 --- a/core/Makefile +++ b/core/Makefile @@ -2238,7 +2238,6 @@ define build-systemimage-target $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \ build/make/tools/releasetools/build_image.py \ $(TARGET_OUT) $(systemimage_intermediates)/system_image_info.txt $(1) $(TARGET_OUT) \ - $(systemimage_intermediates)/generated_system_image_info.txt \ || ( mkdir -p $${DIST_DIR}; cp $(INSTALLED_FILES_FILE) $${DIST_DIR}/installed-files-rescued.txt; \ exit 1 ) endef @@ -2286,9 +2285,7 @@ endif # INSTALLED_BOOTIMAGE_TARGET $(INSTALLED_SYSTEMIMAGE_TARGET): $(BUILT_SYSTEMIMAGE) $(RECOVERY_FROM_BOOT_PATCH) @echo "Install system fs image: $@" $(copy-file-to-target) - $(hide) $(call assert-max-image-size,$@ $(RECOVERY_FROM_BOOT_PATCH),\ - $(call read-image-prop-dictionary,\ - $(systemimage_intermediates)/generated_system_image_info.txt,system_size)) + $(hide) $(call assert-max-image-size,$@ $(RECOVERY_FROM_BOOT_PATCH),$(BOARD_SYSTEMIMAGE_PARTITION_SIZE)) systemimage: $(INSTALLED_SYSTEMIMAGE_TARGET) @@ -2297,9 +2294,7 @@ systemimage-nodeps snod: $(filter-out systemimage-nodeps snod,$(MAKECMDGOALS)) \ | $(INTERNAL_USERIMAGES_DEPS) @echo "make $@: ignoring dependencies" $(call build-systemimage-target,$(INSTALLED_SYSTEMIMAGE_TARGET)) - $(hide) $(call assert-max-image-size,$(INSTALLED_SYSTEMIMAGE_TARGET),\ - $(call read-image-prop-dictionary,\ - $(systemimage_intermediates)/generated_system_image_info.txt,system_size)) + $(hide) $(call assert-max-image-size,$(INSTALLED_SYSTEMIMAGE_TARGET),$(BOARD_SYSTEMIMAGE_PARTITION_SIZE)) ifneq (,$(filter systemimage-nodeps snod, $(MAKECMDGOALS))) ifeq (true,$(WITH_DEXPREOPT)) @@ -2665,11 +2660,8 @@ define build-systemotherimage-target $(call generate-image-prop-dictionary, $(systemotherimage_intermediates)/system_other_image_info.txt,system,skip_fsck=true) $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \ build/make/tools/releasetools/build_image.py \ - $(TARGET_OUT_SYSTEM_OTHER) $(systemotherimage_intermediates)/system_other_image_info.txt $(INSTALLED_SYSTEMOTHERIMAGE_TARGET) $(TARGET_OUT)\ - $(systemotherimage_intermediates)/generated_system_other_image_info.txt - $(hide) $(call assert-max-image-size,$(INSTALLED_SYSTEMOTHERIMAGE_TARGET),\ - $(call read-image-prop-dictionary,\ - $(systemotherimage_intermediates)/generated_system_other_image_info.txt,system_other_size)) + $(TARGET_OUT_SYSTEM_OTHER) $(systemotherimage_intermediates)/system_other_image_info.txt $(INSTALLED_SYSTEMOTHERIMAGE_TARGET) $(TARGET_OUT) + $(hide) $(call assert-max-image-size,$(INSTALLED_SYSTEMOTHERIMAGE_TARGET),$(BOARD_SYSTEMIMAGE_PARTITION_SIZE)) endef # We just build this directly to the install location. @@ -2808,11 +2800,8 @@ define build-vendorimage-target $(call build-image-kernel-modules,$(BOARD_VENDOR_KERNEL_MODULES),$(TARGET_OUT_VENDOR),vendor/,$(call intermediates-dir-for,PACKAGING,depmod_vendor))) $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \ build/make/tools/releasetools/build_image.py \ - $(TARGET_OUT_VENDOR) $(vendorimage_intermediates)/vendor_image_info.txt $(INSTALLED_VENDORIMAGE_TARGET) $(TARGET_OUT) \ - $(vendorimage_intermediates)/generated_vendor_image_info.txt - $(hide) $(call assert-max-image-size,$(INSTALLED_VENDORIMAGE_TARGET),\ - $(call read-image-prop-dictionary,\ - $(vendorimage_intermediates)/generated_vendor_image_info.txt,vendor_size)) + $(TARGET_OUT_VENDOR) $(vendorimage_intermediates)/vendor_image_info.txt $(INSTALLED_VENDORIMAGE_TARGET) $(TARGET_OUT) + $(hide) $(call assert-max-image-size,$(INSTALLED_VENDORIMAGE_TARGET),$(BOARD_VENDORIMAGE_PARTITION_SIZE)) endef # We just build this directly to the install location. @@ -2866,11 +2855,8 @@ define build-productimage-target $(call generate-image-prop-dictionary, $(productimage_intermediates)/product_image_info.txt,product,skip_fsck=true) $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \ ./build/tools/releasetools/build_image.py \ - $(TARGET_OUT_PRODUCT) $(productimage_intermediates)/product_image_info.txt $(INSTALLED_PRODUCTIMAGE_TARGET) $(TARGET_OUT) \ - $(productimage_intermediates)/generated_product_image_info.txt - $(hide) $(call assert-max-image-size,$(INSTALLED_PRODUCTIMAGE_TARGET),\ - $(call read-image-prop-dictionary,\ - $(productimage_intermediates)/generated_product_image_info.txt,product_size)) + $(TARGET_OUT_PRODUCT) $(productimage_intermediates)/product_image_info.txt $(INSTALLED_PRODUCTIMAGE_TARGET) $(TARGET_OUT) + $(hide) $(call assert-max-image-size,$(INSTALLED_PRODUCTIMAGE_TARGET),$(BOARD_PRODUCTIMAGE_PARTITION_SIZE)) endef # We just build this directly to the install location. @@ -2921,11 +2907,8 @@ define build-product_servicesimage-target $(call generate-image-prop-dictionary, $(product_servicesimage_intermediates)/product_services_image_info.txt,product_services, skip_fsck=true) $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \ ./build/tools/releasetools/build_image.py \ - $(TARGET_OUT_PRODUCT_SERVICES) $(product_servicesimage_intermediates)/product_services_image_info.txt $(INSTALLED_PRODUCT_SERVICESIMAGE_TARGET) $(TARGET_OUT) \ - $(product_servicesimage_intermediates)/generated_product_services_image_info.txt - $(hide) $(call assert-max-image-size,$(INSTALLED_PRODUCT_SERVICESIMAGE_TARGET),\ - $(call read-image-prop-dictionary,\ - $(product_servicesimage_intermediates)/generated_product_services_image_info.txt,product_services_size)) + $(TARGET_OUT_PRODUCT_SERVICES) $(productservicesimage_intermediates)/productservices_image_info.txt $(INSTALLED_PRODUCT_SERVICESIMAGE_TARGET) $(TARGET_OUT) + $(hide) $(call assert-max-image-size,$(INSTALLED_PRODUCT_SERVICESIMAGE_TARGET),$(BOARD_PRODUCT_SERVICESIMAGE_PARTITION_SIZE)) endef # We just build this directly to the install location. @@ -2977,11 +2960,8 @@ define build-odmimage-target $(call build-image-kernel-modules,$(BOARD_ODM_KERNEL_MODULES),$(TARGET_OUT_ODM),odm/,$(call intermediates-dir-for,PACKAGING,depmod_odm))) $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \ ./build/tools/releasetools/build_image.py \ - $(TARGET_OUT_ODM) $(odmimage_intermediates)/odm_image_info.txt $(INSTALLED_ODMIMAGE_TARGET) $(TARGET_OUT) \ - $(odmimage_intermediates)/generated_odm_image_info.txt - $(hide) $(call assert-max-image-size,$(INSTALLED_ODMIMAGE_TARGET),\ - $(call read-image-prop-dictionary,\ - $(odmimage_intermediates)/generated_odm_image_info.txt,odm_size)) + $(TARGET_OUT_ODM) $(odmimage_intermediates)/odm_image_info.txt $(INSTALLED_ODMIMAGE_TARGET) $(TARGET_OUT) + $(hide) $(call assert-max-image-size,$(INSTALLED_ODMIMAGE_TARGET),$(BOARD_ODMIMAGE_PARTITION_SIZE)) endef # We just build this directly to the install location. @@ -3351,7 +3331,8 @@ ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION)) # (1): list of items like "system", "vendor", "product", "product_services" # return: map each item into a command ( wrapped in $$() ) that reads the size define read-size-of-partitions -$(foreach p,$(1),$(call read-image-prop-dictionary,$($(p)image_intermediates)/generated_$(p)_image_info.txt,$(p)_size)) +$(foreach image,$(call images-for-partitions,$(1)),$$( \ + build/make/tools/releasetools/sparse_img.py --get_partition_size $(image))) endef # round result to BOARD_SUPER_PARTITION_ALIGNMENT @@ -3375,7 +3356,9 @@ droid_targets: check-all-partition-sizes .PHONY: check-all-partition-sizes check-all-partition-sizes-nodeps # Add image dependencies so that generated_*_image_info.txt are written before checking. -check-all-partition-sizes: $(call images-for-partitions,$(BOARD_SUPER_PARTITION_PARTITION_LIST)) +check-all-partition-sizes: \ + build/make/tools/releasetools/sparse_img.py \ + $(call images-for-partitions,$(BOARD_SUPER_PARTITION_PARTITION_LIST)) ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true) # Check sum(super partition block devices) == super partition diff --git a/tools/releasetools/build_image.py b/tools/releasetools/build_image.py index d2f4e259b5..4136ed432e 100755 --- a/tools/releasetools/build_image.py +++ b/tools/releasetools/build_image.py @@ -18,10 +18,8 @@ Builds output_image from the given input_directory, properties_file, and writes the image to target_output_directory. -If argument generated_prop_file exists, write additional properties to the file. - Usage: build_image.py input_directory properties_file output_image \\ - target_output_directory [generated_prop_file] + target_output_directory """ from __future__ import print_function @@ -735,13 +733,8 @@ def GlobalDictFromImageProp(image_prop, mount_point): return d -def SaveGlobalDict(filename, glob_dict): - with open(filename, "w") as f: - f.writelines(["%s=%s" % (key, value) for (key, value) in glob_dict.items()]) - - def main(argv): - if len(argv) < 4 or len(argv) > 5: + if len(argv) != 4: print(__doc__) sys.exit(1) @@ -751,7 +744,6 @@ def main(argv): glob_dict_file = argv[1] out_file = argv[2] target_out = argv[3] - prop_file_out = argv[4] if len(argv) >= 5 else None glob_dict = LoadGlobalDict(glob_dict_file) if "mount_point" in glob_dict: @@ -791,10 +783,6 @@ def main(argv): logger.error("Failed to build %s from %s", out_file, in_dir) raise - if prop_file_out: - glob_dict_out = GlobalDictFromImageProp(image_properties, mount_point) - SaveGlobalDict(prop_file_out, glob_dict_out) - if __name__ == '__main__': try: