diff --git a/core/Makefile b/core/Makefile index 6ba2be9214..6226d6be9e 100644 --- a/core/Makefile +++ b/core/Makefile @@ -2921,7 +2921,12 @@ vbmetaimage-nodeps: endif # BOARD_AVB_ENABLE # ----------------------------------------------------------------- -# super partition image +# Check image sizes <= size of super partition + +ifeq (,$(TARGET_BUILD_APPS)) +# Do not check for apps-only build + +ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION)) # (1): list of items like "system", "vendor", "product", "product_services" # return: map each item into a command ( wrapped in $$() ) that reads the size @@ -2929,75 +2934,10 @@ define read-size-of-partitions $(foreach p,$(1),$(call read-image-prop-dictionary,$($(p)image_intermediates)/generated_$(p)_image_info.txt,$(p)_size)) endef -ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION)) - -# BOARD_SUPER_PARTITION_SIZE must be defined to build super image. -ifneq ($(BOARD_SUPER_PARTITION_SIZE),) - define super-slot-suffix $(if $(filter true,$(AB_OTA_UPDATER)),$(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)),,_a)) endef -# $(1): slot A suffix (_a or empty) -# $(2): include images or not (true or empty) -define build-superimage-target-args - $(if $(2), --sparse) \ - --metadata-size 65536 \ - --metadata-slots $(if $(filter true,$(AB_OTA_UPDATER)),2,1) \ - --super-name $(BOARD_SUPER_PARTITION_METADATA_DEVICE) \ - $(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)), $(if $(filter true,$(AB_OTA_UPDATER)), --auto-slot-suffixing)) \ - $(foreach device,$(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \ - --device $(device):$(BOARD_SUPER_PARTITION_$(call to-upper,$(device))_DEVICE_SIZE)) \ - $(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \ - --group $(group)$(1):$(BOARD_$(call to-upper,$(group))_SIZE) \ - $(if $(1), --group $(group)_b:$(BOARD_$(call to-upper,$(group))_SIZE)) \ - $(foreach name,$(BOARD_$(call to-upper,$(group))_PARTITION_LIST), \ - --partition $(name)$(1):readonly:$(if $(2),$(call read-size-of-partitions,$(name)),0):$(group)$(1) \ - $(if $(2), --image $(name)$(1)=$(call images-for-partitions,$(name))) \ - $(if $(1), --partition $(name)_b:readonly:0:$(group)_b) \ - )) -endef - -# $(1): output image path -# $(2): slot A suffix (_a or empty) -# $(3): include images or not (true or empty) -define build-superimage-target - $(LPMAKE) \ - $(call build-superimage-target-args,$(2),$(3)) \ - --output $(1) -endef - -# For A/B devices, super partition always contains sub-partitions in the _a slot, because this -# image should only be used for bootstrapping / initializing the device. When flashing the image, -# bootloader fastboot should always mark _a slot as bootable. - -ifneq (true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)) -INSTALLED_SUPERIMAGE_TARGET := $(PRODUCT_OUT)/super.img -$(INSTALLED_SUPERIMAGE_TARGET): $(LPMAKE) $(call images-for-partitions,$(BOARD_SUPER_PARTITION_PARTITION_LIST)) - $(call pretty,"Target super fs image: $@") - $(call build-superimage-target,$@,$(call super-slot-suffix),true) -endif - -$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_TARGET)) - -INSTALLED_SUPERIMAGE_EMPTY_TARGET := $(PRODUCT_OUT)/super_empty.img -$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): $(LPMAKE) - $(call pretty,"Target empty super fs image: $@") - $(call build-superimage-target,$@,$(call super-slot-suffix)) - -$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_EMPTY_TARGET)) - -endif # BOARD_SUPER_PARTITION_SIZE -endif # PRODUCT_BUILD_SUPER_PARTITION - -# ----------------------------------------------------------------- -# Check image sizes <= size of super partition - -ifeq (,$(TARGET_BUILD_APPS)) -# Do not check for apps-only build - -ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION)) - droid_targets: check-all-partition-sizes .PHONY: check-all-partition-sizes check-all-partition-sizes-nodeps @@ -3329,6 +3269,30 @@ define fs_config (cd $(1); find . -type d | sed 's,$$,/,'; find . \! -type d) | cut -c 3- | sort | sed 's,^,$(2),' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) -R "$(2)" endef +# $(1): file +define dump-dynamic-partitions-info + $(if $(filter true,$(PRODUCT_USE_DYNAMIC_PARTITIONS)), \ + echo "use_dynamic_partitions=true" >> $(1)) + $(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)), \ + echo "dynamic_partition_retrofit=true" >> $(1)) + echo "lpmake=$(notdir $(LPMAKE))" >> $(1) + $(if $(filter true,$(PRODUCT_BUILD_SUPER_PARTITION)), $(if $(BOARD_SUPER_PARTITION_SIZE), \ + echo "build_super_partition=true" >> $(1))) + echo "super_metadata_device=$(BOARD_SUPER_PARTITION_METADATA_DEVICE)" >> $(1) + $(if $(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \ + echo "super_block_devices=$(BOARD_SUPER_PARTITION_BLOCK_DEVICES)" >> $(1)) + $(foreach device,$(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \ + echo "super_$(device)_device_size=$(BOARD_SUPER_PARTITION_$(call to-upper,$(device))_DEVICE_SIZE)" >> $(1);) + $(if $(BOARD_SUPER_PARTITION_PARTITION_LIST), \ + echo "dynamic_partition_list=$(BOARD_SUPER_PARTITION_PARTITION_LIST)" >> $(1)) + $(if $(BOARD_SUPER_PARTITION_GROUPS), + echo "super_partition_groups=$(BOARD_SUPER_PARTITION_GROUPS)" >> $(1)) + $(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \ + echo "super_$(group)_group_size=$(BOARD_$(call to-upper,$(group))_SIZE)" >> $(1); \ + $(if $(BOARD_$(call to-upper,$(group))_PARTITION_LIST), \ + echo "super_$(group)_partition_list=$(BOARD_$(call to-upper,$(group))_PARTITION_LIST)" >> $(1);)) +endef + # Depending on the various images guarantees that the underlying # directories are up-to-date. $(BUILT_TARGET_FILES_PACKAGE): \ @@ -3644,6 +3608,7 @@ ifdef BOARD_AVB_DTBO_KEY_PATH endif # BOARD_AVB_DTBO_KEY_PATH endif # BOARD_AVB_ENABLE endif # BOARD_PREBUILT_DTBOIMAGE + $(call dump-dynamic-partitions-info,$(zip_root)/META/misc_info.txt) @# The radio images in BOARD_PACK_RADIOIMAGES will be additionally copied from RADIO/ into @# IMAGES/, which then will be added into -img.zip. Such images must be listed in @# INSTALLED_RADIOIMAGE_TARGET. @@ -3689,24 +3654,6 @@ endif ifdef BUILT_VENDOR_MATRIX $(hide) cp $(BUILT_VENDOR_MATRIX) $(zip_root)/META/vendor_matrix.xml endif -ifeq ($(PRODUCT_USE_DYNAMIC_PARTITIONS),true) - $(hide) echo "use_dynamic_partitions=true" >> $(zip_root)/META/misc_info.txt -endif -ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true) - $(hide) echo "dynamic_partition_retrofit=true" >> $(zip_root)/META/misc_info.txt -endif -ifneq ($(BOARD_SUPER_PARTITION_SIZE),) - $(hide) echo "lpmake=$(notdir $(LPMAKE))" >> $(zip_root)/META/misc_info.txt - $(hide) echo -n "lpmake_args=" >> $(zip_root)/META/misc_info.txt - $(hide) echo $(call build-superimage-target-args,$(call super-slot-suffix)) \ - >> $(zip_root)/META/misc_info.txt -endif -ifneq ($(BOARD_SUPER_PARTITION_BLOCK_DEVICES),) - $(hide) echo "super_block_devices=$(BOARD_SUPER_PARTITION_BLOCK_DEVICES)" >> $(zip_root)/META/misc_info.txt -endif -ifneq ($(BOARD_SUPER_PARTITION_PARTITION_LIST),) - $(hide) echo "dynamic_partition_list=$(BOARD_SUPER_PARTITION_PARTITION_LIST)" >> $(zip_root)/META/misc_info.txt -endif ifneq ($(BOARD_SUPER_PARTITION_GROUPS),) $(hide) echo "super_partition_groups=$(BOARD_SUPER_PARTITION_GROUPS)" > $(zip_root)/META/dynamic_partitions_info.txt $(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \ @@ -3974,6 +3921,41 @@ $(PROGUARD_DICT_ZIP) : endif # TARGET_BUILD_APPS +# ----------------------------------------------------------------- +# super partition image + +ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION)) + +# BOARD_SUPER_PARTITION_SIZE must be defined to build super image. +ifneq ($(BOARD_SUPER_PARTITION_SIZE),) + +ifneq (true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)) +INSTALLED_SUPERIMAGE_TARGET := $(PRODUCT_OUT)/super.img +$(INSTALLED_SUPERIMAGE_TARGET): extracted_input_target_files := $(patsubst %.zip,%,$(BUILT_TARGET_FILES_PACKAGE)) +$(INSTALLED_SUPERIMAGE_TARGET): $(LPMAKE) $(BUILT_TARGET_FILES_PACKAGE) $(BUILD_SUPER_IMAGE) + $(call pretty,"Target super fs image: $@") + $(BUILD_SUPER_IMAGE) -v $(extracted_input_target_files) $@ +endif + +$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_TARGET)) + +INSTALLED_SUPERIMAGE_EMPTY_TARGET := $(PRODUCT_OUT)/super_empty.img +$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): intermediates := $(call intermediates-dir-for,PACKAGING,super_empty) +$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): $(LPMAKE) $(BUILD_SUPER_IMAGE) + $(call pretty,"Target empty super fs image: $@") + mkdir -p $(intermediates) + rm -rf $(intermediates)/misc_info.txt + $(call dump-dynamic-partitions-info,$(intermediates)/misc_info.txt) +ifeq ($(AB_OTA_UPDATER),true) + $(hide) echo "ab_update=true" >> $(intermediates)/misc_info.txt +endif + $(BUILD_SUPER_IMAGE) -v $(intermediates)/misc_info.txt $@ + +$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_EMPTY_TARGET)) + +endif # BOARD_SUPER_PARTITION_SIZE != "" +endif # PRODUCT_BUILD_SUPER_PARTITION == "true" + # ----------------------------------------------------------------- # dalvik something .PHONY: dalvikfiles diff --git a/core/config.mk b/core/config.mk index 99feeb2a2d..b7c2ed1620 100644 --- a/core/config.mk +++ b/core/config.mk @@ -698,6 +698,7 @@ DATA_BINDING_COMPILER := $(HOST_OUT_JAVA_LIBRARIES)/databinding-compiler.jar FAT16COPY := build/make/tools/fat16copy.py CHECK_LINK_TYPE := build/make/tools/check_link_type.py LPMAKE := $(HOST_OUT_EXECUTABLES)/lpmake$(HOST_EXECUTABLE_SUFFIX) +BUILD_SUPER_IMAGE := build/make/tools/releasetools/build_super_image.py PROGUARD := external/proguard/bin/proguard.sh JAVATAGS := build/make/tools/java-event-log-tags.py diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py index 139d25713a..669d87b697 100755 --- a/tools/releasetools/add_img_to_target_files.py +++ b/tools/releasetools/add_img_to_target_files.py @@ -55,6 +55,7 @@ import uuid import zipfile import build_image +import build_super_image import common import rangelib import sparse_img @@ -648,64 +649,15 @@ def AddSuperEmpty(output_zip): """Create a super_empty.img and store it in output_zip.""" img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "super_empty.img") - cmd = [OPTIONS.info_dict['lpmake']] - cmd += shlex.split(OPTIONS.info_dict['lpmake_args'].strip()) - cmd += ['--output', img.name] - common.RunAndCheckOutput(cmd) - + build_super_image.BuildSuperImage(OPTIONS.info_dict, img.name) img.Write() def AddSuperSplit(output_zip): """Create split super_*.img and store it in output_zip.""" - def GetPartitionSizeFromImage(img): - try: - simg = sparse_img.SparseImage(img) - return simg.blocksize * simg.total_blocks - except ValueError: - return os.path.getsize(img) - - def TransformPartitionArg(arg): - lst = arg.split(':') - # Because --auto-slot-suffixing for A/B, there is no need to remove suffix. - name = lst[0] - if name + '_size' in OPTIONS.info_dict: - size = str(OPTIONS.info_dict[name + '_size']) - logger.info("Using %s_size = %s", name, size) - else: - size = str(GetPartitionSizeFromImage( - os.path.join(OPTIONS.input_tmp, "IMAGES", '{}.img'.format(name)))) - logger.info("Using size of prebuilt %s = %s", name, size) - lst[2] = size - return ':'.join(lst) - - def GetLpmakeArgsWithSizes(): - lpmake_args = shlex.split(OPTIONS.info_dict['lpmake_args'].strip()) - - for i, arg in enumerate(lpmake_args): - if arg == '--partition': - assert i + 1 < len(lpmake_args), \ - 'lpmake_args has --partition without value' - lpmake_args[i + 1] = TransformPartitionArg(lpmake_args[i + 1]) - - return lpmake_args - - outdir = OutputFile(output_zip, OPTIONS.input_tmp, "OTA", "") - cmd = [OPTIONS.info_dict['lpmake']] - cmd += GetLpmakeArgsWithSizes() - - source = OPTIONS.info_dict.get('dynamic_partition_list', '').strip() - if source: - cmd.append('--sparse') - for name in shlex.split(source): - img = os.path.join(OPTIONS.input_tmp, "IMAGES", '{}.img'.format(name)) - # Because --auto-slot-suffixing for A/B, there is no need to add suffix. - cmd += ['--image', '{}={}'.format(name, img)] - - cmd += ['--output', outdir.name] - - common.RunAndCheckOutput(cmd) + outdir = os.path.join(OPTIONS.input_tmp, "OTA") + build_super_image.BuildSuperImage(OPTIONS.input_tmp, outdir) for dev in OPTIONS.info_dict['super_block_devices'].strip().split(): img = OutputFile(output_zip, OPTIONS.input_tmp, "OTA", @@ -906,14 +858,13 @@ def AddImagesToTargetFiles(filename): banner("vbmeta") AddVBMeta(output_zip, partitions, "vbmeta", vbmeta_partitions) - if OPTIONS.info_dict.get("lpmake_args"): + if OPTIONS.info_dict.get("build_super_partition"): banner("super_empty") AddSuperEmpty(output_zip) if OPTIONS.info_dict.get("dynamic_partition_retrofit") == "true": banner("super split images") AddSuperSplit(output_zip) - # TODO(b/119322123): Add super.img to target_files for non-retrofit banner("radio") ab_partitions_txt = os.path.join(OPTIONS.input_tmp, "META", diff --git a/tools/releasetools/build_super_image.py b/tools/releasetools/build_super_image.py new file mode 100755 index 0000000000..6efd3f45b6 --- /dev/null +++ b/tools/releasetools/build_super_image.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Usage: build_super_image input_file output_dir_or_file + +input_file: one of the following: + - directory containing extracted target files. It will load info from + META/misc_info.txt and build full super image / split images using source + images from IMAGES/. + - target files package. Same as above, but extracts the archive before + building super image. + - a dictionary file containing input arguments to build. Check + `dump_dynamic_partitions_info' for details. + In addition: + - "ab_update" needs to be true for A/B devices. + - If source images should be included in the output image (for super.img + and super split images), a list of "*_image" should be paths of each + source images. + +output_dir_or_file: + If a single super image is built (for super_empty.img, or super.img for + launch devices), this argument is the output file. + If a collection of split images are built (for retrofit devices), this + argument is the output directory. +""" + +from __future__ import print_function + +import logging +import os.path +import shlex +import sys +import zipfile + +import common +import sparse_img + +if sys.hexversion < 0x02070000: + print("Python 2.7 or newer is required.", file=sys.stderr) + sys.exit(1) + +logger = logging.getLogger(__name__) + + +UNZIP_PATTERN = ["IMAGES/*", "META/*"] + + +def GetPartitionSizeFromImage(img): + try: + simg = sparse_img.SparseImage(img) + return simg.blocksize * simg.total_blocks + except ValueError: + return os.path.getsize(img) + + +def BuildSuperImageFromDict(info_dict, output): + + cmd = [info_dict["lpmake"], + "--metadata-size", "65536", + "--super-name", info_dict["super_metadata_device"]] + + ab_update = info_dict.get("ab_update") == "true" + retrofit = info_dict.get("dynamic_partition_retrofit") == "true" + block_devices = shlex.split(info_dict.get("super_block_devices", "").strip()) + groups = shlex.split(info_dict.get("super_partition_groups", "").strip()) + + if ab_update: + cmd += ["--metadata-slots", "2"] + else: + cmd += ["--metadata-slots", "1"] + + if ab_update and retrofit: + cmd.append("--auto-slot-suffixing") + + for device in block_devices: + size = info_dict["super_{}_device_size".format(device)] + cmd += ["--device", "{}:{}".format(device, size)] + + append_suffix = ab_update and not retrofit + has_image = False + for group in groups: + group_size = info_dict["super_{}_group_size".format(group)] + if append_suffix: + cmd += ["--group", "{}_a:{}".format(group, group_size), + "--group", "{}_b:{}".format(group, group_size)] + else: + cmd += ["--group", "{}:{}".format(group, group_size)] + + partition_list = shlex.split( + info_dict["super_{}_partition_list".format(group)].strip()) + + for partition in partition_list: + image = info_dict.get("{}_image".format(partition)) + image_size = 0 + if image: + image_size = GetPartitionSizeFromImage(image) + has_image = True + if append_suffix: + cmd += ["--partition", + "{}_a:readonly:{}:{}_a".format(partition, image_size, group), + "--partition", + "{}_b:readonly:0:{}_b".format(partition, group)] + if image: + # For A/B devices, super partition always contains sub-partitions in + # the _a slot, because this image should only be used for + # bootstrapping / initializing the device. When flashing the image, + # bootloader fastboot should always mark _a slot as bootable. + cmd += ["--image", "{}_a={}".format(partition, image)] + else: + cmd += ["--partition", + "{}:readonly:{}:{}".format(partition, image_size, group)] + if image: + cmd += ["--image", "{}={}".format(partition, image)] + + if has_image: + cmd.append("--sparse") + + cmd += ["--output", output] + + common.RunAndCheckOutput(cmd) + + if retrofit and has_image: + logger.info("Done writing images to directory %s", output) + else: + logger.info("Done writing image %s", output) + + +def BuildSuperImageFromExtractedTargetFiles(inp, out): + info_dict = common.LoadInfoDict(inp) + partition_list = shlex.split( + info_dict.get("dynamic_partition_list", "").strip()) + for partition in partition_list: + info_dict["{}_image".format(partition)] = os.path.join( + inp, "IMAGES", "{}.img".format(partition)) + return BuildSuperImageFromDict(info_dict, out) + + +def BuildSuperImageFromTargetFiles(inp, out): + input_tmp = common.UnzipTemp(inp, UNZIP_PATTERN) + return BuildSuperImageFromExtractedTargetFiles(input_tmp, out) + + +def BuildSuperImage(inp, out): + + if isinstance(inp, dict): + logger.info("Building super image from info dict...") + return BuildSuperImageFromDict(inp, out) + + if isinstance(inp, str): + if os.path.isdir(inp): + logger.info("Building super image from extracted target files...") + return BuildSuperImageFromExtractedTargetFiles(inp, out) + + if zipfile.is_zipfile(inp): + logger.info("Building super image from target files...") + return BuildSuperImageFromTargetFiles(inp, out) + + if os.path.isfile(inp): + with open(inp) as f: + lines = f.read() + logger.info("Building super image from info dict...") + return BuildSuperImageFromDict(common.LoadDictionaryFromLines(lines.split("\n")), out) + + raise ValueError("{} is not a dictionary or a valid path".format(inp)) + + +def main(argv): + + args = common.ParseOptions(argv, __doc__) + + if len(args) != 2: + common.Usage(__doc__) + sys.exit(1) + + common.InitLogging() + + BuildSuperImage(args[0], args[1]) + + +if __name__ == "__main__": + try: + common.CloseInheritedPipes() + main(sys.argv[1:]) + except common.ExternalError: + logger.exception("\n ERROR:\n") + sys.exit(1) + finally: + common.Cleanup()