diff --git a/tools/releasetools/merge/merge_meta.py b/tools/releasetools/merge/merge_meta.py index 81f672956d..580b3ce648 100644 --- a/tools/releasetools/merge/merge_meta.py +++ b/tools/releasetools/merge/merge_meta.py @@ -142,7 +142,8 @@ def MergeMiscInfo(framework_meta_dir, vendor_meta_dir, merged_meta_dir): merged_dict = OPTIONS.vendor_misc_info for key in OPTIONS.framework_misc_info_keys: - merged_dict[key] = OPTIONS.framework_misc_info[key] + if key in OPTIONS.framework_misc_info: + merged_dict[key] = OPTIONS.framework_misc_info[key] # If AVB is enabled then ensure that we build vbmeta.img. # Partial builds with AVB enabled may set PRODUCT_BUILD_VBMETA_IMAGE=false to diff --git a/tools/releasetools/merge/merge_target_files.py b/tools/releasetools/merge/merge_target_files.py index 67dd6f183f..c06fd4cc4b 100755 --- a/tools/releasetools/merge/merge_target_files.py +++ b/tools/releasetools/merge/merge_target_files.py @@ -31,20 +31,20 @@ Usage: merge_target_files [args] archive. --framework-item-list framework-item-list-file - The optional path to a newline-separated config file that replaces the - contents of DEFAULT_FRAMEWORK_ITEM_LIST if provided. + The optional path to a newline-separated config file of items that + are extracted as-is from the framework target files package. --framework-misc-info-keys framework-misc-info-keys-file - The optional path to a newline-separated config file that replaces the - contents of DEFAULT_FRAMEWORK_MISC_INFO_KEYS if provided. + The optional path to a newline-separated config file of keys to + extract from the framework META/misc_info.txt file. --vendor-target-files vendor-target-files-zip-archive The input target files package containing vendor bits. This is a zip archive. --vendor-item-list vendor-item-list-file - The optional path to a newline-separated config file that replaces the - contents of DEFAULT_VENDOR_ITEM_LIST if provided. + The optional path to a newline-separated config file of items that + are extracted as-is from the vendor target files package. --output-target-files output-target-files-package If provided, the output merged target files package. Also a zip archive. @@ -107,6 +107,7 @@ import os import shutil import subprocess import sys +import zipfile import add_img_to_target_files import build_image @@ -127,13 +128,13 @@ OPTIONS = common.OPTIONS # Always turn on verbose logging. OPTIONS.verbose = True OPTIONS.framework_target_files = None -OPTIONS.framework_item_list = None -OPTIONS.framework_misc_info_keys = None +OPTIONS.framework_item_list = [] +OPTIONS.framework_misc_info_keys = [] OPTIONS.vendor_target_files = None -OPTIONS.vendor_item_list = None +OPTIONS.vendor_item_list = [] OPTIONS.output_target_files = None OPTIONS.output_dir = None -OPTIONS.output_item_list = None +OPTIONS.output_item_list = [] OPTIONS.output_ota = None OPTIONS.output_img = None OPTIONS.output_super_empty = None @@ -147,64 +148,6 @@ OPTIONS.framework_dexpreopt_config = None OPTIONS.framework_dexpreopt_tools = None OPTIONS.vendor_dexpreopt_config = None -# DEFAULT_FRAMEWORK_ITEM_LIST is a list of items to extract from the partial -# framework target files package as is, meaning these items will land in the -# output target files package exactly as they appear in the input partial -# framework target files package. - -DEFAULT_FRAMEWORK_ITEM_LIST = ( - 'META/apkcerts.txt', - 'META/filesystem_config.txt', - 'META/root_filesystem_config.txt', - 'META/update_engine_config.txt', - 'PRODUCT/*', - 'ROOT/*', - 'SYSTEM/*', -) - -# DEFAULT_FRAMEWORK_MISC_INFO_KEYS is a list of keys to obtain from the -# framework instance of META/misc_info.txt. The remaining keys should come -# from the vendor instance. - -DEFAULT_FRAMEWORK_MISC_INFO_KEYS = ( - 'avb_system_hashtree_enable', - 'avb_system_add_hashtree_footer_args', - 'avb_system_key_path', - 'avb_system_algorithm', - 'avb_system_rollback_index_location', - 'avb_product_hashtree_enable', - 'avb_product_add_hashtree_footer_args', - 'avb_system_ext_hashtree_enable', - 'avb_system_ext_add_hashtree_footer_args', - 'system_root_image', - 'root_dir', - 'ab_update', - 'default_system_dev_certificate', - 'system_size', - 'building_system_image', - 'building_system_ext_image', - 'building_product_image', -) - -# DEFAULT_VENDOR_ITEM_LIST is a list of items to extract from the partial -# vendor target files package as is, meaning these items will land in the output -# target files package exactly as they appear in the input partial vendor target -# files package. - -DEFAULT_VENDOR_ITEM_LIST = ( - 'META/boot_filesystem_config.txt', - 'META/otakeys.txt', - 'META/releasetools.py', - 'META/vendor_filesystem_config.txt', - 'BOOT/*', - 'DATA/*', - 'ODM/*', - 'OTA/android-info.txt', - 'PREBUILT_IMAGES/*', - 'RADIO/*', - 'VENDOR/*', -) - def create_merged_package(temp_dir): """Merges two target files packages into one target files structure. @@ -614,16 +557,22 @@ def main(): if (args or OPTIONS.framework_target_files is None or OPTIONS.vendor_target_files is None or (OPTIONS.output_target_files is None and OPTIONS.output_dir is None) or - (OPTIONS.output_dir is not None and OPTIONS.output_item_list is None) or + (OPTIONS.output_dir is not None and not OPTIONS.output_item_list) or (OPTIONS.rebuild_recovery and not OPTIONS.rebuild_sepolicy)): common.Usage(__doc__) sys.exit(1) + with zipfile.ZipFile(OPTIONS.framework_target_files, allowZip64=True) as fz: + framework_namelist = fz.namelist() + with zipfile.ZipFile(OPTIONS.vendor_target_files, allowZip64=True) as vz: + vendor_namelist = vz.namelist() + if OPTIONS.framework_item_list: OPTIONS.framework_item_list = common.LoadListFromFile( OPTIONS.framework_item_list) else: - OPTIONS.framework_item_list = DEFAULT_FRAMEWORK_ITEM_LIST + OPTIONS.framework_item_list = merge_utils.InferItemList( + input_namelist=framework_namelist, framework=True) OPTIONS.framework_partition_set = merge_utils.ItemListToPartitionSet( OPTIONS.framework_item_list) @@ -631,19 +580,19 @@ def main(): OPTIONS.framework_misc_info_keys = common.LoadListFromFile( OPTIONS.framework_misc_info_keys) else: - OPTIONS.framework_misc_info_keys = DEFAULT_FRAMEWORK_MISC_INFO_KEYS + OPTIONS.framework_misc_info_keys = merge_utils.InferFrameworkMiscInfoKeys( + input_namelist=framework_namelist) if OPTIONS.vendor_item_list: OPTIONS.vendor_item_list = common.LoadListFromFile(OPTIONS.vendor_item_list) else: - OPTIONS.vendor_item_list = DEFAULT_VENDOR_ITEM_LIST + OPTIONS.vendor_item_list = merge_utils.InferItemList( + input_namelist=vendor_namelist, framework=False) OPTIONS.vendor_partition_set = merge_utils.ItemListToPartitionSet( OPTIONS.vendor_item_list) if OPTIONS.output_item_list: OPTIONS.output_item_list = common.LoadListFromFile(OPTIONS.output_item_list) - else: - OPTIONS.output_item_list = None if not merge_utils.ValidateConfigLists(): sys.exit(1) diff --git a/tools/releasetools/merge/merge_utils.py b/tools/releasetools/merge/merge_utils.py index e8220610ff..f623ad2d4a 100644 --- a/tools/releasetools/merge/merge_utils.py +++ b/tools/releasetools/merge/merge_utils.py @@ -91,29 +91,6 @@ def WriteSortedData(data, path): output.write(out_str) -# The merge config lists should not attempt to extract items from both -# builds for any of the following partitions. The partitions in -# SINGLE_BUILD_PARTITIONS should come entirely from a single build (either -# framework or vendor, but not both). - -_SINGLE_BUILD_PARTITIONS = ( - 'BOOT/', - 'DATA/', - 'ODM/', - 'PRODUCT/', - 'SYSTEM_EXT/', - 'RADIO/', - 'RECOVERY/', - 'ROOT/', - 'SYSTEM/', - 'SYSTEM_OTHER/', - 'VENDOR/', - 'VENDOR_DLKM/', - 'ODM_DLKM/', - 'SYSTEM_DLKM/', -) - - def ValidateConfigLists(): """Performs validations on the merge config lists. @@ -123,7 +100,7 @@ def ValidateConfigLists(): has_error = False # Check that partitions only come from one input. - for partition in _SINGLE_BUILD_PARTITIONS: + for partition in _FRAMEWORK_PARTITIONS.union(_VENDOR_PARTITIONS): image_path = 'IMAGES/{}.img'.format(partition.lower().replace('/', '')) in_framework = ( any(item.startswith(partition) for item in OPTIONS.framework_item_list) @@ -185,3 +162,76 @@ def ItemListToPartitionSet(item_list): partition_set.add(partition_tag) return partition_set + + +# Partitions that are grabbed from the framework partial build by default. +_FRAMEWORK_PARTITIONS = { + 'system', 'product', 'system_ext', 'system_other', 'root', 'system_dlkm' +} +# Partitions that are grabbed from the vendor partial build by default. +_VENDOR_PARTITIONS = { + 'vendor', 'odm', 'oem', 'boot', 'vendor_boot', 'recovery', + 'prebuilt_images', 'radio', 'data', 'vendor_dlkm', 'odm_dlkm' +} + + +def InferItemList(input_namelist, framework): + item_list = [] + + # Some META items are grabbed from partial builds directly. + # Others are combined in merge_meta.py. + if framework: + item_list.extend([ + 'META/liblz4.so', + 'META/postinstall_config.txt', + 'META/update_engine_config.txt', + 'META/zucchini_config.txt', + ]) + else: # vendor + item_list.extend([ + 'META/kernel_configs.txt', + 'META/kernel_version.txt', + 'META/otakeys.txt', + 'META/releasetools.py', + 'OTA/android-info.txt', + ]) + + # Grab a set of items for the expected partitions in the partial build. + for partition in (_FRAMEWORK_PARTITIONS if framework else _VENDOR_PARTITIONS): + for namelist in input_namelist: + if namelist.startswith('%s/' % partition.upper()): + fs_config_prefix = '' if partition == 'system' else '%s_' % partition + item_list.extend([ + '%s/*' % partition.upper(), + 'IMAGES/%s.img' % partition, + 'IMAGES/%s.map' % partition, + 'META/%sfilesystem_config.txt' % fs_config_prefix, + ]) + break + + return sorted(item_list) + + +def InferFrameworkMiscInfoKeys(input_namelist): + keys = [ + 'ab_update', + 'avb_vbmeta_system', + 'avb_vbmeta_system_algorithm', + 'avb_vbmeta_system_key_path', + 'avb_vbmeta_system_rollback_index_location', + 'default_system_dev_certificate', + ] + + for partition in _FRAMEWORK_PARTITIONS: + for namelist in input_namelist: + if namelist.startswith('%s/' % partition.upper()): + fs_type_prefix = '' if partition == 'system' else '%s_' % partition + keys.extend([ + 'avb_%s_hashtree_enable' % partition, + 'avb_%s_add_hashtree_footer_args' % partition, + '%s_disable_sparse' % partition, + 'building_%s_image' % partition, + '%sfs_type' % fs_type_prefix, + ]) + + return sorted(keys) diff --git a/tools/releasetools/merge/test_merge_utils.py b/tools/releasetools/merge/test_merge_utils.py index d0cd2cf327..194905023c 100644 --- a/tools/releasetools/merge/test_merge_utils.py +++ b/tools/releasetools/merge/test_merge_utils.py @@ -20,20 +20,12 @@ import common import merge_target_files import merge_utils import test_utils -from merge_target_files import ( - DEFAULT_FRAMEWORK_ITEM_LIST, - DEFAULT_VENDOR_ITEM_LIST, - DEFAULT_FRAMEWORK_MISC_INFO_KEYS, -) class MergeUtilsTest(test_utils.ReleaseToolsTestCase): def setUp(self): self.OPTIONS = merge_target_files.OPTIONS - self.OPTIONS.framework_item_list = DEFAULT_FRAMEWORK_ITEM_LIST - self.OPTIONS.framework_misc_info_keys = DEFAULT_FRAMEWORK_MISC_INFO_KEYS - self.OPTIONS.vendor_item_list = DEFAULT_VENDOR_ITEM_LIST def test_CopyItems_CopiesItemsMatchingPatterns(self): @@ -88,21 +80,30 @@ class MergeUtilsTest(test_utils.ReleaseToolsTestCase): os.readlink(os.path.join(output_dir, 'a_link.cpp')), 'a.cpp') def test_ValidateConfigLists_ReturnsFalseIfSharedExtractedPartition(self): - self.OPTIONS.vendor_item_list = list(DEFAULT_VENDOR_ITEM_LIST) + self.OPTIONS.system_item_list = [ + 'SYSTEM/*', + ] + self.OPTIONS.vendor_item_list = [ + 'SYSTEM/my_system_file', + 'VENDOR/*', + ] self.OPTIONS.vendor_item_list.append('SYSTEM/my_system_file') self.assertFalse(merge_utils.ValidateConfigLists()) def test_ValidateConfigLists_ReturnsFalseIfSharedExtractedPartitionImage( self): - self.OPTIONS.vendor_item_list = list(DEFAULT_VENDOR_ITEM_LIST) - self.OPTIONS.vendor_item_list.append('IMAGES/system.img') + self.OPTIONS.system_item_list = [ + 'SYSTEM/*', + ] + self.OPTIONS.vendor_item_list = [ + 'IMAGES/system.img', + 'VENDOR/*', + ] self.assertFalse(merge_utils.ValidateConfigLists()) def test_ValidateConfigLists_ReturnsFalseIfBadSystemMiscInfoKeys(self): for bad_key in ['dynamic_partition_list', 'super_partition_groups']: - self.OPTIONS.framework_misc_info_keys = list( - DEFAULT_FRAMEWORK_MISC_INFO_KEYS) - self.OPTIONS.framework_misc_info_keys.append(bad_key) + self.OPTIONS.framework_misc_info_keys = [bad_key] self.assertFalse(merge_utils.ValidateConfigLists()) def test_ItemListToPartitionSet(self): @@ -116,3 +117,81 @@ class MergeUtilsTest(test_utils.ReleaseToolsTestCase): ] partition_set = merge_utils.ItemListToPartitionSet(item_list) self.assertEqual(set(['product', 'system', 'system_ext']), partition_set) + + def test_InferItemList_Framework(self): + zip_namelist = [ + 'SYSTEM/my_system_file', + 'PRODUCT/my_product_file', + ] + + item_list = merge_utils.InferItemList(zip_namelist, framework=True) + + expected_framework_item_list = [ + 'IMAGES/product.img', + 'IMAGES/product.map', + 'IMAGES/system.img', + 'IMAGES/system.map', + 'META/filesystem_config.txt', + 'META/liblz4.so', + 'META/postinstall_config.txt', + 'META/product_filesystem_config.txt', + 'META/update_engine_config.txt', + 'META/zucchini_config.txt', + 'PRODUCT/*', + 'SYSTEM/*', + ] + + self.assertEqual(item_list, expected_framework_item_list) + + def test_InferItemList_Vendor(self): + zip_namelist = [ + 'VENDOR/my_vendor_file', + 'ODM/my_odm_file', + ] + + item_list = merge_utils.InferItemList(zip_namelist, framework=False) + + expected_vendor_item_list = [ + 'IMAGES/odm.img', + 'IMAGES/odm.map', + 'IMAGES/vendor.img', + 'IMAGES/vendor.map', + 'META/kernel_configs.txt', + 'META/kernel_version.txt', + 'META/odm_filesystem_config.txt', + 'META/otakeys.txt', + 'META/releasetools.py', + 'META/vendor_filesystem_config.txt', + 'ODM/*', + 'OTA/android-info.txt', + 'VENDOR/*', + ] + self.assertEqual(item_list, expected_vendor_item_list) + + def test_InferFrameworkMiscInfoKeys(self): + zip_namelist = [ + 'SYSTEM/my_system_file', + 'SYSTEM_EXT/my_system_ext_file', + ] + + keys = merge_utils.InferFrameworkMiscInfoKeys(zip_namelist) + + expected_keys = [ + 'ab_update', + 'avb_system_add_hashtree_footer_args', + 'avb_system_ext_add_hashtree_footer_args', + 'avb_system_ext_hashtree_enable', + 'avb_system_hashtree_enable', + 'avb_vbmeta_system', + 'avb_vbmeta_system_algorithm', + 'avb_vbmeta_system_key_path', + 'avb_vbmeta_system_rollback_index_location', + 'building_system_ext_image', + 'building_system_image', + 'default_system_dev_certificate', + 'fs_type', + 'system_disable_sparse', + 'system_ext_disable_sparse', + 'system_ext_fs_type', + ] + self.assertEqual(keys, expected_keys)