StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adds validation check that certain partitions come from a single build.

Browse Source 
This is to prevent a user from accidentally including files from the
wrong build. For example, adding any SYSTEM/ line to other_item_list
while keeping SYSTEM/* in system_item_list would cause the other build
to introduce an extra or changed file in the system image.

Bug: 132730710
Test: python -m unittest test_merge_target_files
Change-Id: Ic1178cdc9b991114f293ff3f2b4e6054e06647c6
This commit is contained in:
Daniel Norman
2019-05-22 10:47:08 -07:00
parent cd56c0df95
commit edf124780f
2 changed files with 87 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
tools/releasetools/merge_target_files.py
View File

@@ -27,11 +27,11 @@ Usage: merge_target_files.py [args]
--system-item-list system-item-list-file --system-item-list system-item-list-file
The optional path to a newline-separated config file that replaces the The optional path to a newline-separated config file that replaces the
contents of default_system_item_list if provided. contents of DEFAULT_SYSTEM_ITEM_LIST if provided.
--system-misc-info-keys system-misc-info-keys-file --system-misc-info-keys system-misc-info-keys-file
The optional path to a newline-separated config file that replaces the The optional path to a newline-separated config file that replaces the
contents of default_system_misc_info_keys if provided. contents of DEFAULT_SYSTEM_MISC_INFO_KEYS if provided.
--other-target-files other-target-files-zip-archive --other-target-files other-target-files-zip-archive
The input target files package containing other bits. This is a zip The input target files package containing other bits. This is a zip
@@ -39,7 +39,7 @@ Usage: merge_target_files.py [args]
--other-item-list other-item-list-file --other-item-list other-item-list-file
The optional path to a newline-separated config file that replaces the The optional path to a newline-separated config file that replaces the
contents of default_other_item_list if provided. contents of DEFAULT_OTHER_ITEM_LIST if provided.
--output-target-files output-target-files-package --output-target-files output-target-files-package
If provided, the output merged target files package. Also a zip archive. If provided, the output merged target files package. Also a zip archive.
@@ -107,12 +107,12 @@ OPTIONS.output_super_empty = None
OPTIONS.rebuild_recovery = False OPTIONS.rebuild_recovery = False
OPTIONS.keep_tmp = False OPTIONS.keep_tmp = False
# default_system_item_list is a list of items to extract from the partial # DEFAULT_SYSTEM_ITEM_LIST is a list of items to extract from the partial
# system target files package as is, meaning these items will land in the # system target files package as is, meaning these items will land in the
# output target files package exactly as they appear in the input partial # output target files package exactly as they appear in the input partial
# system target files package. # system target files package.
default_system_item_list = [ DEFAULT_SYSTEM_ITEM_LIST = (
'META/apkcerts.txt', 'META/apkcerts.txt',
'META/filesystem_config.txt', 'META/filesystem_config.txt',
'META/root_filesystem_config.txt', 'META/root_filesystem_config.txt',
@@ -122,21 +122,19 @@ default_system_item_list = [
'PRODUCT/*', 'PRODUCT/*',
'ROOT/*', 'ROOT/*',
'SYSTEM/*', 'SYSTEM/*',
] )
# system_extract_special_item_list is a list of items to extract from the # SYSTEM_EXTRACT_SPECIAL_ITEM_LIST is a list of items to extract from the
# partial system target files package that need some special processing, such # partial system target files package that need some special processing, such
# as some sort of combination with items from the partial other target files # as some sort of combination with items from the partial other target files
# package. # package.
system_extract_special_item_list = [ SYSTEM_EXTRACT_SPECIAL_ITEM_LIST = ('META/*',)
'META/*',
]
# default_system_misc_info_keys is a list of keys to obtain from the system # DEFAULT_SYSTEM_MISC_INFO_KEYS is a list of keys to obtain from the system
# instance of META/misc_info.txt. The remaining keys from the other instance. # instance of META/misc_info.txt. The remaining keys from the other instance.
default_system_misc_info_keys = [ DEFAULT_SYSTEM_MISC_INFO_KEYS = (
'avb_system_hashtree_enable', 'avb_system_hashtree_enable',
'avb_system_add_hashtree_footer_args', 'avb_system_add_hashtree_footer_args',
'avb_system_key_path', 'avb_system_key_path',
@@ -151,14 +149,14 @@ default_system_misc_info_keys = [
'ab_update', 'ab_update',
'default_system_dev_certificate', 'default_system_dev_certificate',
'system_size', 'system_size',
] )
# default_other_item_list is a list of items to extract from the partial # DEFAULT_OTHER_ITEM_LIST is a list of items to extract from the partial
# other target files package as is, meaning these items will land in the output # other target files package as is, meaning these items will land in the output
# target files package exactly as they appear in the input partial other target # target files package exactly as they appear in the input partial other target
# files package. # files package.
default_other_item_list = [ DEFAULT_OTHER_ITEM_LIST = (
'META/boot_filesystem_config.txt', 'META/boot_filesystem_config.txt',
'META/otakeys.txt', 'META/otakeys.txt',
'META/releasetools.py', 'META/releasetools.py',
@@ -172,16 +170,33 @@ default_other_item_list = [
'PREBUILT_IMAGES/*', 'PREBUILT_IMAGES/*',
'RADIO/*', 'RADIO/*',
'VENDOR/*', 'VENDOR/*',
] )
# other_extract_special_item_list is a list of items to extract from the # OTHER_EXTRACT_SPECIAL_ITEM_LIST is a list of items to extract from the
# partial other target files package that need some special processing, such as # partial other target files package that need some special processing, such as
# some sort of combination with items from the partial system target files # some sort of combination with items from the partial system target files
# package. # package.
other_extract_special_item_list = [ OTHER_EXTRACT_SPECIAL_ITEM_LIST = ('META/*',)
'META/*',
] # The merge config lists should not attempt to extract items from both
# builds for any of the following partitions. The partitions in
# SINGLE_BUILD_PARTITIONS should come entirely from a single build (either
# system or other, but not both).
SINGLE_BUILD_PARTITIONS = (
'BOOT/',
'DATA/',
'ODM/',
'PRODUCT/',
'PRODUCT_SERVICES/',
'RADIO/',
'RECOVERY/',
'ROOT/',
'SYSTEM/',
'SYSTEM_OTHER/',
'VENDOR/',
)
def write_sorted_data(data, path): def write_sorted_data(data, path):
@@ -295,8 +310,10 @@ def validate_config_lists(system_item_list, system_misc_info_keys,
Returns: Returns:
False if a validation fails, otherwise true. False if a validation fails, otherwise true.
""" """
default_combined_item_set = set(default_system_item_list) has_error = False
default_combined_item_set.update(default_other_item_list)
default_combined_item_set = set(DEFAULT_SYSTEM_ITEM_LIST)
default_combined_item_set.update(DEFAULT_OTHER_ITEM_LIST)
combined_item_set = set(system_item_list) combined_item_set = set(system_item_list)
combined_item_set.update(other_item_list) combined_item_set.update(other_item_list)
@@ -309,15 +326,25 @@ def validate_config_lists(system_item_list, system_misc_info_keys,
logger.error('Please ensure missing items are in either the ' logger.error('Please ensure missing items are in either the '
'system-item-list or other-item-list files provided to ' 'system-item-list or other-item-list files provided to '
'this script.') 'this script.')
return False has_error = True
for partition in SINGLE_BUILD_PARTITIONS:
in_system = any(item.startswith(partition) for item in system_item_list)
in_other = any(item.startswith(partition) for item in other_item_list)
if in_system and in_other:
logger.error(
'Cannot extract items from {0} for both the system and other builds. '
'Please ensure only one merge config item list includes {0}.'.format(
partition))
has_error = True
if ('dynamic_partition_list' in system_misc_info_keys) or ( if ('dynamic_partition_list' in system_misc_info_keys) or (
'super_partition_groups' in system_misc_info_keys): 'super_partition_groups' in system_misc_info_keys):
logger.error('Dynamic partition misc info keys should come from ' logger.error('Dynamic partition misc info keys should come from '
'the other instance of META/misc_info.txt.') 'the other instance of META/misc_info.txt.')
return False has_error = True
return True return not has_error
def process_ab_partitions_txt(system_target_files_temp_dir, def process_ab_partitions_txt(system_target_files_temp_dir,
@@ -619,18 +646,22 @@ def process_apex_keys_apk_certs_common(system_target_files_dir,
def copy_file_contexts(system_target_files_dir, other_target_files_dir, def copy_file_contexts(system_target_files_dir, other_target_files_dir,
output_target_files_dir): output_target_files_dir):
"""Creates named copies of each build's file_contexts.bin in output META/.""" """Creates named copies of each build's file_contexts.bin in output META/."""
system_fc_path = os.path.join(system_target_files_dir, 'META', 'system_file_contexts.bin') system_fc_path = os.path.join(system_target_files_dir, 'META',
'system_file_contexts.bin')
if not os.path.exists(system_fc_path): if not os.path.exists(system_fc_path):
system_fc_path = os.path.join(system_target_files_dir, 'META', 'file_contexts.bin') system_fc_path = os.path.join(system_target_files_dir, 'META',
'file_contexts.bin')
if not os.path.exists(system_fc_path): if not os.path.exists(system_fc_path):
raise ValueError('Missing system file_contexts.bin.') raise ValueError('Missing system file_contexts.bin.')
shutil.copyfile( shutil.copyfile(
system_fc_path, system_fc_path,
os.path.join(output_target_files_dir, 'META', 'system_file_contexts.bin')) os.path.join(output_target_files_dir, 'META', 'system_file_contexts.bin'))
other_fc_path = os.path.join(other_target_files_dir, 'META', 'other_file_contexts.bin') other_fc_path = os.path.join(other_target_files_dir, 'META',
'other_file_contexts.bin')
if not os.path.exists(other_fc_path): if not os.path.exists(other_fc_path):
other_fc_path = os.path.join(other_target_files_dir, 'META', 'file_contexts.bin') other_fc_path = os.path.join(other_target_files_dir, 'META',
'file_contexts.bin')
if not os.path.exists(other_fc_path): if not os.path.exists(other_fc_path):
raise ValueError('Missing other file_contexts.bin.') raise ValueError('Missing other file_contexts.bin.')
shutil.copyfile( shutil.copyfile(
@@ -776,7 +807,7 @@ def merge_target_files(temp_dir, system_target_files, system_item_list,
extract_items( extract_items(
target_files=system_target_files, target_files=system_target_files,
target_files_temp_dir=system_target_files_temp_dir, target_files_temp_dir=system_target_files_temp_dir,
extract_item_list=system_extract_special_item_list) extract_item_list=SYSTEM_EXTRACT_SPECIAL_ITEM_LIST)
# Extract "special" items from the input other partial target files package. # Extract "special" items from the input other partial target files package.
# We extract these items to different directory since they require special # We extract these items to different directory since they require special
@@ -785,7 +816,7 @@ def merge_target_files(temp_dir, system_target_files, system_item_list,
extract_items( extract_items(
target_files=other_target_files, target_files=other_target_files,
target_files_temp_dir=other_target_files_temp_dir, target_files_temp_dir=other_target_files_temp_dir,
extract_item_list=other_extract_special_item_list) extract_item_list=OTHER_EXTRACT_SPECIAL_ITEM_LIST)
# Now that the temporary directories contain all the extracted files, perform # Now that the temporary directories contain all the extracted files, perform
# special case processing on any items that need it. After this function # special case processing on any items that need it. After this function
@@ -1004,17 +1035,17 @@ def main():
if OPTIONS.system_item_list: if OPTIONS.system_item_list:
system_item_list = read_config_list(OPTIONS.system_item_list) system_item_list = read_config_list(OPTIONS.system_item_list)
else: else:
system_item_list = default_system_item_list system_item_list = DEFAULT_SYSTEM_ITEM_LIST
if OPTIONS.system_misc_info_keys: if OPTIONS.system_misc_info_keys:
system_misc_info_keys = read_config_list(OPTIONS.system_misc_info_keys) system_misc_info_keys = read_config_list(OPTIONS.system_misc_info_keys)
else: else:
system_misc_info_keys = default_system_misc_info_keys system_misc_info_keys = DEFAULT_SYSTEM_MISC_INFO_KEYS
if OPTIONS.other_item_list: if OPTIONS.other_item_list:
other_item_list = read_config_list(OPTIONS.other_item_list) other_item_list = read_config_list(OPTIONS.other_item_list)
else: else:
other_item_list = default_other_item_list other_item_list = DEFAULT_OTHER_ITEM_LIST
if OPTIONS.output_item_list: if OPTIONS.output_item_list:
output_item_list = read_config_list(OPTIONS.output_item_list) output_item_list = read_config_list(OPTIONS.output_item_list)

37
tools/releasetools/test_merge_target_files.py
View File

@@ -19,9 +19,9 @@ import os.path
import common import common
import test_utils import test_utils
from merge_target_files import (read_config_list, validate_config_lists, from merge_target_files import (read_config_list, validate_config_lists,
default_system_item_list, DEFAULT_SYSTEM_ITEM_LIST,
default_other_item_list, DEFAULT_OTHER_ITEM_LIST,
default_system_misc_info_keys, copy_items, DEFAULT_SYSTEM_MISC_INFO_KEYS, copy_items,
merge_dynamic_partition_info_dicts, merge_dynamic_partition_info_dicts,
process_apex_keys_apk_certs_common) process_apex_keys_apk_certs_common)
@@ -101,35 +101,42 @@ class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase):
self.assertItemsEqual(system_item_list, expected_system_item_list) self.assertItemsEqual(system_item_list, expected_system_item_list)
def test_validate_config_lists_ReturnsFalseIfMissingDefaultItem(self): def test_validate_config_lists_ReturnsFalseIfMissingDefaultItem(self):
system_item_list = default_system_item_list[:] system_item_list = list(DEFAULT_SYSTEM_ITEM_LIST)
system_item_list.remove('SYSTEM/*') system_item_list.remove('SYSTEM/*')
self.assertFalse( self.assertFalse(
validate_config_lists(system_item_list, default_system_misc_info_keys, validate_config_lists(system_item_list, DEFAULT_SYSTEM_MISC_INFO_KEYS,
default_other_item_list)) DEFAULT_OTHER_ITEM_LIST))
def test_validate_config_lists_ReturnsTrueIfDefaultItemInDifferentList(self): def test_validate_config_lists_ReturnsTrueIfDefaultItemInDifferentList(self):
system_item_list = default_system_item_list[:] system_item_list = list(DEFAULT_SYSTEM_ITEM_LIST)
system_item_list.remove('ROOT/*') system_item_list.remove('ROOT/*')
other_item_list = default_other_item_list[:] other_item_list = list(DEFAULT_OTHER_ITEM_LIST)
other_item_list.append('ROOT/*') other_item_list.append('ROOT/*')
self.assertTrue( self.assertTrue(
validate_config_lists(system_item_list, default_system_misc_info_keys, validate_config_lists(system_item_list, DEFAULT_SYSTEM_MISC_INFO_KEYS,
other_item_list)) other_item_list))
def test_validate_config_lists_ReturnsTrueIfExtraItem(self): def test_validate_config_lists_ReturnsTrueIfExtraItem(self):
system_item_list = default_system_item_list[:] system_item_list = list(DEFAULT_SYSTEM_ITEM_LIST)
system_item_list.append('MY_NEW_PARTITION/*') system_item_list.append('MY_NEW_PARTITION/*')
self.assertTrue( self.assertTrue(
validate_config_lists(system_item_list, default_system_misc_info_keys, validate_config_lists(system_item_list, DEFAULT_SYSTEM_MISC_INFO_KEYS,
default_other_item_list)) DEFAULT_OTHER_ITEM_LIST))
def test_validate_config_lists_ReturnsFalseIfSharedExtractedPartition(self):
other_item_list = list(DEFAULT_OTHER_ITEM_LIST)
other_item_list.append('SYSTEM/my_system_file')
self.assertFalse(
validate_config_lists(DEFAULT_SYSTEM_ITEM_LIST,
DEFAULT_SYSTEM_MISC_INFO_KEYS, other_item_list))
def test_validate_config_lists_ReturnsFalseIfBadSystemMiscInfoKeys(self): def test_validate_config_lists_ReturnsFalseIfBadSystemMiscInfoKeys(self):
for bad_key in ['dynamic_partition_list', 'super_partition_groups']: for bad_key in ['dynamic_partition_list', 'super_partition_groups']:
system_misc_info_keys = default_system_misc_info_keys[:] system_misc_info_keys = list(DEFAULT_SYSTEM_MISC_INFO_KEYS)
system_misc_info_keys.append(bad_key) system_misc_info_keys.append(bad_key)
self.assertFalse( self.assertFalse(
validate_config_lists(default_system_item_list, system_misc_info_keys, validate_config_lists(DEFAULT_SYSTEM_ITEM_LIST, system_misc_info_keys,
default_other_item_list)) DEFAULT_OTHER_ITEM_LIST))
def test_merge_dynamic_partition_info_dicts_ReturnsMergedDict(self): def test_merge_dynamic_partition_info_dicts_ReturnsMergedDict(self):
system_dict = { system_dict = {