diff --git a/tools/releasetools/merge_target_files.py b/tools/releasetools/merge_target_files.py index 3b72551a37..77c774a966 100755 --- a/tools/releasetools/merge_target_files.py +++ b/tools/releasetools/merge_target_files.py @@ -42,7 +42,25 @@ Usage: merge_target_files.py [args] contents of default_other_item_list if provided. --output-target-files output-target-files-package - The output merged target files package. Also a zip archive. + If provided, the output merged target files package. Also a zip archive. + + --output-dir output-directory + If provided, the destination directory for saving merged files. Requires + the --output-item-list flag. + Can be provided alongside --output-target-files, or by itself. + + --output-item-list output-item-list-file. + The optional path to a newline-separated config file that specifies the + file patterns to copy into the --output-dir. Required if providing + the --output-dir flag. + + --output-ota output-ota-package + The output ota package. This is a zip archive. Use of this flag may + require passing the --path common flag; see common.py. + + --output-super-empty output-super-empty-image + If provided, creates a super_empty.img file from the merged target + files package and saves it at this path. --rebuild_recovery Rebuild the recovery patch used by non-A/B devices and write it to the @@ -57,11 +75,14 @@ from __future__ import print_function import fnmatch import logging import os +import shutil import sys import zipfile -import common import add_img_to_target_files +import build_super_image +import common +import ota_from_target_files logger = logging.getLogger(__name__) OPTIONS = common.OPTIONS @@ -72,6 +93,10 @@ OPTIONS.system_misc_info_keys = None OPTIONS.other_target_files = None OPTIONS.other_item_list = None OPTIONS.output_target_files = None +OPTIONS.output_dir = None +OPTIONS.output_item_list = None +OPTIONS.output_ota = None +OPTIONS.output_super_empty = None OPTIONS.rebuild_recovery = False OPTIONS.keep_tmp = False @@ -195,6 +220,29 @@ def extract_items(target_files, target_files_temp_dir, extract_item_list): filtered_extract_item_list) +def copy_items(from_dir, to_dir, patterns): + """Similar to extract_items() except uses an input dir instead of zip.""" + file_paths = [] + for dirpath, _, filenames in os.walk(from_dir): + file_paths.extend(os.path.relpath(path=os.path.join(dirpath, filename), + start=from_dir) for filename in filenames) + + filtered_file_paths = set() + for pattern in patterns: + filtered_file_paths.update(fnmatch.filter(file_paths, pattern)) + + for file_path in filtered_file_paths: + original_file_path = os.path.join(from_dir, file_path) + copied_file_path = os.path.join(to_dir, file_path) + copied_file_dir = os.path.dirname(copied_file_path) + if not os.path.exists(copied_file_dir): + os.makedirs(copied_file_dir) + if os.path.islink(original_file_path): + os.symlink(os.readlink(original_file_path), copied_file_path) + else: + shutil.copyfile(original_file_path, copied_file_path) + + def read_config_list(config_file_path): """Reads a config file into a list of strings. @@ -388,7 +436,7 @@ def process_misc_info_txt( # a shared system image. for partition_group in merged_info_dict['super_partition_groups'].split(' '): if ('super_%s_group_size' % partition_group) not in merged_info_dict: - raise common.ExternalError( + raise ValueError( 'Other META/misc_info.txt does not contain required key ' 'super_%s_group_size.' % partition_group) key = 'super_%s_partition_list' % partition_group @@ -546,6 +594,10 @@ def merge_target_files( other_target_files, other_item_list, output_target_files, + output_dir, + output_item_list, + output_ota, + output_super_empty, rebuild_recovery): """Merge two target files packages together. @@ -580,6 +632,11 @@ def merge_target_files( output_target_files: The name of the output zip archive target files package created by merging system and other. + output_ota: The name of the output zip archive ota package. + + output_super_empty: If provided, creates a super_empty.img file from the + merged target files package and saves it at this path. + rebuild_recovery: If true, rebuild the recovery patch used by non-A/B devices and write it to the system image. """ @@ -646,6 +703,27 @@ def merge_target_files( system_misc_info_keys=system_misc_info_keys, rebuild_recovery=rebuild_recovery) + # Create super_empty.img using the merged misc_info.txt. + + if output_super_empty: + misc_info_txt = os.path.join(output_target_files_temp_dir, + 'META', 'misc_info.txt') + def read_helper(): + with open(misc_info_txt) as f: + return list(f.read().splitlines()) + + misc_info_dict = common.LoadDictionaryFromLines(read_helper()) + if misc_info_dict.get('use_dynamic_partitions') != 'true': + raise ValueError( + 'Building super_empty.img requires use_dynamic_partitions=true.') + + build_super_image_args = [ + '--verbose', + misc_info_txt, + output_super_empty, + ] + build_super_image.main(build_super_image_args) + # Regenerate IMAGES in the temporary directory. add_img_args = ['--verbose'] @@ -655,7 +733,14 @@ def merge_target_files( add_img_to_target_files.main(add_img_args) - # Finally, create the output target files zip archive. + # Finally, create the output target files zip archive and/or copy the + # output items to the output target files directory. + + if output_dir: + copy_items(output_target_files_temp_dir, output_dir, output_item_list) + + if not output_target_files: + return output_zip = os.path.abspath(output_target_files) output_target_files_list = os.path.join(temp_dir, 'output.list') @@ -694,6 +779,15 @@ def merge_target_files( logger.info('creating %s', output_target_files) common.RunAndWait(command, verbose=True) + # Create the OTA package from the merged target files package. + + if output_ota: + ota_from_target_files_args = [ + output_zip, + output_ota, + ] + ota_from_target_files.main(ota_from_target_files_args) + def call_func_with_temp_dir(func, keep_tmp): """Manage the creation and cleanup of the temporary directory. @@ -747,6 +841,14 @@ def main(): OPTIONS.other_item_list = a elif o == '--output-target-files': OPTIONS.output_target_files = a + elif o == '--output-dir': + OPTIONS.output_dir = a + elif o == '--output-item-list': + OPTIONS.output_item_list = a + elif o == '--output-ota': + OPTIONS.output_ota = a + elif o == '--output-super-empty': + OPTIONS.output_super_empty = a elif o == '--rebuild_recovery': OPTIONS.rebuild_recovery = True elif o == '--keep-tmp': @@ -764,6 +866,10 @@ def main(): 'other-target-files=', 'other-item-list=', 'output-target-files=', + 'output-dir=', + 'output-item-list=', + 'output-ota=', + 'output-super-empty=', 'rebuild_recovery', 'keep-tmp', ], @@ -771,8 +877,11 @@ def main(): if (len(args) != 0 or OPTIONS.system_target_files is None or - OPTIONS.other_target_files is None or - OPTIONS.output_target_files is None): + OPTIONS.other_target_files is None or ( + OPTIONS.output_target_files is None and + OPTIONS.output_dir is None) or ( + OPTIONS.output_dir is not None and + OPTIONS.output_item_list is None)): common.Usage(__doc__) sys.exit(1) @@ -791,6 +900,11 @@ def main(): else: other_item_list = default_other_item_list + if OPTIONS.output_item_list: + output_item_list = read_config_list(OPTIONS.output_item_list) + else: + output_item_list = None + if not validate_config_lists( system_item_list=system_item_list, system_misc_info_keys=system_misc_info_keys, @@ -806,6 +920,10 @@ def main(): other_target_files=OPTIONS.other_target_files, other_item_list=other_item_list, output_target_files=OPTIONS.output_target_files, + output_dir=OPTIONS.output_dir, + output_item_list=output_item_list, + output_ota=OPTIONS.output_ota, + output_super_empty=OPTIONS.output_super_empty, rebuild_recovery=OPTIONS.rebuild_recovery), OPTIONS.keep_tmp) diff --git a/tools/releasetools/test_merge_target_files.py b/tools/releasetools/test_merge_target_files.py index ec452a8cc6..7e18a3425d 100644 --- a/tools/releasetools/test_merge_target_files.py +++ b/tools/releasetools/test_merge_target_files.py @@ -16,10 +16,12 @@ import os.path +import common import test_utils -from merge_target_files import ( - read_config_list, validate_config_lists, default_system_item_list, - default_other_item_list, default_system_misc_info_keys) +from merge_target_files import (read_config_list, validate_config_lists, + default_system_item_list, + default_other_item_list, + default_system_misc_info_keys, copy_items) class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase): @@ -27,6 +29,58 @@ class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase): def setUp(self): self.testdata_dir = test_utils.get_testdata_dir() + def test_copy_items_CopiesItemsMatchingPatterns(self): + + def createEmptyFile(path): + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + open(path, 'a').close() + return path + + def createSymLink(source, dest): + os.symlink(source, dest) + return dest + + def getRelPaths(start, filepaths): + return set( + os.path.relpath(path=filepath, start=start) for filepath in filepaths) + + input_dir = common.MakeTempDir() + output_dir = common.MakeTempDir() + expected_copied_items = [] + actual_copied_items = [] + patterns = ['*.cpp', 'subdir/*.txt'] + + # Create various files that we expect to get copied because they + # match one of the patterns. + expected_copied_items.extend([ + createEmptyFile(os.path.join(input_dir, 'a.cpp')), + createEmptyFile(os.path.join(input_dir, 'b.cpp')), + createEmptyFile(os.path.join(input_dir, 'subdir', 'c.txt')), + createEmptyFile(os.path.join(input_dir, 'subdir', 'd.txt')), + createEmptyFile( + os.path.join(input_dir, 'subdir', 'subsubdir', 'e.txt')), + createSymLink('a.cpp', os.path.join(input_dir, 'a_link.cpp')), + ]) + # Create some more files that we expect to not get copied. + createEmptyFile(os.path.join(input_dir, 'a.h')) + createEmptyFile(os.path.join(input_dir, 'b.h')) + createEmptyFile(os.path.join(input_dir, 'subdir', 'subsubdir', 'f.gif')) + createSymLink('a.h', os.path.join(input_dir, 'a_link.h')) + + # Copy items. + copy_items(input_dir, output_dir, patterns) + + # Assert the actual copied items match the ones we expected. + for dirpath, _, filenames in os.walk(output_dir): + actual_copied_items.extend( + os.path.join(dirpath, filename) for filename in filenames) + self.assertEqual( + getRelPaths(output_dir, actual_copied_items), + getRelPaths(input_dir, expected_copied_items)) + self.assertEqual( + os.readlink(os.path.join(output_dir, 'a_link.cpp')), 'a.cpp') + def test_read_config_list(self): system_item_list_file = os.path.join(self.testdata_dir, 'merge_config_system_item_list')