Merge changes from topic "merge_target_files_new_flags"

* changes:
  Adds --output-ota flag to enable building the OTA package.
  Adds --output-super-empty flag to enable building super_empty.img.
  Adds output-dir and output-item-list for copying only certain files.

tools/releasetools/merge_target_files.py

@@ -42,7 +42,25 @@ Usage: merge_target_files.py [args]
       contents of default_other_item_list if provided.
 
   --output-target-files output-target-files-package
-      The output merged target files package. Also a zip archive.
+      If provided, the output merged target files package. Also a zip archive.
+
+  --output-dir output-directory
+      If provided, the destination directory for saving merged files. Requires
+      the --output-item-list flag.
+      Can be provided alongside --output-target-files, or by itself.
+
+  --output-item-list output-item-list-file.
+      The optional path to a newline-separated config file that specifies the
+      file patterns to copy into the --output-dir. Required if providing
+      the --output-dir flag.
+
+  --output-ota output-ota-package
+      The output ota package. This is a zip archive. Use of this flag may
+      require passing the --path common flag; see common.py.
+
+  --output-super-empty output-super-empty-image
+      If provided, creates a super_empty.img file from the merged target
+      files package and saves it at this path.
 
   --rebuild_recovery
       Rebuild the recovery patch used by non-A/B devices and write it to the
@@ -57,11 +75,14 @@ from __future__ import print_function
 import fnmatch
 import logging
 import os
+import shutil
 import sys
 import zipfile
 
-import common
 import add_img_to_target_files
+import build_super_image
+import common
+import ota_from_target_files
 
 logger = logging.getLogger(__name__)
 OPTIONS = common.OPTIONS
@@ -72,6 +93,10 @@ OPTIONS.system_misc_info_keys = None
 OPTIONS.other_target_files = None
 OPTIONS.other_item_list = None
 OPTIONS.output_target_files = None
+OPTIONS.output_dir = None
+OPTIONS.output_item_list = None
+OPTIONS.output_ota = None
+OPTIONS.output_super_empty = None
 OPTIONS.rebuild_recovery = False
 OPTIONS.keep_tmp = False
 
@@ -195,6 +220,29 @@ def extract_items(target_files, target_files_temp_dir, extract_item_list):
       filtered_extract_item_list)
 
 
+def copy_items(from_dir, to_dir, patterns):
+  """Similar to extract_items() except uses an input dir instead of zip."""
+  file_paths = []
+  for dirpath, _, filenames in os.walk(from_dir):
+    file_paths.extend(os.path.relpath(path=os.path.join(dirpath, filename),
+                                      start=from_dir) for filename in filenames)
+
+  filtered_file_paths = set()
+  for pattern in patterns:
+    filtered_file_paths.update(fnmatch.filter(file_paths, pattern))
+
+  for file_path in filtered_file_paths:
+    original_file_path = os.path.join(from_dir, file_path)
+    copied_file_path = os.path.join(to_dir, file_path)
+    copied_file_dir = os.path.dirname(copied_file_path)
+    if not os.path.exists(copied_file_dir):
+      os.makedirs(copied_file_dir)
+    if os.path.islink(original_file_path):
+      os.symlink(os.readlink(original_file_path), copied_file_path)
+    else:
+      shutil.copyfile(original_file_path, copied_file_path)
+
+
 def read_config_list(config_file_path):
   """Reads a config file into a list of strings.
 
@@ -388,7 +436,7 @@ def process_misc_info_txt(
     # a shared system image.
     for partition_group in merged_info_dict['super_partition_groups'].split(' '):
       if ('super_%s_group_size' % partition_group) not in merged_info_dict:
-        raise common.ExternalError(
+        raise ValueError(
             'Other META/misc_info.txt does not contain required key '
             'super_%s_group_size.' % partition_group)
       key = 'super_%s_partition_list' % partition_group
@@ -546,6 +594,10 @@ def merge_target_files(
     other_target_files,
     other_item_list,
     output_target_files,
+    output_dir,
+    output_item_list,
+    output_ota,
+    output_super_empty,
     rebuild_recovery):
   """Merge two target files packages together.
 
@@ -580,6 +632,11 @@ def merge_target_files(
     output_target_files: The name of the output zip archive target files
     package created by merging system and other.
 
+    output_ota: The name of the output zip archive ota package.
+
+    output_super_empty: If provided, creates a super_empty.img file from the
+    merged target files package and saves it at this path.
+
     rebuild_recovery: If true, rebuild the recovery patch used by non-A/B
     devices and write it to the system image.
   """
@@ -646,6 +703,27 @@ def merge_target_files(
       system_misc_info_keys=system_misc_info_keys,
       rebuild_recovery=rebuild_recovery)
 
+  # Create super_empty.img using the merged misc_info.txt.
+
+  if output_super_empty:
+    misc_info_txt = os.path.join(output_target_files_temp_dir,
+                                 'META', 'misc_info.txt')
+    def read_helper():
+      with open(misc_info_txt) as f:
+        return list(f.read().splitlines())
+
+    misc_info_dict = common.LoadDictionaryFromLines(read_helper())
+    if misc_info_dict.get('use_dynamic_partitions') != 'true':
+      raise ValueError(
+          'Building super_empty.img requires use_dynamic_partitions=true.')
+
+    build_super_image_args = [
+      '--verbose',
+      misc_info_txt,
+      output_super_empty,
+    ]
+    build_super_image.main(build_super_image_args)
+
   # Regenerate IMAGES in the temporary directory.
 
   add_img_args = ['--verbose']
@@ -655,7 +733,14 @@ def merge_target_files(
 
   add_img_to_target_files.main(add_img_args)
 
-  # Finally, create the output target files zip archive.
+  # Finally, create the output target files zip archive and/or copy the
+  # output items to the output target files directory.
+
+  if output_dir:
+    copy_items(output_target_files_temp_dir, output_dir, output_item_list)
+
+  if not output_target_files:
+    return
 
   output_zip = os.path.abspath(output_target_files)
   output_target_files_list = os.path.join(temp_dir, 'output.list')
@@ -694,6 +779,15 @@ def merge_target_files(
   logger.info('creating %s', output_target_files)
   common.RunAndWait(command, verbose=True)
 
+  # Create the OTA package from the merged target files package.
+
+  if output_ota:
+    ota_from_target_files_args = [
+        output_zip,
+        output_ota,
+    ]
+    ota_from_target_files.main(ota_from_target_files_args)
+
 
 def call_func_with_temp_dir(func, keep_tmp):
   """Manage the creation and cleanup of the temporary directory.
@@ -747,6 +841,14 @@ def main():
       OPTIONS.other_item_list = a
     elif o == '--output-target-files':
       OPTIONS.output_target_files = a
+    elif o == '--output-dir':
+      OPTIONS.output_dir = a
+    elif o == '--output-item-list':
+      OPTIONS.output_item_list = a
+    elif o == '--output-ota':
+      OPTIONS.output_ota = a
+    elif o == '--output-super-empty':
+      OPTIONS.output_super_empty = a
     elif o == '--rebuild_recovery':
       OPTIONS.rebuild_recovery = True
     elif o == '--keep-tmp':
@@ -764,6 +866,10 @@ def main():
           'other-target-files=',
           'other-item-list=',
           'output-target-files=',
+          'output-dir=',
+          'output-item-list=',
+          'output-ota=',
+          'output-super-empty=',
           'rebuild_recovery',
           'keep-tmp',
       ],
@@ -771,8 +877,11 @@ def main():
 
   if (len(args) != 0 or
       OPTIONS.system_target_files is None or
-      OPTIONS.other_target_files is None or
+      OPTIONS.other_target_files is None or (
-      OPTIONS.output_target_files is None):
+        OPTIONS.output_target_files is None and
+        OPTIONS.output_dir is None) or (
+        OPTIONS.output_dir is not None and
+        OPTIONS.output_item_list is None)):
     common.Usage(__doc__)
     sys.exit(1)
 
@@ -791,6 +900,11 @@ def main():
   else:
     other_item_list = default_other_item_list
 
+  if OPTIONS.output_item_list:
+    output_item_list = read_config_list(OPTIONS.output_item_list)
+  else:
+    output_item_list = None
+
   if not validate_config_lists(
       system_item_list=system_item_list,
       system_misc_info_keys=system_misc_info_keys,
@@ -806,6 +920,10 @@ def main():
           other_target_files=OPTIONS.other_target_files,
           other_item_list=other_item_list,
           output_target_files=OPTIONS.output_target_files,
+          output_dir=OPTIONS.output_dir,
+          output_item_list=output_item_list,
+          output_ota=OPTIONS.output_ota,
+          output_super_empty=OPTIONS.output_super_empty,
           rebuild_recovery=OPTIONS.rebuild_recovery),
       OPTIONS.keep_tmp)
 

tools/releasetools/test_merge_target_files.py

@@ -16,10 +16,12 @@
 
 import os.path
 
+import common
 import test_utils
-from merge_target_files import (
+from merge_target_files import (read_config_list, validate_config_lists,
-    read_config_list, validate_config_lists, default_system_item_list,
+                                default_system_item_list,
-    default_other_item_list, default_system_misc_info_keys)
+                                default_other_item_list,
+                                default_system_misc_info_keys, copy_items)
 
 
 class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase):
@@ -27,6 +29,58 @@ class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase):
   def setUp(self):
     self.testdata_dir = test_utils.get_testdata_dir()
 
+  def test_copy_items_CopiesItemsMatchingPatterns(self):
+
+    def createEmptyFile(path):
+      if not os.path.exists(os.path.dirname(path)):
+        os.makedirs(os.path.dirname(path))
+      open(path, 'a').close()
+      return path
+
+    def createSymLink(source, dest):
+      os.symlink(source, dest)
+      return dest
+
+    def getRelPaths(start, filepaths):
+      return set(
+          os.path.relpath(path=filepath, start=start) for filepath in filepaths)
+
+    input_dir = common.MakeTempDir()
+    output_dir = common.MakeTempDir()
+    expected_copied_items = []
+    actual_copied_items = []
+    patterns = ['*.cpp', 'subdir/*.txt']
+
+    # Create various files that we expect to get copied because they
+    # match one of the patterns.
+    expected_copied_items.extend([
+        createEmptyFile(os.path.join(input_dir, 'a.cpp')),
+        createEmptyFile(os.path.join(input_dir, 'b.cpp')),
+        createEmptyFile(os.path.join(input_dir, 'subdir', 'c.txt')),
+        createEmptyFile(os.path.join(input_dir, 'subdir', 'd.txt')),
+        createEmptyFile(
+            os.path.join(input_dir, 'subdir', 'subsubdir', 'e.txt')),
+        createSymLink('a.cpp', os.path.join(input_dir, 'a_link.cpp')),
+    ])
+    # Create some more files that we expect to not get copied.
+    createEmptyFile(os.path.join(input_dir, 'a.h'))
+    createEmptyFile(os.path.join(input_dir, 'b.h'))
+    createEmptyFile(os.path.join(input_dir, 'subdir', 'subsubdir', 'f.gif'))
+    createSymLink('a.h', os.path.join(input_dir, 'a_link.h'))
+
+    # Copy items.
+    copy_items(input_dir, output_dir, patterns)
+
+    # Assert the actual copied items match the ones we expected.
+    for dirpath, _, filenames in os.walk(output_dir):
+      actual_copied_items.extend(
+          os.path.join(dirpath, filename) for filename in filenames)
+    self.assertEqual(
+        getRelPaths(output_dir, actual_copied_items),
+        getRelPaths(input_dir, expected_copied_items))
+    self.assertEqual(
+        os.readlink(os.path.join(output_dir, 'a_link.cpp')), 'a.cpp')
+
   def test_read_config_list(self):
     system_item_list_file = os.path.join(self.testdata_dir,
                                          'merge_config_system_item_list')