Infer merge configs if not provided.

Bug: 221858722
Test: Create a merged package using inferred configs.
Test: atest --host releasetools_test
Change-Id: I93d67ca0f00be3f0e0424ed0a1e44c39ca2f3094

tools/releasetools/merge/merge_meta.py

@@ -142,7 +142,8 @@ def MergeMiscInfo(framework_meta_dir, vendor_meta_dir, merged_meta_dir):
 
   merged_dict = OPTIONS.vendor_misc_info
   for key in OPTIONS.framework_misc_info_keys:
-    merged_dict[key] = OPTIONS.framework_misc_info[key]
+    if key in OPTIONS.framework_misc_info:
+      merged_dict[key] = OPTIONS.framework_misc_info[key]
 
   # If AVB is enabled then ensure that we build vbmeta.img.
   # Partial builds with AVB enabled may set PRODUCT_BUILD_VBMETA_IMAGE=false to

tools/releasetools/merge/merge_target_files.py

@@ -31,20 +31,20 @@ Usage: merge_target_files [args]
       archive.
 
   --framework-item-list framework-item-list-file
-      The optional path to a newline-separated config file that replaces the
+      The optional path to a newline-separated config file of items that
-      contents of DEFAULT_FRAMEWORK_ITEM_LIST if provided.
+      are extracted as-is from the framework target files package.
 
   --framework-misc-info-keys framework-misc-info-keys-file
-      The optional path to a newline-separated config file that replaces the
+      The optional path to a newline-separated config file of keys to
-      contents of DEFAULT_FRAMEWORK_MISC_INFO_KEYS if provided.
+      extract from the framework META/misc_info.txt file.
 
   --vendor-target-files vendor-target-files-zip-archive
       The input target files package containing vendor bits. This is a zip
       archive.
 
   --vendor-item-list vendor-item-list-file
-      The optional path to a newline-separated config file that replaces the
+      The optional path to a newline-separated config file of items that
-      contents of DEFAULT_VENDOR_ITEM_LIST if provided.
+      are extracted as-is from the vendor target files package.
 
   --output-target-files output-target-files-package
       If provided, the output merged target files package. Also a zip archive.
@@ -107,6 +107,7 @@ import os
 import shutil
 import subprocess
 import sys
+import zipfile
 
 import add_img_to_target_files
 import build_image
@@ -127,13 +128,13 @@ OPTIONS = common.OPTIONS
 # Always turn on verbose logging.
 OPTIONS.verbose = True
 OPTIONS.framework_target_files = None
-OPTIONS.framework_item_list = None
+OPTIONS.framework_item_list = []
-OPTIONS.framework_misc_info_keys = None
+OPTIONS.framework_misc_info_keys = []
 OPTIONS.vendor_target_files = None
-OPTIONS.vendor_item_list = None
+OPTIONS.vendor_item_list = []
 OPTIONS.output_target_files = None
 OPTIONS.output_dir = None
-OPTIONS.output_item_list = None
+OPTIONS.output_item_list = []
 OPTIONS.output_ota = None
 OPTIONS.output_img = None
 OPTIONS.output_super_empty = None
@@ -147,64 +148,6 @@ OPTIONS.framework_dexpreopt_config = None
 OPTIONS.framework_dexpreopt_tools = None
 OPTIONS.vendor_dexpreopt_config = None
 
-# DEFAULT_FRAMEWORK_ITEM_LIST is a list of items to extract from the partial
-# framework target files package as is, meaning these items will land in the
-# output target files package exactly as they appear in the input partial
-# framework target files package.
-
-DEFAULT_FRAMEWORK_ITEM_LIST = (
-    'META/apkcerts.txt',
-    'META/filesystem_config.txt',
-    'META/root_filesystem_config.txt',
-    'META/update_engine_config.txt',
-    'PRODUCT/*',
-    'ROOT/*',
-    'SYSTEM/*',
-)
-
-# DEFAULT_FRAMEWORK_MISC_INFO_KEYS is a list of keys to obtain from the
-# framework instance of META/misc_info.txt. The remaining keys should come
-# from the vendor instance.
-
-DEFAULT_FRAMEWORK_MISC_INFO_KEYS = (
-    'avb_system_hashtree_enable',
-    'avb_system_add_hashtree_footer_args',
-    'avb_system_key_path',
-    'avb_system_algorithm',
-    'avb_system_rollback_index_location',
-    'avb_product_hashtree_enable',
-    'avb_product_add_hashtree_footer_args',
-    'avb_system_ext_hashtree_enable',
-    'avb_system_ext_add_hashtree_footer_args',
-    'system_root_image',
-    'root_dir',
-    'ab_update',
-    'default_system_dev_certificate',
-    'system_size',
-    'building_system_image',
-    'building_system_ext_image',
-    'building_product_image',
-)
-
-# DEFAULT_VENDOR_ITEM_LIST is a list of items to extract from the partial
-# vendor target files package as is, meaning these items will land in the output
-# target files package exactly as they appear in the input partial vendor target
-# files package.
-
-DEFAULT_VENDOR_ITEM_LIST = (
-    'META/boot_filesystem_config.txt',
-    'META/otakeys.txt',
-    'META/releasetools.py',
-    'META/vendor_filesystem_config.txt',
-    'BOOT/*',
-    'DATA/*',
-    'ODM/*',
-    'OTA/android-info.txt',
-    'PREBUILT_IMAGES/*',
-    'RADIO/*',
-    'VENDOR/*',
-)
-
 
 def create_merged_package(temp_dir):
   """Merges two target files packages into one target files structure.
@@ -614,16 +557,22 @@ def main():
   if (args or OPTIONS.framework_target_files is None or
       OPTIONS.vendor_target_files is None or
       (OPTIONS.output_target_files is None and OPTIONS.output_dir is None) or
-      (OPTIONS.output_dir is not None and OPTIONS.output_item_list is None) or
+      (OPTIONS.output_dir is not None and not OPTIONS.output_item_list) or
       (OPTIONS.rebuild_recovery and not OPTIONS.rebuild_sepolicy)):
     common.Usage(__doc__)
     sys.exit(1)
 
+  with zipfile.ZipFile(OPTIONS.framework_target_files, allowZip64=True) as fz:
+    framework_namelist = fz.namelist()
+  with zipfile.ZipFile(OPTIONS.vendor_target_files, allowZip64=True) as vz:
+    vendor_namelist = vz.namelist()
+
   if OPTIONS.framework_item_list:
     OPTIONS.framework_item_list = common.LoadListFromFile(
         OPTIONS.framework_item_list)
   else:
-    OPTIONS.framework_item_list = DEFAULT_FRAMEWORK_ITEM_LIST
+    OPTIONS.framework_item_list = merge_utils.InferItemList(
+        input_namelist=framework_namelist, framework=True)
   OPTIONS.framework_partition_set = merge_utils.ItemListToPartitionSet(
       OPTIONS.framework_item_list)
 
@@ -631,19 +580,19 @@ def main():
     OPTIONS.framework_misc_info_keys = common.LoadListFromFile(
         OPTIONS.framework_misc_info_keys)
   else:
-    OPTIONS.framework_misc_info_keys = DEFAULT_FRAMEWORK_MISC_INFO_KEYS
+    OPTIONS.framework_misc_info_keys = merge_utils.InferFrameworkMiscInfoKeys(
+        input_namelist=framework_namelist)
 
   if OPTIONS.vendor_item_list:
     OPTIONS.vendor_item_list = common.LoadListFromFile(OPTIONS.vendor_item_list)
   else:
-    OPTIONS.vendor_item_list = DEFAULT_VENDOR_ITEM_LIST
+    OPTIONS.vendor_item_list = merge_utils.InferItemList(
+        input_namelist=vendor_namelist, framework=False)
   OPTIONS.vendor_partition_set = merge_utils.ItemListToPartitionSet(
       OPTIONS.vendor_item_list)
 
   if OPTIONS.output_item_list:
     OPTIONS.output_item_list = common.LoadListFromFile(OPTIONS.output_item_list)
-  else:
-    OPTIONS.output_item_list = None
 
   if not merge_utils.ValidateConfigLists():
     sys.exit(1)

tools/releasetools/merge/merge_utils.py

@@ -91,29 +91,6 @@ def WriteSortedData(data, path):
       output.write(out_str)
 
 
-# The merge config lists should not attempt to extract items from both
-# builds for any of the following partitions. The partitions in
-# SINGLE_BUILD_PARTITIONS should come entirely from a single build (either
-# framework or vendor, but not both).
-
-_SINGLE_BUILD_PARTITIONS = (
-    'BOOT/',
-    'DATA/',
-    'ODM/',
-    'PRODUCT/',
-    'SYSTEM_EXT/',
-    'RADIO/',
-    'RECOVERY/',
-    'ROOT/',
-    'SYSTEM/',
-    'SYSTEM_OTHER/',
-    'VENDOR/',
-    'VENDOR_DLKM/',
-    'ODM_DLKM/',
-    'SYSTEM_DLKM/',
-)
-
-
 def ValidateConfigLists():
   """Performs validations on the merge config lists.
 
@@ -123,7 +100,7 @@ def ValidateConfigLists():
   has_error = False
 
   # Check that partitions only come from one input.
-  for partition in _SINGLE_BUILD_PARTITIONS:
+  for partition in _FRAMEWORK_PARTITIONS.union(_VENDOR_PARTITIONS):
     image_path = 'IMAGES/{}.img'.format(partition.lower().replace('/', ''))
     in_framework = (
         any(item.startswith(partition) for item in OPTIONS.framework_item_list)
@@ -185,3 +162,76 @@ def ItemListToPartitionSet(item_list):
       partition_set.add(partition_tag)
 
   return partition_set
+
+
+# Partitions that are grabbed from the framework partial build by default.
+_FRAMEWORK_PARTITIONS = {
+    'system', 'product', 'system_ext', 'system_other', 'root', 'system_dlkm'
+}
+# Partitions that are grabbed from the vendor partial build by default.
+_VENDOR_PARTITIONS = {
+    'vendor', 'odm', 'oem', 'boot', 'vendor_boot', 'recovery',
+    'prebuilt_images', 'radio', 'data', 'vendor_dlkm', 'odm_dlkm'
+}
+
+
+def InferItemList(input_namelist, framework):
+  item_list = []
+
+  # Some META items are grabbed from partial builds directly.
+  # Others are combined in merge_meta.py.
+  if framework:
+    item_list.extend([
+        'META/liblz4.so',
+        'META/postinstall_config.txt',
+        'META/update_engine_config.txt',
+        'META/zucchini_config.txt',
+    ])
+  else:  # vendor
+    item_list.extend([
+        'META/kernel_configs.txt',
+        'META/kernel_version.txt',
+        'META/otakeys.txt',
+        'META/releasetools.py',
+        'OTA/android-info.txt',
+    ])
+
+  # Grab a set of items for the expected partitions in the partial build.
+  for partition in (_FRAMEWORK_PARTITIONS if framework else _VENDOR_PARTITIONS):
+    for namelist in input_namelist:
+      if namelist.startswith('%s/' % partition.upper()):
+        fs_config_prefix = '' if partition == 'system' else '%s_' % partition
+        item_list.extend([
+            '%s/*' % partition.upper(),
+            'IMAGES/%s.img' % partition,
+            'IMAGES/%s.map' % partition,
+            'META/%sfilesystem_config.txt' % fs_config_prefix,
+        ])
+        break
+
+  return sorted(item_list)
+
+
+def InferFrameworkMiscInfoKeys(input_namelist):
+  keys = [
+      'ab_update',
+      'avb_vbmeta_system',
+      'avb_vbmeta_system_algorithm',
+      'avb_vbmeta_system_key_path',
+      'avb_vbmeta_system_rollback_index_location',
+      'default_system_dev_certificate',
+  ]
+
+  for partition in _FRAMEWORK_PARTITIONS:
+    for namelist in input_namelist:
+      if namelist.startswith('%s/' % partition.upper()):
+        fs_type_prefix = '' if partition == 'system' else '%s_' % partition
+        keys.extend([
+            'avb_%s_hashtree_enable' % partition,
+            'avb_%s_add_hashtree_footer_args' % partition,
+            '%s_disable_sparse' % partition,
+            'building_%s_image' % partition,
+            '%sfs_type' % fs_type_prefix,
+        ])
+
+  return sorted(keys)

tools/releasetools/merge/test_merge_utils.py

@@ -20,20 +20,12 @@ import common
 import merge_target_files
 import merge_utils
 import test_utils
-from merge_target_files import (
-    DEFAULT_FRAMEWORK_ITEM_LIST,
-    DEFAULT_VENDOR_ITEM_LIST,
-    DEFAULT_FRAMEWORK_MISC_INFO_KEYS,
-)
 
 
 class MergeUtilsTest(test_utils.ReleaseToolsTestCase):
 
   def setUp(self):
     self.OPTIONS = merge_target_files.OPTIONS
-    self.OPTIONS.framework_item_list = DEFAULT_FRAMEWORK_ITEM_LIST
-    self.OPTIONS.framework_misc_info_keys = DEFAULT_FRAMEWORK_MISC_INFO_KEYS
-    self.OPTIONS.vendor_item_list = DEFAULT_VENDOR_ITEM_LIST
 
   def test_CopyItems_CopiesItemsMatchingPatterns(self):
 
@@ -88,21 +80,30 @@ class MergeUtilsTest(test_utils.ReleaseToolsTestCase):
         os.readlink(os.path.join(output_dir, 'a_link.cpp')), 'a.cpp')
 
   def test_ValidateConfigLists_ReturnsFalseIfSharedExtractedPartition(self):
-    self.OPTIONS.vendor_item_list = list(DEFAULT_VENDOR_ITEM_LIST)
+    self.OPTIONS.system_item_list = [
+        'SYSTEM/*',
+    ]
+    self.OPTIONS.vendor_item_list = [
+        'SYSTEM/my_system_file',
+        'VENDOR/*',
+    ]
     self.OPTIONS.vendor_item_list.append('SYSTEM/my_system_file')
     self.assertFalse(merge_utils.ValidateConfigLists())
 
   def test_ValidateConfigLists_ReturnsFalseIfSharedExtractedPartitionImage(
       self):
-    self.OPTIONS.vendor_item_list = list(DEFAULT_VENDOR_ITEM_LIST)
+    self.OPTIONS.system_item_list = [
-    self.OPTIONS.vendor_item_list.append('IMAGES/system.img')
+        'SYSTEM/*',
+    ]
+    self.OPTIONS.vendor_item_list = [
+        'IMAGES/system.img',
+        'VENDOR/*',
+    ]
     self.assertFalse(merge_utils.ValidateConfigLists())
 
   def test_ValidateConfigLists_ReturnsFalseIfBadSystemMiscInfoKeys(self):
     for bad_key in ['dynamic_partition_list', 'super_partition_groups']:
-      self.OPTIONS.framework_misc_info_keys = list(
+      self.OPTIONS.framework_misc_info_keys = [bad_key]
-          DEFAULT_FRAMEWORK_MISC_INFO_KEYS)
-      self.OPTIONS.framework_misc_info_keys.append(bad_key)
       self.assertFalse(merge_utils.ValidateConfigLists())
 
   def test_ItemListToPartitionSet(self):
@@ -116,3 +117,81 @@ class MergeUtilsTest(test_utils.ReleaseToolsTestCase):
     ]
     partition_set = merge_utils.ItemListToPartitionSet(item_list)
     self.assertEqual(set(['product', 'system', 'system_ext']), partition_set)
+
+  def test_InferItemList_Framework(self):
+    zip_namelist = [
+        'SYSTEM/my_system_file',
+        'PRODUCT/my_product_file',
+    ]
+
+    item_list = merge_utils.InferItemList(zip_namelist, framework=True)
+
+    expected_framework_item_list = [
+        'IMAGES/product.img',
+        'IMAGES/product.map',
+        'IMAGES/system.img',
+        'IMAGES/system.map',
+        'META/filesystem_config.txt',
+        'META/liblz4.so',
+        'META/postinstall_config.txt',
+        'META/product_filesystem_config.txt',
+        'META/update_engine_config.txt',
+        'META/zucchini_config.txt',
+        'PRODUCT/*',
+        'SYSTEM/*',
+    ]
+
+    self.assertEqual(item_list, expected_framework_item_list)
+
+  def test_InferItemList_Vendor(self):
+    zip_namelist = [
+        'VENDOR/my_vendor_file',
+        'ODM/my_odm_file',
+    ]
+
+    item_list = merge_utils.InferItemList(zip_namelist, framework=False)
+
+    expected_vendor_item_list = [
+        'IMAGES/odm.img',
+        'IMAGES/odm.map',
+        'IMAGES/vendor.img',
+        'IMAGES/vendor.map',
+        'META/kernel_configs.txt',
+        'META/kernel_version.txt',
+        'META/odm_filesystem_config.txt',
+        'META/otakeys.txt',
+        'META/releasetools.py',
+        'META/vendor_filesystem_config.txt',
+        'ODM/*',
+        'OTA/android-info.txt',
+        'VENDOR/*',
+    ]
+    self.assertEqual(item_list, expected_vendor_item_list)
+
+  def test_InferFrameworkMiscInfoKeys(self):
+    zip_namelist = [
+        'SYSTEM/my_system_file',
+        'SYSTEM_EXT/my_system_ext_file',
+    ]
+
+    keys = merge_utils.InferFrameworkMiscInfoKeys(zip_namelist)
+
+    expected_keys = [
+        'ab_update',
+        'avb_system_add_hashtree_footer_args',
+        'avb_system_ext_add_hashtree_footer_args',
+        'avb_system_ext_hashtree_enable',
+        'avb_system_hashtree_enable',
+        'avb_vbmeta_system',
+        'avb_vbmeta_system_algorithm',
+        'avb_vbmeta_system_key_path',
+        'avb_vbmeta_system_rollback_index_location',
+        'building_system_ext_image',
+        'building_system_image',
+        'default_system_dev_certificate',
+        'fs_type',
+        'system_disable_sparse',
+        'system_ext_disable_sparse',
+        'system_ext_fs_type',
+    ]
+    self.assertEqual(keys, expected_keys)