Merge changes from topic "dp_retrofit_gen"

* changes:
  Generate retrofit OTA.
  Add split super images to target_files.

core/Makefile

@@ -3671,13 +3671,24 @@ endif
 ifdef BUILT_VENDOR_MATRIX
 	$(hide) cp $(BUILT_VENDOR_MATRIX) $(zip_root)/META/vendor_matrix.xml
 endif
+ifeq ($(PRODUCT_USE_DYNAMIC_PARTITIONS),true)
+	$(hide) echo "dynamic_partition_use=true" >> $(zip_root)/META/misc_info.txt
+endif
+ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true)
+	$(hide) echo "dynamic_partition_retrofit=true" >> $(zip_root)/META/misc_info.txt
+endif
 ifneq ($(BOARD_SUPER_PARTITION_SIZE),)
-	$(hide) echo "super_size=$(BOARD_SUPER_PARTITION_SIZE)" >> $(zip_root)/META/misc_info.txt
 	$(hide) echo "lpmake=$(notdir $(LPMAKE))" >> $(zip_root)/META/misc_info.txt
 	$(hide) echo -n "lpmake_args=" >> $(zip_root)/META/misc_info.txt
 	$(hide) echo $(call build-superimage-target-args,$(call super-slot-suffix)) \
 	    >> $(zip_root)/META/misc_info.txt
 endif
+ifneq ($(BOARD_SUPER_PARTITION_BLOCK_DEVICES),)
+	$(hide) echo "super_block_devices=$(BOARD_SUPER_PARTITION_BLOCK_DEVICES)" >> $(zip_root)/META/misc_info.txt
+endif
+ifneq ($(BOARD_SUPER_PARTITION_PARTITION_LIST),)
+	$(hide) echo "dynamic_partition_list=$(BOARD_SUPER_PARTITION_PARTITION_LIST)" >> $(zip_root)/META/misc_info.txt
+endif
 ifneq ($(BOARD_SUPER_PARTITION_GROUPS),)
 	$(hide) echo "super_partition_groups=$(BOARD_SUPER_PARTITION_GROUPS)" > $(zip_root)/META/dynamic_partitions_info.txt
 	$(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \
@@ -3714,6 +3725,19 @@ ifeq ($(build_ota_package),true)
 # -----------------------------------------------------------------
 # OTA update package
 
+# $(1): output file
+# $(2): additional args
+define build-ota-package-target
+PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \
+   build/make/tools/releasetools/ota_from_target_files -v \
+   --block \
+   --extracted_input_target_files $(patsubst %.zip,%,$(BUILT_TARGET_FILES_PACKAGE)) \
+   -p $(HOST_OUT) \
+   $(if $(OEM_OTA_CONFIG), -o $(OEM_OTA_CONFIG)) \
+   $(2) \
+   $(BUILT_TARGET_FILES_PACKAGE) $(1)
+endef
+
 name := $(TARGET_PRODUCT)
 ifeq ($(TARGET_BUILD_TYPE),debug)
   name := $(name)_debug
@@ -3733,18 +3757,39 @@ endif
 $(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) \
 		build/make/tools/releasetools/ota_from_target_files
 	@echo "Package OTA: $@"
-	$(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \
+	$(call build-ota-package-target,$@,-k $(KEY_CERT_PAIR))
-	   build/make/tools/releasetools/ota_from_target_files -v \
-	   --block \
-	   --extracted_input_target_files $(patsubst %.zip,%,$(BUILT_TARGET_FILES_PACKAGE)) \
-	   -p $(HOST_OUT) \
-	   -k $(KEY_CERT_PAIR) \
-	   $(if $(OEM_OTA_CONFIG), -o $(OEM_OTA_CONFIG)) \
-	   $(BUILT_TARGET_FILES_PACKAGE) $@
 
 .PHONY: otapackage
 otapackage: $(INTERNAL_OTA_PACKAGE_TARGET)
 
+ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true)
+name := $(TARGET_PRODUCT)
+ifeq ($(TARGET_BUILD_TYPE),debug)
+  name := $(name)_debug
+endif
+name := $(name)-ota-retrofit-$(FILE_NAME_TAG)
+
+INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip
+
+$(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET): KEY_CERT_PAIR := $(DEFAULT_KEY_CERT_PAIR)
+
+ifeq ($(AB_OTA_UPDATER),true)
+$(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET): $(BRILLO_UPDATE_PAYLOAD)
+else
+$(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET): $(BROTLI)
+endif
+
+$(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) \
+		build/make/tools/releasetools/ota_from_target_files
+	@echo "Package OTA (retrofit dynamic partitions): $@"
+	$(call build-ota-package-target,$@,-k $(KEY_CERT_PAIR) --retrofit_dynamic_partitions)
+
+.PHONY: otardppackage
+
+otapackage otardppackage: $(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET)
+
+endif # PRODUCT_RETROFIT_DYNAMIC_PARTITIONS
+
 endif    # build_ota_package
 
 # -----------------------------------------------------------------

core/config.mk

@@ -1041,8 +1041,13 @@ ifdef BOARD_SUPER_PARTITION_SIZE
 ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true)
 
 # The metadata device must be specified manually for retrofitting.
-ifndef BOARD_SUPER_PARTITION_METADATA_DEVICE
+ifeq ($(BOARD_SUPER_PARTITION_METADATA_DEVICE),)
-$(error Must specify BOARD_SUPER_PARTITION_METADATA_DEVICE if BOARD_SUPER_PARTITION_BLOCK_DEVICES is used.)
+$(error Must specify BOARD_SUPER_PARTITION_METADATA_DEVICE if PRODUCT_RETROFIT_DYNAMIC_PARTITIONS=true.)
+endif
+
+# The super partition block device list must be specified manually for retrofitting.
+ifeq ($(BOARD_SUPER_PARTITION_BLOCK_DEVICES),)
+$(error Must specify BOARD_SUPER_PARTITION_BLOCK_DEVICES if PRODUCT_RETROFIT_DYNAMIC_PARTITIONS=true.)
 endif
 
 # The metadata device must be included in the super partition block device list.

core/main.mk

@@ -1391,6 +1391,7 @@ else # TARGET_BUILD_APPS
   $(call dist-for-goals, droidcore, \
     $(INTERNAL_UPDATE_PACKAGE_TARGET) \
     $(INTERNAL_OTA_PACKAGE_TARGET) \
+    $(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET) \
     $(BUILT_OTATOOLS_PACKAGE) \
     $(SYMBOLS_ZIP) \
     $(COVERAGE_ZIP) \

tools/releasetools/add_img_to_target_files.py

@@ -330,6 +330,12 @@ def CreateImage(input_dir, info_dict, what, output_file, block_list=None):
       image_blocks_key = what + "_image_blocks"
       info_dict[image_blocks_key] = int(image_size) / 4096 - 1
 
+  use_dynamic_size = (
+    info_dict.get("use_dynamic_partition_size") == "true" and
+    what in shlex.split(info_dict.get("dynamic_partition_list", "").strip()))
+  if use_dynamic_size:
+    info_dict.update(build_image.GlobalDictFromImageProp(image_props, what))
+
 
 def AddUserdata(output_zip):
   """Create a userdata image and store it in output_zip.
@@ -656,8 +662,8 @@ def AddSuperEmpty(output_zip):
   """Create a super_empty.img and store it in output_zip."""
 
   img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "super_empty.img")
-  cmd = [OPTIONS.info_dict.get('lpmake')]
+  cmd = [OPTIONS.info_dict['lpmake']]
-  cmd += shlex.split(OPTIONS.info_dict.get('lpmake_args').strip())
+  cmd += shlex.split(OPTIONS.info_dict['lpmake_args'].strip())
   cmd += ['--output', img.name]
 
   proc = common.Run(cmd)
@@ -668,6 +674,58 @@ def AddSuperEmpty(output_zip):
   img.Write()
 
 
+def AddSuperSplit(output_zip):
+  """Create split super_*.img and store it in output_zip."""
+
+  def TransformPartitionArg(arg):
+    lst = arg.split(':')
+    # Because --auto-slot-suffixing for A/B, there is no need to remove suffix.
+    name = lst[0]
+    assert name + '_size' in OPTIONS.info_dict, (
+        "{} is a prebuilt. Dynamic partitions with prebuilt images "
+        "are not supported yet.".format(name))
+    size = OPTIONS.info_dict[name + '_size']
+    assert size is not None, \
+        '{0}_size is not found; is {0} built?'.format(name)
+    lst[2] = str(size)
+    return ':'.join(lst)
+
+  def GetLpmakeArgsWithSizes():
+    lpmake_args = shlex.split(OPTIONS.info_dict['lpmake_args'].strip())
+
+    for i, arg in enumerate(lpmake_args):
+      if arg == '--partition':
+        assert i + 1 < len(lpmake_args), \
+          'lpmake_args has --partition without value'
+        lpmake_args[i + 1] = TransformPartitionArg(lpmake_args[i + 1])
+
+    return lpmake_args
+
+  outdir = OutputFile(output_zip, OPTIONS.input_tmp, "OTA", "")
+  cmd = [OPTIONS.info_dict['lpmake']]
+  cmd += GetLpmakeArgsWithSizes()
+
+  source = OPTIONS.info_dict.get('dynamic_partition_list', '').strip()
+  if source:
+    cmd.append('--sparse')
+    for name in shlex.split(source):
+      img = os.path.join(OPTIONS.input_tmp, "IMAGES", '{}.img'.format(name))
+      # Because --auto-slot-suffixing for A/B, there is no need to add suffix.
+      cmd += ['--image', '{}={}'.format(name, img)]
+
+  cmd += ['--output', outdir.name]
+
+  proc = common.Run(cmd)
+  stdoutdata, _ = proc.communicate()
+  assert proc.returncode == 0, \
+      "lpmake tool failed:\n{}".format(stdoutdata)
+
+  for dev in OPTIONS.info_dict['super_block_devices'].strip().split():
+    img = OutputFile(output_zip, OPTIONS.input_tmp, "OTA",
+                     "super_" + dev + ".img")
+    img.Write()
+
+
 def ReplaceUpdatedFiles(zip_filename, files_list):
   """Updates all the ZIP entries listed in files_list.
 
@@ -861,10 +919,15 @@ def AddImagesToTargetFiles(filename):
     banner("vbmeta")
     AddVBMeta(output_zip, partitions, "vbmeta", vbmeta_partitions)
 
-  if OPTIONS.info_dict.get("super_size"):
+  if OPTIONS.info_dict.get("lpmake_args"):
     banner("super_empty")
     AddSuperEmpty(output_zip)
 
+    if OPTIONS.info_dict.get("dynamic_partition_retrofit") == "true":
+      banner("super split images")
+      AddSuperSplit(output_zip)
+    # TODO(b/119322123): Add super.img to target_files for non-retrofit
+
   banner("radio")
   ab_partitions_txt = os.path.join(OPTIONS.input_tmp, "META",
                                    "ab_partitions.txt")

tools/releasetools/ota_from_target_files.py

@@ -64,6 +64,13 @@ Common options that apply to both of non-A/B and A/B OTAs
       Generate an OTA package that will wipe the user data partition when
       installed.
 
+  --retrofit_dynamic_partitions
+      Generates an OTA package that updates a device to support dynamic
+      partitions (default False). This flag is implied when generating
+      an incremental OTA where the base build does not support dynamic
+      partitions but the target build does. For A/B, when this flag is set,
+      --skip_postinstall is implied.
+
 Non-A/B OTA specific options
 
   -b  (--binary) <file>
@@ -213,11 +220,14 @@ OPTIONS.payload_signer_args = []
 OPTIONS.extracted_input = None
 OPTIONS.key_passwords = []
 OPTIONS.skip_postinstall = False
+OPTIONS.retrofit_dynamic_partitions = False
 
 
 METADATA_NAME = 'META-INF/com/android/metadata'
 POSTINSTALL_CONFIG = 'META/postinstall_config.txt'
+DYNAMIC_PARTITION_INFO = 'META/dynamic_partitions_info.txt'
 UNZIP_PATTERN = ['IMAGES/*', 'META/*']
+SUPER_SPLIT_PATTERN = ['OTA/super_*.img']
 
 
 class BuildInfo(object):
@@ -1717,6 +1727,59 @@ def GetTargetFilesZipWithoutPostinstallConfig(input_file):
   return target_file
 
 
+def GetTargetFilesZipForRetrofitDynamicPartitions(input_file,
+                                                  super_block_devices):
+  """Returns a target-files.zip for retrofitting dynamic partitions.
+
+  This allows brillo_update_payload to generate an OTA based on the exact
+  bits on the block devices. Postinstall is disabled.
+
+  Args:
+    input_file: The input target-files.zip filename.
+    super_block_devices: The list of super block devices
+
+  Returns:
+    The filename of target-files.zip with *.img replaced with super_*.img for
+    each block device in super_block_devices.
+  """
+  assert super_block_devices, "No super_block_devices are specified."
+
+  replace = {'OTA/super_{}.img'.format(dev): 'IMAGES/{}.img'.format(dev)
+      for dev in super_block_devices}
+
+  target_file = common.MakeTempFile(prefix="targetfiles-", suffix=".zip")
+  shutil.copyfile(input_file, target_file)
+
+  with zipfile.ZipFile(input_file, 'r') as input_zip:
+    namelist = input_zip.namelist()
+
+  # Always skip postinstall for a retrofit update.
+  to_delete = [POSTINSTALL_CONFIG]
+
+  # Delete dynamic_partitions_info.txt so that brillo_update_payload thinks this
+  # is a regular update on devices without dynamic partitions support.
+  to_delete += [DYNAMIC_PARTITION_INFO]
+
+  # Remove the existing partition images.
+  to_delete += replace.values()
+
+  common.ZipDelete(target_file, to_delete)
+
+  input_tmp = common.UnzipTemp(input_file, SUPER_SPLIT_PATTERN)
+  target_zip = zipfile.ZipFile(target_file, 'a', allowZip64=True)
+
+  # Write super_{foo}.img as {foo}.img.
+  for src, dst in replace.items():
+    assert src in namelist, \
+          'Missing {} in {}; {} cannot be written'.format(src, input_file, dst)
+    unzipped_file = os.path.join(input_tmp, *src.split('/'))
+    common.ZipWrite(target_zip, unzipped_file, arcname=dst)
+
+  common.ZipClose(target_zip)
+
+  return target_file
+
+
 def WriteABOTAPackageWithBrilloScript(target_file, output_file,
                                       source_file=None):
   """Generates an Android OTA package that has A/B update payload."""
@@ -1738,7 +1801,10 @@ def WriteABOTAPackageWithBrilloScript(target_file, output_file,
   # Metadata to comply with Android OTA package format.
   metadata = GetPackageMetadata(target_info, source_info)
 
-  if OPTIONS.skip_postinstall:
+  if OPTIONS.retrofit_dynamic_partitions:
+    target_file = GetTargetFilesZipForRetrofitDynamicPartitions(
+        target_file, target_info.get("super_block_devices").strip().split())
+  elif OPTIONS.skip_postinstall:
     target_file = GetTargetFilesZipWithoutPostinstallConfig(target_file)
 
   # Generate payload.
@@ -1870,6 +1936,8 @@ def main(argv):
       OPTIONS.extracted_input = a
     elif o == "--skip_postinstall":
       OPTIONS.skip_postinstall = True
+    elif o == "--retrofit_dynamic_partitions":
+      OPTIONS.retrofit_dynamic_partitions = True
     else:
       return False
     return True
@@ -1900,6 +1968,7 @@ def main(argv):
                                  "payload_signer_args=",
                                  "extracted_input_target_files=",
                                  "skip_postinstall",
+                                 "retrofit_dynamic_partitions",
                              ], extra_option_handler=option_handler)
 
   if len(args) != 2:
@@ -1943,6 +2012,23 @@ def main(argv):
   # Load OEM dicts if provided.
   OPTIONS.oem_dicts = _LoadOemDicts(OPTIONS.oem_source)
 
+  # Assume retrofitting dynamic partitions when base build does not set
+  # dynamic_partition_use but target build does.
+  if (OPTIONS.source_info_dict and
+      OPTIONS.source_info_dict.get("dynamic_partition_use") != "true" and
+      OPTIONS.target_info_dict.get("dynamic_partition_use") == "true"):
+    if OPTIONS.target_info_dict.get("dynamic_partition_retrofit") != "true":
+      raise common.ExternalError(
+          "Expect to generate incremental OTA for retrofitting dynamic "
+          "partitions, but dynamic_partition_retrofit is not set in target "
+          "build.")
+    logger.info("Implicitly generating retrofit incremental OTA.")
+    OPTIONS.retrofit_dynamic_partitions = True
+
+  # Skip postinstall for retrofitting dynamic partitions.
+  if OPTIONS.retrofit_dynamic_partitions:
+    OPTIONS.skip_postinstall = True
+
   ab_update = OPTIONS.info_dict.get("ab_update") == "true"
 
   # Use the default key to sign the package if not specified with package_key.