Merge changes from topic "super_target_files"

am: 5df29604b9

Change-Id: Iba83bad72030a543ea4d653ac04d075a8b1b161b

core/Makefile

@@ -2921,7 +2921,12 @@ vbmetaimage-nodeps:
 endif # BOARD_AVB_ENABLE
 
 # -----------------------------------------------------------------
-# super partition image
+# Check image sizes <= size of super partition
+
+ifeq (,$(TARGET_BUILD_APPS))
+# Do not check for apps-only build
+
+ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION))
 
 # (1): list of items like "system", "vendor", "product", "product_services"
 # return: map each item into a command ( wrapped in $$() ) that reads the size
@@ -2929,75 +2934,10 @@ define read-size-of-partitions
 $(foreach p,$(1),$(call read-image-prop-dictionary,$($(p)image_intermediates)/generated_$(p)_image_info.txt,$(p)_size))
 endef
 
-ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION))
-
-# BOARD_SUPER_PARTITION_SIZE must be defined to build super image.
-ifneq ($(BOARD_SUPER_PARTITION_SIZE),)
-
 define super-slot-suffix
 $(if $(filter true,$(AB_OTA_UPDATER)),$(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)),,_a))
 endef
 
-# $(1): slot A suffix (_a or empty)
-# $(2): include images or not (true or empty)
-define build-superimage-target-args
-  $(if $(2), --sparse) \
-  --metadata-size 65536 \
-  --metadata-slots $(if $(filter true,$(AB_OTA_UPDATER)),2,1) \
-  --super-name $(BOARD_SUPER_PARTITION_METADATA_DEVICE) \
-  $(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)), $(if $(filter true,$(AB_OTA_UPDATER)), --auto-slot-suffixing)) \
-  $(foreach device,$(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \
-    --device $(device):$(BOARD_SUPER_PARTITION_$(call to-upper,$(device))_DEVICE_SIZE)) \
-  $(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \
-    --group $(group)$(1):$(BOARD_$(call to-upper,$(group))_SIZE) \
-    $(if $(1), --group $(group)_b:$(BOARD_$(call to-upper,$(group))_SIZE)) \
-    $(foreach name,$(BOARD_$(call to-upper,$(group))_PARTITION_LIST), \
-      --partition $(name)$(1):readonly:$(if $(2),$(call read-size-of-partitions,$(name)),0):$(group)$(1) \
-      $(if $(2), --image $(name)$(1)=$(call images-for-partitions,$(name))) \
-      $(if $(1), --partition $(name)_b:readonly:0:$(group)_b) \
-  ))
-endef
-
-# $(1): output image path
-# $(2): slot A suffix (_a or empty)
-# $(3): include images or not (true or empty)
-define build-superimage-target
-  $(LPMAKE) \
-    $(call build-superimage-target-args,$(2),$(3)) \
-    --output $(1)
-endef
-
-# For A/B devices, super partition always contains sub-partitions in the _a slot, because this
-# image should only be used for bootstrapping / initializing the device. When flashing the image,
-# bootloader fastboot should always mark _a slot as bootable.
-
-ifneq (true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS))
-INSTALLED_SUPERIMAGE_TARGET := $(PRODUCT_OUT)/super.img
-$(INSTALLED_SUPERIMAGE_TARGET): $(LPMAKE) $(call images-for-partitions,$(BOARD_SUPER_PARTITION_PARTITION_LIST))
-	$(call pretty,"Target super fs image: $@")
-	$(call build-superimage-target,$@,$(call super-slot-suffix),true)
-endif
-
-$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_TARGET))
-
-INSTALLED_SUPERIMAGE_EMPTY_TARGET := $(PRODUCT_OUT)/super_empty.img
-$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): $(LPMAKE)
-	$(call pretty,"Target empty super fs image: $@")
-	$(call build-superimage-target,$@,$(call super-slot-suffix))
-
-$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_EMPTY_TARGET))
-
-endif # BOARD_SUPER_PARTITION_SIZE
-endif # PRODUCT_BUILD_SUPER_PARTITION
-
-# -----------------------------------------------------------------
-# Check image sizes <= size of super partition
-
-ifeq (,$(TARGET_BUILD_APPS))
-# Do not check for apps-only build
-
-ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION))
-
 droid_targets: check-all-partition-sizes
 
 .PHONY: check-all-partition-sizes check-all-partition-sizes-nodeps
@@ -3329,6 +3269,30 @@ define fs_config
 (cd $(1); find . -type d | sed 's,$$,/,'; find . \! -type d) | cut -c 3- | sort | sed 's,^,$(2),' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) -R "$(2)"
 endef
 
+# $(1): file
+define dump-dynamic-partitions-info
+  $(if $(filter true,$(PRODUCT_USE_DYNAMIC_PARTITIONS)), \
+    echo "use_dynamic_partitions=true" >> $(1))
+  $(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)), \
+    echo "dynamic_partition_retrofit=true" >> $(1))
+  echo "lpmake=$(notdir $(LPMAKE))" >> $(1)
+  $(if $(filter true,$(PRODUCT_BUILD_SUPER_PARTITION)), $(if $(BOARD_SUPER_PARTITION_SIZE), \
+    echo "build_super_partition=true" >> $(1)))
+  echo "super_metadata_device=$(BOARD_SUPER_PARTITION_METADATA_DEVICE)" >> $(1)
+  $(if $(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \
+    echo "super_block_devices=$(BOARD_SUPER_PARTITION_BLOCK_DEVICES)" >> $(1))
+  $(foreach device,$(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \
+    echo "super_$(device)_device_size=$(BOARD_SUPER_PARTITION_$(call to-upper,$(device))_DEVICE_SIZE)" >> $(1);)
+  $(if $(BOARD_SUPER_PARTITION_PARTITION_LIST), \
+    echo "dynamic_partition_list=$(BOARD_SUPER_PARTITION_PARTITION_LIST)" >> $(1))
+  $(if $(BOARD_SUPER_PARTITION_GROUPS),
+    echo "super_partition_groups=$(BOARD_SUPER_PARTITION_GROUPS)" >> $(1))
+  $(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \
+    echo "super_$(group)_group_size=$(BOARD_$(call to-upper,$(group))_SIZE)" >> $(1); \
+    $(if $(BOARD_$(call to-upper,$(group))_PARTITION_LIST), \
+      echo "super_$(group)_partition_list=$(BOARD_$(call to-upper,$(group))_PARTITION_LIST)" >> $(1);))
+endef
+
 # Depending on the various images guarantees that the underlying
 # directories are up-to-date.
 $(BUILT_TARGET_FILES_PACKAGE): \
@@ -3644,6 +3608,7 @@ ifdef BOARD_AVB_DTBO_KEY_PATH
 endif # BOARD_AVB_DTBO_KEY_PATH
 endif # BOARD_AVB_ENABLE
 endif # BOARD_PREBUILT_DTBOIMAGE
+	$(call dump-dynamic-partitions-info,$(zip_root)/META/misc_info.txt)
 	@# The radio images in BOARD_PACK_RADIOIMAGES will be additionally copied from RADIO/ into
 	@# IMAGES/, which then will be added into <product>-img.zip. Such images must be listed in
 	@# INSTALLED_RADIOIMAGE_TARGET.
@@ -3689,24 +3654,6 @@ endif
 ifdef BUILT_VENDOR_MATRIX
 	$(hide) cp $(BUILT_VENDOR_MATRIX) $(zip_root)/META/vendor_matrix.xml
 endif
-ifeq ($(PRODUCT_USE_DYNAMIC_PARTITIONS),true)
-	$(hide) echo "use_dynamic_partitions=true" >> $(zip_root)/META/misc_info.txt
-endif
-ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true)
-	$(hide) echo "dynamic_partition_retrofit=true" >> $(zip_root)/META/misc_info.txt
-endif
-ifneq ($(BOARD_SUPER_PARTITION_SIZE),)
-	$(hide) echo "lpmake=$(notdir $(LPMAKE))" >> $(zip_root)/META/misc_info.txt
-	$(hide) echo -n "lpmake_args=" >> $(zip_root)/META/misc_info.txt
-	$(hide) echo $(call build-superimage-target-args,$(call super-slot-suffix)) \
-	    >> $(zip_root)/META/misc_info.txt
-endif
-ifneq ($(BOARD_SUPER_PARTITION_BLOCK_DEVICES),)
-	$(hide) echo "super_block_devices=$(BOARD_SUPER_PARTITION_BLOCK_DEVICES)" >> $(zip_root)/META/misc_info.txt
-endif
-ifneq ($(BOARD_SUPER_PARTITION_PARTITION_LIST),)
-	$(hide) echo "dynamic_partition_list=$(BOARD_SUPER_PARTITION_PARTITION_LIST)" >> $(zip_root)/META/misc_info.txt
-endif
 ifneq ($(BOARD_SUPER_PARTITION_GROUPS),)
 	$(hide) echo "super_partition_groups=$(BOARD_SUPER_PARTITION_GROUPS)" > $(zip_root)/META/dynamic_partitions_info.txt
 	$(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \
@@ -3974,6 +3921,41 @@ $(PROGUARD_DICT_ZIP) :
 
 endif # TARGET_BUILD_APPS
 
+# -----------------------------------------------------------------
+# super partition image
+
+ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION))
+
+# BOARD_SUPER_PARTITION_SIZE must be defined to build super image.
+ifneq ($(BOARD_SUPER_PARTITION_SIZE),)
+
+ifneq (true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS))
+INSTALLED_SUPERIMAGE_TARGET := $(PRODUCT_OUT)/super.img
+$(INSTALLED_SUPERIMAGE_TARGET): extracted_input_target_files := $(patsubst %.zip,%,$(BUILT_TARGET_FILES_PACKAGE))
+$(INSTALLED_SUPERIMAGE_TARGET): $(LPMAKE) $(BUILT_TARGET_FILES_PACKAGE) $(BUILD_SUPER_IMAGE)
+	$(call pretty,"Target super fs image: $@")
+	$(BUILD_SUPER_IMAGE) -v $(extracted_input_target_files) $@
+endif
+
+$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_TARGET))
+
+INSTALLED_SUPERIMAGE_EMPTY_TARGET := $(PRODUCT_OUT)/super_empty.img
+$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): intermediates := $(call intermediates-dir-for,PACKAGING,super_empty)
+$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): $(LPMAKE) $(BUILD_SUPER_IMAGE)
+	$(call pretty,"Target empty super fs image: $@")
+	mkdir -p $(intermediates)
+	rm -rf $(intermediates)/misc_info.txt
+	$(call dump-dynamic-partitions-info,$(intermediates)/misc_info.txt)
+ifeq ($(AB_OTA_UPDATER),true)
+	$(hide) echo "ab_update=true" >> $(intermediates)/misc_info.txt
+endif
+	$(BUILD_SUPER_IMAGE) -v $(intermediates)/misc_info.txt $@
+
+$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_EMPTY_TARGET))
+
+endif # BOARD_SUPER_PARTITION_SIZE != ""
+endif # PRODUCT_BUILD_SUPER_PARTITION == "true"
+
 # -----------------------------------------------------------------
 # dalvik something
 .PHONY: dalvikfiles

core/config.mk

@@ -698,6 +698,7 @@ DATA_BINDING_COMPILER := $(HOST_OUT_JAVA_LIBRARIES)/databinding-compiler.jar
 FAT16COPY := build/make/tools/fat16copy.py
 CHECK_LINK_TYPE := build/make/tools/check_link_type.py
 LPMAKE := $(HOST_OUT_EXECUTABLES)/lpmake$(HOST_EXECUTABLE_SUFFIX)
+BUILD_SUPER_IMAGE := build/make/tools/releasetools/build_super_image.py
 
 PROGUARD := external/proguard/bin/proguard.sh
 JAVATAGS := build/make/tools/java-event-log-tags.py

tools/releasetools/add_img_to_target_files.py

@@ -55,6 +55,7 @@ import uuid
 import zipfile
 
 import build_image
+import build_super_image
 import common
 import rangelib
 import sparse_img
@@ -648,64 +649,15 @@ def AddSuperEmpty(output_zip):
   """Create a super_empty.img and store it in output_zip."""
 
   img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "super_empty.img")
-  cmd = [OPTIONS.info_dict['lpmake']]
+  build_super_image.BuildSuperImage(OPTIONS.info_dict, img.name)
-  cmd += shlex.split(OPTIONS.info_dict['lpmake_args'].strip())
-  cmd += ['--output', img.name]
-  common.RunAndCheckOutput(cmd)
-
   img.Write()
 
 
 def AddSuperSplit(output_zip):
   """Create split super_*.img and store it in output_zip."""
 
-  def GetPartitionSizeFromImage(img):
+  outdir = os.path.join(OPTIONS.input_tmp, "OTA")
-    try:
+  build_super_image.BuildSuperImage(OPTIONS.input_tmp, outdir)
-      simg = sparse_img.SparseImage(img)
-      return simg.blocksize * simg.total_blocks
-    except ValueError:
-      return os.path.getsize(img)
-
-  def TransformPartitionArg(arg):
-    lst = arg.split(':')
-    # Because --auto-slot-suffixing for A/B, there is no need to remove suffix.
-    name = lst[0]
-    if name + '_size' in OPTIONS.info_dict:
-      size = str(OPTIONS.info_dict[name + '_size'])
-      logger.info("Using %s_size = %s", name, size)
-    else:
-      size = str(GetPartitionSizeFromImage(
-          os.path.join(OPTIONS.input_tmp, "IMAGES", '{}.img'.format(name))))
-      logger.info("Using size of prebuilt %s = %s", name, size)
-    lst[2] = size
-    return ':'.join(lst)
-
-  def GetLpmakeArgsWithSizes():
-    lpmake_args = shlex.split(OPTIONS.info_dict['lpmake_args'].strip())
-
-    for i, arg in enumerate(lpmake_args):
-      if arg == '--partition':
-        assert i + 1 < len(lpmake_args), \
-          'lpmake_args has --partition without value'
-        lpmake_args[i + 1] = TransformPartitionArg(lpmake_args[i + 1])
-
-    return lpmake_args
-
-  outdir = OutputFile(output_zip, OPTIONS.input_tmp, "OTA", "")
-  cmd = [OPTIONS.info_dict['lpmake']]
-  cmd += GetLpmakeArgsWithSizes()
-
-  source = OPTIONS.info_dict.get('dynamic_partition_list', '').strip()
-  if source:
-    cmd.append('--sparse')
-    for name in shlex.split(source):
-      img = os.path.join(OPTIONS.input_tmp, "IMAGES", '{}.img'.format(name))
-      # Because --auto-slot-suffixing for A/B, there is no need to add suffix.
-      cmd += ['--image', '{}={}'.format(name, img)]
-
-  cmd += ['--output', outdir.name]
-
-  common.RunAndCheckOutput(cmd)
 
   for dev in OPTIONS.info_dict['super_block_devices'].strip().split():
     img = OutputFile(output_zip, OPTIONS.input_tmp, "OTA",
@@ -906,14 +858,13 @@ def AddImagesToTargetFiles(filename):
     banner("vbmeta")
     AddVBMeta(output_zip, partitions, "vbmeta", vbmeta_partitions)
 
-  if OPTIONS.info_dict.get("lpmake_args"):
+  if OPTIONS.info_dict.get("build_super_partition"):
     banner("super_empty")
     AddSuperEmpty(output_zip)
 
     if OPTIONS.info_dict.get("dynamic_partition_retrofit") == "true":
       banner("super split images")
       AddSuperSplit(output_zip)
-    # TODO(b/119322123): Add super.img to target_files for non-retrofit
 
   banner("radio")
   ab_partitions_txt = os.path.join(OPTIONS.input_tmp, "META",

tools/releasetools/build_super_image.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Usage: build_super_image input_file output_dir_or_file
+
+input_file: one of the following:
+  - directory containing extracted target files. It will load info from
+    META/misc_info.txt and build full super image / split images using source
+    images from IMAGES/.
+  - target files package. Same as above, but extracts the archive before
+    building super image.
+  - a dictionary file containing input arguments to build. Check
+    `dump_dynamic_partitions_info' for details.
+    In addition:
+    - "ab_update" needs to be true for A/B devices.
+    - If source images should be included in the output image (for super.img
+      and super split images), a list of "*_image" should be paths of each
+      source images.
+
+output_dir_or_file:
+    If a single super image is built (for super_empty.img, or super.img for
+    launch devices), this argument is the output file.
+    If a collection of split images are built (for retrofit devices), this
+    argument is the output directory.
+"""
+
+from __future__ import print_function
+
+import logging
+import os.path
+import shlex
+import sys
+import zipfile
+
+import common
+import sparse_img
+
+if sys.hexversion < 0x02070000:
+  print("Python 2.7 or newer is required.", file=sys.stderr)
+  sys.exit(1)
+
+logger = logging.getLogger(__name__)
+
+
+UNZIP_PATTERN = ["IMAGES/*", "META/*"]
+
+
+def GetPartitionSizeFromImage(img):
+  try:
+    simg = sparse_img.SparseImage(img)
+    return simg.blocksize * simg.total_blocks
+  except ValueError:
+    return os.path.getsize(img)
+
+
+def BuildSuperImageFromDict(info_dict, output):
+
+  cmd = [info_dict["lpmake"],
+         "--metadata-size", "65536",
+         "--super-name", info_dict["super_metadata_device"]]
+
+  ab_update = info_dict.get("ab_update") == "true"
+  retrofit = info_dict.get("dynamic_partition_retrofit") == "true"
+  block_devices = shlex.split(info_dict.get("super_block_devices", "").strip())
+  groups = shlex.split(info_dict.get("super_partition_groups", "").strip())
+
+  if ab_update:
+    cmd += ["--metadata-slots", "2"]
+  else:
+    cmd += ["--metadata-slots", "1"]
+
+  if ab_update and retrofit:
+    cmd.append("--auto-slot-suffixing")
+
+  for device in block_devices:
+    size = info_dict["super_{}_device_size".format(device)]
+    cmd += ["--device", "{}:{}".format(device, size)]
+
+  append_suffix = ab_update and not retrofit
+  has_image = False
+  for group in groups:
+    group_size = info_dict["super_{}_group_size".format(group)]
+    if append_suffix:
+      cmd += ["--group", "{}_a:{}".format(group, group_size),
+              "--group", "{}_b:{}".format(group, group_size)]
+    else:
+      cmd += ["--group", "{}:{}".format(group, group_size)]
+
+    partition_list = shlex.split(
+        info_dict["super_{}_partition_list".format(group)].strip())
+
+    for partition in partition_list:
+      image = info_dict.get("{}_image".format(partition))
+      image_size = 0
+      if image:
+        image_size = GetPartitionSizeFromImage(image)
+        has_image = True
+      if append_suffix:
+        cmd += ["--partition",
+                "{}_a:readonly:{}:{}_a".format(partition, image_size, group),
+                "--partition",
+                "{}_b:readonly:0:{}_b".format(partition, group)]
+        if image:
+          # For A/B devices, super partition always contains sub-partitions in
+          # the _a slot, because this image should only be used for
+          # bootstrapping / initializing the device. When flashing the image,
+          # bootloader fastboot should always mark _a slot as bootable.
+          cmd += ["--image", "{}_a={}".format(partition, image)]
+      else:
+        cmd += ["--partition",
+                "{}:readonly:{}:{}".format(partition, image_size, group)]
+        if image:
+          cmd += ["--image", "{}={}".format(partition, image)]
+
+  if has_image:
+    cmd.append("--sparse")
+
+  cmd += ["--output", output]
+
+  common.RunAndCheckOutput(cmd)
+
+  if retrofit and has_image:
+    logger.info("Done writing images to directory %s", output)
+  else:
+    logger.info("Done writing image %s", output)
+
+
+def BuildSuperImageFromExtractedTargetFiles(inp, out):
+  info_dict = common.LoadInfoDict(inp)
+  partition_list = shlex.split(
+      info_dict.get("dynamic_partition_list", "").strip())
+  for partition in partition_list:
+    info_dict["{}_image".format(partition)] = os.path.join(
+        inp, "IMAGES", "{}.img".format(partition))
+  return BuildSuperImageFromDict(info_dict, out)
+
+
+def BuildSuperImageFromTargetFiles(inp, out):
+  input_tmp = common.UnzipTemp(inp, UNZIP_PATTERN)
+  return BuildSuperImageFromExtractedTargetFiles(input_tmp, out)
+
+
+def BuildSuperImage(inp, out):
+
+  if isinstance(inp, dict):
+    logger.info("Building super image from info dict...")
+    return BuildSuperImageFromDict(inp, out)
+
+  if isinstance(inp, str):
+    if os.path.isdir(inp):
+      logger.info("Building super image from extracted target files...")
+      return BuildSuperImageFromExtractedTargetFiles(inp, out)
+
+    if zipfile.is_zipfile(inp):
+      logger.info("Building super image from target files...")
+      return BuildSuperImageFromTargetFiles(inp, out)
+
+    if os.path.isfile(inp):
+      with open(inp) as f:
+        lines = f.read()
+      logger.info("Building super image from info dict...")
+      return BuildSuperImageFromDict(common.LoadDictionaryFromLines(lines.split("\n")), out)
+
+  raise ValueError("{} is not a dictionary or a valid path".format(inp))
+
+
+def main(argv):
+
+  args = common.ParseOptions(argv, __doc__)
+
+  if len(args) != 2:
+    common.Usage(__doc__)
+    sys.exit(1)
+
+  common.InitLogging()
+
+  BuildSuperImage(args[0], args[1])
+
+
+if __name__ == "__main__":
+  try:
+    common.CloseInheritedPipes()
+    main(sys.argv[1:])
+  except common.ExternalError:
+    logger.exception("\n   ERROR:\n")
+    sys.exit(1)
+  finally:
+    common.Cleanup()