From 2ee00d5f66ad789b7d97bffd2e4b1365597b848d Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Sun, 5 Mar 2017 19:51:56 -0800 Subject: [PATCH] Allow add_img_to_target_files to work on a directory In addition to the current behavior of add_img_to_target_files working on an existing zip file, allow passing in a directory where the target files have already been extracted. When in this mode, it writes the images out to that directory instead of the zip file. This allows us to call add_img_to_target_files on the temp directory used during the build to create the target files package, saving the time and space of unzipping what we just zipped. This also allows us to use the parallel soong_zip, which compresses the images much faster. Test: aosp_marlin target_files zip is the same before/after this change Test: marlin target_files zip is the same before/after this change Test: bullhead target_files zip is the same before/after this change Change-Id: I155654cdc7ad7754ba4ef76ec69c31f504a58198 --- tools/releasetools/add_img_to_target_files.py | 224 ++++++++++-------- tools/releasetools/common.py | 4 + 2 files changed, 128 insertions(+), 100 deletions(-) diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py index 9403a77f24..0b8f3a0d36 100755 --- a/tools/releasetools/add_img_to_target_files.py +++ b/tools/releasetools/add_img_to_target_files.py @@ -73,6 +73,25 @@ OPTIONS.replace_verity_public_key = False OPTIONS.replace_verity_private_key = False OPTIONS.is_signing = False + +class OutputFile(object): + def __init__(self, output_zip, input_dir, prefix, name): + self._output_zip = output_zip + self.input_name = os.path.join(input_dir, prefix, name) + + if self._output_zip: + self._zip_name = os.path.join(prefix, name) + + root, suffix = os.path.splitext(name) + self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix) + else: + self.name = self.input_name + + def Write(self): + if self._output_zip: + common.ZipWrite(self._output_zip, self.name, self._zip_name) + + def GetCareMap(which, imgname): """Generate care_map of system (or vendor) partition""" @@ -98,10 +117,10 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): """Turn the contents of SYSTEM into a system image and store it in output_zip. Returns the name of the system image file.""" - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img") + if os.path.exists(img.input_name): print("system.img already exists in %s, no need to rebuild..." % (prefix,)) - return prebuilt_path + return img.input_name def output_sink(fn, data): ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w") @@ -113,74 +132,52 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img, boot_img, info_dict=OPTIONS.info_dict) - block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map") - imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict, - block_list=block_list) + block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map") + CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img, + block_list=block_list) - common.ZipWrite(output_zip, imgname, prefix + "system.img") - common.ZipWrite(output_zip, block_list, prefix + "system.map") - return imgname - - -def BuildSystem(input_dir, info_dict, block_list=None): - """Build the (sparse) system image and return the name of a temp - file containing it.""" - return CreateImage(input_dir, info_dict, "system", block_list=block_list) + return img.name def AddSystemOther(output_zip, prefix="IMAGES/"): """Turn the contents of SYSTEM_OTHER into a system_other image and store it in output_zip.""" - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img") + if os.path.exists(img.input_name): print("system_other.img already exists in %s, no need to rebuild..." % ( prefix,)) return - imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict) - common.ZipWrite(output_zip, imgname, prefix + "system_other.img") - -def BuildSystemOther(input_dir, info_dict): - """Build the (sparse) system_other image and return the name of a temp - file containing it.""" - return CreateImage(input_dir, info_dict, "system_other", block_list=None) + CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img) def AddVendor(output_zip, prefix="IMAGES/"): """Turn the contents of VENDOR into a vendor image and store in it output_zip.""" - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img") + if os.path.exists(img.input_name): print("vendor.img already exists in %s, no need to rebuild..." % (prefix,)) - return prebuilt_path + return img.input_name - block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map") - imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict, - block_list=block_list) - common.ZipWrite(output_zip, imgname, prefix + "vendor.img") - common.ZipWrite(output_zip, block_list, prefix + "vendor.map") - return imgname + block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map") + CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img, + block_list=block_list) + return img.name -def BuildVendor(input_dir, info_dict, block_list=None): - """Build the (sparse) vendor image and return the name of a temp - file containing it.""" - return CreateImage(input_dir, info_dict, "vendor", block_list=block_list) - - -def CreateImage(input_dir, info_dict, what, block_list=None): +def CreateImage(input_dir, info_dict, what, output_file, block_list=None): print("creating " + what + ".img...") - img = common.MakeTempFile(prefix=what + "-", suffix=".img") - # The name of the directory it is making an image out of matters to # mkyaffs2image. It wants "system" but we have a directory named # "SYSTEM", so create a symlink. + temp_dir = tempfile.mkdtemp() + OPTIONS.tempfiles.append(temp_dir) try: os.symlink(os.path.join(input_dir, what.upper()), - os.path.join(input_dir, what)) + os.path.join(temp_dir, what)) except OSError as e: # bogus error on my mac version? # File "./build/tools/releasetools/img_from_target_files" @@ -215,12 +212,16 @@ def CreateImage(input_dir, info_dict, what, block_list=None): if fs_config: image_props["fs_config"] = fs_config if block_list: - image_props["block_list"] = block_list + image_props["block_list"] = block_list.name - succ = build_image.BuildImage(os.path.join(input_dir, what), - image_props, img) + succ = build_image.BuildImage(os.path.join(temp_dir, what), + image_props, output_file.name) assert succ, "build " + what + ".img image failed" + output_file.Write() + if block_list: + block_list.Write() + is_verity_partition = "verity_block_device" in image_props verity_supported = image_props.get("verity") == "true" if is_verity_partition and verity_supported: @@ -229,8 +230,6 @@ def CreateImage(input_dir, info_dict, what, block_list=None): adjusted_blocks_key = what + "_adjusted_partition_size" info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1 - return img - def AddUserdata(output_zip, prefix="IMAGES/"): """Create a userdata image and store it in output_zip. @@ -241,8 +240,8 @@ def AddUserdata(output_zip, prefix="IMAGES/"): in OPTIONS.info_dict. """ - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img") + if os.path.exists(img.input_name): print("userdata.img already exists in %s, no need to rebuild..." % ( prefix,)) return @@ -265,6 +264,7 @@ def AddUserdata(output_zip, prefix="IMAGES/"): # empty dir named "data", or a symlink to the DATA dir, # and build the image from that. temp_dir = tempfile.mkdtemp() + OPTIONS.tempfiles.append(temp_dir) user_dir = os.path.join(temp_dir, "data") empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true") if empty: @@ -275,8 +275,6 @@ def AddUserdata(output_zip, prefix="IMAGES/"): os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"), user_dir) - img = tempfile.NamedTemporaryFile() - fstab = OPTIONS.info_dict["fstab"] if fstab: image_props["fs_type"] = fstab["/data"].fs_type @@ -284,17 +282,15 @@ def AddUserdata(output_zip, prefix="IMAGES/"): assert succ, "build userdata.img image failed" common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict) - common.ZipWrite(output_zip, img.name, prefix + "userdata.img") - img.close() - shutil.rmtree(temp_dir) + img.Write() def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"): """Create a VBMeta image and store it in output_zip.""" - _, img_file_name = tempfile.mkstemp() + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img") avbtool = os.getenv('AVBTOOL') or "avbtool" cmd = [avbtool, "make_vbmeta_image", - "--output", img_file_name, + "--output", img.name, "--include_descriptors_from_image", boot_img_path, "--include_descriptors_from_image", system_img_path, "--generate_dm_verity_cmdline_from_hashtree", system_img_path] @@ -305,19 +301,19 @@ def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"): p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() assert p.returncode == 0, "avbtool make_vbmeta_image failed" - common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img") + img.Write() def AddPartitionTable(output_zip, prefix="IMAGES/"): """Create a partition table image and store it in output_zip.""" - _, img_file_name = tempfile.mkstemp() - _, bpt_file_name = tempfile.mkstemp() + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img") + bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt") # use BPTTOOL from environ, or "bpttool" if empty or not set. bpttool = os.getenv("BPTTOOL") or "bpttool" - cmd = [bpttool, "make_table", "--output_json", bpt_file_name, - "--output_gpt", img_file_name] + cmd = [bpttool, "make_table", "--output_json", bpt.name, + "--output_gpt", img.name] input_files_str = OPTIONS.info_dict["board_bpt_input_files"] input_files = input_files_str.split(" ") for i in input_files: @@ -333,15 +329,15 @@ def AddPartitionTable(output_zip, prefix="IMAGES/"): p.communicate() assert p.returncode == 0, "bpttool make_table failed" - common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img") - common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt") + img.Write() + bpt.Write() def AddCache(output_zip, prefix="IMAGES/"): """Create an empty cache image and store it in output_zip.""" - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img") + if os.path.exists(img.input_name): print("cache.img already exists in %s, no need to rebuild..." % (prefix,)) return @@ -362,9 +358,9 @@ def AddCache(output_zip, prefix="IMAGES/"): # mkyaffs2image. So we create a temp dir, and within it we create an # empty dir named "cache", and build the image from that. temp_dir = tempfile.mkdtemp() + OPTIONS.tempfiles.append(temp_dir) user_dir = os.path.join(temp_dir, "cache") os.mkdir(user_dir) - img = tempfile.NamedTemporaryFile() fstab = OPTIONS.info_dict["fstab"] if fstab: @@ -373,35 +369,39 @@ def AddCache(output_zip, prefix="IMAGES/"): assert succ, "build cache.img image failed" common.CheckSize(img.name, "cache.img", OPTIONS.info_dict) - common.ZipWrite(output_zip, img.name, prefix + "cache.img") - img.close() - os.rmdir(user_dir) - os.rmdir(temp_dir) + img.Write() def AddImagesToTargetFiles(filename): - OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) + if os.path.isdir(filename): + OPTIONS.input_tmp = os.path.abspath(filename) + input_zip = None + else: + OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) if not OPTIONS.add_missing: - for n in input_zip.namelist(): - if n.startswith("IMAGES/"): - print("target_files appears to already contain images.") - sys.exit(1) + if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")): + print("target_files appears to already contain images.") + sys.exit(1) - try: - input_zip.getinfo("VENDOR/") - has_vendor = True - except KeyError: - has_vendor = False + has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR")) + has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp, + "SYSTEM_OTHER")) - has_system_other = "SYSTEM_OTHER/" in input_zip.namelist() + if input_zip: + OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) - OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) - - common.ZipClose(input_zip) - output_zip = zipfile.ZipFile(filename, "a", - compression=zipfile.ZIP_DEFLATED, - allowZip64=True) + common.ZipClose(input_zip) + output_zip = zipfile.ZipFile(filename, "a", + compression=zipfile.ZIP_DEFLATED, + allowZip64=True) + else: + OPTIONS.info_dict = common.LoadInfoDict(filename, filename) + output_zip = None + images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES") + if not os.path.isdir(images_dir): + os.makedirs(images_dir) + images_dir = None has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true") system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true") @@ -422,7 +422,10 @@ def AddImagesToTargetFiles(filename): boot_image = common.GetBootableImage( "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") if boot_image: - boot_image.AddToZip(output_zip) + if output_zip: + boot_image.AddToZip(output_zip) + else: + boot_image.WriteToDir(OPTIONS.input_tmp) recovery_image = None if has_recovery: @@ -438,7 +441,10 @@ def AddImagesToTargetFiles(filename): recovery_image = common.GetBootableImage( "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") if recovery_image: - recovery_image.AddToZip(output_zip) + if output_zip: + recovery_image.AddToZip(output_zip) + else: + recovery_image.WriteToDir(OPTIONS.input_tmp) banner("recovery (two-step image)") # The special recovery.img for two-step package use. @@ -446,7 +452,10 @@ def AddImagesToTargetFiles(filename): "IMAGES/recovery-two-step.img", "recovery-two-step.img", OPTIONS.input_tmp, "RECOVERY", two_step_image=True) if recovery_two_step_image: - recovery_two_step_image.AddToZip(output_zip) + if output_zip: + recovery_two_step_image.AddToZip(output_zip) + else: + recovery_two_step_image.WriteToDir(OPTIONS.input_tmp) banner("system") system_img_path = AddSystem( @@ -502,24 +511,39 @@ def AddImagesToTargetFiles(filename): img_vendor_dir = os.path.join( OPTIONS.input_tmp, "VENDOR_IMAGES") if os.path.exists(img_radio_path): - common.ZipWrite(output_zip, img_radio_path, - os.path.join("IMAGES", img_name)) + if output_zip: + common.ZipWrite(output_zip, img_radio_path, + os.path.join("IMAGES", img_name)) + else: + shutil.copy(img_radio_path, prebuilt_path) else: for root, _, files in os.walk(img_vendor_dir): if img_name in files: - common.ZipWrite(output_zip, os.path.join(root, img_name), - os.path.join("IMAGES", img_name)) + if output_zip: + common.ZipWrite(output_zip, os.path.join(root, img_name), + os.path.join("IMAGES", img_name)) + else: + shutil.copy(os.path.join(root, img_name), prebuilt_path) break - # Zip spec says: All slashes MUST be forward slashes. - img_path = 'IMAGES/' + img_name - assert img_path in output_zip.namelist(), "cannot find " + img_name + if output_zip: + # Zip spec says: All slashes MUST be forward slashes. + img_path = 'IMAGES/' + img_name + assert img_path in output_zip.namelist(), "cannot find " + img_name + else: + img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name) + assert os.path.exists(img_path), "cannot find " + img_name if care_map_list: file_path = "META/care_map.txt" - common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) + if output_zip: + common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) + else: + with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp: + fp.write('\n'.join(care_map_list)) - common.ZipClose(output_zip) + if output_zip: + common.ZipClose(output_zip) def main(argv): def option_handler(o, a): diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 2b975181b3..322433313a 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -1191,6 +1191,10 @@ class File(object): t.flush() return t + def WriteToDir(self, d): + with open(os.path.join(d, self.name), "wb") as fp: + fp.write(self.data) + def AddToZip(self, z, compression=None): ZipWriteStr(z, self.name, self.data, compress_type=compression)