StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow add_img_to_target_files to work on a directory

Browse Source 
In addition to the current behavior of add_img_to_target_files working
on an existing zip file, allow passing in a directory where the target
files have already been extracted. When in this mode, it writes the
images out to that directory instead of the zip file.

This allows us to call add_img_to_target_files on the temp directory
used during the build to create the target files package, saving the
time and space of unzipping what we just zipped. This also allows us to
use the parallel soong_zip, which compresses the images much faster.

Test: aosp_marlin target_files zip is the same before/after this change
Test: marlin target_files zip is the same before/after this change
Test: bullhead target_files zip is the same before/after this change
Change-Id: I155654cdc7ad7754ba4ef76ec69c31f504a58198
This commit is contained in:
Dan Willemsen
2017-03-05 19:51:56 -08:00
parent 7dc54e9761
commit 2ee00d5f66
2 changed files with 128 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
tools/releasetools/add_img_to_target_files.py
View File

@@ -73,6 +73,25 @@ OPTIONS.replace_verity_public_key = False
OPTIONS.replace_verity_private_key = False OPTIONS.replace_verity_private_key = False
OPTIONS.is_signing = False OPTIONS.is_signing = False
class OutputFile(object):
def __init__(self, output_zip, input_dir, prefix, name):
self._output_zip = output_zip
self.input_name = os.path.join(input_dir, prefix, name)
if self._output_zip:
self._zip_name = os.path.join(prefix, name)
root, suffix = os.path.splitext(name)
self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix)
else:
self.name = self.input_name
def Write(self):
if self._output_zip:
common.ZipWrite(self._output_zip, self.name, self._zip_name)
def GetCareMap(which, imgname): def GetCareMap(which, imgname):
"""Generate care_map of system (or vendor) partition""" """Generate care_map of system (or vendor) partition"""
@@ -98,10 +117,10 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
"""Turn the contents of SYSTEM into a system image and store it in """Turn the contents of SYSTEM into a system image and store it in
output_zip. Returns the name of the system image file.""" output_zip. Returns the name of the system image file."""
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img") img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img")
if os.path.exists(prebuilt_path): if os.path.exists(img.input_name):
print("system.img already exists in %s, no need to rebuild..." % (prefix,)) print("system.img already exists in %s, no need to rebuild..." % (prefix,))
return prebuilt_path return img.input_name
def output_sink(fn, data): def output_sink(fn, data):
ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w") ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w")
@@ -113,74 +132,52 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img, common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,
boot_img, info_dict=OPTIONS.info_dict) boot_img, info_dict=OPTIONS.info_dict)
block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map") block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map")
imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict, CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img,
block_list=block_list) block_list=block_list)
common.ZipWrite(output_zip, imgname, prefix + "system.img") return img.name
common.ZipWrite(output_zip, block_list, prefix + "system.map")
return imgname
def BuildSystem(input_dir, info_dict, block_list=None):
"""Build the (sparse) system image and return the name of a temp
file containing it."""
return CreateImage(input_dir, info_dict, "system", block_list=block_list)
def AddSystemOther(output_zip, prefix="IMAGES/"): def AddSystemOther(output_zip, prefix="IMAGES/"):
"""Turn the contents of SYSTEM_OTHER into a system_other image """Turn the contents of SYSTEM_OTHER into a system_other image
and store it in output_zip.""" and store it in output_zip."""
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img") img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img")
if os.path.exists(prebuilt_path): if os.path.exists(img.input_name):
print("system_other.img already exists in %s, no need to rebuild..." % ( print("system_other.img already exists in %s, no need to rebuild..." % (
prefix,)) prefix,))
return return
imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict) CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img)
common.ZipWrite(output_zip, imgname, prefix + "system_other.img")
def BuildSystemOther(input_dir, info_dict):
"""Build the (sparse) system_other image and return the name of a temp
file containing it."""
return CreateImage(input_dir, info_dict, "system_other", block_list=None)
def AddVendor(output_zip, prefix="IMAGES/"): def AddVendor(output_zip, prefix="IMAGES/"):
"""Turn the contents of VENDOR into a vendor image and store in it """Turn the contents of VENDOR into a vendor image and store in it
output_zip.""" output_zip."""
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img") img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img")
if os.path.exists(prebuilt_path): if os.path.exists(img.input_name):
print("vendor.img already exists in %s, no need to rebuild..." % (prefix,)) print("vendor.img already exists in %s, no need to rebuild..." % (prefix,))
return prebuilt_path return img.input_name
block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map") block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map")
imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict, CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img,
block_list=block_list) block_list=block_list)
common.ZipWrite(output_zip, imgname, prefix + "vendor.img") return img.name
common.ZipWrite(output_zip, block_list, prefix + "vendor.map")
return imgname
def BuildVendor(input_dir, info_dict, block_list=None): def CreateImage(input_dir, info_dict, what, output_file, block_list=None):
"""Build the (sparse) vendor image and return the name of a temp
file containing it."""
return CreateImage(input_dir, info_dict, "vendor", block_list=block_list)
def CreateImage(input_dir, info_dict, what, block_list=None):
print("creating " + what + ".img...") print("creating " + what + ".img...")
img = common.MakeTempFile(prefix=what + "-", suffix=".img")
# The name of the directory it is making an image out of matters to # The name of the directory it is making an image out of matters to
# mkyaffs2image. It wants "system" but we have a directory named # mkyaffs2image. It wants "system" but we have a directory named
# "SYSTEM", so create a symlink. # "SYSTEM", so create a symlink.
temp_dir = tempfile.mkdtemp()
OPTIONS.tempfiles.append(temp_dir)
try: try:
os.symlink(os.path.join(input_dir, what.upper()), os.symlink(os.path.join(input_dir, what.upper()),
os.path.join(input_dir, what)) os.path.join(temp_dir, what))
except OSError as e: except OSError as e:
# bogus error on my mac version? # bogus error on my mac version?
# File "./build/tools/releasetools/img_from_target_files" # File "./build/tools/releasetools/img_from_target_files"
@@ -215,12 +212,16 @@ def CreateImage(input_dir, info_dict, what, block_list=None):
if fs_config: if fs_config:
image_props["fs_config"] = fs_config image_props["fs_config"] = fs_config
if block_list: if block_list:
image_props["block_list"] = block_list image_props["block_list"] = block_list.name
succ = build_image.BuildImage(os.path.join(input_dir, what), succ = build_image.BuildImage(os.path.join(temp_dir, what),
image_props, img) image_props, output_file.name)
assert succ, "build " + what + ".img image failed" assert succ, "build " + what + ".img image failed"
output_file.Write()
if block_list:
block_list.Write()
is_verity_partition = "verity_block_device" in image_props is_verity_partition = "verity_block_device" in image_props
verity_supported = image_props.get("verity") == "true" verity_supported = image_props.get("verity") == "true"
if is_verity_partition and verity_supported: if is_verity_partition and verity_supported:
@@ -229,8 +230,6 @@ def CreateImage(input_dir, info_dict, what, block_list=None):
adjusted_blocks_key = what + "_adjusted_partition_size" adjusted_blocks_key = what + "_adjusted_partition_size"
info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1 info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1
return img
def AddUserdata(output_zip, prefix="IMAGES/"): def AddUserdata(output_zip, prefix="IMAGES/"):
"""Create a userdata image and store it in output_zip. """Create a userdata image and store it in output_zip.
@@ -241,8 +240,8 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
in OPTIONS.info_dict. in OPTIONS.info_dict.
""" """
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img") img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img")
if os.path.exists(prebuilt_path): if os.path.exists(img.input_name):
print("userdata.img already exists in %s, no need to rebuild..." % ( print("userdata.img already exists in %s, no need to rebuild..." % (
prefix,)) prefix,))
return return
@@ -265,6 +264,7 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
# empty dir named "data", or a symlink to the DATA dir, # empty dir named "data", or a symlink to the DATA dir,
# and build the image from that. # and build the image from that.
temp_dir = tempfile.mkdtemp() temp_dir = tempfile.mkdtemp()
OPTIONS.tempfiles.append(temp_dir)
user_dir = os.path.join(temp_dir, "data") user_dir = os.path.join(temp_dir, "data")
empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true") empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true")
if empty: if empty:
@@ -275,8 +275,6 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"), os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"),
user_dir) user_dir)
img = tempfile.NamedTemporaryFile()
fstab = OPTIONS.info_dict["fstab"] fstab = OPTIONS.info_dict["fstab"]
if fstab: if fstab:
image_props["fs_type"] = fstab["/data"].fs_type image_props["fs_type"] = fstab["/data"].fs_type
@@ -284,17 +282,15 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
assert succ, "build userdata.img image failed" assert succ, "build userdata.img image failed"
common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict) common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
common.ZipWrite(output_zip, img.name, prefix + "userdata.img") img.Write()
img.close()
shutil.rmtree(temp_dir)
def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"): def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):
"""Create a VBMeta image and store it in output_zip.""" """Create a VBMeta image and store it in output_zip."""
_, img_file_name = tempfile.mkstemp() img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img")
avbtool = os.getenv('AVBTOOL') or "avbtool" avbtool = os.getenv('AVBTOOL') or "avbtool"
cmd = [avbtool, "make_vbmeta_image", cmd = [avbtool, "make_vbmeta_image",
"--output", img_file_name, "--output", img.name,
"--include_descriptors_from_image", boot_img_path, "--include_descriptors_from_image", boot_img_path,
"--include_descriptors_from_image", system_img_path, "--include_descriptors_from_image", system_img_path,
"--generate_dm_verity_cmdline_from_hashtree", system_img_path] "--generate_dm_verity_cmdline_from_hashtree", system_img_path]
@@ -305,19 +301,19 @@ def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):
p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.communicate() p.communicate()
assert p.returncode == 0, "avbtool make_vbmeta_image failed" assert p.returncode == 0, "avbtool make_vbmeta_image failed"
common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img") img.Write()
def AddPartitionTable(output_zip, prefix="IMAGES/"): def AddPartitionTable(output_zip, prefix="IMAGES/"):
"""Create a partition table image and store it in output_zip.""" """Create a partition table image and store it in output_zip."""
_, img_file_name = tempfile.mkstemp() img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img")
_, bpt_file_name = tempfile.mkstemp() bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt")
# use BPTTOOL from environ, or "bpttool" if empty or not set. # use BPTTOOL from environ, or "bpttool" if empty or not set.
bpttool = os.getenv("BPTTOOL") or "bpttool" bpttool = os.getenv("BPTTOOL") or "bpttool"
cmd = [bpttool, "make_table", "--output_json", bpt_file_name, cmd = [bpttool, "make_table", "--output_json", bpt.name,
"--output_gpt", img_file_name] "--output_gpt", img.name]
input_files_str = OPTIONS.info_dict["board_bpt_input_files"] input_files_str = OPTIONS.info_dict["board_bpt_input_files"]
input_files = input_files_str.split(" ") input_files = input_files_str.split(" ")
for i in input_files: for i in input_files:
@@ -333,15 +329,15 @@ def AddPartitionTable(output_zip, prefix="IMAGES/"):
p.communicate() p.communicate()
assert p.returncode == 0, "bpttool make_table failed" assert p.returncode == 0, "bpttool make_table failed"
common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img") img.Write()
common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt") bpt.Write()
def AddCache(output_zip, prefix="IMAGES/"): def AddCache(output_zip, prefix="IMAGES/"):
"""Create an empty cache image and store it in output_zip.""" """Create an empty cache image and store it in output_zip."""
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img") img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img")
if os.path.exists(prebuilt_path): if os.path.exists(img.input_name):
print("cache.img already exists in %s, no need to rebuild..." % (prefix,)) print("cache.img already exists in %s, no need to rebuild..." % (prefix,))
return return
@@ -362,9 +358,9 @@ def AddCache(output_zip, prefix="IMAGES/"):
# mkyaffs2image. So we create a temp dir, and within it we create an # mkyaffs2image. So we create a temp dir, and within it we create an
# empty dir named "cache", and build the image from that. # empty dir named "cache", and build the image from that.
temp_dir = tempfile.mkdtemp() temp_dir = tempfile.mkdtemp()
OPTIONS.tempfiles.append(temp_dir)
user_dir = os.path.join(temp_dir, "cache") user_dir = os.path.join(temp_dir, "cache")
os.mkdir(user_dir) os.mkdir(user_dir)
img = tempfile.NamedTemporaryFile()
fstab = OPTIONS.info_dict["fstab"] fstab = OPTIONS.info_dict["fstab"]
if fstab: if fstab:
@@ -373,35 +369,39 @@ def AddCache(output_zip, prefix="IMAGES/"):
assert succ, "build cache.img image failed" assert succ, "build cache.img image failed"
common.CheckSize(img.name, "cache.img", OPTIONS.info_dict) common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)
common.ZipWrite(output_zip, img.name, prefix + "cache.img") img.Write()
img.close()
os.rmdir(user_dir)
os.rmdir(temp_dir)
def AddImagesToTargetFiles(filename): def AddImagesToTargetFiles(filename):
OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) if os.path.isdir(filename):
OPTIONS.input_tmp = os.path.abspath(filename)
input_zip = None
else:
OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)
if not OPTIONS.add_missing: if not OPTIONS.add_missing:
for n in input_zip.namelist(): if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")):
if n.startswith("IMAGES/"): print("target_files appears to already contain images.")
print("target_files appears to already contain images.") sys.exit(1)
sys.exit(1)
try: has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR"))
input_zip.getinfo("VENDOR/") has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp,
has_vendor = True "SYSTEM_OTHER"))
except KeyError:
has_vendor = False
has_system_other = "SYSTEM_OTHER/" in input_zip.namelist() if input_zip:
OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)
OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) common.ZipClose(input_zip)
output_zip = zipfile.ZipFile(filename, "a",
common.ZipClose(input_zip) compression=zipfile.ZIP_DEFLATED,
output_zip = zipfile.ZipFile(filename, "a", allowZip64=True)
compression=zipfile.ZIP_DEFLATED, else:
allowZip64=True) OPTIONS.info_dict = common.LoadInfoDict(filename, filename)
output_zip = None
images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES")
if not os.path.isdir(images_dir):
os.makedirs(images_dir)
images_dir = None
has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true") has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true")
system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true") system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true")
@@ -422,7 +422,10 @@ def AddImagesToTargetFiles(filename):
boot_image = common.GetBootableImage( boot_image = common.GetBootableImage(
"IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
if boot_image: if boot_image:
boot_image.AddToZip(output_zip) if output_zip:
boot_image.AddToZip(output_zip)
else:
boot_image.WriteToDir(OPTIONS.input_tmp)
recovery_image = None recovery_image = None
if has_recovery: if has_recovery:
@@ -438,7 +441,10 @@ def AddImagesToTargetFiles(filename):
recovery_image = common.GetBootableImage( recovery_image = common.GetBootableImage(
"IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY")
if recovery_image: if recovery_image:
recovery_image.AddToZip(output_zip) if output_zip:
recovery_image.AddToZip(output_zip)
else:
recovery_image.WriteToDir(OPTIONS.input_tmp)
banner("recovery (two-step image)") banner("recovery (two-step image)")
# The special recovery.img for two-step package use. # The special recovery.img for two-step package use.
@@ -446,7 +452,10 @@ def AddImagesToTargetFiles(filename):
"IMAGES/recovery-two-step.img", "recovery-two-step.img", "IMAGES/recovery-two-step.img", "recovery-two-step.img",
OPTIONS.input_tmp, "RECOVERY", two_step_image=True) OPTIONS.input_tmp, "RECOVERY", two_step_image=True)
if recovery_two_step_image: if recovery_two_step_image:
recovery_two_step_image.AddToZip(output_zip) if output_zip:
recovery_two_step_image.AddToZip(output_zip)
else:
recovery_two_step_image.WriteToDir(OPTIONS.input_tmp)
banner("system") banner("system")
system_img_path = AddSystem( system_img_path = AddSystem(
@@ -502,24 +511,39 @@ def AddImagesToTargetFiles(filename):
img_vendor_dir = os.path.join( img_vendor_dir = os.path.join(
OPTIONS.input_tmp, "VENDOR_IMAGES") OPTIONS.input_tmp, "VENDOR_IMAGES")
if os.path.exists(img_radio_path): if os.path.exists(img_radio_path):
common.ZipWrite(output_zip, img_radio_path, if output_zip:
os.path.join("IMAGES", img_name)) common.ZipWrite(output_zip, img_radio_path,
os.path.join("IMAGES", img_name))
else:
shutil.copy(img_radio_path, prebuilt_path)
else: else:
for root, _, files in os.walk(img_vendor_dir): for root, _, files in os.walk(img_vendor_dir):
if img_name in files: if img_name in files:
common.ZipWrite(output_zip, os.path.join(root, img_name), if output_zip:
os.path.join("IMAGES", img_name)) common.ZipWrite(output_zip, os.path.join(root, img_name),
os.path.join("IMAGES", img_name))
else:
shutil.copy(os.path.join(root, img_name), prebuilt_path)
break break
# Zip spec says: All slashes MUST be forward slashes. if output_zip:
img_path = 'IMAGES/' + img_name # Zip spec says: All slashes MUST be forward slashes.
assert img_path in output_zip.namelist(), "cannot find " + img_name img_path = 'IMAGES/' + img_name
assert img_path in output_zip.namelist(), "cannot find " + img_name
else:
img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)
assert os.path.exists(img_path), "cannot find " + img_name
if care_map_list: if care_map_list:
file_path = "META/care_map.txt" file_path = "META/care_map.txt"
common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) if output_zip:
common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list))
else:
with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp:
fp.write('\n'.join(care_map_list))
common.ZipClose(output_zip) if output_zip:
common.ZipClose(output_zip)
def main(argv): def main(argv):
def option_handler(o, a): def option_handler(o, a):

4
tools/releasetools/common.py
View File

@@ -1191,6 +1191,10 @@ class File(object):
t.flush() t.flush()
return t return t
def WriteToDir(self, d):
with open(os.path.join(d, self.name), "wb") as fp:
fp.write(self.data)
def AddToZip(self, z, compression=None): def AddToZip(self, z, compression=None):
ZipWriteStr(z, self.name, self.data, compress_type=compression) ZipWriteStr(z, self.name, self.data, compress_type=compression)