diff --git a/tools/releasetools/ota_from_target_files b/tools/releasetools/ota_from_target_files index fe63c3a0d3..cbae34c351 100755 --- a/tools/releasetools/ota_from_target_files +++ b/tools/releasetools/ota_from_target_files @@ -62,6 +62,7 @@ import re import sha import subprocess import tempfile +import threading import time import zipfile @@ -80,6 +81,7 @@ OPTIONS.wipe_user_data = False OPTIONS.omit_prereq = False OPTIONS.extra_script = None OPTIONS.script_mode = 'auto' +OPTIONS.worker_threads = 3 def MostPopularKey(d, default): """Given a dict, return the key corresponding to the largest @@ -297,7 +299,8 @@ def MakeRecoveryPatch(output_zip, recovery_img, boot_img): executable. """ - patch = Difference(recovery_img, boot_img, "imgdiff") + d = Difference(recovery_img, boot_img) + _, _, patch = d.ComputePatch() common.ZipWriteStr(output_zip, "system/recovery-from-boot.p", patch) Item.Get("system/recovery-from-boot.p", dir=False) @@ -420,38 +423,111 @@ def LoadSystemFiles(z): return out -def Difference(tf, sf, diff_program): - """Return the patch (as a string of data) needed to turn sf into tf. - diff_program is the name of an external program (or list, if - additional arguments are desired) to run to generate the diff. - """ +DIFF_PROGRAM_BY_EXT = { + ".gz" : "imgdiff", + ".zip" : ["imgdiff", "-z"], + ".jar" : ["imgdiff", "-z"], + ".apk" : ["imgdiff", "-z"], + ".img" : "imgdiff", + } - ttemp = tf.WriteToTemp() - stemp = sf.WriteToTemp() - ext = os.path.splitext(tf.name)[1] +class Difference(object): + def __init__(self, tf, sf): + self.tf = tf + self.sf = sf + self.patch = None - try: - ptemp = tempfile.NamedTemporaryFile() - if isinstance(diff_program, list): - cmd = copy.copy(diff_program) - else: - cmd = [diff_program] - cmd.append(stemp.name) - cmd.append(ttemp.name) - cmd.append(ptemp.name) - p = common.Run(cmd) - _, err = p.communicate() - if err or p.returncode != 0: - print "WARNING: failure running %s:\n%s\n" % (diff_program, err) - return None - diff = ptemp.read() - finally: - ptemp.close() - stemp.close() - ttemp.close() + def ComputePatch(self): + """Compute the patch (as a string of data) needed to turn sf into + tf. Returns the same tuple as GetPatch().""" - return diff + tf = self.tf + sf = self.sf + + ext = os.path.splitext(tf.name)[1] + diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") + + ttemp = tf.WriteToTemp() + stemp = sf.WriteToTemp() + + ext = os.path.splitext(tf.name)[1] + + try: + ptemp = tempfile.NamedTemporaryFile() + if isinstance(diff_program, list): + cmd = copy.copy(diff_program) + else: + cmd = [diff_program] + cmd.append(stemp.name) + cmd.append(ttemp.name) + cmd.append(ptemp.name) + p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, err = p.communicate() + if err or p.returncode != 0: + print "WARNING: failure running %s:\n%s\n" % (diff_program, err) + return None + diff = ptemp.read() + finally: + ptemp.close() + stemp.close() + ttemp.close() + + self.patch = diff + return self.tf, self.sf, self.patch + + + def GetPatch(self): + """Return a tuple (target_file, source_file, patch_data). + patch_data may be None if ComputePatch hasn't been called, or if + computing the patch failed.""" + return self.tf, self.sf, self.patch + + +def ComputeDifferences(diffs): + """Call ComputePatch on all the Difference objects in 'diffs'.""" + print len(diffs), "diffs to compute" + + # Do the largest files first, to try and reduce the long-pole effect. + by_size = [(i.tf.size, i) for i in diffs] + by_size.sort(reverse=True) + by_size = [i[1] for i in by_size] + + lock = threading.Lock() + diff_iter = iter(by_size) # accessed under lock + + def worker(): + try: + lock.acquire() + for d in diff_iter: + lock.release() + start = time.time() + d.ComputePatch() + dur = time.time() - start + lock.acquire() + + tf, sf, patch = d.GetPatch() + if sf.name == tf.name: + name = tf.name + else: + name = "%s (%s)" % (tf.name, sf.name) + if patch is None: + print "patching failed! %s" % (name,) + else: + print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( + dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) + lock.release() + except e: + print e + raise + + # start worker threads; wait for them all to finish. + threads = [threading.Thread(target=worker) + for i in range(OPTIONS.worker_threads)] + for th in threads: + th.start() + while threads: + threads.pop().join() def GetBuildProp(property, z): @@ -482,14 +558,6 @@ def GetRecoveryAPIVersion(zip): return 0 -DIFF_METHOD_BY_EXT = { - ".gz" : "imgdiff", - ".zip" : ["imgdiff", "-z"], - ".jar" : ["imgdiff", "-z"], - ".apk" : ["imgdiff", "-z"], - } - - def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): source_version = GetRecoveryAPIVersion(source_zip) @@ -521,9 +589,11 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): verbatim_targets = [] patch_list = [] + diffs = [] largest_source_size = 0 for fn in sorted(target_data.keys()): tf = target_data[fn] + assert fn == tf.name sf = source_data.get(fn, None) if sf is None or fn in OPTIONS.require_verbatim: @@ -535,25 +605,23 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): verbatim_targets.append((fn, tf.size)) elif tf.sha1 != sf.sha1: # File is different; consider sending as a patch - ext = os.path.splitext(tf.name)[1] - diff_method = DIFF_METHOD_BY_EXT.get(ext, "bsdiff") - d = Difference(tf, sf, diff_method) - if d is not None: - print fn, tf.size, len(d), (float(len(d)) / tf.size) - if d is None or len(d) > tf.size * OPTIONS.patch_threshold: - # patch is almost as big as the file; don't bother patching - tf.AddToZip(output_zip) - verbatim_targets.append((fn, tf.size)) - else: - common.ZipWriteStr(output_zip, "patch/" + fn + ".p", d) - patch_list.append((fn, tf, sf, tf.size)) - largest_source_size = max(largest_source_size, sf.size) + diffs.append(Difference(tf, sf)) else: # Target file identical to source. pass - total_verbatim_size = sum([i[1] for i in verbatim_targets]) - total_patched_size = sum([i[3] for i in patch_list]) + ComputeDifferences(diffs) + + for diff in diffs: + tf, sf, d = diff.GetPatch() + if d is None or len(d) > tf.size * OPTIONS.patch_threshold: + # patch is almost as big as the file; don't bother patching + tf.AddToZip(output_zip) + verbatim_targets.append((tf.name, tf.size)) + else: + common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d) + patch_list.append((tf.name, tf, sf, tf.size)) + largest_source_size = max(largest_source_size, sf.size) source_fp = GetBuildProp("ro.build.fingerprint", source_zip) target_fp = GetBuildProp("ro.build.fingerprint", target_zip) @@ -600,7 +668,8 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): script.SetProgress(so_far / total_verify_size) if updating_boot: - d = Difference(target_boot, source_boot, "imgdiff") + d = Difference(target_boot, source_boot) + _, _, d = d.ComputePatch() print "boot target: %d source: %d diff: %d" % ( target_boot.size, source_boot.size, len(d)) @@ -755,6 +824,8 @@ def main(argv): OPTIONS.extra_script = a elif o in ("-m", "--script_mode"): OPTIONS.script_mode = a + elif o in ("--worker_threads"): + OPTIONS.worker_threads = int(a) else: return False return True @@ -767,7 +838,8 @@ def main(argv): "wipe_user_data", "no_prereq", "extra_script=", - "script_mode="], + "script_mode=", + "worker_threads="], extra_option_handler=option_handler) if len(args) != 2: