tequilaOS/platform_build
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge change 27066 into eclair

Browse source 
* changes:
  parallellize computation of binary patches
This commit is contained in:
Android (Google) Code Review 2009-09-25 15:40:43 -04:00
commit ea4a3fa3d4
1 changed files with 126 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

180
tools/releasetools/ota_from_target_files
View file

@ -62,6 +62,7 @@ import re
import sha
import subprocess
import tempfile
import threading
import time
import zipfile
@ -80,6 +81,7 @@ OPTIONS.wipe_user_data = False
OPTIONS.omit_prereq = False
OPTIONS.extra_script = None
OPTIONS.script_mode = 'auto'
OPTIONS.worker_threads = 3
def MostPopularKey(d, default):
"""Given a dict, return the key corresponding to the largest
@ -297,7 +299,8 @@ def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
executable.
"""
patch = Difference(recovery_img, boot_img, "imgdiff")
d = Difference(recovery_img, boot_img)
_, _, patch = d.ComputePatch()
common.ZipWriteStr(output_zip, "system/recovery-from-boot.p", patch)
Item.Get("system/recovery-from-boot.p", dir=False)
@ -420,38 +423,111 @@ def LoadSystemFiles(z):
return out
def Difference(tf, sf, diff_program):
"""Return the patch (as a string of data) needed to turn sf into tf.
diff_program is the name of an external program (or list, if
additional arguments are desired) to run to generate the diff.
"""
DIFF_PROGRAM_BY_EXT = {
".gz" : "imgdiff",
".zip" : ["imgdiff", "-z"],
".jar" : ["imgdiff", "-z"],
".apk" : ["imgdiff", "-z"],
".img" : "imgdiff",
}
ttemp = tf.WriteToTemp()
stemp = sf.WriteToTemp()
ext = os.path.splitext(tf.name)[1]
class Difference(object):
def __init__(self, tf, sf):
self.tf = tf
self.sf = sf
self.patch = None
try:
ptemp = tempfile.NamedTemporaryFile()
if isinstance(diff_program, list):
cmd = copy.copy(diff_program)
else:
cmd = [diff_program]
cmd.append(stemp.name)
cmd.append(ttemp.name)
cmd.append(ptemp.name)
p = common.Run(cmd)
_, err = p.communicate()
if err or p.returncode != 0:
print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
return None
diff = ptemp.read()
finally:
ptemp.close()
stemp.close()
ttemp.close()
def ComputePatch(self):
"""Compute the patch (as a string of data) needed to turn sf into
tf. Returns the same tuple as GetPatch()."""
return diff
tf = self.tf
sf = self.sf
ext = os.path.splitext(tf.name)[1]
diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
ttemp = tf.WriteToTemp()
stemp = sf.WriteToTemp()
ext = os.path.splitext(tf.name)[1]
try:
ptemp = tempfile.NamedTemporaryFile()
if isinstance(diff_program, list):
cmd = copy.copy(diff_program)
else:
cmd = [diff_program]
cmd.append(stemp.name)
cmd.append(ttemp.name)
cmd.append(ptemp.name)
p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
_, err = p.communicate()
if err or p.returncode != 0:
print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
return None
diff = ptemp.read()
finally:
ptemp.close()
stemp.close()
ttemp.close()
self.patch = diff
return self.tf, self.sf, self.patch
def GetPatch(self):
"""Return a tuple (target_file, source_file, patch_data).
patch_data may be None if ComputePatch hasn't been called, or if
computing the patch failed."""
return self.tf, self.sf, self.patch
def ComputeDifferences(diffs):
"""Call ComputePatch on all the Difference objects in 'diffs'."""
print len(diffs), "diffs to compute"
# Do the largest files first, to try and reduce the long-pole effect.
by_size = [(i.tf.size, i) for i in diffs]
by_size.sort(reverse=True)
by_size = [i[1] for i in by_size]
lock = threading.Lock()
diff_iter = iter(by_size) # accessed under lock
def worker():
try:
lock.acquire()
for d in diff_iter:
lock.release()
start = time.time()
d.ComputePatch()
dur = time.time() - start
lock.acquire()
tf, sf, patch = d.GetPatch()
if sf.name == tf.name:
name = tf.name
else:
name = "%s (%s)" % (tf.name, sf.name)
if patch is None:
print "patching failed! %s" % (name,)
else:
print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
lock.release()
except e:
print e
raise
# start worker threads; wait for them all to finish.
threads = [threading.Thread(target=worker)
for i in range(OPTIONS.worker_threads)]
for th in threads:
th.start()
while threads:
threads.pop().join()
def GetBuildProp(property, z):
@ -482,14 +558,6 @@ def GetRecoveryAPIVersion(zip):
return 0
DIFF_METHOD_BY_EXT = {
".gz" : "imgdiff",
".zip" : ["imgdiff", "-z"],
".jar" : ["imgdiff", "-z"],
".apk" : ["imgdiff", "-z"],
}
def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
source_version = GetRecoveryAPIVersion(source_zip)
@ -521,9 +589,11 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
verbatim_targets = []
patch_list = []
diffs = []
largest_source_size = 0
for fn in sorted(target_data.keys()):
tf = target_data[fn]
assert fn == tf.name
sf = source_data.get(fn, None)
if sf is None or fn in OPTIONS.require_verbatim:
@ -535,25 +605,23 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
verbatim_targets.append((fn, tf.size))
elif tf.sha1 != sf.sha1:
# File is different; consider sending as a patch
ext = os.path.splitext(tf.name)[1]
diff_method = DIFF_METHOD_BY_EXT.get(ext, "bsdiff")
d = Difference(tf, sf, diff_method)
if d is not None:
print fn, tf.size, len(d), (float(len(d)) / tf.size)
if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
# patch is almost as big as the file; don't bother patching
tf.AddToZip(output_zip)
verbatim_targets.append((fn, tf.size))
else:
common.ZipWriteStr(output_zip, "patch/" + fn + ".p", d)
patch_list.append((fn, tf, sf, tf.size))
largest_source_size = max(largest_source_size, sf.size)
diffs.append(Difference(tf, sf))
else:
# Target file identical to source.
pass
total_verbatim_size = sum([i[1] for i in verbatim_targets])
total_patched_size = sum([i[3] for i in patch_list])
ComputeDifferences(diffs)
for diff in diffs:
tf, sf, d = diff.GetPatch()
if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
# patch is almost as big as the file; don't bother patching
tf.AddToZip(output_zip)
verbatim_targets.append((tf.name, tf.size))
else:
common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d)
patch_list.append((tf.name, tf, sf, tf.size))
largest_source_size = max(largest_source_size, sf.size)
source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
@ -600,7 +668,8 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
script.SetProgress(so_far / total_verify_size)
if updating_boot:
d = Difference(target_boot, source_boot, "imgdiff")
d = Difference(target_boot, source_boot)
_, _, d = d.ComputePatch()
print "boot target: %d source: %d diff: %d" % (
target_boot.size, source_boot.size, len(d))
@ -755,6 +824,8 @@ def main(argv):
OPTIONS.extra_script = a
elif o in ("-m", "--script_mode"):
OPTIONS.script_mode = a
elif o in ("--worker_threads"):
OPTIONS.worker_threads = int(a)
else:
return False
return True
@ -767,7 +838,8 @@ def main(argv):
"wipe_user_data",
"no_prereq",
"extra_script=",
"script_mode="],
"script_mode=",
"worker_threads="],
extra_option_handler=option_handler)
if len(args) != 2: