Merge change 27066 into eclair

* changes:
  parallellize computation of binary patches

tools/releasetools/ota_from_target_files

@@ -62,6 +62,7 @@ import re
 import sha
 import subprocess
 import tempfile
+import threading
 import time
 import zipfile
 
@@ -80,6 +81,7 @@ OPTIONS.wipe_user_data = False
 OPTIONS.omit_prereq = False
 OPTIONS.extra_script = None
 OPTIONS.script_mode = 'auto'
+OPTIONS.worker_threads = 3
 
 def MostPopularKey(d, default):
   """Given a dict, return the key corresponding to the largest
@@ -297,7 +299,8 @@ def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
   executable.
   """
 
-  patch = Difference(recovery_img, boot_img, "imgdiff")
+  d = Difference(recovery_img, boot_img)
+  _, _, patch = d.ComputePatch()
   common.ZipWriteStr(output_zip, "system/recovery-from-boot.p", patch)
   Item.Get("system/recovery-from-boot.p", dir=False)
 
@@ -420,38 +423,111 @@ def LoadSystemFiles(z):
   return out
 
 
-def Difference(tf, sf, diff_program):
+DIFF_PROGRAM_BY_EXT = {
-  """Return the patch (as a string of data) needed to turn sf into tf.
+    ".gz" : "imgdiff",
-  diff_program is the name of an external program (or list, if
+    ".zip" : ["imgdiff", "-z"],
-  additional arguments are desired) to run to generate the diff.
+    ".jar" : ["imgdiff", "-z"],
-  """
+    ".apk" : ["imgdiff", "-z"],
+    ".img" : "imgdiff",
+    }
 
-  ttemp = tf.WriteToTemp()
-  stemp = sf.WriteToTemp()
 
-  ext = os.path.splitext(tf.name)[1]
+class Difference(object):
+  def __init__(self, tf, sf):
+    self.tf = tf
+    self.sf = sf
+    self.patch = None
 
-  try:
+  def ComputePatch(self):
-    ptemp = tempfile.NamedTemporaryFile()
+    """Compute the patch (as a string of data) needed to turn sf into
-    if isinstance(diff_program, list):
+    tf.  Returns the same tuple as GetPatch()."""
-      cmd = copy.copy(diff_program)
-    else:
-      cmd = [diff_program]
-    cmd.append(stemp.name)
-    cmd.append(ttemp.name)
-    cmd.append(ptemp.name)
-    p = common.Run(cmd)
-    _, err = p.communicate()
-    if err or p.returncode != 0:
-      print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
-      return None
-    diff = ptemp.read()
-  finally:
-    ptemp.close()
-    stemp.close()
-    ttemp.close()
 
-  return diff
+    tf = self.tf
+    sf = self.sf
+
+    ext = os.path.splitext(tf.name)[1]
+    diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
+
+    ttemp = tf.WriteToTemp()
+    stemp = sf.WriteToTemp()
+
+    ext = os.path.splitext(tf.name)[1]
+
+    try:
+      ptemp = tempfile.NamedTemporaryFile()
+      if isinstance(diff_program, list):
+        cmd = copy.copy(diff_program)
+      else:
+        cmd = [diff_program]
+      cmd.append(stemp.name)
+      cmd.append(ttemp.name)
+      cmd.append(ptemp.name)
+      p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      _, err = p.communicate()
+      if err or p.returncode != 0:
+        print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
+        return None
+      diff = ptemp.read()
+    finally:
+      ptemp.close()
+      stemp.close()
+      ttemp.close()
+
+    self.patch = diff
+    return self.tf, self.sf, self.patch
+
+
+  def GetPatch(self):
+    """Return a tuple (target_file, source_file, patch_data).
+    patch_data may be None if ComputePatch hasn't been called, or if
+    computing the patch failed."""
+    return self.tf, self.sf, self.patch
+
+
+def ComputeDifferences(diffs):
+  """Call ComputePatch on all the Difference objects in 'diffs'."""
+  print len(diffs), "diffs to compute"
+
+  # Do the largest files first, to try and reduce the long-pole effect.
+  by_size = [(i.tf.size, i) for i in diffs]
+  by_size.sort(reverse=True)
+  by_size = [i[1] for i in by_size]
+
+  lock = threading.Lock()
+  diff_iter = iter(by_size)   # accessed under lock
+
+  def worker():
+    try:
+      lock.acquire()
+      for d in diff_iter:
+        lock.release()
+        start = time.time()
+        d.ComputePatch()
+        dur = time.time() - start
+        lock.acquire()
+
+        tf, sf, patch = d.GetPatch()
+        if sf.name == tf.name:
+          name = tf.name
+        else:
+          name = "%s (%s)" % (tf.name, sf.name)
+        if patch is None:
+          print "patching failed!                                  %s" % (name,)
+        else:
+          print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
+              dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
+      lock.release()
+    except e:
+      print e
+      raise
+
+  # start worker threads; wait for them all to finish.
+  threads = [threading.Thread(target=worker)
+             for i in range(OPTIONS.worker_threads)]
+  for th in threads:
+    th.start()
+  while threads:
+    threads.pop().join()
 
 
 def GetBuildProp(property, z):
@@ -482,14 +558,6 @@ def GetRecoveryAPIVersion(zip):
       return 0
 
 
-DIFF_METHOD_BY_EXT = {
-    ".gz" : "imgdiff",
-    ".zip" : ["imgdiff", "-z"],
-    ".jar" : ["imgdiff", "-z"],
-    ".apk" : ["imgdiff", "-z"],
-    }
-
-
 def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
   source_version = GetRecoveryAPIVersion(source_zip)
 
@@ -521,9 +589,11 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
 
   verbatim_targets = []
   patch_list = []
+  diffs = []
   largest_source_size = 0
   for fn in sorted(target_data.keys()):
     tf = target_data[fn]
+    assert fn == tf.name
     sf = source_data.get(fn, None)
 
     if sf is None or fn in OPTIONS.require_verbatim:
@@ -535,25 +605,23 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
       verbatim_targets.append((fn, tf.size))
     elif tf.sha1 != sf.sha1:
       # File is different; consider sending as a patch
-      ext = os.path.splitext(tf.name)[1]
+      diffs.append(Difference(tf, sf))
-      diff_method = DIFF_METHOD_BY_EXT.get(ext, "bsdiff")
-      d = Difference(tf, sf, diff_method)
-      if d is not None:
-        print fn, tf.size, len(d), (float(len(d)) / tf.size)
-      if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
-        # patch is almost as big as the file; don't bother patching
-        tf.AddToZip(output_zip)
-        verbatim_targets.append((fn, tf.size))
-      else:
-        common.ZipWriteStr(output_zip, "patch/" + fn + ".p", d)
-        patch_list.append((fn, tf, sf, tf.size))
-        largest_source_size = max(largest_source_size, sf.size)
     else:
       # Target file identical to source.
       pass
 
-  total_verbatim_size = sum([i[1] for i in verbatim_targets])
+  ComputeDifferences(diffs)
-  total_patched_size = sum([i[3] for i in patch_list])
+
+  for diff in diffs:
+    tf, sf, d = diff.GetPatch()
+    if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
+      # patch is almost as big as the file; don't bother patching
+      tf.AddToZip(output_zip)
+      verbatim_targets.append((tf.name, tf.size))
+    else:
+      common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d)
+      patch_list.append((tf.name, tf, sf, tf.size))
+      largest_source_size = max(largest_source_size, sf.size)
 
   source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
   target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
@@ -600,7 +668,8 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
     script.SetProgress(so_far / total_verify_size)
 
   if updating_boot:
-    d = Difference(target_boot, source_boot, "imgdiff")
+    d = Difference(target_boot, source_boot)
+    _, _, d = d.ComputePatch()
     print "boot      target: %d  source: %d  diff: %d" % (
         target_boot.size, source_boot.size, len(d))
 
@@ -755,6 +824,8 @@ def main(argv):
       OPTIONS.extra_script = a
     elif o in ("-m", "--script_mode"):
       OPTIONS.script_mode = a
+    elif o in ("--worker_threads"):
+      OPTIONS.worker_threads = int(a)
     else:
       return False
     return True
@@ -767,7 +838,8 @@ def main(argv):
                                               "wipe_user_data",
                                               "no_prereq",
                                               "extra_script=",
-                                              "script_mode="],
+                                              "script_mode=",
+                                              "worker_threads="],
                              extra_option_handler=option_handler)
 
   if len(args) != 2: