Merge change 2594 into donut

* changes:
  new image diffing tool and support for image patches in applypatch

tools/applypatch/Android.mk

@@ -12,18 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ifneq ($(TARGET_SIMULATOR),true)
+
 LOCAL_PATH := $(call my-dir)
 include $(CLEAR_VARS)
 
-ifneq ($(TARGET_SIMULATOR),true)
+LOCAL_SRC_FILES := applypatch.c bsdiff.c freecache.c imgpatch.c
-
-LOCAL_SRC_FILES := applypatch.c bsdiff.c freecache.c
 LOCAL_MODULE := applypatch
 LOCAL_FORCE_STATIC_EXECUTABLE := true
 LOCAL_MODULE_TAGS := eng
-LOCAL_C_INCLUDES += external/bzip2
+LOCAL_C_INCLUDES += external/bzip2 external/zlib
-LOCAL_STATIC_LIBRARIES += libmincrypt libbz libc
+LOCAL_STATIC_LIBRARIES += libmincrypt libbz libz libc
 
 include $(BUILD_EXECUTABLE)
 
+include $(CLEAR_VARS)
+
+LOCAL_SRC_FILES := imgdiff.c
+LOCAL_MODULE := imgdiff
+LOCAL_FORCE_STATIC_EXECUTABLE := true
+LOCAL_MODULE_TAGS := eng
+LOCAL_C_INCLUDES += external/zlib
+LOCAL_STATIC_LIBRARIES += libz
+
+include $(BUILD_HOST_EXECUTABLE)
+
 endif  # !TARGET_SIMULATOR

tools/applypatch/applypatch.c

@@ -431,11 +431,19 @@ int main(int argc, char** argv) {
   } else if (header_bytes_read >= 8 &&
              memcmp(header, "BSDIFF40", 8) == 0) {
     int result = ApplyBSDiffPatch(source_to_use->data, source_to_use->size,
-                                  patch_filename, output, &ctx);
+                                  patch_filename, 0, output, &ctx);
     if (result != 0) {
       fprintf(stderr, "ApplyBSDiffPatch failed\n");
       return result;
     }
+  } else if (header_bytes_read >= 8 &&
+             memcmp(header, "IMGDIFF1", 8) == 0) {
+    int result = ApplyImagePatch(source_to_use->data, source_to_use->size,
+                                 patch_filename, output, &ctx);
+    if (result != 0) {
+      fprintf(stderr, "ApplyImagePatch failed\n");
+      return result;
+    }
   } else {
     fprintf(stderr, "Unknown patch file format");
     return 1;

tools/applypatch/applypatch.h

@@ -44,6 +44,14 @@ size_t FreeSpaceForFile(const char* filename);
 // bsdiff.c
 void ShowBSDiffLicense();
 int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
+                     const char* patch_filename, ssize_t offset,
+                     FILE* output, SHA_CTX* ctx);
+int ApplyBSDiffPatchMem(const unsigned char* old_data, ssize_t old_size,
+                        const char* patch_filename, ssize_t patch_offset,
+                        unsigned char** new_data, ssize_t* new_size);
+
+// imgpatch.c
+int ApplyImagePatch(const unsigned char* old_data, ssize_t old_size,
                      const char* patch_filename,
                      FILE* output, SHA_CTX* ctx);
 

tools/applypatch/bsdiff.c

@@ -29,6 +29,7 @@
 #include <bzlib.h>
 
 #include "mincrypt/sha.h"
+#include "applypatch.h"
 
 void ShowBSDiffLicense() {
   puts("The bsdiff library used herein is:\n"
@@ -80,10 +81,34 @@ static off_t offtin(u_char *buf)
   return y;
 }
 
+
 int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
-                     const char* patch_filename,
+                     const char* patch_filename, ssize_t patch_offset,
                      FILE* output, SHA_CTX* ctx) {
 
+  unsigned char* new_data;
+  ssize_t new_size;
+  if (ApplyBSDiffPatchMem(old_data, old_size, patch_filename, patch_offset,
+                          &new_data, &new_size) != 0) {
+    return -1;
+  }
+
+  if (fwrite(new_data, 1, new_size, output) < new_size) {
+    fprintf(stderr, "short write of output: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+  if (ctx) {
+    SHA_update(ctx, new_data, new_size);
+  }
+  free(new_data);
+
+  return 0;
+}
+
+int ApplyBSDiffPatchMem(const unsigned char* old_data, ssize_t old_size,
+                        const char* patch_filename, ssize_t patch_offset,
+                        unsigned char** new_data, ssize_t* new_size) {
+
   FILE* f;
   if ((f = fopen(patch_filename, "rb")) == NULL) {
     fprintf(stderr, "failed to open patch file\n");
@@ -102,6 +127,8 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
   // from oldfile to x bytes from the diff block; copy y bytes from the
   // extra block; seek forwards in oldfile by z bytes".
 
+  fseek(f, patch_offset, SEEK_SET);
+
   unsigned char header[32];
   if (fread(header, 1, 32, f) < 32) {
     fprintf(stderr, "failed to read patch file header\n");
@@ -109,17 +136,16 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
   }
 
   if (memcmp(header, "BSDIFF40", 8) != 0) {
-    fprintf(stderr, "corrupt patch file header (magic number)\n");
+    fprintf(stderr, "corrupt bsdiff patch file header (magic number)\n");
     return 1;
   }
 
   ssize_t ctrl_len, data_len;
-  ssize_t new_size;
   ctrl_len = offtin(header+8);
   data_len = offtin(header+16);
-  new_size = offtin(header+24);
+  *new_size = offtin(header+24);
 
-  if (ctrl_len < 0 || data_len < 0 || new_size < 0) {
+  if (ctrl_len < 0 || data_len < 0 || *new_size < 0) {
     fprintf(stderr, "corrupt patch file header (data lengths)\n");
     return 1;
   }
@@ -135,7 +161,7 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
     fprintf(stderr, "failed to open patch file\n");                      \
     return 1;                                                            \
   }                                                                      \
-  if (fseeko(f, offset, SEEK_SET)) {                                     \
+  if (fseeko(f, offset+patch_offset, SEEK_SET)) {                        \
     fprintf(stderr, "failed to seek in patch file\n");                   \
     return 1;                                                            \
   }                                                                      \
@@ -150,9 +176,10 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
 
 #undef OPEN_AT
 
-  unsigned char* new_data = malloc(new_size);
+  *new_data = malloc(*new_size);
-  if (new_data == NULL) {
+  if (*new_data == NULL) {
-    fprintf(stderr, "failed to allocate memory for output file\n");
+    fprintf(stderr, "failed to allocate %d bytes of memory for output file\n",
+            (int)*new_size);
     return 1;
   }
 
@@ -161,7 +188,7 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
   off_t len_read;
   int i;
   unsigned char buf[8];
-  while (newpos < new_size) {
+  while (newpos < *new_size) {
     // Read control data
     for (i = 0; i < 3; ++i) {
       len_read = BZ2_bzRead(&bzerr, cpfbz2, buf, 8);
@@ -173,13 +200,13 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
     }
 
     // Sanity check
-    if (newpos + ctrl[0] > new_size) {
+    if (newpos + ctrl[0] > *new_size) {
       fprintf(stderr, "corrupt patch (new file overrun)\n");
       return 1;
     }
 
     // Read diff string
-    len_read = BZ2_bzRead(&bzerr, dpfbz2, new_data + newpos, ctrl[0]);
+    len_read = BZ2_bzRead(&bzerr, dpfbz2, *new_data + newpos, ctrl[0]);
     if (len_read < ctrl[0] || !(bzerr == BZ_OK || bzerr == BZ_STREAM_END)) {
       fprintf(stderr, "corrupt patch (read diff)\n");
       return 1;
@@ -188,7 +215,7 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
     // Add old data to diff string
     for (i = 0; i < ctrl[0]; ++i) {
       if ((oldpos+i >= 0) && (oldpos+i < old_size)) {
-        new_data[newpos+i] += old_data[oldpos+i];
+        (*new_data)[newpos+i] += old_data[oldpos+i];
       }
     }
 
@@ -197,13 +224,13 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
     oldpos += ctrl[0];
 
     // Sanity check
-    if (newpos + ctrl[1] > new_size) {
+    if (newpos + ctrl[1] > *new_size) {
       fprintf(stderr, "corrupt patch (new file overrun)\n");
       return 1;
     }
 
     // Read extra string
-    len_read = BZ2_bzRead(&bzerr, epfbz2, new_data + newpos, ctrl[1]);
+    len_read = BZ2_bzRead(&bzerr, epfbz2, *new_data + newpos, ctrl[1]);
     if (len_read < ctrl[1] || !(bzerr == BZ_OK || bzerr == BZ_STREAM_END)) {
       fprintf(stderr, "corrupt patch (read extra)\n");
       return 1;
@@ -221,12 +248,5 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size,
   fclose(dpf);
   fclose(epf);
 
-  if (fwrite(new_data, 1, new_size, output) < new_size) {
-    fprintf(stderr, "short write of output: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-  SHA_update(ctx, new_data, new_size);
-  free(new_data);
-
   return 0;
 }

tools/applypatch/imgdiff.c

@@ -0,0 +1,560 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This program constructs binary patches for images -- such as boot.img
+ * and recovery.img -- that consist primarily of large chunks of gzipped
+ * data interspersed with uncompressed data.  Doing a naive bsdiff of
+ * these files is not useful because small changes in the data lead to
+ * large changes in the compressed bitstream; bsdiff patches of gzipped
+ * data are typically as large as the data itself.
+ *
+ * To patch these usefully, we break the source and target images up into
+ * chunks of two types: "normal" and "gzip".  Normal chunks are simply
+ * patched using a plain bsdiff.  Gzip chunks are first expanded, then a
+ * bsdiff is applied to the uncompressed data, then the patched data is
+ * gzipped using the same encoder parameters.  Patched chunks are
+ * concatenated together to create the output file; the output image
+ * should be *exactly* the same series of bytes as the target image used
+ * originally to generate the patch.
+ *
+ * To work well with this tool, the gzipped sections of the target
+ * image must have been generated using the same deflate encoder that
+ * is available in applypatch, namely, the one in the zlib library.
+ * In practice this means that images should be compressed using the
+ * "minigzip" tool included in the zlib distribution, not the GNU gzip
+ * program.
+ *
+ * An "imgdiff" patch consists of a header describing the chunk structure
+ * of the file and any encoding parameters needed for the gzipped
+ * chunks, followed by N bsdiff patches, one per chunk.
+ *
+ * For a diff to be generated, the source and target images must have the
+ * same "chunk" structure: that is, the same number of gzipped and normal
+ * chunks in the same order.  Android boot and recovery images currently
+ * consist of five chunks:  a small normal header, a gzipped kernel, a
+ * small normal section, a gzipped ramdisk, and finally a small normal
+ * footer.
+ *
+ * Caveats:  we locate gzipped sections within the source and target
+ * images by searching for the byte sequence 1f8b0800:  1f8b is the gzip
+ * magic number; 08 specifies the "deflate" encoding [the only encoding
+ * supported by the gzip standard]; and 00 is the flags byte.  We do not
+ * currently support any extra header fields (which would be indicated by
+ * a nonzero flags byte).  We also don't handle the case when that byte
+ * sequence appears spuriously in the file.  (Note that it would have to
+ * occur spuriously within a normal chunk to be a problem.)
+ *
+ *
+ * The imgdiff patch header looks like this:
+ *
+ *    "IMGDIFF1"                  (8)   [magic number and version]
+ *    chunk count                 (4)
+ *    for each chunk:
+ *        chunk type              (4)   [CHUNK_NORMAL or CHUNK_GZIP]
+ *        source start            (8)
+ *        source len              (8)
+ *        bsdiff patch offset     (8)   [from start of patch file]
+ *        if chunk type == CHUNK_GZIP:
+ *           source expanded len  (8)   [size of uncompressed source]
+ *           target expected len  (8)   [size of uncompressed target]
+ *           gzip level           (4)
+ *                method          (4)
+ *                windowBits      (4)
+ *                memLevel        (4)
+ *                strategy        (4)
+ *           gzip header len      (4)
+ *           gzip header          (gzip header len)
+ *           gzip footer          (8)
+ *
+ * All integers are little-endian.  "source start" and "source len"
+ * specify the section of the input image that comprises this chunk,
+ * including the gzip header and footer for gzip chunks.  "source
+ * expanded len" is the size of the uncompressed source data.  "target
+ * expected len" is the size of the uncompressed data after applying
+ * the bsdiff patch.  The next five parameters specify the zlib
+ * parameters to be used when compressing the patched data, and the
+ * next three specify the header and footer to be wrapped around the
+ * compressed data to create the output chunk (so that header contents
+ * like the timestamp are recreated exactly).
+ *
+ * After the header there are 'chunk count' bsdiff patches; the offset
+ * of each from the beginning of the file is specified in the header.
+ */
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "zlib.h"
+#include "imgdiff.h"
+
+typedef struct {
+  int type;             // CHUNK_NORMAL or CHUNK_GZIP
+  size_t start;         // offset of chunk in original image file
+
+  size_t len;
+  unsigned char* data;  // data to be patched (ie, uncompressed, for
+                        // gzip chunks)
+
+  // everything else is for CHUNK_GZIP chunks only:
+
+  size_t gzip_header_len;
+  unsigned char* gzip_header;
+  unsigned char* gzip_footer;
+
+  // original (compressed) gzip data, including header and footer
+  size_t gzip_len;
+  unsigned char* gzip_data;
+
+  // deflate encoder parameters
+  int level, method, windowBits, memLevel, strategy;
+} ImageChunk;
+
+/*
+ * Read the given file and break it up into chunks, putting the number
+ * of chunks and their info in *num_chunks and **chunks,
+ * respectively.  Returns a malloc'd block of memory containing the
+ * contents of the file; various pointers in the output chunk array
+ * will point into this block of memory.  The caller should free the
+ * return value when done with all the chunks.  Returns NULL on
+ * failure.
+ */
+unsigned char* ReadImage(const char* filename,
+                         int* num_chunks, ImageChunk** chunks) {
+  struct stat st;
+  if (stat(filename, &st) != 0) {
+    fprintf(stderr, "failed to stat \"%s\": %s\n", filename, strerror(errno));
+    return NULL;
+  }
+
+  unsigned char* img = malloc(st.st_size + 4);
+  FILE* f = fopen(filename, "rb");
+  if (fread(img, 1, st.st_size, f) != st.st_size) {
+    fprintf(stderr, "failed to read \"%s\" %s\n", filename, strerror(errno));
+    fclose(f);
+    return NULL;
+  }
+  fclose(f);
+
+  // append 4 zero bytes to the data so we can always search for the
+  // four-byte string 1f8b0800 starting at any point in the actual
+  // file data, without special-casing the end of the data.
+  memset(img+st.st_size, 0, 4);
+
+  size_t pos = 0;
+
+  *num_chunks = 0;
+  *chunks = NULL;
+
+  while (pos < st.st_size) {
+    unsigned char* p = img+pos;
+
+    // Reallocate the list for every chunk; we expect the number of
+    // chunks to be small (5 for typical boot and recovery images).
+    ++*num_chunks;
+    *chunks = realloc(*chunks, *num_chunks * sizeof(ImageChunk));
+    ImageChunk* curr = *chunks + (*num_chunks-1);
+    curr->start = pos;
+
+    if (st.st_size - pos >= 4 &&
+        p[0] == 0x1f && p[1] == 0x8b &&
+        p[2] == 0x08 &&    // deflate compression
+        p[3] == 0x00) {    // no header flags
+      // 'pos' is the offset of the start of a gzip chunk.
+
+      curr->type = CHUNK_GZIP;
+      curr->gzip_header_len = GZIP_HEADER_LEN;
+      curr->gzip_header = p;
+
+      // We must decompress this chunk in order to discover where it
+      // ends, and so we can put the uncompressed data and its length
+      // into curr->data and curr->len;
+
+      size_t allocated = 32768;
+      curr->len = 0;
+      curr->data = malloc(allocated);
+      curr->gzip_data = p;
+
+      z_stream strm;
+      strm.zalloc = Z_NULL;
+      strm.zfree = Z_NULL;
+      strm.opaque = Z_NULL;
+      strm.avail_in = st.st_size - (pos + curr->gzip_header_len);
+      strm.next_in = p + GZIP_HEADER_LEN;
+
+      // -15 means we are decoding a 'raw' deflate stream; zlib will
+      // not expect zlib headers.
+      int ret = inflateInit2(&strm, -15);
+
+      do {
+        strm.avail_out = allocated - curr->len;
+        strm.next_out = curr->data + curr->len;
+        ret = inflate(&strm, Z_NO_FLUSH);
+        curr->len = allocated - strm.avail_out;
+        if (strm.avail_out == 0) {
+          allocated *= 2;
+          curr->data = realloc(curr->data, allocated);
+        }
+      } while (ret != Z_STREAM_END);
+
+      curr->gzip_len = st.st_size - strm.avail_in - pos + GZIP_FOOTER_LEN;
+      pos = st.st_size - strm.avail_in;
+      inflateEnd(&strm);
+
+      // consume the gzip footer.
+      curr->gzip_footer = img+pos;
+      pos += GZIP_FOOTER_LEN;
+      p = img+pos;
+
+      // The footer (that we just skipped over) contains the size of
+      // the uncompressed data.  Double-check to make sure that it
+      // matches the size of the data we got when we actually did
+      // the decompression.
+      size_t footer_size = p[-4] + (p[-3] << 8) + (p[-2] << 16) + (p[-1] << 24);
+      if (footer_size != curr->len) {
+        fprintf(stderr, "Error: footer size %d != decompressed size %d\n",
+                footer_size, curr->len);
+        free(img);
+        return NULL;
+      }
+    } else {
+      // 'pos' is not the offset of the start of a gzip chunk, so scan
+      // forward until we find a gzip header.
+      curr->type = CHUNK_NORMAL;
+      curr->data = p;
+
+      for (curr->len = 0; curr->len < (st.st_size - pos); ++curr->len) {
+        if (p[curr->len] == 0x1f &&
+            p[curr->len+1] == 0x8b &&
+            p[curr->len+2] == 0x08 &&
+            p[curr->len+3] == 0x00) {
+          break;
+        }
+      }
+      pos += curr->len;
+    }
+  }
+
+  return img;
+}
+
+#define BUFFER_SIZE 32768
+
+/*
+ * Takes the uncompressed data stored in the chunk, compresses it
+ * using the zlib parameters stored in the chunk, and checks that it
+ * matches exactly the compressed data we started with (also stored in
+ * the chunk).  Return 0 on success.
+ */
+int TryReconstruction(ImageChunk* chunk, unsigned char* out) {
+  size_t p = chunk->gzip_header_len;
+
+  z_stream strm;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+  strm.avail_in = chunk->len;
+  strm.next_in = chunk->data;
+  int ret;
+  ret = deflateInit2(&strm, chunk->level, chunk->method, chunk->windowBits,
+                     chunk->memLevel, chunk->strategy);
+  do {
+    strm.avail_out = BUFFER_SIZE;
+    strm.next_out = out;
+    ret = deflate(&strm, Z_FINISH);
+    size_t have = BUFFER_SIZE - strm.avail_out;
+
+    if (memcmp(out, chunk->gzip_data+p, have) != 0) {
+      // mismatch; data isn't the same.
+      deflateEnd(&strm);
+      return -1;
+    }
+    p += have;
+  } while (ret != Z_STREAM_END);
+  deflateEnd(&strm);
+  if (p + GZIP_FOOTER_LEN != chunk->gzip_len) {
+    // mismatch; ran out of data before we should have.
+    return -1;
+  }
+  return 0;
+}
+
+/*
+ * Verify that we can reproduce exactly the same compressed data that
+ * we started with.  Sets the level, method, windowBits, memLevel, and
+ * strategy fields in the chunk to the encoding parameters needed to
+ * produce the right output.  Returns 0 on success.
+ */
+int ReconstructGzipChunk(ImageChunk* chunk) {
+  if (chunk->type != CHUNK_GZIP) {
+    fprintf(stderr, "attempt to reconstruct non-gzip chunk\n");
+    return -1;
+  }
+
+  size_t p = 0;
+  unsigned char* out = malloc(BUFFER_SIZE);
+
+  // We only check two combinations of encoder parameters:  level 6
+  // (the default) and level 9 (the maximum).
+  for (chunk->level = 6; chunk->level <= 9; chunk->level += 3) {
+    chunk->windowBits = -15;  // 32kb window; negative to indicate a raw stream.
+    chunk->memLevel = 8;      // the default value.
+    chunk->method = Z_DEFLATED;
+    chunk->strategy = Z_DEFAULT_STRATEGY;
+
+    if (TryReconstruction(chunk, out) == 0) {
+      free(out);
+      return 0;
+    }
+  }
+
+  free(out);
+  return -1;
+}
+
+/** Write a 4-byte value to f in little-endian order. */
+void Write4(int value, FILE* f) {
+  fputc(value & 0xff, f);
+  fputc((value >> 8) & 0xff, f);
+  fputc((value >> 16) & 0xff, f);
+  fputc((value >> 24) & 0xff, f);
+}
+
+/** Write an 8-byte value to f in little-endian order. */
+void Write8(long long value, FILE* f) {
+  fputc(value & 0xff, f);
+  fputc((value >> 8) & 0xff, f);
+  fputc((value >> 16) & 0xff, f);
+  fputc((value >> 24) & 0xff, f);
+  fputc((value >> 32) & 0xff, f);
+  fputc((value >> 40) & 0xff, f);
+  fputc((value >> 48) & 0xff, f);
+  fputc((value >> 56) & 0xff, f);
+}
+
+
+/*
+ * Given source and target chunks, compute a bsdiff patch between them
+ * by running bsdiff in a subprocess.  Return the patch data, placing
+ * its length in *size.  Return NULL on failure.  We expect the bsdiff
+ * program to be in the path.
+ */
+unsigned char* MakePatch(ImageChunk* src, ImageChunk* tgt, size_t* size) {
+  char stemp[] = "/tmp/imgdiff-src-XXXXXX";
+  char ttemp[] = "/tmp/imgdiff-tgt-XXXXXX";
+  char ptemp[] = "/tmp/imgdiff-patch-XXXXXX";
+  mkstemp(stemp);
+  mkstemp(ttemp);
+  mkstemp(ptemp);
+
+  FILE* f = fopen(stemp, "wb");
+  if (f == NULL) {
+    fprintf(stderr, "failed to open src chunk %s: %s\n",
+            stemp, strerror(errno));
+    return NULL;
+  }
+  if (fwrite(src->data, 1, src->len, f) != src->len) {
+    fprintf(stderr, "failed to write src chunk to %s: %s\n",
+            stemp, strerror(errno));
+    return NULL;
+  }
+  fclose(f);
+
+  f = fopen(ttemp, "wb");
+  if (f == NULL) {
+    fprintf(stderr, "failed to open tgt chunk %s: %s\n",
+            ttemp, strerror(errno));
+    return NULL;
+  }
+  if (fwrite(tgt->data, 1, tgt->len, f) != tgt->len) {
+    fprintf(stderr, "failed to write tgt chunk to %s: %s\n",
+            ttemp, strerror(errno));
+    return NULL;
+  }
+  fclose(f);
+
+  char cmd[200];
+  sprintf(cmd, "bsdiff %s %s %s", stemp, ttemp, ptemp);
+  if (system(cmd) != 0) {
+    fprintf(stderr, "failed to run bsdiff: %s\n", strerror(errno));
+    return NULL;
+  }
+
+  struct stat st;
+  if (stat(ptemp, &st) != 0) {
+    fprintf(stderr, "failed to stat patch file %s: %s\n",
+            ptemp, strerror(errno));
+    return NULL;
+  }
+
+  unsigned char* data = malloc(st.st_size);
+  *size = st.st_size;
+
+  f = fopen(ptemp, "rb");
+  if (f == NULL) {
+    fprintf(stderr, "failed to open patch %s: %s\n", ptemp, strerror(errno));
+    return NULL;
+  }
+  if (fread(data, 1, st.st_size, f) != st.st_size) {
+    fprintf(stderr, "failed to read patch %s: %s\n", ptemp, strerror(errno));
+    return NULL;
+  }
+  fclose(f);
+
+  unlink(stemp);
+  unlink(ttemp);
+  unlink(ptemp);
+
+  return data;
+}
+
+/*
+ * Cause a gzip chunk to be treated as a normal chunk (ie, as a blob
+ * of uninterpreted data).  The resulting patch will likely be about
+ * as big as the target file, but it lets us handle the case of images
+ * where some gzip chunks are reconstructible but others aren't (by
+ * treating the ones that aren't as normal chunks).
+ */
+void ChangeGzipChunkToNormal(ImageChunk* ch) {
+  ch->type = CHUNK_NORMAL;
+  free(ch->data);
+  ch->data = ch->gzip_data;
+  ch->len = ch->gzip_len;
+}
+
+int main(int argc, char** argv) {
+  if (argc != 4) {
+    fprintf(stderr, "usage: %s <src-img> <tgt-img> <patch-file>\n", argv[0]);
+    return 2;
+  }
+
+  int num_src_chunks;
+  ImageChunk* src_chunks;
+  if (ReadImage(argv[1], &num_src_chunks, &src_chunks) == NULL) {
+    fprintf(stderr, "failed to break apart source image\n");
+    return 1;
+  }
+
+  int num_tgt_chunks;
+  ImageChunk* tgt_chunks;
+  if (ReadImage(argv[2], &num_tgt_chunks, &tgt_chunks) == NULL) {
+    fprintf(stderr, "failed to break apart target image\n");
+    return 1;
+  }
+
+  // Verify that the source and target images have the same chunk
+  // structure (ie, the same sequence of gzip and normal chunks).
+
+  if (num_src_chunks != num_tgt_chunks) {
+    fprintf(stderr, "source and target don't have same number of chunks!\n");
+    return 1;
+  }
+  int i;
+  for (i = 0; i < num_src_chunks; ++i) {
+    if (src_chunks[i].type != tgt_chunks[i].type) {
+      fprintf(stderr, "source and target don't have same chunk "
+              "structure! (chunk %d)\n", i);
+      return 1;
+    }
+  }
+
+  // Confirm that given the uncompressed chunk data in the target, we
+  // can recompress it and get exactly the same bits as are in the
+  // input target image.  If this fails, treat the chunk as a normal
+  // non-gzipped chunk.
+
+  for (i = 0; i < num_tgt_chunks; ++i) {
+    if (tgt_chunks[i].type == CHUNK_GZIP) {
+      if (ReconstructGzipChunk(tgt_chunks+i) < 0) {
+        printf("failed to reconstruct target gzip chunk %d; "
+               "treating as normal chunk\n", i);
+        ChangeGzipChunkToNormal(tgt_chunks+i);
+        ChangeGzipChunkToNormal(src_chunks+i);
+      } else {
+        printf("reconstructed target gzip chunk %d\n", i);
+      }
+    }
+  }
+
+  // Compute bsdiff patches for each chunk's data (the uncompressed
+  // data, in the case of gzip chunks).
+
+  unsigned char** patch_data = malloc(num_src_chunks * sizeof(unsigned char*));
+  size_t* patch_size = malloc(num_src_chunks * sizeof(size_t));
+  for (i = 0; i < num_src_chunks; ++i) {
+    patch_data[i] = MakePatch(src_chunks+i, tgt_chunks+i, patch_size+i);
+    printf("patch %d is %d bytes (of %d)\n", i, patch_size[i],
+           tgt_chunks[i].type == CHUNK_NORMAL ? tgt_chunks[i].len : tgt_chunks[i].gzip_len);
+
+  }
+
+  // Figure out how big the imgdiff file header is going to be, so
+  // that we can correctly compute the offset of each bsdiff patch
+  // within the file.
+
+  size_t total_header_size = 12;
+  for (i = 0; i < num_src_chunks; ++i) {
+    total_header_size += 4 + 8*3;
+    if (src_chunks[i].type == CHUNK_GZIP) {
+      total_header_size += 8*2 + 4*6 + tgt_chunks[i].gzip_header_len + 8;
+    }
+  }
+
+  size_t offset = total_header_size;
+
+  FILE* f = fopen(argv[3], "wb");
+
+  // Write out the headers.
+
+  fwrite("IMGDIFF1", 1, 8, f);
+  Write4(num_src_chunks, f);
+  for (i = 0; i < num_tgt_chunks; ++i) {
+    Write4(tgt_chunks[i].type, f);
+    Write8(src_chunks[i].start, f);
+    Write8(src_chunks[i].type == CHUNK_NORMAL ? src_chunks[i].len :
+           (src_chunks[i].gzip_len + src_chunks[i].gzip_header_len + 8), f);
+    Write8(offset, f);
+
+    if (tgt_chunks[i].type == CHUNK_GZIP) {
+      Write8(src_chunks[i].len, f);
+      Write8(tgt_chunks[i].len, f);
+      Write4(tgt_chunks[i].level, f);
+      Write4(tgt_chunks[i].method, f);
+      Write4(tgt_chunks[i].windowBits, f);
+      Write4(tgt_chunks[i].memLevel, f);
+      Write4(tgt_chunks[i].strategy, f);
+      Write4(tgt_chunks[i].gzip_header_len, f);
+      fwrite(tgt_chunks[i].gzip_header, 1, tgt_chunks[i].gzip_header_len, f);
+      fwrite(tgt_chunks[i].gzip_footer, 1, GZIP_FOOTER_LEN, f);
+    }
+
+    offset += patch_size[i];
+  }
+
+  // Append each chunk's bsdiff patch, in order.
+
+  for (i = 0; i < num_tgt_chunks; ++i) {
+    fwrite(patch_data[i], 1, patch_size[i], f);
+  }
+
+  fclose(f);
+
+  return 0;
+}

tools/applypatch/imgdiff.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Image patch chunk types
+#define CHUNK_NORMAL 0
+#define CHUNK_GZIP   1
+
+// The gzip header size is actually variable, but we currently don't
+// support gzipped data with any of the optional fields, so for now it
+// will always be ten bytes.  See RFC 1952 for the definition of the
+// gzip format.
+#define GZIP_HEADER_LEN   10
+
+// The gzip footer size really is fixed.
+#define GZIP_FOOTER_LEN   8

tools/applypatch/imgpatch.c

@@ -0,0 +1,227 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// See imgdiff.c in this directory for a description of the patch file
+// format.
+
+#include <stdio.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "zlib.h"
+#include "mincrypt/sha.h"
+#include "applypatch.h"
+#include "imgdiff.h"
+
+int Read4(unsigned char* p) {
+  return (int)(((unsigned int)p[3] << 24) |
+               ((unsigned int)p[2] << 16) |
+               ((unsigned int)p[1] << 8) |
+               (unsigned int)p[0]);
+}
+
+long long Read8(unsigned char* p) {
+  return (long long)(((unsigned long long)p[7] << 56) |
+                     ((unsigned long long)p[6] << 48) |
+                     ((unsigned long long)p[5] << 40) |
+                     ((unsigned long long)p[4] << 32) |
+                     ((unsigned long long)p[3] << 24) |
+                     ((unsigned long long)p[2] << 16) |
+                     ((unsigned long long)p[1] << 8) |
+                     (unsigned long long)p[0]);
+}
+
+/*
+ * Apply the patch given in 'patch_filename' to the source data given
+ * by (old_data, old_size).  Write the patched output to the 'output'
+ * file, and update the SHA context with the output data as well.
+ * Return 0 on success.
+ */
+int ApplyImagePatch(const unsigned char* old_data, ssize_t old_size,
+                    const char* patch_filename,
+                    FILE* output, SHA_CTX* ctx) {
+  FILE* f;
+  if ((f = fopen(patch_filename, "rb")) == NULL) {
+    fprintf(stderr, "failed to open patch file\n");
+    return -1;
+  }
+
+  unsigned char header[12];
+  if (fread(header, 1, 12, f) != 12) {
+    fprintf(stderr, "failed to read patch file header\n");
+    return -1;
+  }
+
+  if (memcmp(header, "IMGDIFF1", 8) != 0) {
+    fprintf(stderr, "corrupt patch file header (magic number)\n");
+    return -1;
+  }
+
+  int num_chunks = Read4(header+8);
+
+  int i;
+  for (i = 0; i < num_chunks; ++i) {
+    // each chunk's header record starts with 28 bytes (4 + 8*3).
+    unsigned char chunk[28];
+    if (fread(chunk, 1, 28, f) != 28) {
+      fprintf(stderr, "failed to read chunk %d record\n", i);
+      return -1;
+    }
+
+    int type = Read4(chunk);
+    size_t src_start = Read8(chunk+4);
+    size_t src_len = Read8(chunk+12);
+    size_t patch_offset = Read8(chunk+20);
+
+    if (type == CHUNK_NORMAL) {
+      fprintf(stderr, "CHUNK %d:  normal   patch offset %d\n", i, patch_offset);
+
+      ApplyBSDiffPatch(old_data + src_start, src_len,
+                       patch_filename, patch_offset,
+                       output, ctx);
+    } else if (type == CHUNK_GZIP) {
+      fprintf(stderr, "CHUNK %d:  gzip     patch offset %d\n", i, patch_offset);
+
+      // gzip chunks have an additional 40 + gzip_header_len + 8 bytes
+      // in their chunk header.
+      unsigned char* gzip = malloc(40);
+      if (fread(gzip, 1, 40, f) != 40) {
+        fprintf(stderr, "failed to read chunk %d initial gzip data\n", i);
+        return -1;
+      }
+      size_t gzip_header_len = Read4(gzip+36);
+      gzip = realloc(gzip, 40 + gzip_header_len + 8);
+      if (fread(gzip+40, 1, gzip_header_len+8, f) != gzip_header_len+8) {
+        fprintf(stderr, "failed to read chunk %d remaining gzip data\n", i);
+        return -1;
+      }
+
+      size_t expanded_len = Read8(gzip);
+      size_t target_len = Read8(gzip);
+      int gz_level = Read4(gzip+16);
+      int gz_method = Read4(gzip+20);
+      int gz_windowBits = Read4(gzip+24);
+      int gz_memLevel = Read4(gzip+28);
+      int gz_strategy = Read4(gzip+32);
+
+      // Decompress the source data; the chunk header tells us exactly
+      // how big we expect it to be when decompressed.
+
+      unsigned char* expanded_source = malloc(expanded_len);
+      if (expanded_source == NULL) {
+        fprintf(stderr, "failed to allocate %d bytes for expanded_source\n", expanded_len);
+        return -1;
+      }
+
+      z_stream strm;
+      strm.zalloc = Z_NULL;
+      strm.zfree = Z_NULL;
+      strm.opaque = Z_NULL;
+      strm.avail_in = src_len - (gzip_header_len + 8);
+      strm.next_in = (unsigned char*)(old_data + src_start + gzip_header_len);
+      strm.avail_out = expanded_len;
+      strm.next_out = expanded_source;
+
+      int ret;
+      ret = inflateInit2(&strm, -15);
+      if (ret != Z_OK) {
+        fprintf(stderr, "failed to init source inflation: %d\n", ret);
+        return -1;
+      }
+
+      // Because we've provided enough room to accommodate the output
+      // data, we expect one call to inflate() to suffice.
+      ret = inflate(&strm, Z_SYNC_FLUSH);
+      if (ret != Z_STREAM_END) {
+        fprintf(stderr, "source inflation returned %d\n", ret);
+        return -1;
+      }
+      // We should have filled the output buffer exactly.
+      if (strm.avail_out != 0) {
+        fprintf(stderr, "source inflation short by %d bytes\n", strm.avail_out);
+        return -1;
+      }
+      inflateEnd(&strm);
+
+      // Next, apply the bsdiff patch (in memory) to the uncompressed
+      // data.
+      unsigned char* uncompressed_target_data;
+      ssize_t uncompressed_target_size;
+      if (ApplyBSDiffPatchMem(expanded_source, expanded_len,
+                              patch_filename, patch_offset,
+                              &uncompressed_target_data,
+                              &uncompressed_target_size) != 0) {
+        return -1;
+      }
+
+      // Now compress the target data and append it to the output.
+
+      // start with the gzip header.
+      fwrite(gzip+40, 1, gzip_header_len, output);
+      SHA_update(ctx, gzip+40, gzip_header_len);
+
+      // we're done with the expanded_source data buffer, so we'll
+      // reuse that memory to receive the output of deflate.
+      unsigned char* temp_data = expanded_source;
+      ssize_t temp_size = expanded_len;
+      if (temp_size < 32768) {
+        // ... unless the buffer is too small, in which case we'll
+        // allocate a fresh one.
+        free(temp_data);
+        temp_data = malloc(32768);
+        temp_size = 32768;
+      }
+
+      // now the deflate stream
+      strm.zalloc = Z_NULL;
+      strm.zfree = Z_NULL;
+      strm.opaque = Z_NULL;
+      strm.avail_in = uncompressed_target_size;
+      strm.next_in = uncompressed_target_data;
+      ret = deflateInit2(&strm, gz_level, gz_method, gz_windowBits,
+                         gz_memLevel, gz_strategy);
+      do {
+        strm.avail_out = temp_size;
+        strm.next_out = temp_data;
+        ret = deflate(&strm, Z_FINISH);
+        size_t have = temp_size - strm.avail_out;
+
+        if (fwrite(temp_data, 1, have, output) != have) {
+          fprintf(stderr, "failed to write %d compressed bytes to output\n",
+                  have);
+          return -1;
+        }
+        SHA_update(ctx, temp_data, have);
+      } while (ret != Z_STREAM_END);
+      deflateEnd(&strm);
+
+      // lastly, the gzip footer.
+      fwrite(gzip+40+gzip_header_len, 1, 8, output);
+      SHA_update(ctx, gzip+40+gzip_header_len, 8);
+
+      free(temp_data);
+      free(uncompressed_target_data);
+      free(gzip);
+    } else {
+      fprintf(stderr, "patch chunk %d is unknown type %d\n", i, type);
+      return -1;
+    }
+  }
+
+  return 0;
+}