From 1ea84d6da97f4c4269ac16c4d332bbff3466b1b8 Mon Sep 17 00:00:00 2001 From: Tianjie Xu Date: Wed, 22 Feb 2017 18:23:58 -0800 Subject: [PATCH] Refractor the code for imgdiff Put ImageChunk and some helper functions into a class. Also switch to using std::vector instead of malloc. Bug: 18606652 Test: imgdiff_test passed on host. Also generate a complete incremental OTA package. The file content is the same and time consumption is similar. Change-Id: Id603ada4e130ef521218400761a119001a86ca79 --- applypatch/Android.mk | 3 + applypatch/imgdiff.cpp | 1285 +++++++++++++++--------------- tests/Android.mk | 1 + tests/component/imgdiff_test.cpp | 56 ++ 4 files changed, 690 insertions(+), 655 deletions(-) diff --git a/applypatch/Android.mk b/applypatch/Android.mk index 47ddad6c..8be5c36b 100644 --- a/applypatch/Android.mk +++ b/applypatch/Android.mk @@ -136,6 +136,9 @@ libimgdiff_static_libraries := \ libbsdiff \ libdivsufsort \ libdivsufsort64 \ + libziparchive \ + libutils \ + liblog \ libbase \ libz diff --git a/applypatch/imgdiff.cpp b/applypatch/imgdiff.cpp index 2f0e1651..fba74e83 100644 --- a/applypatch/imgdiff.cpp +++ b/applypatch/imgdiff.cpp @@ -132,502 +132,611 @@ #include #include +#include +#include +#include + #include +#include +#include #include +#include #include #include #include "utils.h" -typedef struct { - int type; // CHUNK_NORMAL, CHUNK_DEFLATE - size_t start; // offset of chunk in original image file +using android::base::get_unaligned; - size_t len; - unsigned char* data; // data to be patched (uncompressed, for deflate chunks) +static constexpr auto BUFFER_SIZE = 0x8000; - size_t source_start; - size_t source_len; +class ImageChunk { + public: + static constexpr auto WINDOWBITS = -15; // 32kb window; negative to indicate a raw stream. + static constexpr auto MEMLEVEL = 8; // the default value. + static constexpr auto METHOD = Z_DEFLATED; + static constexpr auto STRATEGY = Z_DEFAULT_STRATEGY; + + ImageChunk(int type, size_t start, const std::vector* file_content, size_t raw_data_len) + : type_(type), + start_(start), + input_file_ptr_(file_content), + raw_data_len_(raw_data_len), + entry_name_(""), + compress_level_(6), + source_start_(0), + source_len_(0), + source_uncompressed_len_(0) {} + + int GetType() const { + return type_; + } + size_t GetRawDataLength() const { + return raw_data_len_; + } + const std::string& GetEntryName() const { + return entry_name_; + } + + // CHUNK_DEFLATE will return the uncompressed data for diff, while other types will simply return + // the raw data. + const uint8_t * DataForPatch() const; + size_t DataLengthForPatch() const; + + void Dump() const { + printf("type %d start %zu len %zu\n", type_, start_, DataLengthForPatch()); + } + + void SetSourceInfo(const ImageChunk& other); + void SetEntryName(std::string entryname); + void SetUncompressedData(std::vector data); + bool SetBonusData(const std::vector& bonus_data); + + bool operator==(const ImageChunk& other) const; + bool operator!=(const ImageChunk& other) const { + return !(*this == other); + } + + size_t GetHeaderSize(size_t patch_size) const; + size_t WriteHeaderToFile(FILE* f, const std::vector patch, size_t offset); + + /* + * Cause a gzip chunk to be treated as a normal chunk (ie, as a blob + * of uninterpreted data). The resulting patch will likely be about + * as big as the target file, but it lets us handle the case of images + * where some gzip chunks are reconstructible but others aren't (by + * treating the ones that aren't as normal chunks). + */ + void ChangeDeflateChunkToNormal(); + bool ChangeChunkToRaw(size_t patch_size); + + /* + * Verify that we can reproduce exactly the same compressed data that + * we started with. Sets the level, method, windowBits, memLevel, and + * strategy fields in the chunk to the encoding parameters needed to + * produce the right output. + */ + bool ReconstructDeflateChunk(); + bool IsAdjacentNormal(const ImageChunk& other) const; + void MergeAdjacentNormal(const ImageChunk& other); + + private: + int type_; // CHUNK_NORMAL, CHUNK_DEFLATE, CHUNK_RAW + size_t start_; // offset of chunk in the original input file + const std::vector* input_file_ptr_; // pointer to the full content of original input file + size_t raw_data_len_; // --- for CHUNK_DEFLATE chunks only: --- - - // original (compressed) deflate data - size_t deflate_len; - unsigned char* deflate_data; - - char* filename; // used for zip entries + std::vector uncompressed_data_; + std::string entry_name_; // used for zip entries // deflate encoder parameters - int level, method, windowBits, memLevel, strategy; + int compress_level_; - size_t source_uncompressed_len; -} ImageChunk; + size_t source_start_; + size_t source_len_; + size_t source_uncompressed_len_; -typedef struct { - int data_offset; - int deflate_len; - int uncomp_len; - char* filename; -} ZipFileEntry; + const uint8_t* GetRawData() const; + bool TryReconstruction(int level); +}; -static int fileentry_compare(const void* a, const void* b) { - int ao = ((ZipFileEntry*)a)->data_offset; - int bo = ((ZipFileEntry*)b)->data_offset; - if (ao < bo) { - return -1; - } else if (ao > bo) { - return 1; - } else { - return 0; +const uint8_t* ImageChunk::GetRawData() const { + CHECK_LE(start_ + raw_data_len_, input_file_ptr_->size()); + return input_file_ptr_->data() + start_; +} + +const uint8_t * ImageChunk::DataForPatch() const { + if (type_ == CHUNK_DEFLATE) { + return uncompressed_data_.data(); + } + return GetRawData(); +} + +size_t ImageChunk::DataLengthForPatch() const { + if (type_ == CHUNK_DEFLATE) { + return uncompressed_data_.size(); + } + return raw_data_len_; +} + +bool ImageChunk::operator==(const ImageChunk& other) const { + if (type_ != other.type_) { + return false; + } + return (raw_data_len_ == other.raw_data_len_ && + memcmp(GetRawData(), other.GetRawData(), raw_data_len_) == 0); +} + +void ImageChunk::SetSourceInfo(const ImageChunk& src) { + source_start_ = src.start_; + if (type_ == CHUNK_NORMAL) { + source_len_ = src.raw_data_len_; + } else if (type_ == CHUNK_DEFLATE) { + source_len_ = src.raw_data_len_; + source_uncompressed_len_ = src.uncompressed_data_.size(); } } -unsigned char* ReadZip(const char* filename, - int* num_chunks, ImageChunk** chunks, - int include_pseudo_chunk) { - struct stat st; - if (stat(filename, &st) != 0) { - printf("failed to stat \"%s\": %s\n", filename, strerror(errno)); - return NULL; +void ImageChunk::SetEntryName(std::string entryname) { + entry_name_ = entryname; +} + +void ImageChunk::SetUncompressedData(std::vector data) { + uncompressed_data_ = data; +} + +bool ImageChunk::SetBonusData(const std::vector& bonus_data) { + if (type_ != CHUNK_DEFLATE) { + return false; + } + uncompressed_data_.insert(uncompressed_data_.end(), bonus_data.begin(), bonus_data.end()); + return true; +} + +// Convert CHUNK_NORMAL & CHUNK_DEFLATE to CHUNK_RAW if the terget size is +// smaller. Also take the header size into account during size comparison. +bool ImageChunk::ChangeChunkToRaw(size_t patch_size) { + if (type_ == CHUNK_RAW) { + return true; + } else if (type_ == CHUNK_NORMAL && (raw_data_len_ <= 160 || raw_data_len_ < patch_size)) { + type_ = CHUNK_RAW; + return true; + } + return false; +} + +void ImageChunk::ChangeDeflateChunkToNormal() { + if (type_ != CHUNK_DEFLATE) return; + type_ = CHUNK_NORMAL; + uncompressed_data_.clear(); +} + +// Header size: +// header_type 4 bytes +// CHUNK_NORMAL 8*3 = 24 bytes +// CHUNK_DEFLATE 8*5 + 4*5 = 60 bytes +// CHUNK_RAW 4 bytes +size_t ImageChunk::GetHeaderSize(size_t patch_size) const { + switch (type_) { + case CHUNK_NORMAL: + return 4 + 8 * 3; + case CHUNK_DEFLATE: + return 4 + 8 * 5 + 4 * 5; + case CHUNK_RAW: + return 4 + 4 + patch_size; + default: + printf("unexpected chunk type: %d\n", type_); // should not reach here. + CHECK(false); + return 0; + } +} + +size_t ImageChunk::WriteHeaderToFile(FILE* f, const std::vector patch, size_t offset) { + Write4(type_, f); + switch (type_) { + case CHUNK_NORMAL: + printf("normal (%10zu, %10zu) %10zu\n", start_, raw_data_len_, patch.size()); + Write8(source_start_, f); + Write8(source_len_, f); + Write8(offset, f); + return offset + patch.size(); + case CHUNK_DEFLATE: + printf("deflate (%10zu, %10zu) %10zu %s\n", start_, raw_data_len_, patch.size(), + entry_name_.c_str()); + Write8(source_start_, f); + Write8(source_len_, f); + Write8(offset, f); + Write8(source_uncompressed_len_, f); + Write8(uncompressed_data_.size(), f); + Write4(compress_level_, f); + Write4(METHOD, f); + Write4(WINDOWBITS, f); + Write4(MEMLEVEL, f); + Write4(STRATEGY, f); + return offset + patch.size(); + case CHUNK_RAW: + printf("raw (%10zu, %10zu)\n", start_, raw_data_len_); + Write4(patch.size(), f); + fwrite(patch.data(), 1, patch.size(), f); + return offset; + default: + printf("unexpected chunk type: %d\n", type_); + CHECK(false); + return offset; + } +} + +bool ImageChunk::IsAdjacentNormal(const ImageChunk& other) const { + if (type_ != CHUNK_NORMAL || other.type_ != CHUNK_NORMAL) { + return false; + } + return (other.start_ == start_ + raw_data_len_); +} + +void ImageChunk::MergeAdjacentNormal(const ImageChunk& other) { + CHECK(IsAdjacentNormal(other)); + raw_data_len_ = raw_data_len_ + other.raw_data_len_; +} + +bool ImageChunk::ReconstructDeflateChunk() { + if (type_ != CHUNK_DEFLATE) { + printf("attempt to reconstruct non-deflate chunk\n"); + return false; } - size_t sz = static_cast(st.st_size); - unsigned char* img = static_cast(malloc(sz)); - FILE* f = fopen(filename, "rb"); - if (fread(img, 1, sz, f) != sz) { - printf("failed to read \"%s\" %s\n", filename, strerror(errno)); - fclose(f); - free(img); - return NULL; - } - fclose(f); - - // look for the end-of-central-directory record. - - int i; - for (i = st.st_size-20; i >= 0 && i > st.st_size - 65600; --i) { - if (img[i] == 0x50 && img[i+1] == 0x4b && - img[i+2] == 0x05 && img[i+3] == 0x06) { - break; + // We only check two combinations of encoder parameters: level 6 + // (the default) and level 9 (the maximum). + for (int level = 6; level <= 9; level += 3) { + if (TryReconstruction(level)) { + compress_level_ = level; + return true; } } - // double-check: this archive consists of a single "disk" - if (!(img[i+4] == 0 && img[i+5] == 0 && img[i+6] == 0 && img[i+7] == 0)) { - printf("can't process multi-disk archive\n"); - return NULL; - } - int cdcount = Read2(img+i+8); - int cdoffset = Read4(img+i+16); - - ZipFileEntry* temp_entries = static_cast(malloc( - cdcount * sizeof(ZipFileEntry))); - int entrycount = 0; - - unsigned char* cd = img+cdoffset; - for (i = 0; i < cdcount; ++i) { - if (!(cd[0] == 0x50 && cd[1] == 0x4b && cd[2] == 0x01 && cd[3] == 0x02)) { - printf("bad central directory entry %d\n", i); - free(temp_entries); - return NULL; - } - - int clen = Read4(cd+20); // compressed len - int ulen = Read4(cd+24); // uncompressed len - int nlen = Read2(cd+28); // filename len - int xlen = Read2(cd+30); // extra field len - int mlen = Read2(cd+32); // file comment len - int hoffset = Read4(cd+42); // local header offset - - char* filename = static_cast(malloc(nlen+1)); - memcpy(filename, cd+46, nlen); - filename[nlen] = '\0'; - - int method = Read2(cd+10); - - cd += 46 + nlen + xlen + mlen; - - if (method != 8) { // 8 == deflate - free(filename); - continue; - } - - unsigned char* lh = img + hoffset; - - if (!(lh[0] == 0x50 && lh[1] == 0x4b && lh[2] == 0x03 && lh[3] == 0x04)) { - printf("bad local file header entry %d\n", i); - return NULL; - } - - if (Read2(lh+26) != nlen || memcmp(lh+30, filename, nlen) != 0) { - printf("central dir filename doesn't match local header\n"); - return NULL; - } - - xlen = Read2(lh+28); // extra field len; might be different from CD entry? - - temp_entries[entrycount].data_offset = hoffset+30+nlen+xlen; - temp_entries[entrycount].deflate_len = clen; - temp_entries[entrycount].uncomp_len = ulen; - temp_entries[entrycount].filename = filename; - ++entrycount; - } - - qsort(temp_entries, entrycount, sizeof(ZipFileEntry), fileentry_compare); - -#if 0 - printf("found %d deflated entries\n", entrycount); - for (i = 0; i < entrycount; ++i) { - printf("off %10d len %10d unlen %10d %p %s\n", - temp_entries[i].data_offset, - temp_entries[i].deflate_len, - temp_entries[i].uncomp_len, - temp_entries[i].filename, - temp_entries[i].filename); - } -#endif - - *num_chunks = 0; - *chunks = static_cast(malloc((entrycount*2+2) * sizeof(ImageChunk))); - ImageChunk* curr = *chunks; - - if (include_pseudo_chunk) { - curr->type = CHUNK_NORMAL; - curr->start = 0; - curr->len = st.st_size; - curr->data = img; - curr->filename = NULL; - ++curr; - ++*num_chunks; - } - - int pos = 0; - int nextentry = 0; - - while (pos < st.st_size) { - if (nextentry < entrycount && pos == temp_entries[nextentry].data_offset) { - curr->type = CHUNK_DEFLATE; - curr->start = pos; - curr->deflate_len = temp_entries[nextentry].deflate_len; - curr->deflate_data = img + pos; - curr->filename = temp_entries[nextentry].filename; - - curr->len = temp_entries[nextentry].uncomp_len; - curr->data = static_cast(malloc(curr->len)); - - z_stream strm; - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = curr->deflate_len; - strm.next_in = curr->deflate_data; - - // -15 means we are decoding a 'raw' deflate stream; zlib will - // not expect zlib headers. - int ret = inflateInit2(&strm, -15); - if (ret < 0) { - printf("failed to initialize inflate: %d\n", ret); - return NULL; - } - - strm.avail_out = curr->len; - strm.next_out = curr->data; - ret = inflate(&strm, Z_NO_FLUSH); - if (ret != Z_STREAM_END) { - printf("failed to inflate \"%s\"; %d\n", curr->filename, ret); - return NULL; - } - - inflateEnd(&strm); - - pos += curr->deflate_len; - ++nextentry; - ++*num_chunks; - ++curr; - continue; - } - - // use a normal chunk to take all the data up to the start of the - // next deflate section. - - curr->type = CHUNK_NORMAL; - curr->start = pos; - if (nextentry < entrycount) { - curr->len = temp_entries[nextentry].data_offset - pos; - } else { - curr->len = st.st_size - pos; - } - curr->data = img + pos; - curr->filename = NULL; - pos += curr->len; - - ++*num_chunks; - ++curr; - } - - free(temp_entries); - return img; + return false; } /* - * Read the given file and break it up into chunks, putting the number - * of chunks and their info in *num_chunks and **chunks, - * respectively. Returns a malloc'd block of memory containing the - * contents of the file; various pointers in the output chunk array - * will point into this block of memory. The caller should free the - * return value when done with all the chunks. Returns NULL on - * failure. + * Takes the uncompressed data stored in the chunk, compresses it + * using the zlib parameters stored in the chunk, and checks that it + * matches exactly the compressed data we started with (also stored in + * the chunk). */ -unsigned char* ReadImage(const char* filename, int* num_chunks, ImageChunk** chunks) { +bool ImageChunk::TryReconstruction(int level) { + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = uncompressed_data_.size(); + strm.next_in = uncompressed_data_.data(); + int ret = deflateInit2(&strm, level, METHOD, WINDOWBITS, MEMLEVEL, STRATEGY); + if (ret < 0) { + printf("failed to initialize deflate: %d\n", ret); + return false; + } + + std::vector buffer(BUFFER_SIZE); + size_t offset = 0; + do { + strm.avail_out = buffer.size(); + strm.next_out = buffer.data(); + ret = deflate(&strm, Z_FINISH); + if (ret < 0) { + printf("failed to deflate: %d\n", ret); + return false; + } + + size_t compressed_size = buffer.size() - strm.avail_out; + if (memcmp(buffer.data(), input_file_ptr_->data() + start_ + offset, compressed_size) != 0) { + // mismatch; data isn't the same. + deflateEnd(&strm); + return false; + } + offset += compressed_size; + } while (ret != Z_STREAM_END); + deflateEnd(&strm); + + if (offset != raw_data_len_) { + // mismatch; ran out of data before we should have. + return false; + } + return true; +} + +// EOCD record +// offset 0: signature 0x06054b50, 4 bytes +// offset 4: number of this disk, 2 bytes +// ... +// offset 20: comment length, 2 bytes +// offset 22: comment, n bytes +static bool GetZipFileSize(const std::vector& zip_file, size_t* input_file_size) { + if (zip_file.size() < 22) { + printf("file is too small to be a zip file\n"); + return false; + } + + // Look for End of central directory record of the zip file, and calculate the actual + // zip_file size. + for (int i = zip_file.size() - 22; i >= 0; i--) { + if (zip_file[i] == 0x50) { + if (get_unaligned(&zip_file[i]) == 0x06054b50) { + // double-check: this archive consists of a single "disk". + CHECK_EQ(get_unaligned(&zip_file[i + 4]), 0); + + uint16_t comment_length = get_unaligned(&zip_file[i + 20]); + size_t file_size = i + 22 + comment_length; + CHECK_LE(file_size, zip_file.size()); + *input_file_size = file_size; + return true; + } + } + } + + // EOCD not found, this file is likely not a valid zip file. + return false; +} + +static bool ReadZip(const char* filename, std::vector* chunks, + std::vector* zip_file, bool include_pseudo_chunk) { + CHECK(zip_file != nullptr); struct stat st; if (stat(filename, &st) != 0) { printf("failed to stat \"%s\": %s\n", filename, strerror(errno)); - return NULL; + return false; } size_t sz = static_cast(st.st_size); - unsigned char* img = static_cast(malloc(sz)); + zip_file->resize(sz); android::base::unique_fd fd(open(filename, O_RDONLY)); - if (!android::base::ReadFully(fd, img, sz)) { + if (fd == -1) { + printf("failed to open \"%s\" %s\n", filename, strerror(errno)); + return false; + } + if (!android::base::ReadFully(fd, zip_file->data(), sz)) { printf("failed to read \"%s\" %s\n", filename, strerror(errno)); - return nullptr; + return false; + } + fd.reset(); + + // Trim the trailing zeros before we pass the file to ziparchive handler. + size_t zipfile_size; + if (!GetZipFileSize(*zip_file, &zipfile_size)) { + printf("failed to parse the actual size of %s\n", filename); + return false; + } + ZipArchiveHandle handle; + int err = OpenArchiveFromMemory(zip_file->data(), zipfile_size, filename, &handle); + if (err != 0) { + printf("failed to open zip file %s: %s\n", filename, ErrorCodeString(err)); + CloseArchive(handle); + return false; + } + + // Create a list of deflated zip entries, sorted by offset. + std::vector> temp_entries; + void* cookie; + int ret = StartIteration(handle, &cookie, nullptr, nullptr); + if (ret != 0) { + printf("failed to iterate over entries in %s: %s\n", filename, ErrorCodeString(ret)); + CloseArchive(handle); + return false; + } + + ZipString name; + ZipEntry entry; + while ((ret = Next(cookie, &entry, &name)) == 0) { + if (entry.method == kCompressDeflated) { + std::string entryname(name.name, name.name + name.name_length); + temp_entries.push_back(std::make_pair(entryname, entry)); + } + } + + if (ret != -1) { + printf("Error while iterating over zip entries: %s\n", ErrorCodeString(ret)); + CloseArchive(handle); + return false; + } + std::sort(temp_entries.begin(), temp_entries.end(), + [](auto& entry1, auto& entry2) { + return entry1.second.offset < entry2.second.offset; + }); + + EndIteration(cookie); + + if (include_pseudo_chunk) { + chunks->emplace_back(CHUNK_NORMAL, 0, zip_file, zip_file->size()); + } + + size_t pos = 0; + size_t nextentry = 0; + while (pos < zip_file->size()) { + if (nextentry < temp_entries.size() && + static_cast(pos) == temp_entries[nextentry].second.offset) { + // compose the next deflate chunk. + std::string entryname = temp_entries[nextentry].first; + size_t uncompressed_len = temp_entries[nextentry].second.uncompressed_length; + std::vector uncompressed_data(uncompressed_len); + if ((ret = ExtractToMemory(handle, &temp_entries[nextentry].second, uncompressed_data.data(), + uncompressed_len)) != 0) { + printf("failed to extract %s with size %zu: %s\n", entryname.c_str(), uncompressed_len, + ErrorCodeString(ret)); + CloseArchive(handle); + return false; + } + + size_t compressed_len = temp_entries[nextentry].second.compressed_length; + ImageChunk curr(CHUNK_DEFLATE, pos, zip_file, compressed_len); + curr.SetEntryName(std::move(entryname)); + curr.SetUncompressedData(std::move(uncompressed_data)); + chunks->push_back(curr); + + pos += compressed_len; + ++nextentry; + continue; + } + + // Use a normal chunk to take all the data up to the start of the next deflate section. + size_t raw_data_len; + if (nextentry < temp_entries.size()) { + raw_data_len = temp_entries[nextentry].second.offset - pos; + } else { + raw_data_len = zip_file->size() - pos; + } + chunks->emplace_back(CHUNK_NORMAL, pos, zip_file, raw_data_len); + + pos += raw_data_len; + } + + CloseArchive(handle); + return true; +} + +// Read the given file and break it up into chunks, and putting the data in to a vector. +static bool ReadImage(const char* filename, std::vector* chunks, + std::vector* img) { + CHECK(img != nullptr); + struct stat st; + if (stat(filename, &st) != 0) { + printf("failed to stat \"%s\": %s\n", filename, strerror(errno)); + return false; + } + + size_t sz = static_cast(st.st_size); + img->resize(sz); + android::base::unique_fd fd(open(filename, O_RDONLY)); + if (fd == -1) { + printf("failed to open \"%s\" %s\n", filename, strerror(errno)); + return false; + } + if (!android::base::ReadFully(fd, img->data(), sz)) { + printf("failed to read \"%s\" %s\n", filename, strerror(errno)); + return false; } size_t pos = 0; - *num_chunks = 0; - *chunks = NULL; - while (pos < sz) { - unsigned char* p = img + pos; - - if (sz - pos >= 4 && - p[0] == 0x1f && p[1] == 0x8b && - p[2] == 0x08 && // deflate compression - p[3] == 0x00) { // no header flags + if (sz - pos >= 4 && img->at(pos) == 0x1f && img->at(pos + 1) == 0x8b && + img->at(pos + 2) == 0x08 && // deflate compression + img->at(pos + 3) == 0x00) { // no header flags // 'pos' is the offset of the start of a gzip chunk. size_t chunk_offset = pos; - *num_chunks += 3; - *chunks = static_cast(realloc(*chunks, *num_chunks * sizeof(ImageChunk))); - ImageChunk* curr = *chunks + (*num_chunks-3); + // The remaining data is too small to be a gzip chunk; treat them as a normal chunk. + if (sz - pos < GZIP_HEADER_LEN + GZIP_FOOTER_LEN) { + chunks->emplace_back(CHUNK_NORMAL, pos, img, sz - pos); + break; + } - // create a normal chunk for the header. - curr->start = pos; - curr->type = CHUNK_NORMAL; - curr->len = GZIP_HEADER_LEN; - curr->data = p; + // We need three chunks for the deflated image in total, one normal chunk for the header, + // one deflated chunk for the body, and another normal chunk for the footer. + chunks->emplace_back(CHUNK_NORMAL, pos, img, GZIP_HEADER_LEN); + pos += GZIP_HEADER_LEN; - pos += curr->len; - p += curr->len; - ++curr; - - curr->type = CHUNK_DEFLATE; - curr->filename = NULL; - - // We must decompress this chunk in order to discover where it - // ends, and so we can put the uncompressed data and its length - // into curr->data and curr->len. - - size_t allocated = 32768; - curr->len = 0; - curr->data = static_cast(malloc(allocated)); - curr->start = pos; - curr->deflate_data = p; + // We must decompress this chunk in order to discover where it ends, and so we can update + // the uncompressed_data of the image body and its length. z_stream strm; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = sz - pos; - strm.next_in = p; + strm.next_in = img->data() + pos; // -15 means we are decoding a 'raw' deflate stream; zlib will // not expect zlib headers. int ret = inflateInit2(&strm, -15); if (ret < 0) { printf("failed to initialize inflate: %d\n", ret); - return NULL; + return false; } + size_t allocated = BUFFER_SIZE; + std::vector uncompressed_data(allocated); + size_t uncompressed_len = 0, raw_data_len = 0; do { - strm.avail_out = allocated - curr->len; - strm.next_out = curr->data + curr->len; + strm.avail_out = allocated - uncompressed_len; + strm.next_out = uncompressed_data.data() + uncompressed_len; ret = inflate(&strm, Z_NO_FLUSH); if (ret < 0) { - printf("Warning: inflate failed [%s] at offset [%zu]," - " treating as a normal chunk\n", + printf("Warning: inflate failed [%s] at offset [%zu], treating as a normal chunk\n", strm.msg, chunk_offset); break; } - curr->len = allocated - strm.avail_out; + uncompressed_len = allocated - strm.avail_out; if (strm.avail_out == 0) { allocated *= 2; - curr->data = static_cast(realloc(curr->data, allocated)); + uncompressed_data.resize(allocated); } } while (ret != Z_STREAM_END); - curr->deflate_len = sz - strm.avail_in - pos; + raw_data_len = sz - strm.avail_in - pos; inflateEnd(&strm); if (ret < 0) { - free(curr->data); - *num_chunks -= 2; continue; } - pos += curr->deflate_len; - p += curr->deflate_len; - ++curr; + ImageChunk body(CHUNK_DEFLATE, pos, img, raw_data_len); + uncompressed_data.resize(uncompressed_len); + body.SetUncompressedData(std::move(uncompressed_data)); + chunks->push_back(body); + + pos += raw_data_len; // create a normal chunk for the footer + chunks->emplace_back(CHUNK_NORMAL, pos, img, GZIP_FOOTER_LEN); - curr->type = CHUNK_NORMAL; - curr->start = pos; - curr->len = GZIP_FOOTER_LEN; - curr->data = img+pos; - - pos += curr->len; - p += curr->len; - ++curr; + pos += GZIP_FOOTER_LEN; // The footer (that we just skipped over) contains the size of // the uncompressed data. Double-check to make sure that it // matches the size of the data we got when we actually did // the decompression. - size_t footer_size = Read4(p-4); - if (footer_size != curr[-2].len) { - printf("Error: footer size %zu != decompressed size %zu\n", footer_size, curr[-2].len); - free(img); - return NULL; + size_t footer_size = Read4(img->data() + pos - 4); + if (footer_size != body.DataLengthForPatch()) { + printf("Error: footer size %zu != decompressed size %zu\n", footer_size, + body.GetRawDataLength()); + return false; } } else { - // Reallocate the list for every chunk; we expect the number of - // chunks to be small (5 for typical boot and recovery images). - ++*num_chunks; - *chunks = static_cast(realloc(*chunks, *num_chunks * sizeof(ImageChunk))); - ImageChunk* curr = *chunks + (*num_chunks-1); - curr->start = pos; + // Use a normal chunk to take all the contents until the next gzip chunk (or EOF); we expect + // the number of chunks to be small (5 for typical boot and recovery images). - // 'pos' is not the offset of the start of a gzip chunk, so scan - // forward until we find a gzip header. - curr->type = CHUNK_NORMAL; - curr->data = p; - - for (curr->len = 0; curr->len < (sz - pos); ++curr->len) { - if (sz - pos >= 4 && p[curr->len] == 0x1f && p[curr->len + 1] == 0x8b && - p[curr->len + 2] == 0x08 && p[curr->len + 3] == 0x00) { + // Scan forward until we find a gzip header. + size_t data_len = 0; + while (data_len + pos < sz) { + if (data_len + pos + 4 <= sz && img->at(pos + data_len) == 0x1f && + img->at(pos + data_len + 1) == 0x8b && img->at(pos + data_len + 2) == 0x08 && + img->at(pos + data_len + 3) == 0x00) { break; } + data_len++; } - pos += curr->len; + chunks->emplace_back(CHUNK_NORMAL, pos, img, data_len); + + pos += data_len; } } - return img; -} - -#define BUFFER_SIZE 32768 - -/* - * Takes the uncompressed data stored in the chunk, compresses it - * using the zlib parameters stored in the chunk, and checks that it - * matches exactly the compressed data we started with (also stored in - * the chunk). Return 0 on success. - */ -int TryReconstruction(ImageChunk* chunk, unsigned char* out) { - size_t p = 0; - -#if 0 - printf("trying %d %d %d %d %d\n", - chunk->level, chunk->method, chunk->windowBits, - chunk->memLevel, chunk->strategy); -#endif - - z_stream strm; - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = chunk->len; - strm.next_in = chunk->data; - int ret; - ret = deflateInit2(&strm, chunk->level, chunk->method, chunk->windowBits, - chunk->memLevel, chunk->strategy); - if (ret < 0) { - printf("failed to initialize deflate: %d\n", ret); - return -1; - } - do { - strm.avail_out = BUFFER_SIZE; - strm.next_out = out; - ret = deflate(&strm, Z_FINISH); - if (ret < 0) { - printf("failed to deflate: %d\n", ret); - return -1; - } - size_t have = BUFFER_SIZE - strm.avail_out; - - if (memcmp(out, chunk->deflate_data+p, have) != 0) { - // mismatch; data isn't the same. - deflateEnd(&strm); - return -1; - } - p += have; - } while (ret != Z_STREAM_END); - deflateEnd(&strm); - if (p != chunk->deflate_len) { - // mismatch; ran out of data before we should have. - return -1; - } - return 0; -} - -/* - * Verify that we can reproduce exactly the same compressed data that - * we started with. Sets the level, method, windowBits, memLevel, and - * strategy fields in the chunk to the encoding parameters needed to - * produce the right output. Returns 0 on success. - */ -int ReconstructDeflateChunk(ImageChunk* chunk) { - if (chunk->type != CHUNK_DEFLATE) { - printf("attempt to reconstruct non-deflate chunk\n"); - return -1; - } - - unsigned char* out = static_cast(malloc(BUFFER_SIZE)); - - // We only check two combinations of encoder parameters: level 6 - // (the default) and level 9 (the maximum). - for (chunk->level = 6; chunk->level <= 9; chunk->level += 3) { - chunk->windowBits = -15; // 32kb window; negative to indicate a raw stream. - chunk->memLevel = 8; // the default value. - chunk->method = Z_DEFLATED; - chunk->strategy = Z_DEFAULT_STRATEGY; - - if (TryReconstruction(chunk, out) == 0) { - free(out); - return 0; - } - } - - free(out); - return -1; + return true; } /* * Given source and target chunks, compute a bsdiff patch between them. - * Return the patch data, placing its length in *size. Return NULL on failure. + * Store the result in the patch_data. * |bsdiff_cache| can be used to cache the suffix array if the same |src| chunk * is used repeatedly, pass nullptr if not needed. */ -unsigned char* MakePatch(ImageChunk* src, ImageChunk* tgt, size_t* size, saidx_t** bsdiff_cache) { - if (tgt->type == CHUNK_NORMAL) { - if (tgt->len <= 160) { - tgt->type = CHUNK_RAW; - *size = tgt->len; - return tgt->data; - } +static bool MakePatch(const ImageChunk* src, ImageChunk* tgt, std::vector* patch_data, + saidx_t** bsdiff_cache) { + if (tgt->ChangeChunkToRaw(0)) { + size_t patch_size = tgt->DataLengthForPatch(); + patch_data->resize(patch_size); + std::copy(tgt->DataForPatch(), tgt->DataForPatch() + patch_size, patch_data->begin()); + return true; } #if defined(__ANDROID__) @@ -635,104 +744,51 @@ unsigned char* MakePatch(ImageChunk* src, ImageChunk* tgt, size_t* size, saidx_t #else char ptemp[] = "/tmp/imgdiff-patch-XXXXXX"; #endif + int fd = mkstemp(ptemp); - if (fd == -1) { - printf("MakePatch failed to create a temporary file: %s\n", - strerror(errno)); - return NULL; + printf("MakePatch failed to create a temporary file: %s\n", strerror(errno)); + return false; } - close(fd); // temporary file is created and we don't need its file - // descriptor + close(fd); - int r = bsdiff::bsdiff(src->data, src->len, tgt->data, tgt->len, ptemp, bsdiff_cache); + int r = bsdiff::bsdiff(src->DataForPatch(), src->DataLengthForPatch(), tgt->DataForPatch(), + tgt->DataLengthForPatch(), ptemp, bsdiff_cache); if (r != 0) { printf("bsdiff() failed: %d\n", r); - return NULL; + return false; } struct stat st; if (stat(ptemp, &st) != 0) { - printf("failed to stat patch file %s: %s\n", - ptemp, strerror(errno)); - return NULL; + printf("failed to stat patch file %s: %s\n", ptemp, strerror(errno)); + return false; } size_t sz = static_cast(st.st_size); - // TODO: Memory leak on error return. - unsigned char* data = static_cast(malloc(sz)); - - if (tgt->type == CHUNK_NORMAL && tgt->len <= sz) { + if (tgt->ChangeChunkToRaw(sz)) { unlink(ptemp); - - tgt->type = CHUNK_RAW; - *size = tgt->len; - return tgt->data; + size_t patch_size = tgt->DataLengthForPatch(); + patch_data->resize(patch_size); + std::copy(tgt->DataForPatch(), tgt->DataForPatch() + patch_size, patch_data->begin()); + return true; } - *size = sz; - - FILE* f = fopen(ptemp, "rb"); - if (f == NULL) { - printf("failed to open patch %s: %s\n", ptemp, strerror(errno)); - return NULL; + android::base::unique_fd patch_fd(open(ptemp, O_RDONLY)); + if (patch_fd == -1) { + printf("failed to open %s: %s\n", ptemp, strerror(errno)); + return false; } - if (fread(data, 1, sz, f) != sz) { - printf("failed to read patch %s: %s\n", ptemp, strerror(errno)); - return NULL; + patch_data->resize(sz); + if (!android::base::ReadFully(patch_fd, patch_data->data(), sz)) { + printf("failed to read \"%s\" %s\n", ptemp, strerror(errno)); + return false; } - fclose(f); unlink(ptemp); + tgt->SetSourceInfo(*src); - tgt->source_start = src->start; - switch (tgt->type) { - case CHUNK_NORMAL: - tgt->source_len = src->len; - break; - case CHUNK_DEFLATE: - tgt->source_len = src->deflate_len; - tgt->source_uncompressed_len = src->len; - break; - } - - return data; -} - -/* - * Cause a gzip chunk to be treated as a normal chunk (ie, as a blob - * of uninterpreted data). The resulting patch will likely be about - * as big as the target file, but it lets us handle the case of images - * where some gzip chunks are reconstructible but others aren't (by - * treating the ones that aren't as normal chunks). - */ -void ChangeDeflateChunkToNormal(ImageChunk* ch) { - if (ch->type != CHUNK_DEFLATE) return; - ch->type = CHUNK_NORMAL; - free(ch->data); - ch->data = ch->deflate_data; - ch->len = ch->deflate_len; -} - -/* - * Return true if the data in the chunk is identical (including the - * compressed representation, for gzip chunks). - */ -int AreChunksEqual(ImageChunk* a, ImageChunk* b) { - if (a->type != b->type) return 0; - - switch (a->type) { - case CHUNK_NORMAL: - return a->len == b->len && memcmp(a->data, b->data, a->len) == 0; - - case CHUNK_DEFLATE: - return a->deflate_len == b->deflate_len && - memcmp(a->deflate_data, b->deflate_data, a->deflate_len) == 0; - - default: - printf("unknown chunk type %d\n", a->type); - return 0; - } + return true; } /* @@ -740,71 +796,42 @@ int AreChunksEqual(ImageChunk* a, ImageChunk* b) { * a single chunk. (Such runs can be produced when deflate chunks are * changed to normal chunks.) */ -void MergeAdjacentNormalChunks(ImageChunk* chunks, int* num_chunks) { - int out = 0; - int in_start = 0, in_end; - while (in_start < *num_chunks) { - if (chunks[in_start].type != CHUNK_NORMAL) { - in_end = in_start+1; - } else { - // in_start is a normal chunk. Look for a run of normal chunks - // that constitute a solid block of data (ie, each chunk begins - // where the previous one ended). - for (in_end = in_start+1; - in_end < *num_chunks && chunks[in_end].type == CHUNK_NORMAL && - (chunks[in_end].start == - chunks[in_end-1].start + chunks[in_end-1].len && - chunks[in_end].data == - chunks[in_end-1].data + chunks[in_end-1].len); - ++in_end); +static void MergeAdjacentNormalChunks(std::vector* chunks) { + size_t merged_last = 0, cur = 0; + while (cur < chunks->size()) { + // Look for normal chunks adjacent to the current one. If such chunk exists, extend the + // length of the current normal chunk. + size_t to_check = cur + 1; + while (to_check < chunks->size() && chunks->at(cur).IsAdjacentNormal(chunks->at(to_check))) { + chunks->at(cur).MergeAdjacentNormal(chunks->at(to_check)); + to_check++; } - if (in_end == in_start+1) { -#if 0 - printf("chunk %d is now %d\n", in_start, out); -#endif - if (out != in_start) { - memcpy(chunks+out, chunks+in_start, sizeof(ImageChunk)); - } - } else { -#if 0 - printf("collapse normal chunks %d-%d into %d\n", in_start, in_end-1, out); -#endif - - // Merge chunks [in_start, in_end-1] into one chunk. Since the - // data member of each chunk is just a pointer into an in-memory - // copy of the file, this can be done without recopying (the - // output chunk has the first chunk's start location and data - // pointer, and length equal to the sum of the input chunk - // lengths). - chunks[out].type = CHUNK_NORMAL; - chunks[out].start = chunks[in_start].start; - chunks[out].data = chunks[in_start].data; - chunks[out].len = chunks[in_end-1].len + - (chunks[in_end-1].start - chunks[in_start].start); + if (merged_last != cur) { + chunks->at(merged_last) = std::move(chunks->at(cur)); } - - ++out; - in_start = in_end; + merged_last++; + cur = to_check; + } + if (merged_last < chunks->size()) { + chunks->erase(chunks->begin() + merged_last, chunks->end()); } - *num_chunks = out; } -ImageChunk* FindChunkByName(const char* name, ImageChunk* chunks, int num_chunks) { - for (int i = 0; i < num_chunks; ++i) { - if (chunks[i].type == CHUNK_DEFLATE && chunks[i].filename && - strcmp(name, chunks[i].filename) == 0) { - return chunks+i; +static ImageChunk* FindChunkByName(const std::string& name, std::vector& chunks) { + for (size_t i = 0; i < chunks.size(); ++i) { + if (chunks[i].GetType() == CHUNK_DEFLATE && chunks[i].GetEntryName() == name) { + return &chunks[i]; } } - return NULL; + return nullptr; } -void DumpChunks(ImageChunk* chunks, int num_chunks) { - for (int i = 0; i < num_chunks; ++i) { - printf("chunk %d: type %d start %zu len %zu\n", - i, chunks[i].type, chunks[i].start, chunks[i].len); - } +static void DumpChunks(const std::vector& chunks) { + for (size_t i = 0; i < chunks.size(); ++i) { + printf("chunk %zu: ", i); + chunks[i].Dump(); + } } int imgdiff(int argc, const char** argv) { @@ -816,26 +843,24 @@ int imgdiff(int argc, const char** argv) { ++argv; } - size_t bonus_size = 0; - unsigned char* bonus_data = NULL; + std::vector bonus_data; if (argc >= 3 && strcmp(argv[1], "-b") == 0) { struct stat st; if (stat(argv[2], &st) != 0) { printf("failed to stat bonus file %s: %s\n", argv[2], strerror(errno)); return 1; } - bonus_size = st.st_size; - bonus_data = static_cast(malloc(bonus_size)); - FILE* f = fopen(argv[2], "rb"); - if (f == NULL) { + size_t bonus_size = st.st_size; + bonus_data.resize(bonus_size); + android::base::unique_fd fd(open(argv[2], O_RDONLY)); + if (fd == -1) { printf("failed to open bonus file %s: %s\n", argv[2], strerror(errno)); return 1; } - if (fread(bonus_data, 1, bonus_size, f) != bonus_size) { + if (!android::base::ReadFully(fd, bonus_data.data(), bonus_size)) { printf("failed to read bonus file %s: %s\n", argv[2], strerror(errno)); return 1; } - fclose(f); argc -= 2; argv += 2; @@ -847,27 +872,26 @@ int imgdiff(int argc, const char** argv) { return 2; } - int num_src_chunks; - ImageChunk* src_chunks; - int num_tgt_chunks; - ImageChunk* tgt_chunks; - int i; + std::vector src_chunks; + std::vector tgt_chunks; + std::vector src_file; + std::vector tgt_file; if (zip_mode) { - if (ReadZip(argv[1], &num_src_chunks, &src_chunks, 1) == NULL) { + if (!ReadZip(argv[1], &src_chunks, &src_file, true)) { printf("failed to break apart source zip file\n"); return 1; } - if (ReadZip(argv[2], &num_tgt_chunks, &tgt_chunks, 0) == NULL) { + if (!ReadZip(argv[2], &tgt_chunks, &tgt_file, false)) { printf("failed to break apart target zip file\n"); return 1; } } else { - if (ReadImage(argv[1], &num_src_chunks, &src_chunks) == NULL) { + if (!ReadImage(argv[1], &src_chunks, &src_file)) { printf("failed to break apart source image\n"); return 1; } - if (ReadImage(argv[2], &num_tgt_chunks, &tgt_chunks) == NULL) { + if (!ReadImage(argv[2], &tgt_chunks, &tgt_file)) { printf("failed to break apart target image\n"); return 1; } @@ -875,48 +899,47 @@ int imgdiff(int argc, const char** argv) { // Verify that the source and target images have the same chunk // structure (ie, the same sequence of deflate and normal chunks). - // Merge the gzip header and footer in with any adjacent - // normal chunks. - MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks); - MergeAdjacentNormalChunks(src_chunks, &num_src_chunks); + // Merge the gzip header and footer in with any adjacent normal chunks. + MergeAdjacentNormalChunks(&tgt_chunks); + MergeAdjacentNormalChunks(&src_chunks); - if (num_src_chunks != num_tgt_chunks) { + if (src_chunks.size() != tgt_chunks.size()) { printf("source and target don't have same number of chunks!\n"); printf("source chunks:\n"); - DumpChunks(src_chunks, num_src_chunks); + DumpChunks(src_chunks); printf("target chunks:\n"); - DumpChunks(tgt_chunks, num_tgt_chunks); + DumpChunks(tgt_chunks); return 1; } - for (i = 0; i < num_src_chunks; ++i) { - if (src_chunks[i].type != tgt_chunks[i].type) { - printf("source and target don't have same chunk structure! (chunk %d)\n", i); + for (size_t i = 0; i < src_chunks.size(); ++i) { + if (src_chunks[i].GetType() != tgt_chunks[i].GetType()) { + printf("source and target don't have same chunk structure! (chunk %zu)\n", i); printf("source chunks:\n"); - DumpChunks(src_chunks, num_src_chunks); + DumpChunks(src_chunks); printf("target chunks:\n"); - DumpChunks(tgt_chunks, num_tgt_chunks); + DumpChunks(tgt_chunks); return 1; } } } - for (i = 0; i < num_tgt_chunks; ++i) { - if (tgt_chunks[i].type == CHUNK_DEFLATE) { + for (size_t i = 0; i < tgt_chunks.size(); ++i) { + if (tgt_chunks[i].GetType() == CHUNK_DEFLATE) { // Confirm that given the uncompressed chunk data in the target, we // can recompress it and get exactly the same bits as are in the // input target image. If this fails, treat the chunk as a normal // non-deflated chunk. - if (ReconstructDeflateChunk(tgt_chunks+i) < 0) { - printf("failed to reconstruct target deflate chunk %d [%s]; " - "treating as normal\n", i, tgt_chunks[i].filename); - ChangeDeflateChunkToNormal(tgt_chunks+i); + if (!tgt_chunks[i].ReconstructDeflateChunk()) { + printf("failed to reconstruct target deflate chunk %zu [%s]; treating as normal\n", i, + tgt_chunks[i].GetEntryName().c_str()); + tgt_chunks[i].ChangeDeflateChunkToNormal(); if (zip_mode) { - ImageChunk* src = FindChunkByName(tgt_chunks[i].filename, src_chunks, num_src_chunks); - if (src) { - ChangeDeflateChunkToNormal(src); + ImageChunk* src = FindChunkByName(tgt_chunks[i].GetEntryName(), src_chunks); + if (src != nullptr) { + src->ChangeDeflateChunkToNormal(); } } else { - ChangeDeflateChunkToNormal(src_chunks+i); + src_chunks[i].ChangeDeflateChunkToNormal(); } continue; } @@ -929,16 +952,16 @@ int imgdiff(int argc, const char** argv) { // data. ImageChunk* src; if (zip_mode) { - src = FindChunkByName(tgt_chunks[i].filename, src_chunks, num_src_chunks); + src = FindChunkByName(tgt_chunks[i].GetEntryName(), src_chunks); } else { - src = src_chunks+i; + src = &src_chunks[i]; } - if (src == NULL || AreChunksEqual(tgt_chunks+i, src)) { - ChangeDeflateChunkToNormal(tgt_chunks+i); - if (src) { - ChangeDeflateChunkToNormal(src); - } + if (src == nullptr) { + tgt_chunks[i].ChangeDeflateChunkToNormal(); + } else if (tgt_chunks[i] == *src) { + tgt_chunks[i].ChangeDeflateChunkToNormal(); + src->ChangeDeflateChunkToNormal(); } } } @@ -948,14 +971,15 @@ int imgdiff(int argc, const char** argv) { // For zips, we only need to do this to the target: deflated // chunks are matched via filename, and normal chunks are patched // using the entire source file as the source. - MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks); + MergeAdjacentNormalChunks(&tgt_chunks); + } else { // For images, we need to maintain the parallel structure of the // chunk lists, so do the merging in both the source and target // lists. - MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks); - MergeAdjacentNormalChunks(src_chunks, &num_src_chunks); - if (num_src_chunks != num_tgt_chunks) { + MergeAdjacentNormalChunks(&tgt_chunks); + MergeAdjacentNormalChunks(&src_chunks); + if (src_chunks.size() != tgt_chunks.size()) { // This shouldn't happen. printf("merging normal chunks went awry\n"); return 1; @@ -965,117 +989,68 @@ int imgdiff(int argc, const char** argv) { // Compute bsdiff patches for each chunk's data (the uncompressed // data, in the case of deflate chunks). - DumpChunks(src_chunks, num_src_chunks); + DumpChunks(src_chunks); - printf("Construct patches for %d chunks...\n", num_tgt_chunks); - unsigned char** patch_data = static_cast(malloc( - num_tgt_chunks * sizeof(unsigned char*))); - size_t* patch_size = static_cast(malloc(num_tgt_chunks * sizeof(size_t))); + printf("Construct patches for %zu chunks...\n", tgt_chunks.size()); + std::vector> patch_data(tgt_chunks.size()); saidx_t* bsdiff_cache = nullptr; - for (i = 0; i < num_tgt_chunks; ++i) { + for (size_t i = 0; i < tgt_chunks.size(); ++i) { if (zip_mode) { ImageChunk* src; - if (tgt_chunks[i].type == CHUNK_DEFLATE && - (src = FindChunkByName(tgt_chunks[i].filename, src_chunks, num_src_chunks))) { - patch_data[i] = MakePatch(src, tgt_chunks + i, patch_size + i, nullptr); + if (tgt_chunks[i].GetType() == CHUNK_DEFLATE && + (src = FindChunkByName(tgt_chunks[i].GetEntryName(), src_chunks))) { + MakePatch(src, &tgt_chunks[i], &patch_data[i], nullptr); } else { - patch_data[i] = MakePatch(src_chunks, tgt_chunks + i, patch_size + i, &bsdiff_cache); + MakePatch(&src_chunks[0], &tgt_chunks[i], &patch_data[i], &bsdiff_cache); } } else { - if (i == 1 && bonus_data) { - printf(" using %zu bytes of bonus data for chunk %d\n", bonus_size, i); - src_chunks[i].data = - static_cast(realloc(src_chunks[i].data, src_chunks[i].len + bonus_size)); - memcpy(src_chunks[i].data + src_chunks[i].len, bonus_data, bonus_size); - src_chunks[i].len += bonus_size; + if (i == 1 && !bonus_data.empty()) { + printf(" using %zu bytes of bonus data for chunk %zu\n", bonus_data.size(), i); + src_chunks[i].SetBonusData(bonus_data); } - patch_data[i] = MakePatch(src_chunks + i, tgt_chunks + i, patch_size + i, nullptr); + MakePatch(&src_chunks[i], &tgt_chunks[i], &patch_data[i], nullptr); } - printf("patch %3d is %zu bytes (of %zu)\n", i, patch_size[i], tgt_chunks[i].source_len); + printf("patch %3zu is %zu bytes (of %zu)\n", i, patch_data[i].size(), + src_chunks[i].GetRawDataLength()); + } + + if (bsdiff_cache != nullptr) { + free(bsdiff_cache); } - free(bsdiff_cache); - free(src_chunks); // Figure out how big the imgdiff file header is going to be, so // that we can correctly compute the offset of each bsdiff patch // within the file. size_t total_header_size = 12; - for (i = 0; i < num_tgt_chunks; ++i) { - total_header_size += 4; - switch (tgt_chunks[i].type) { - case CHUNK_NORMAL: - total_header_size += 8*3; - break; - case CHUNK_DEFLATE: - total_header_size += 8*5 + 4*5; - break; - case CHUNK_RAW: - total_header_size += 4 + patch_size[i]; - break; - } + for (size_t i = 0; i < tgt_chunks.size(); ++i) { + total_header_size += tgt_chunks[i].GetHeaderSize(patch_data[i].size()); } size_t offset = total_header_size; FILE* f = fopen(argv[3], "wb"); + if (f == nullptr) { + printf("failed to open \"%s\": %s\n", argv[3], strerror(errno)); + } // Write out the headers. fwrite("IMGDIFF2", 1, 8, f); - Write4(num_tgt_chunks, f); - for (i = 0; i < num_tgt_chunks; ++i) { - Write4(tgt_chunks[i].type, f); - - switch (tgt_chunks[i].type) { - case CHUNK_NORMAL: - printf("chunk %3d: normal (%10zu, %10zu) %10zu\n", i, - tgt_chunks[i].start, tgt_chunks[i].len, patch_size[i]); - Write8(tgt_chunks[i].source_start, f); - Write8(tgt_chunks[i].source_len, f); - Write8(offset, f); - offset += patch_size[i]; - break; - - case CHUNK_DEFLATE: - printf("chunk %3d: deflate (%10zu, %10zu) %10zu %s\n", i, - tgt_chunks[i].start, tgt_chunks[i].deflate_len, patch_size[i], - tgt_chunks[i].filename); - Write8(tgt_chunks[i].source_start, f); - Write8(tgt_chunks[i].source_len, f); - Write8(offset, f); - Write8(tgt_chunks[i].source_uncompressed_len, f); - Write8(tgt_chunks[i].len, f); - Write4(tgt_chunks[i].level, f); - Write4(tgt_chunks[i].method, f); - Write4(tgt_chunks[i].windowBits, f); - Write4(tgt_chunks[i].memLevel, f); - Write4(tgt_chunks[i].strategy, f); - offset += patch_size[i]; - break; - - case CHUNK_RAW: - printf("chunk %3d: raw (%10zu, %10zu)\n", i, - tgt_chunks[i].start, tgt_chunks[i].len); - Write4(patch_size[i], f); - fwrite(patch_data[i], 1, patch_size[i], f); - break; - } + Write4(static_cast(tgt_chunks.size()), f); + for (size_t i = 0; i < tgt_chunks.size(); ++i) { + printf("chunk %zu: ", i); + offset = tgt_chunks[i].WriteHeaderToFile(f, patch_data[i], offset); } // Append each chunk's bsdiff patch, in order. - - for (i = 0; i < num_tgt_chunks; ++i) { - if (tgt_chunks[i].type != CHUNK_RAW) { - fwrite(patch_data[i], 1, patch_size[i], f); + for (size_t i = 0; i < tgt_chunks.size(); ++i) { + if (tgt_chunks[i].GetType() != CHUNK_RAW) { + fwrite(patch_data[i].data(), 1, patch_data[i].size(), f); } } - free(tgt_chunks); - free(patch_data); - free(patch_size); - fclose(f); return 0; diff --git a/tests/Android.mk b/tests/Android.mk index 3ab5ea6c..9ee27a06 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -176,6 +176,7 @@ LOCAL_STATIC_LIBRARIES := \ libbsdiff \ libbspatch \ libziparchive \ + libutils \ libbase \ libcrypto \ libbz \ diff --git a/tests/component/imgdiff_test.cpp b/tests/component/imgdiff_test.cpp index 7ad33078..be2dd385 100644 --- a/tests/component/imgdiff_test.cpp +++ b/tests/component/imgdiff_test.cpp @@ -236,6 +236,62 @@ TEST(ImgdiffTest, zip_mode_smoke_compressed) { ASSERT_EQ(tgt, patched); } +TEST(ImgdiffTest, zip_mode_smoke_trailer_zeros) { + // Construct src and tgt zip files. + TemporaryFile src_file; + FILE* src_file_ptr = fdopen(src_file.fd, "wb"); + ZipWriter src_writer(src_file_ptr); + ASSERT_EQ(0, src_writer.StartEntry("file1.txt", ZipWriter::kCompress)); + const std::string src_content("abcdefg"); + ASSERT_EQ(0, src_writer.WriteBytes(src_content.data(), src_content.size())); + ASSERT_EQ(0, src_writer.FinishEntry()); + ASSERT_EQ(0, src_writer.Finish()); + ASSERT_EQ(0, fclose(src_file_ptr)); + + TemporaryFile tgt_file; + FILE* tgt_file_ptr = fdopen(tgt_file.fd, "wb"); + ZipWriter tgt_writer(tgt_file_ptr); + ASSERT_EQ(0, tgt_writer.StartEntry("file1.txt", ZipWriter::kCompress)); + const std::string tgt_content("abcdefgxyz"); + ASSERT_EQ(0, tgt_writer.WriteBytes(tgt_content.data(), tgt_content.size())); + ASSERT_EQ(0, tgt_writer.FinishEntry()); + ASSERT_EQ(0, tgt_writer.Finish()); + // Add trailing zeros to the target zip file. + std::vector zeros(10); + ASSERT_EQ(zeros.size(), fwrite(zeros.data(), sizeof(uint8_t), zeros.size(), tgt_file_ptr)); + ASSERT_EQ(0, fclose(tgt_file_ptr)); + + // Compute patch. + TemporaryFile patch_file; + std::vector args = { + "imgdiff", "-z", src_file.path, tgt_file.path, patch_file.path, + }; + ASSERT_EQ(0, imgdiff(args.size(), args.data())); + + // Verify. + std::string tgt; + ASSERT_TRUE(android::base::ReadFileToString(tgt_file.path, &tgt)); + std::string src; + ASSERT_TRUE(android::base::ReadFileToString(src_file.path, &src)); + std::string patch; + ASSERT_TRUE(android::base::ReadFileToString(patch_file.path, &patch)); + + // Expect three entries: CHUNK_RAW (header) + CHUNK_DEFLATE (data) + CHUNK_RAW (footer). + size_t num_normal; + size_t num_raw; + size_t num_deflate; + verify_patch_header(patch, &num_normal, &num_raw, &num_deflate); + ASSERT_EQ(0U, num_normal); + ASSERT_EQ(1U, num_deflate); + ASSERT_EQ(2U, num_raw); + + std::string patched; + ASSERT_EQ(0, ApplyImagePatch(reinterpret_cast(src.data()), src.size(), + reinterpret_cast(patch.data()), patch.size(), + MemorySink, &patched)); + ASSERT_EQ(tgt, patched); +} + TEST(ImgdiffTest, image_mode_simple) { // src: "abcdefgh" + gzipped "xyz" (echo -n "xyz" | gzip -f | hd). const std::vector src_data = { 'a', 'b', 'c', 'd', 'e', 'f', 'g',