tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Support streaming of compressed assets > 1 megabyte

Browse source 
Compressed assets larger than one megabyte are now decompressed on demand
rather than being decompressed in their entirety and held in memory.  Reading
the data in order is relatively efficient, as is seeking forward in the stream.
Seeking backwards is supported, but requires reprocessing the compressed data
from the beginning, so is very inefficient.

In addition, the size limit on compressed assets has been eliminated.

Change-Id: I6e68247957e6c53e7e8ba70d12764695f1723bad
This commit is contained in:
Christopher Tate 2010-07-26 11:24:18 -07:00 committed by Alex Ray
parent 35a154e57f
commit c28f214f6c
5 changed files with 356 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
include/utils/Asset.h
View file

@ -61,15 +61,6 @@ public:
ACCESS_BUFFER,
} AccessMode;
enum {
/* data larger than this does not get uncompressed into a buffer */
#ifdef HAVE_ANDROID_OS
UNCOMPRESS_DATA_MAX = 1 * 1024 * 1024
#else
UNCOMPRESS_DATA_MAX = 2 * 1024 * 1024
#endif
};
/*
* Read data from the current offset. Returns the actual number of
* bytes read, 0 on EOF, or -1 on error.
@ -317,6 +308,8 @@ private:
FileMap* mMap; // for memory-mapped input
int mFd; // for file input
class StreamingZipInflater* mZipInflater; // for streaming large compressed assets
unsigned char* mBuf; // for getBuffer()
};

82
include/utils/StreamingZipInflater.h Normal file
View file

@ -0,0 +1,82 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __LIBS_STREAMINGZIPINFLATER_H
#define __LIBS_STREAMINGZIPINFLATER_H
#include <unistd.h>
#include <inttypes.h>
#include <zlib.h>
namespace android {
class StreamingZipInflater {
public:
static const size_t INPUT_CHUNK_SIZE = 64 * 1024;
static const size_t OUTPUT_CHUNK_SIZE = 64 * 1024;
// Flavor that pages in the compressed data from a fd
StreamingZipInflater(int fd, off_t compDataStart, size_t uncompSize, size_t compSize);
// Flavor that gets the compressed data from an in-memory buffer
StreamingZipInflater(class FileMap* dataMap, size_t uncompSize);
~StreamingZipInflater();
// read 'count' bytes of uncompressed data from the current position. outBuf may
// be NULL, in which case the data is consumed and discarded.
ssize_t read(void* outBuf, size_t count);
// seeking backwards requires uncompressing fom the beginning, so is very
// expensive. seeking forwards only requires uncompressing from the current
// position to the destination.
off_t seekAbsolute(off_t absoluteInputPosition);
private:
void initInflateState();
int readNextChunk();
// where to find the uncompressed data
int mFd;
off_t mInFileStart; // where the compressed data lives in the file
class FileMap* mDataMap;
z_stream mInflateState;
bool mStreamNeedsInit;
// output invariants for this asset
uint8_t* mOutBuf; // output buf for decompressed bytes
size_t mOutBufSize; // allocated size of mOutBuf
size_t mOutTotalSize; // total uncompressed size of the blob
// current output state bookkeeping
off_t mOutCurPosition; // current position in total offset
size_t mOutLastDecoded; // last decoded byte + 1 in mOutbuf
size_t mOutDeliverable; // next undelivered byte of decoded output in mOutBuf
// input invariants
uint8_t* mInBuf;
size_t mInBufSize; // allocated size of mInBuf;
size_t mInTotalSize; // total size of compressed data for this blob
// input state bookkeeping
size_t mInNextChunkOffset; // offset from start of blob at which the next input chunk lies
// the z_stream contains state about input block consumption
};
}
#endif

3
libs/utils/Android.mk
View file

@ -33,6 +33,7 @@ commonSources:= \
SharedBuffer.cpp \
Static.cpp \
StopWatch.cpp \
StreamingZipInflater.cpp \
String8.cpp \
String16.cpp \
StringArray.cpp \
@ -131,4 +132,4 @@ endif
# team really wants is to build the stuff defined by this makefile.
ifeq (,$(ONE_SHOT_MAKEFILE))
include $(call first-makefiles-under,$(LOCAL_PATH))
endif
endif

76
libs/utils/Asset.cpp
View file

@ -24,6 +24,7 @@
#include <utils/Asset.h>
#include <utils/Atomic.h>
#include <utils/FileMap.h>
#include <utils/StreamingZipInflater.h>
#include <utils/ZipUtils.h>
#include <utils/ZipFileRO.h>
#include <utils/Log.h>
@ -659,7 +660,7 @@ const void* _FileAsset::ensureAlignment(FileMap* map)
*/
_CompressedAsset::_CompressedAsset(void)
: mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0),
mMap(NULL), mFd(-1), mBuf(NULL)
mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL)
{
}
@ -698,6 +699,10 @@ status_t _CompressedAsset::openChunk(int fd, off_t offset,
mFd = fd;
assert(mBuf == NULL);
if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen);
}
return NO_ERROR;
}
@ -724,6 +729,9 @@ status_t _CompressedAsset::openChunk(FileMap* dataMap, int compressionMethod,
mUncompressedLen = uncompressedLen;
assert(mOffset == 0);
if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen);
}
return NO_ERROR;
}
@ -739,26 +747,29 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)
assert(mOffset >= 0 && mOffset <= mUncompressedLen);
// TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly
/* If we're relying on a streaming inflater, go through that */
if (mZipInflater) {
actual = mZipInflater->read(buf, count);
} else {
if (mBuf == NULL) {
if (getBuffer(false) == NULL)
return -1;
}
assert(mBuf != NULL);
if (mBuf == NULL) {
if (getBuffer(false) == NULL)
return -1;
/* adjust count if we're near EOF */
maxLen = mUncompressedLen - mOffset;
if (count > maxLen)
count = maxLen;
if (!count)
return 0;
/* copy from buffer */
//printf("comp buf read\n");
memcpy(buf, (char*)mBuf + mOffset, count);
actual = count;
}
assert(mBuf != NULL);
/* adjust count if we're near EOF */
maxLen = mUncompressedLen - mOffset;
if (count > maxLen)
count = maxLen;
if (!count)
return 0;
/* copy from buffer */
//printf("comp buf read\n");
memcpy(buf, (char*)mBuf + mOffset, count);
actual = count;
mOffset += actual;
return actual;
@ -780,6 +791,9 @@ off_t _CompressedAsset::seek(off_t offset, int whence)
if (newPosn == (off_t) -1)
return newPosn;
if (mZipInflater) {
mZipInflater->seekAbsolute(newPosn);
}
mOffset = newPosn;
return mOffset;
}
@ -793,10 +807,12 @@ void _CompressedAsset::close(void)
mMap->release();
mMap = NULL;
}
if (mBuf != NULL) {
delete[] mBuf;
mBuf = NULL;
}
delete[] mBuf;
mBuf = NULL;
delete mZipInflater;
mZipInflater = NULL;
if (mFd > 0) {
::close(mFd);
@ -817,12 +833,6 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
if (mBuf != NULL)
return mBuf;
if (mUncompressedLen > UNCOMPRESS_DATA_MAX) {
LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n",
(long) mUncompressedLen, UNCOMPRESS_DATA_MAX);
goto bail;
}
/*
* Allocate a buffer and read the file into it.
*/
@ -853,7 +863,13 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
goto bail;
}
/* success! */
/*
* Success - now that we have the full asset in RAM we
* no longer need the streaming inflater
*/
delete mZipInflater;
mZipInflater = NULL;
mBuf = buf;
buf = NULL;

224
libs/utils/StreamingZipInflater.cpp Normal file
View file

@ -0,0 +1,224 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "szipinf"
#include <utils/Log.h>
#include <utils/FileMap.h>
#include <utils/StreamingZipInflater.h>
#include <string.h>
#include <assert.h>
static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; }
using namespace android;
/*
* Streaming access to compressed asset data in an open fd
*/
StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart,
size_t uncompSize, size_t compSize) {
mFd = fd;
mDataMap = NULL;
mInFileStart = compDataStart;
mOutTotalSize = uncompSize;
mInTotalSize = compSize;
mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE;
mInBuf = new uint8_t[mInBufSize];
mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
mOutBuf = new uint8_t[mOutBufSize];
initInflateState();
}
/*
* Streaming access to compressed data held in an mmapped region of memory
*/
StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) {
mFd = -1;
mDataMap = dataMap;
mOutTotalSize = uncompSize;
mInTotalSize = dataMap->getDataLength();
mInBuf = (uint8_t*) dataMap->getDataPtr();
mInBufSize = mInTotalSize;
mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
mOutBuf = new uint8_t[mOutBufSize];
initInflateState();
}
StreamingZipInflater::~StreamingZipInflater() {
// tear down the in-flight zip state just in case
::inflateEnd(&mInflateState);
if (mDataMap == NULL) {
delete [] mInBuf;
}
delete [] mOutBuf;
}
void StreamingZipInflater::initInflateState() {
LOGD("Initializing inflate state");
memset(&mInflateState, 0, sizeof(mInflateState));
mInflateState.zalloc = Z_NULL;
mInflateState.zfree = Z_NULL;
mInflateState.opaque = Z_NULL;
mInflateState.next_in = (Bytef*)mInBuf;
mInflateState.next_out = (Bytef*) mOutBuf;
mInflateState.avail_out = mOutBufSize;
mInflateState.data_type = Z_UNKNOWN;
mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0;
mInNextChunkOffset = 0;
mStreamNeedsInit = true;
if (mDataMap == NULL) {
::lseek(mFd, mInFileStart, SEEK_SET);
mInflateState.avail_in = 0; // set when a chunk is read in
} else {
mInflateState.avail_in = mInBufSize;
}
}
/*
* Basic approach:
*
* 1. If we have undelivered uncompressed data, send it. At this point
* either we've satisfied the request, or we've exhausted the available
* output data in mOutBuf.
*
* 2. While we haven't sent enough data to satisfy the request:
* 0. if the request is for more data than exists, bail.
* a. if there is no input data to decode, read some into the input buffer
* and readjust the z_stream input pointers
* b. point the output to the start of the output buffer and decode what we can
* c. deliver whatever output data we can
*/
ssize_t StreamingZipInflater::read(void* outBuf, size_t count) {
uint8_t* dest = (uint8_t*) outBuf;
size_t bytesRead = 0;
size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition));
while (toRead > 0) {
// First, write from whatever we already have decoded and ready to go
size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable);
if (deliverable > 0) {
if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable);
mOutDeliverable += deliverable;
mOutCurPosition += deliverable;
dest += deliverable;
bytesRead += deliverable;
toRead -= deliverable;
}
// need more data? time to decode some.
if (toRead > 0) {
// if we don't have any data to decode, read some in. If we're working
// from mmapped data this won't happen, because the clipping to total size
// will prevent reading off the end of the mapped input chunk.
if (mInflateState.avail_in == 0) {
int err = readNextChunk();
if (err < 0) {
LOGE("Unable to access asset data: %d", err);
if (!mStreamNeedsInit) {
::inflateEnd(&mInflateState);
initInflateState();
}
return -1;
}
}
// we know we've drained whatever is in the out buffer now, so just
// start from scratch there, reading all the input we have at present.
mInflateState.next_out = (Bytef*) mOutBuf;
mInflateState.avail_out = mOutBufSize;
/*
LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p",
mInflateState.avail_in, mInflateState.avail_out,
mInflateState.next_in, mInflateState.next_out);
*/
int result = Z_OK;
if (mStreamNeedsInit) {
LOGI("Initializing zlib to inflate");
result = inflateInit2(&mInflateState, -MAX_WBITS);
mStreamNeedsInit = false;
}
if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH);
if (result < 0) {
// Whoops, inflation failed
LOGE("Error inflating asset: %d", result);
::inflateEnd(&mInflateState);
initInflateState();
return -1;
} else {
if (result == Z_STREAM_END) {
// we know we have to have reached the target size here and will
// not try to read any further, so just wind things up.
::inflateEnd(&mInflateState);
}
// Note how much data we got, and off we go
mOutDeliverable = 0;
mOutLastDecoded = mOutBufSize - mInflateState.avail_out;
}
}
}
return bytesRead;
}
int StreamingZipInflater::readNextChunk() {
assert(mDataMap == NULL);
if (mInNextChunkOffset < mInTotalSize) {
size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset);
if (toRead > 0) {
ssize_t didRead = ::read(mFd, mInBuf, toRead);
//LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead);
if (didRead < 0) {
// TODO: error
LOGE("Error reading asset data");
return didRead;
} else {
mInNextChunkOffset += didRead;
mInflateState.next_in = (Bytef*) mInBuf;
mInflateState.avail_in = didRead;
}
}
}
return 0;
}
// seeking backwards requires uncompressing fom the beginning, so is very
// expensive. seeking forwards only requires uncompressing from the current
// position to the destination.
off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) {
if (absoluteInputPosition < mOutCurPosition) {
// rewind and reprocess the data from the beginning
if (!mStreamNeedsInit) {
::inflateEnd(&mInflateState);
}
initInflateState();
read(NULL, absoluteInputPosition);
} else if (absoluteInputPosition > mOutCurPosition) {
read(NULL, absoluteInputPosition - mOutCurPosition);
}
// else if the target position *is* our current position, do nothing
return absoluteInputPosition;
}