tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "libsnapshot: take in compression_level" into main

Browse source
This commit is contained in:
Daniel Zheng 2023-08-09 20:03:44 +00:00 committed by Gerrit Code Review
commit 73846d68cf
4 changed files with 83 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
View file

@ -110,16 +110,17 @@ class ICowWriter {
class CompressWorker {
public:
CompressWorker(CowCompressionAlgorithm compression, uint32_t block_size);
CompressWorker(CowCompression compression, uint32_t block_size);
bool RunThread();
void EnqueueCompressBlocks(const void* buffer, size_t num_blocks);
bool GetCompressedBuffers(std::vector<std::basic_string<uint8_t>>* compressed_buf);
void Finalize();
static std::basic_string<uint8_t> Compress(CowCompressionAlgorithm compression,
const void* data, size_t length);
static uint32_t GetDefaultCompressionLevel(CowCompressionAlgorithm compression);
static std::basic_string<uint8_t> Compress(CowCompression compression, const void* data,
size_t length);
static bool CompressBlocks(CowCompressionAlgorithm compression, size_t block_size,
const void* buffer, size_t num_blocks,
static bool CompressBlocks(CowCompression compression, size_t block_size, const void* buffer,
size_t num_blocks,
std::vector<std::basic_string<uint8_t>>* compressed_data);
private:
@ -130,7 +131,7 @@ class CompressWorker {
std::vector<std::basic_string<uint8_t>> compressed_data;
};
CowCompressionAlgorithm compression_;
CowCompression compression_;
uint32_t block_size_;
std::queue<CompressWork> work_queue_;
@ -139,7 +140,6 @@ class CompressWorker {
std::condition_variable cv_;
bool stopped_ = false;
std::basic_string<uint8_t> Compress(const void* data, size_t length);
bool CompressBlocks(const void* buffer, size_t num_blocks,
std::vector<std::basic_string<uint8_t>>* compressed_data);
};

49
fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp
View file

@ -46,24 +46,47 @@ std::optional<CowCompressionAlgorithm> CompressionAlgorithmFromString(std::strin
} else if (name == "none" || name.empty()) {
return {kCowCompressNone};
} else {
LOG(ERROR) << "unable to determine default compression algorithm for: " << name;
return {};
}
}
std::basic_string<uint8_t> CompressWorker::Compress(const void* data, size_t length) {
return Compress(compression_, data, length);
// 1. Default compression level is determined by compression algorithm
// 2. There might be compatibility issues if a value is changed here, as some older versions of
// Android will assume a different compression level, causing cow_size estimation differences that
// will lead to OTA failure. Ensure that the device and OTA package use the same compression level
// for OTA to succeed.
uint32_t CompressWorker::GetDefaultCompressionLevel(CowCompressionAlgorithm compression) {
switch (compression) {
case kCowCompressGz: {
return Z_BEST_COMPRESSION;
}
case kCowCompressBrotli: {
return BROTLI_DEFAULT_QUALITY;
}
case kCowCompressLz4: {
break;
}
case kCowCompressZstd: {
return ZSTD_defaultCLevel();
}
case kCowCompressNone: {
break;
}
}
return 0;
}
std::basic_string<uint8_t> CompressWorker::Compress(CowCompressionAlgorithm compression,
const void* data, size_t length) {
switch (compression) {
std::basic_string<uint8_t> CompressWorker::Compress(CowCompression compression, const void* data,
size_t length) {
switch (compression.algorithm) {
case kCowCompressGz: {
const auto bound = compressBound(length);
std::basic_string<uint8_t> buffer(bound, '\0');
uLongf dest_len = bound;
auto rv = compress2(buffer.data(), &dest_len, reinterpret_cast<const Bytef*>(data),
length, Z_BEST_COMPRESSION);
length, compression.compression_level);
if (rv != Z_OK) {
LOG(ERROR) << "compress2 returned: " << rv;
return {};
@ -81,8 +104,8 @@ std::basic_string<uint8_t> CompressWorker::Compress(CowCompressionAlgorithm comp
size_t encoded_size = bound;
auto rv = BrotliEncoderCompress(
BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, length,
reinterpret_cast<const uint8_t*>(data), &encoded_size, buffer.data());
compression.compression_level, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE,
length, reinterpret_cast<const uint8_t*>(data), &encoded_size, buffer.data());
if (!rv) {
LOG(ERROR) << "BrotliEncoderCompress failed";
return {};
@ -117,8 +140,8 @@ std::basic_string<uint8_t> CompressWorker::Compress(CowCompressionAlgorithm comp
}
case kCowCompressZstd: {
std::basic_string<uint8_t> buffer(ZSTD_compressBound(length), '\0');
const auto compressed_size =
ZSTD_compress(buffer.data(), buffer.size(), data, length, 0);
const auto compressed_size = ZSTD_compress(buffer.data(), buffer.size(), data, length,
compression.compression_level);
if (compressed_size <= 0) {
LOG(ERROR) << "ZSTD compression failed " << compressed_size;
return {};
@ -133,7 +156,7 @@ std::basic_string<uint8_t> CompressWorker::Compress(CowCompressionAlgorithm comp
return buffer;
}
default:
LOG(ERROR) << "unhandled compression type: " << compression;
LOG(ERROR) << "unhandled compression type: " << compression.algorithm;
break;
}
return {};
@ -143,7 +166,7 @@ bool CompressWorker::CompressBlocks(const void* buffer, size_t num_blocks,
return CompressBlocks(compression_, block_size_, buffer, num_blocks, compressed_data);
}
bool CompressWorker::CompressBlocks(CowCompressionAlgorithm compression, size_t block_size,
bool CompressWorker::CompressBlocks(CowCompression compression, size_t block_size,
const void* buffer, size_t num_blocks,
std::vector<std::basic_string<uint8_t>>* compressed_data) {
const uint8_t* iter = reinterpret_cast<const uint8_t*>(buffer);
@ -255,7 +278,7 @@ void CompressWorker::Finalize() {
cv_.notify_all();
}
CompressWorker::CompressWorker(CowCompressionAlgorithm compression, uint32_t block_size)
CompressWorker::CompressWorker(CowCompression compression, uint32_t block_size)
: compression_(compression), block_size_(block_size) {}
} // namespace snapshot

14
fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
View file

@ -480,7 +480,7 @@ TEST_P(CompressionTest, HorribleStream) {
std::string expected = "The quick brown fox jumps over the lazy dog.";
expected.resize(4096, '\0');
auto result = CompressWorker::Compress(*algorithm, expected.data(), expected.size());
auto result = CompressWorker::Compress(compression, expected.data(), expected.size());
ASSERT_FALSE(result.empty());
HorribleStream<uint8_t> stream(result);
@ -1409,6 +1409,18 @@ TEST_F(CowTest, RevMergeOpItrTest) {
ASSERT_TRUE(iter->AtEnd());
}
TEST_F(CowTest, ParseOptionsTest) {
CowOptions options;
std::vector<std::pair<std::string, bool>> testcases = {
{"gz,4", true}, {"gz,4,4", false}, {"lz4,4", true}, {"brotli,4", true},
{"zstd,4", true}, {"zstd,x", false}, {"zs,4", false}, {"zstd.4", false}};
for (size_t i = 0; i < testcases.size(); i++) {
options.compression = testcases[i].first;
CowWriterV2 writer(options, GetCowFd());
ASSERT_EQ(writer.Initialize(), testcases[i].second);
}
}
TEST_F(CowTest, LegacyRevMergeOpItrTest) {
CowOptions options;
options.cluster_ops = 5;

35
fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp
View file

@ -39,6 +39,8 @@
#include <sys/ioctl.h>
#include <unistd.h>
#include "android-base/parseint.h"
#include "android-base/strings.h"
#include "parser_v2.h"
// The info messages here are spammy, but as useful for update_engine. Disable
@ -119,11 +121,28 @@ void CowWriterV2::SetupHeaders() {
}
bool CowWriterV2::ParseOptions() {
auto algorithm = CompressionAlgorithmFromString(options_.compression);
auto parts = android::base::Split(options_.compression, ",");
if (parts.size() > 2) {
LOG(ERROR) << "failed to parse compression parameters: invalid argument count: "
<< parts.size() << " " << options_.compression;
return false;
}
auto algorithm = CompressionAlgorithmFromString(parts[0]);
if (!algorithm) {
LOG(ERROR) << "unrecognized compression: " << options_.compression;
return false;
}
if (parts.size() > 1) {
if (!android::base::ParseUint(parts[1], &compression_.compression_level)) {
LOG(ERROR) << "failed to parse compression level invalid type: " << parts[1];
return false;
}
} else {
compression_.compression_level =
CompressWorker::GetDefaultCompressionLevel(algorithm.value());
}
compression_.algorithm = *algorithm;
if (options_.cluster_ops == 1) {
@ -165,7 +184,7 @@ void CowWriterV2::InitWorkers() {
return;
}
for (int i = 0; i < num_compress_threads_; i++) {
auto wt = std::make_unique<CompressWorker>(compression_.algorithm, header_.block_size);
auto wt = std::make_unique<CompressWorker>(compression_, header_.block_size);
threads_.emplace_back(std::async(std::launch::async, &CompressWorker::RunThread, wt.get()));
compress_threads_.push_back(std::move(wt));
}
@ -320,8 +339,8 @@ bool CowWriterV2::CompressBlocks(size_t num_blocks, const void* data) {
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
compressed_buf_.clear();
if (num_threads <= 1) {
return CompressWorker::CompressBlocks(compression_.algorithm, options_.block_size, data,
num_blocks, &compressed_buf_);
return CompressWorker::CompressBlocks(compression_, options_.block_size, data, num_blocks,
&compressed_buf_);
}
// Submit the blocks per thread. The retrieval of
@ -393,8 +412,8 @@ bool CowWriterV2::EmitBlocks(uint64_t new_block_start, const void* data, size_t
buf_iter_++;
return data;
} else {
auto data = CompressWorker::Compress(compression_.algorithm, iter,
header_.block_size);
auto data =
CompressWorker::Compress(compression_, iter, header_.block_size);
return data;
}
}();
@ -507,8 +526,8 @@ bool CowWriterV2::Finalize() {
}
}
// Footer should be at the end of a file, so if there is data after the current block, end it
// and start a new cluster.
// Footer should be at the end of a file, so if there is data after the current block, end
// it and start a new cluster.
if (cluster_size_ && current_data_size_ > 0) {
EmitCluster();
extra_cluster = true;