diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h index d6194eb93..755050fa1 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h @@ -110,16 +110,17 @@ class ICowWriter { class CompressWorker { public: - CompressWorker(CowCompressionAlgorithm compression, uint32_t block_size); + CompressWorker(CowCompression compression, uint32_t block_size); bool RunThread(); void EnqueueCompressBlocks(const void* buffer, size_t num_blocks); bool GetCompressedBuffers(std::vector>* compressed_buf); void Finalize(); - static std::basic_string Compress(CowCompressionAlgorithm compression, - const void* data, size_t length); + static uint32_t GetDefaultCompressionLevel(CowCompressionAlgorithm compression); + static std::basic_string Compress(CowCompression compression, const void* data, + size_t length); - static bool CompressBlocks(CowCompressionAlgorithm compression, size_t block_size, - const void* buffer, size_t num_blocks, + static bool CompressBlocks(CowCompression compression, size_t block_size, const void* buffer, + size_t num_blocks, std::vector>* compressed_data); private: @@ -130,7 +131,7 @@ class CompressWorker { std::vector> compressed_data; }; - CowCompressionAlgorithm compression_; + CowCompression compression_; uint32_t block_size_; std::queue work_queue_; @@ -139,7 +140,6 @@ class CompressWorker { std::condition_variable cv_; bool stopped_ = false; - std::basic_string Compress(const void* data, size_t length); bool CompressBlocks(const void* buffer, size_t num_blocks, std::vector>* compressed_data); }; diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp index a4a0ad661..96d601612 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp @@ -46,24 +46,47 @@ std::optional CompressionAlgorithmFromString(std::strin } else if (name == "none" || name.empty()) { return {kCowCompressNone}; } else { + LOG(ERROR) << "unable to determine default compression algorithm for: " << name; return {}; } } -std::basic_string CompressWorker::Compress(const void* data, size_t length) { - return Compress(compression_, data, length); +// 1. Default compression level is determined by compression algorithm +// 2. There might be compatibility issues if a value is changed here, as some older versions of +// Android will assume a different compression level, causing cow_size estimation differences that +// will lead to OTA failure. Ensure that the device and OTA package use the same compression level +// for OTA to succeed. +uint32_t CompressWorker::GetDefaultCompressionLevel(CowCompressionAlgorithm compression) { + switch (compression) { + case kCowCompressGz: { + return Z_BEST_COMPRESSION; + } + case kCowCompressBrotli: { + return BROTLI_DEFAULT_QUALITY; + } + case kCowCompressLz4: { + break; + } + case kCowCompressZstd: { + return ZSTD_defaultCLevel(); + } + case kCowCompressNone: { + break; + } + } + return 0; } -std::basic_string CompressWorker::Compress(CowCompressionAlgorithm compression, - const void* data, size_t length) { - switch (compression) { +std::basic_string CompressWorker::Compress(CowCompression compression, const void* data, + size_t length) { + switch (compression.algorithm) { case kCowCompressGz: { const auto bound = compressBound(length); std::basic_string buffer(bound, '\0'); uLongf dest_len = bound; auto rv = compress2(buffer.data(), &dest_len, reinterpret_cast(data), - length, Z_BEST_COMPRESSION); + length, compression.compression_level); if (rv != Z_OK) { LOG(ERROR) << "compress2 returned: " << rv; return {}; @@ -81,8 +104,8 @@ std::basic_string CompressWorker::Compress(CowCompressionAlgorithm comp size_t encoded_size = bound; auto rv = BrotliEncoderCompress( - BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, length, - reinterpret_cast(data), &encoded_size, buffer.data()); + compression.compression_level, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, + length, reinterpret_cast(data), &encoded_size, buffer.data()); if (!rv) { LOG(ERROR) << "BrotliEncoderCompress failed"; return {}; @@ -117,8 +140,8 @@ std::basic_string CompressWorker::Compress(CowCompressionAlgorithm comp } case kCowCompressZstd: { std::basic_string buffer(ZSTD_compressBound(length), '\0'); - const auto compressed_size = - ZSTD_compress(buffer.data(), buffer.size(), data, length, 0); + const auto compressed_size = ZSTD_compress(buffer.data(), buffer.size(), data, length, + compression.compression_level); if (compressed_size <= 0) { LOG(ERROR) << "ZSTD compression failed " << compressed_size; return {}; @@ -133,7 +156,7 @@ std::basic_string CompressWorker::Compress(CowCompressionAlgorithm comp return buffer; } default: - LOG(ERROR) << "unhandled compression type: " << compression; + LOG(ERROR) << "unhandled compression type: " << compression.algorithm; break; } return {}; @@ -143,7 +166,7 @@ bool CompressWorker::CompressBlocks(const void* buffer, size_t num_blocks, return CompressBlocks(compression_, block_size_, buffer, num_blocks, compressed_data); } -bool CompressWorker::CompressBlocks(CowCompressionAlgorithm compression, size_t block_size, +bool CompressWorker::CompressBlocks(CowCompression compression, size_t block_size, const void* buffer, size_t num_blocks, std::vector>* compressed_data) { const uint8_t* iter = reinterpret_cast(buffer); @@ -255,7 +278,7 @@ void CompressWorker::Finalize() { cv_.notify_all(); } -CompressWorker::CompressWorker(CowCompressionAlgorithm compression, uint32_t block_size) +CompressWorker::CompressWorker(CowCompression compression, uint32_t block_size) : compression_(compression), block_size_(block_size) {} } // namespace snapshot diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp index ab275d4eb..2258d9fb7 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp @@ -480,7 +480,7 @@ TEST_P(CompressionTest, HorribleStream) { std::string expected = "The quick brown fox jumps over the lazy dog."; expected.resize(4096, '\0'); - auto result = CompressWorker::Compress(*algorithm, expected.data(), expected.size()); + auto result = CompressWorker::Compress(compression, expected.data(), expected.size()); ASSERT_FALSE(result.empty()); HorribleStream stream(result); @@ -1409,6 +1409,18 @@ TEST_F(CowTest, RevMergeOpItrTest) { ASSERT_TRUE(iter->AtEnd()); } +TEST_F(CowTest, ParseOptionsTest) { + CowOptions options; + std::vector> testcases = { + {"gz,4", true}, {"gz,4,4", false}, {"lz4,4", true}, {"brotli,4", true}, + {"zstd,4", true}, {"zstd,x", false}, {"zs,4", false}, {"zstd.4", false}}; + for (size_t i = 0; i < testcases.size(); i++) { + options.compression = testcases[i].first; + CowWriterV2 writer(options, GetCowFd()); + ASSERT_EQ(writer.Initialize(), testcases[i].second); + } +} + TEST_F(CowTest, LegacyRevMergeOpItrTest) { CowOptions options; options.cluster_ops = 5; diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp index cbd7569d2..4ab5713b4 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp @@ -39,6 +39,8 @@ #include #include +#include "android-base/parseint.h" +#include "android-base/strings.h" #include "parser_v2.h" // The info messages here are spammy, but as useful for update_engine. Disable @@ -119,11 +121,28 @@ void CowWriterV2::SetupHeaders() { } bool CowWriterV2::ParseOptions() { - auto algorithm = CompressionAlgorithmFromString(options_.compression); + auto parts = android::base::Split(options_.compression, ","); + + if (parts.size() > 2) { + LOG(ERROR) << "failed to parse compression parameters: invalid argument count: " + << parts.size() << " " << options_.compression; + return false; + } + auto algorithm = CompressionAlgorithmFromString(parts[0]); if (!algorithm) { LOG(ERROR) << "unrecognized compression: " << options_.compression; return false; } + if (parts.size() > 1) { + if (!android::base::ParseUint(parts[1], &compression_.compression_level)) { + LOG(ERROR) << "failed to parse compression level invalid type: " << parts[1]; + return false; + } + } else { + compression_.compression_level = + CompressWorker::GetDefaultCompressionLevel(algorithm.value()); + } + compression_.algorithm = *algorithm; if (options_.cluster_ops == 1) { @@ -165,7 +184,7 @@ void CowWriterV2::InitWorkers() { return; } for (int i = 0; i < num_compress_threads_; i++) { - auto wt = std::make_unique(compression_.algorithm, header_.block_size); + auto wt = std::make_unique(compression_, header_.block_size); threads_.emplace_back(std::async(std::launch::async, &CompressWorker::RunThread, wt.get())); compress_threads_.push_back(std::move(wt)); } @@ -320,8 +339,8 @@ bool CowWriterV2::CompressBlocks(size_t num_blocks, const void* data) { const uint8_t* iter = reinterpret_cast(data); compressed_buf_.clear(); if (num_threads <= 1) { - return CompressWorker::CompressBlocks(compression_.algorithm, options_.block_size, data, - num_blocks, &compressed_buf_); + return CompressWorker::CompressBlocks(compression_, options_.block_size, data, num_blocks, + &compressed_buf_); } // Submit the blocks per thread. The retrieval of @@ -393,8 +412,8 @@ bool CowWriterV2::EmitBlocks(uint64_t new_block_start, const void* data, size_t buf_iter_++; return data; } else { - auto data = CompressWorker::Compress(compression_.algorithm, iter, - header_.block_size); + auto data = + CompressWorker::Compress(compression_, iter, header_.block_size); return data; } }(); @@ -507,8 +526,8 @@ bool CowWriterV2::Finalize() { } } - // Footer should be at the end of a file, so if there is data after the current block, end it - // and start a new cluster. + // Footer should be at the end of a file, so if there is data after the current block, end + // it and start a new cluster. if (cluster_size_ && current_data_size_ > 0) { EmitCluster(); extra_cluster = true;