tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "libsnapshot: replace non-character basic_string[_view]<T>" into main am: 18744c17c1

Browse source 
Original change: https://android-review.googlesource.com/c/platform/system/core/+/2956695

Change-Id: I2f5f9e60cc8f0421e25764d7652be05e54dc2f49
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Ryan Prichard 2024-02-09 20:00:33 +00:00 committed by Automerger Merge Worker
commit 2499c6d958
9 changed files with 38 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
fs_mgr/libsnapshot/include/libsnapshot/cow_compress.h
View file

@ -17,6 +17,7 @@
#pragma once
#include <memory>
#include <vector>
#include "libsnapshot/cow_format.h"
namespace android {
@ -40,8 +41,7 @@ class ICompressor {
uint32_t GetCompressionLevel() const { return compression_level_; }
uint32_t GetBlockSize() const { return block_size_; }
[[nodiscard]] virtual std::basic_string<uint8_t> Compress(const void* data,
size_t length) const = 0;
[[nodiscard]] virtual std::vector<uint8_t> Compress(const void* data, size_t length) const = 0;
private:
uint32_t compression_level_;

8
fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
View file

@ -122,13 +122,13 @@ class CompressWorker {
CompressWorker(std::unique_ptr<ICompressor>&& compressor);
bool RunThread();
void EnqueueCompressBlocks(const void* buffer, size_t block_size, size_t num_blocks);
bool GetCompressedBuffers(std::vector<std::basic_string<uint8_t>>* compressed_buf);
bool GetCompressedBuffers(std::vector<std::vector<uint8_t>>* compressed_buf);
void Finalize();
static uint32_t GetDefaultCompressionLevel(CowCompressionAlgorithm compression);
static bool CompressBlocks(ICompressor* compressor, size_t block_size, const void* buffer,
size_t num_blocks,
std::vector<std::basic_string<uint8_t>>* compressed_data);
std::vector<std::vector<uint8_t>>* compressed_data);
private:
struct CompressWork {
@ -136,7 +136,7 @@ class CompressWorker {
size_t num_blocks;
size_t block_size;
bool compression_status = false;
std::vector<std::basic_string<uint8_t>> compressed_data;
std::vector<std::vector<uint8_t>> compressed_data;
};
std::unique_ptr<ICompressor> compressor_;
@ -150,7 +150,7 @@ class CompressWorker {
size_t total_processed_ = 0;
bool CompressBlocks(const void* buffer, size_t num_blocks, size_t block_size,
std::vector<std::basic_string<uint8_t>>* compressed_data);
std::vector<std::vector<uint8_t>>* compressed_data);
};
// Create an ICowWriter not backed by any file. This is useful for estimating

23
fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp
View file

@ -20,6 +20,7 @@
#include <limits>
#include <memory>
#include <queue>
#include <vector>
#include <android-base/file.h>
#include <android-base/logging.h>
@ -103,9 +104,9 @@ class GzCompressor final : public ICompressor {
GzCompressor(uint32_t compression_level, const uint32_t block_size)
: ICompressor(compression_level, block_size){};
std::basic_string<uint8_t> Compress(const void* data, size_t length) const override {
std::vector<uint8_t> Compress(const void* data, size_t length) const override {
const auto bound = compressBound(length);
std::basic_string<uint8_t> buffer(bound, '\0');
std::vector<uint8_t> buffer(bound, '\0');
uLongf dest_len = bound;
auto rv = compress2(buffer.data(), &dest_len, reinterpret_cast<const Bytef*>(data), length,
@ -124,13 +125,13 @@ class Lz4Compressor final : public ICompressor {
Lz4Compressor(uint32_t compression_level, const uint32_t block_size)
: ICompressor(compression_level, block_size){};
std::basic_string<uint8_t> Compress(const void* data, size_t length) const override {
std::vector<uint8_t> Compress(const void* data, size_t length) const override {
const auto bound = LZ4_compressBound(length);
if (!bound) {
LOG(ERROR) << "LZ4_compressBound returned 0";
return {};
}
std::basic_string<uint8_t> buffer(bound, '\0');
std::vector<uint8_t> buffer(bound, '\0');
const auto compressed_size =
LZ4_compress_default(static_cast<const char*>(data),
@ -156,13 +157,13 @@ class BrotliCompressor final : public ICompressor {
BrotliCompressor(uint32_t compression_level, const uint32_t block_size)
: ICompressor(compression_level, block_size){};
std::basic_string<uint8_t> Compress(const void* data, size_t length) const override {
std::vector<uint8_t> Compress(const void* data, size_t length) const override {
const auto bound = BrotliEncoderMaxCompressedSize(length);
if (!bound) {
LOG(ERROR) << "BrotliEncoderMaxCompressedSize returned 0";
return {};
}
std::basic_string<uint8_t> buffer(bound, '\0');
std::vector<uint8_t> buffer(bound, '\0');
size_t encoded_size = bound;
auto rv = BrotliEncoderCompress(
@ -186,8 +187,8 @@ class ZstdCompressor final : public ICompressor {
ZSTD_CCtx_setParameter(zstd_context_.get(), ZSTD_c_windowLog, log2(GetBlockSize()));
};
std::basic_string<uint8_t> Compress(const void* data, size_t length) const override {
std::basic_string<uint8_t> buffer(ZSTD_compressBound(length), '\0');
std::vector<uint8_t> Compress(const void* data, size_t length) const override {
std::vector<uint8_t> buffer(ZSTD_compressBound(length), '\0');
const auto compressed_size =
ZSTD_compress2(zstd_context_.get(), buffer.data(), buffer.size(), data, length);
if (compressed_size <= 0) {
@ -209,13 +210,13 @@ class ZstdCompressor final : public ICompressor {
};
bool CompressWorker::CompressBlocks(const void* buffer, size_t num_blocks, size_t block_size,
std::vector<std::basic_string<uint8_t>>* compressed_data) {
std::vector<std::vector<uint8_t>>* compressed_data) {
return CompressBlocks(compressor_.get(), block_size, buffer, num_blocks, compressed_data);
}
bool CompressWorker::CompressBlocks(ICompressor* compressor, size_t block_size, const void* buffer,
size_t num_blocks,
std::vector<std::basic_string<uint8_t>>* compressed_data) {
std::vector<std::vector<uint8_t>>* compressed_data) {
const uint8_t* iter = reinterpret_cast<const uint8_t*>(buffer);
while (num_blocks) {
auto data = compressor->Compress(iter, block_size);
@ -289,7 +290,7 @@ void CompressWorker::EnqueueCompressBlocks(const void* buffer, size_t block_size
cv_.notify_all();
}
bool CompressWorker::GetCompressedBuffers(std::vector<std::basic_string<uint8_t>>* compressed_buf) {
bool CompressWorker::GetCompressedBuffers(std::vector<std::vector<uint8_t>>* compressed_buf) {
while (true) {
std::unique_lock<std::mutex> lock(lock_);
while ((total_submitted_ != total_processed_) && compressed_queue_.empty() && !stopped_) {

10
fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
View file

@ -18,6 +18,7 @@
#include <iostream>
#include <memory>
#include <string_view>
#include <vector>
#include <android-base/file.h>
#include <android-base/logging.h>
@ -429,7 +430,7 @@ TEST_P(CompressionTest, NoBatchWrites) {
template <typename T>
class HorribleStream : public IByteStream {
public:
HorribleStream(const std::basic_string<T>& input) : input_(input) {}
HorribleStream(const std::vector<T>& input) : input_(input) {}
ssize_t Read(void* buffer, size_t length) override {
if (pos_ >= input_.size()) {
@ -444,16 +445,17 @@ class HorribleStream : public IByteStream {
size_t Size() const override { return input_.size(); }
private:
std::basic_string<T> input_;
std::vector<T> input_;
size_t pos_ = 0;
};
TEST(HorribleStream, ReadFully) {
std::string expected = "this is some data";
std::string expected_str = "this is some data";
std::vector<char> expected{expected_str.begin(), expected_str.end()};
HorribleStream<char> stream(expected);
std::string buffer(expected.size(), '\0');
std::vector<char> buffer(expected.size(), '\0');
ASSERT_TRUE(stream.ReadFully(buffer.data(), buffer.size()));
ASSERT_EQ(buffer, expected);
}

4
fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h
View file

@ -81,8 +81,8 @@ class CowWriterV2 : public CowWriterBase {
int num_compress_threads_ = 1;
std::vector<std::unique_ptr<CompressWorker>> compress_threads_;
std::vector<std::future<bool>> threads_;
std::vector<std::basic_string<uint8_t>> compressed_buf_;
std::vector<std::basic_string<uint8_t>>::iterator buf_iter_;
std::vector<std::vector<uint8_t>> compressed_buf_;
std::vector<std::vector<uint8_t>>::iterator buf_iter_;
std::vector<std::unique_ptr<CowOperationV2>> opbuffer_vec_;
std::vector<std::unique_ptr<uint8_t[]>> databuffer_vec_;

9
fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp
View file

@ -512,8 +512,7 @@ bool CowWriterV3::FlushCacheOps() {
}
bytes_written += op.data_length;
}
if (!WriteOperation({cached_ops_.data(), cached_ops_.size()},
{data_vec_.data(), data_vec_.size()})) {
if (!WriteOperation(cached_ops_, data_vec_)) {
LOG(ERROR) << "Failed to flush " << cached_ops_.size() << " ops to disk";
return false;
}
@ -632,7 +631,7 @@ std::vector<CowWriterV3::CompressedBuffer> CowWriterV3::ProcessBlocksWithThreade
}
// Fetch compressed buffers from the threads
std::vector<std::basic_string<uint8_t>> compressed_buf;
std::vector<std::vector<uint8_t>> compressed_buf;
compressed_buf.clear();
for (size_t i = 0; i < num_threads; i++) {
CompressWorker* worker = compress_threads_[i].get();
@ -684,8 +683,8 @@ std::vector<CowWriterV3::CompressedBuffer> CowWriterV3::CompressBlocks(const siz
return ProcessBlocksWithThreadedCompression(num_blocks, data, type);
}
bool CowWriterV3::WriteOperation(std::basic_string_view<CowOperationV3> ops,
std::basic_string_view<struct iovec> data) {
bool CowWriterV3::WriteOperation(std::span<const CowOperationV3> ops,
std::span<const struct iovec> data) {
const auto total_data_size =
std::transform_reduce(data.begin(), data.end(), 0, std::plus<size_t>{},
[](const struct iovec& a) { return a.iov_len; });

8
fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.h
View file

@ -15,6 +15,7 @@
#pragma once
#include <android-base/logging.h>
#include <span>
#include <string_view>
#include <thread>
#include <vector>
@ -49,15 +50,14 @@ class CowWriterV3 : public CowWriterBase {
private:
struct CompressedBuffer {
size_t compression_factor;
std::basic_string<uint8_t> compressed_data;
std::vector<uint8_t> compressed_data;
};
void SetupHeaders();
bool NeedsFlush() const;
bool ParseOptions();
bool OpenForWrite();
bool OpenForAppend(uint64_t label);
bool WriteOperation(std::basic_string_view<CowOperationV3> op,
std::basic_string_view<struct iovec> data);
bool WriteOperation(std::span<const CowOperationV3> op, std::span<const struct iovec> data);
bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block,
uint16_t offset, CowOperationType type);
bool ConstructCowOpCompressedBuffers(uint64_t new_block_start, const void* data,
@ -111,7 +111,7 @@ class CowWriterV3 : public CowWriterBase {
int num_compress_threads_ = 1;
size_t batch_size_ = 1;
std::vector<CowOperationV3> cached_ops_;
std::vector<std::basic_string<uint8_t>> cached_data_;
std::vector<std::vector<uint8_t>> cached_data_;
std::vector<struct iovec> data_vec_;
std::vector<std::thread> threads_;

2
fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.h
View file

@ -69,7 +69,7 @@ class ReadWorker : public Worker, public IBlockServer::Delegate {
std::shared_ptr<IBlockServerOpener> block_server_opener_;
std::unique_ptr<IBlockServer> block_server_;
std::basic_string<uint8_t> xor_buffer_;
std::vector<uint8_t> xor_buffer_;
std::unique_ptr<void, decltype(&::free)> aligned_buffer_;
std::unique_ptr<uint8_t[]> decompressed_buffer_;
};

6
fs_mgr/libsnapshot/tools/cow_benchmark.cpp
View file

@ -78,7 +78,7 @@ void OneShotCompressionTest() {
for (size_t i = 0; i < compressors.size(); i++) {
const auto start = std::chrono::steady_clock::now();
std::basic_string<uint8_t> compressed_data =
std::vector<uint8_t> compressed_data =
compressors[i]->Compress(buffer.data(), buffer.size());
const auto end = std::chrono::steady_clock::now();
const auto latency =
@ -141,13 +141,13 @@ void IncrementalCompressionTest() {
std::vector<std::pair<double, std::string>> ratios;
for (size_t i = 0; i < compressors.size(); i++) {
std::vector<std::basic_string<uint8_t>> compressed_data_vec;
std::vector<std::vector<uint8_t>> compressed_data_vec;
int num_blocks = buffer.size() / BLOCK_SZ;
const uint8_t* iter = reinterpret_cast<const uint8_t*>(buffer.data());
const auto start = std::chrono::steady_clock::now();
while (num_blocks > 0) {
std::basic_string<uint8_t> compressed_data = compressors[i]->Compress(iter, BLOCK_SZ);
std::vector<uint8_t> compressed_data = compressors[i]->Compress(iter, BLOCK_SZ);
compressed_data_vec.emplace_back(compressed_data);
num_blocks--;
iter += BLOCK_SZ;