tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "libsnapshot: Add XorOp to v3 Cow" into main

Browse source
This commit is contained in:
Daniel Zheng 2023-11-07 05:41:08 +00:00 committed by Gerrit Code Review
commit 5c8c768c10
10 changed files with 118 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
View file

@ -183,7 +183,7 @@ class CowReader final : public ICowReader {
uint64_t num_total_data_ops_{};
uint64_t num_ordered_ops_to_merge_{};
bool has_seq_ops_{};
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc_;
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> xor_data_loc_;
ReaderFlags reader_flag_;
bool is_merge_{};
};

8
fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp
View file

@ -82,7 +82,7 @@ std::unique_ptr<CowReader> CowReader::CloneCowReader() {
cow->merge_op_start_ = merge_op_start_;
cow->num_total_data_ops_ = num_total_data_ops_;
cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_;
cow->data_loc_ = data_loc_;
cow->xor_data_loc_ = xor_data_loc_;
cow->block_pos_index_ = block_pos_index_;
cow->is_merge_ = is_merge_;
return cow;
@ -139,7 +139,6 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> lab
LOG(ERROR) << "Unknown version: " << header_.prefix.major_version;
return false;
}
if (!parser->Parse(fd, header_, label)) {
return false;
}
@ -154,7 +153,7 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> lab
footer_ = parser->footer();
fd_size_ = parser->fd_size();
last_label_ = parser->last_label();
data_loc_ = parser->data_loc();
xor_data_loc_ = parser->xor_data_loc();
// If we're resuming a write, we're not ready to merge
if (label.has_value()) return true;
@ -682,11 +681,10 @@ ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_
uint64_t offset;
if (op->type == kCowXorOp) {
offset = data_loc_->at(op->new_block);
offset = xor_data_loc_->at(op->new_block);
} else {
offset = GetCowOpSourceInfoData(*op);
}
if (!decompressor) {
CowDataStream stream(this, offset + ignore_bytes, op->data_length - ignore_bytes);
return stream.ReadFully(buffer, buffer_size);

3
fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp
View file

@ -82,7 +82,8 @@ static bool ShowRawOpStreamV2(borrowed_fd fd, const CowHeaderV3& header) {
}
for (const auto& op : *parser.get_v2ops()) {
std::cout << op << "\n";
if (auto iter = parser.data_loc()->find(op.new_block); iter != parser.data_loc()->end()) {
if (auto iter = parser.xor_data_loc()->find(op.new_block);
iter != parser.xor_data_loc()->end()) {
std::cout << " data loc: " << iter->second << "\n";
}
}

5
fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h
View file

@ -38,7 +38,9 @@ class CowParserBase {
std::optional<uint64_t> label = {}) = 0;
virtual bool Translate(TranslatedCowOps* out) = 0;
virtual std::optional<CowFooter> footer() const { return std::nullopt; }
virtual std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc() const = 0;
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> xor_data_loc() {
return xor_data_loc_;
};
uint64_t fd_size() const { return fd_size_; }
const std::optional<uint64_t>& last_label() const { return last_label_; }
@ -47,6 +49,7 @@ class CowParserBase {
CowHeaderV3 header_ = {};
uint64_t fd_size_;
std::optional<uint64_t> last_label_;
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> xor_data_loc_ = {};
};
} // namespace snapshot

10
fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp
View file

@ -63,7 +63,7 @@ bool CowParserV2::Parse(borrowed_fd fd, const CowHeaderV3& header, std::optional
bool CowParserV2::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) {
uint64_t pos;
auto data_loc = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();
auto xor_data_loc = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();
// Skip the scratch space
if (header_.prefix.major_version >= 2 && (header_.buffer_size > 0)) {
@ -111,7 +111,7 @@ bool CowParserV2::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) {
auto& current_op = ops_buffer->data()[current_op_num];
current_op_num++;
if (current_op.type == kCowXorOp) {
data_loc->insert({current_op.new_block, data_pos});
xor_data_loc->insert({current_op.new_block, data_pos});
}
pos += sizeof(CowOperationV2) + GetNextOpOffset(current_op, header_.cluster_ops);
data_pos += current_op.data_length + GetNextDataOffset(current_op, header_.cluster_ops);
@ -193,7 +193,7 @@ bool CowParserV2::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) {
v2_ops_ = ops_buffer;
v2_ops_->shrink_to_fit();
data_loc_ = data_loc;
xor_data_loc_ = xor_data_loc;
return true;
}
@ -239,9 +239,5 @@ bool CowParserV2::Translate(TranslatedCowOps* out) {
return true;
}
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> CowParserV2::data_loc() const {
return data_loc_;
}
} // namespace snapshot
} // namespace android

2
fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h
View file

@ -35,10 +35,8 @@ class CowParserV2 final : public CowParserBase {
const CowHeader& header() const { return header_; }
std::shared_ptr<std::vector<CowOperationV2>> get_v2ops() { return v2_ops_; }
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc() const override;
private:
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc_;
bool ParseOps(android::base::borrowed_fd fd, std::optional<uint64_t> label);
std::shared_ptr<std::vector<CowOperationV2>> v2_ops_;
std::optional<CowFooter> footer_;

16
fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp
View file

@ -58,6 +58,10 @@ bool CowParserV3::Parse(borrowed_fd fd, const CowHeaderV3& header, std::optional
return ParseOps(fd, label);
}
off_t CowParserV3::GetDataOffset() const {
return sizeof(CowHeaderV3) + header_.buffer_size + header_.op_count_max * sizeof(CowOperation);
}
bool CowParserV3::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) {
ops_ = std::make_shared<std::vector<CowOperationV3>>();
ops_->resize(header_.op_count);
@ -69,12 +73,24 @@ bool CowParserV3::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) {
return false;
}
// fill out mapping of XOR op data location
uint64_t data_pos = GetDataOffset();
xor_data_loc_ = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();
for (auto op : *ops_) {
if (op.type == kCowXorOp) {
xor_data_loc_->insert({op.new_block, data_pos});
}
data_pos += op.data_length;
}
// :TODO: sequence buffer & resume buffer follow
// Once we implement labels, we'll have to discard unused ops and adjust
// the header as needed.
CHECK(!label);
ops_->shrink_to_fit();
return true;
}

5
fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h
View file

@ -45,13 +45,10 @@ class CowParserV3 final : public CowParserBase {
bool Parse(android::base::borrowed_fd fd, const CowHeaderV3& header,
std::optional<uint64_t> label = {}) override;
bool Translate(TranslatedCowOps* out) override;
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc() const override {
return nullptr;
};
private:
bool ParseOps(android::base::borrowed_fd fd, std::optional<uint64_t> label);
off_t GetDataOffset() const;
CowHeaderV3 header_ = {};
std::shared_ptr<std::vector<CowOperationV3>> ops_;
};

81
fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp
View file

@ -218,7 +218,7 @@ TEST_F(CowTestV3, CopyOp) {
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
const auto& header = reader.GetHeader();
const auto& header = reader.header_v3();
ASSERT_EQ(header.prefix.magic, kCowMagicNumber);
ASSERT_EQ(header.prefix.major_version, 3);
ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor);
@ -242,5 +242,84 @@ TEST_F(CowTestV3, CopyOp) {
ASSERT_EQ(i, 100);
}
TEST_F(CowTestV3, XorOp) {
CowOptions options;
options.op_count_max = 100;
auto writer = CreateCowWriter(3, options, GetCowFd());
std::string data = "This is test data-1. Testing xor";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddXorBlocks(50, data.data(), data.size(), 24, 10));
ASSERT_TRUE(writer->Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
const auto& header = reader.header_v3();
ASSERT_EQ(header.op_count, 1);
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->AtEnd());
auto op = iter->Get();
std::string sink(data.size(), '\0');
ASSERT_EQ(op->type, kCowXorOp);
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(GetCowOpSourceInfoData(*op), 98314); // 4096 * 24 + 10
ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
ASSERT_EQ(sink, data);
}
TEST_F(CowTestV3, ConsecutiveXorOp) {
CowOptions options;
options.op_count_max = 100;
auto writer = CreateCowWriter(3, options, GetCowFd());
std::string data;
data.resize(options.block_size * 5);
for (int i = 0; i < data.size(); i++) {
data[i] = char(rand() % 256);
}
ASSERT_TRUE(writer->AddXorBlocks(50, data.data(), data.size(), 24, 10));
ASSERT_TRUE(writer->Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
const auto& header = reader.header_v3();
ASSERT_EQ(header.op_count, 5);
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->AtEnd());
std::string sink(data.size(), '\0');
size_t i = 0;
while (!iter->AtEnd()) {
auto op = iter->Get();
ASSERT_EQ(op->type, kCowXorOp);
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50 + i);
ASSERT_EQ(GetCowOpSourceInfoData(*op), 98314 + (i * options.block_size)); // 4096 * 24 + 10
ASSERT_TRUE(
ReadData(reader, op, sink.data() + (i * options.block_size), options.block_size));
iter->Next();
i++;
}
ASSERT_EQ(sink, data);
ASSERT_EQ(i, 5);
}
} // namespace snapshot
} // namespace android

18
fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp
View file

@ -163,7 +163,6 @@ bool CowWriterV3::OpenForWrite() {
}
next_data_pos_ =
sizeof(CowHeaderV3) + header_.buffer_size + header_.op_count_max * sizeof(CowOperation);
return true;
}
@ -187,19 +186,12 @@ bool CowWriterV3::EmitRawBlocks(uint64_t new_block_start, const void* data, size
bool CowWriterV3::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
uint32_t old_block, uint16_t offset) {
LOG(ERROR) << __LINE__ << " " << __FILE__ << " <- function here should never be called";
if (new_block_start || old_block || offset || data || size) return false;
return false;
return EmitBlocks(new_block_start, data, size, old_block, offset, kCowXorOp);
}
bool CowWriterV3::EmitBlocks(uint64_t new_block_start, const void* data, size_t size,
uint64_t old_block, uint16_t offset, uint8_t type) {
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
// Placing here until we support XOR ops
CHECK_EQ(old_block, 0);
CHECK_EQ(offset, 0);
const size_t num_blocks = (size / header_.block_size);
for (size_t i = 0; i < num_blocks; i++) {
@ -207,12 +199,18 @@ bool CowWriterV3::EmitBlocks(uint64_t new_block_start, const void* data, size_t
op.new_block = new_block_start + i;
op.type = type;
op.source_info = next_data_pos_;
op.data_length = static_cast<uint16_t>(header_.block_size);
if (type == kCowXorOp) {
op.source_info = (old_block + i) * header_.block_size + offset;
} else {
op.source_info = next_data_pos_;
}
if (!WriteOperation(op, iter, header_.block_size)) {
LOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
iter += header_.block_size;
}