diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h index 755a3afa2..1ab6ada6e 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h @@ -183,7 +183,7 @@ class CowReader final : public ICowReader { uint64_t num_total_data_ops_{}; uint64_t num_ordered_ops_to_merge_{}; bool has_seq_ops_{}; - std::shared_ptr> data_loc_; + std::shared_ptr> xor_data_loc_; ReaderFlags reader_flag_; bool is_merge_{}; }; diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp index 93c31bbc9..296987a03 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp @@ -82,7 +82,7 @@ std::unique_ptr CowReader::CloneCowReader() { cow->merge_op_start_ = merge_op_start_; cow->num_total_data_ops_ = num_total_data_ops_; cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_; - cow->data_loc_ = data_loc_; + cow->xor_data_loc_ = xor_data_loc_; cow->block_pos_index_ = block_pos_index_; cow->is_merge_ = is_merge_; return cow; @@ -139,7 +139,6 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional lab LOG(ERROR) << "Unknown version: " << header_.prefix.major_version; return false; } - if (!parser->Parse(fd, header_, label)) { return false; } @@ -154,7 +153,7 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional lab footer_ = parser->footer(); fd_size_ = parser->fd_size(); last_label_ = parser->last_label(); - data_loc_ = parser->data_loc(); + xor_data_loc_ = parser->xor_data_loc(); // If we're resuming a write, we're not ready to merge if (label.has_value()) return true; @@ -682,11 +681,10 @@ ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_ uint64_t offset; if (op->type == kCowXorOp) { - offset = data_loc_->at(op->new_block); + offset = xor_data_loc_->at(op->new_block); } else { offset = GetCowOpSourceInfoData(*op); } - if (!decompressor) { CowDataStream stream(this, offset + ignore_bytes, op->data_length - ignore_bytes); return stream.ReadFully(buffer, buffer_size); diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp index 76509de78..993630b04 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp @@ -82,7 +82,8 @@ static bool ShowRawOpStreamV2(borrowed_fd fd, const CowHeaderV3& header) { } for (const auto& op : *parser.get_v2ops()) { std::cout << op << "\n"; - if (auto iter = parser.data_loc()->find(op.new_block); iter != parser.data_loc()->end()) { + if (auto iter = parser.xor_data_loc()->find(op.new_block); + iter != parser.xor_data_loc()->end()) { std::cout << " data loc: " << iter->second << "\n"; } } diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h b/fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h index 278340740..837b33edb 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h +++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h @@ -38,7 +38,9 @@ class CowParserBase { std::optional label = {}) = 0; virtual bool Translate(TranslatedCowOps* out) = 0; virtual std::optional footer() const { return std::nullopt; } - virtual std::shared_ptr> data_loc() const = 0; + std::shared_ptr> xor_data_loc() { + return xor_data_loc_; + }; uint64_t fd_size() const { return fd_size_; } const std::optional& last_label() const { return last_label_; } @@ -47,6 +49,7 @@ class CowParserBase { CowHeaderV3 header_ = {}; uint64_t fd_size_; std::optional last_label_; + std::shared_ptr> xor_data_loc_ = {}; }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp index 51489c83a..08a43a488 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp @@ -63,7 +63,7 @@ bool CowParserV2::Parse(borrowed_fd fd, const CowHeaderV3& header, std::optional bool CowParserV2::ParseOps(borrowed_fd fd, std::optional label) { uint64_t pos; - auto data_loc = std::make_shared>(); + auto xor_data_loc = std::make_shared>(); // Skip the scratch space if (header_.prefix.major_version >= 2 && (header_.buffer_size > 0)) { @@ -111,7 +111,7 @@ bool CowParserV2::ParseOps(borrowed_fd fd, std::optional label) { auto& current_op = ops_buffer->data()[current_op_num]; current_op_num++; if (current_op.type == kCowXorOp) { - data_loc->insert({current_op.new_block, data_pos}); + xor_data_loc->insert({current_op.new_block, data_pos}); } pos += sizeof(CowOperationV2) + GetNextOpOffset(current_op, header_.cluster_ops); data_pos += current_op.data_length + GetNextDataOffset(current_op, header_.cluster_ops); @@ -193,7 +193,7 @@ bool CowParserV2::ParseOps(borrowed_fd fd, std::optional label) { v2_ops_ = ops_buffer; v2_ops_->shrink_to_fit(); - data_loc_ = data_loc; + xor_data_loc_ = xor_data_loc; return true; } @@ -239,9 +239,5 @@ bool CowParserV2::Translate(TranslatedCowOps* out) { return true; } -std::shared_ptr> CowParserV2::data_loc() const { - return data_loc_; -} - } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h index 318a3ac14..f9ee2e526 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h +++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h @@ -35,10 +35,8 @@ class CowParserV2 final : public CowParserBase { const CowHeader& header() const { return header_; } std::shared_ptr> get_v2ops() { return v2_ops_; } - std::shared_ptr> data_loc() const override; private: - std::shared_ptr> data_loc_; bool ParseOps(android::base::borrowed_fd fd, std::optional label); std::shared_ptr> v2_ops_; std::optional footer_; diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp index 23ddaae71..a8a63d87a 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp @@ -58,6 +58,10 @@ bool CowParserV3::Parse(borrowed_fd fd, const CowHeaderV3& header, std::optional return ParseOps(fd, label); } +off_t CowParserV3::GetDataOffset() const { + return sizeof(CowHeaderV3) + header_.buffer_size + header_.op_count_max * sizeof(CowOperation); +} + bool CowParserV3::ParseOps(borrowed_fd fd, std::optional label) { ops_ = std::make_shared>(); ops_->resize(header_.op_count); @@ -69,12 +73,24 @@ bool CowParserV3::ParseOps(borrowed_fd fd, std::optional label) { return false; } + // fill out mapping of XOR op data location + uint64_t data_pos = GetDataOffset(); + + xor_data_loc_ = std::make_shared>(); + + for (auto op : *ops_) { + if (op.type == kCowXorOp) { + xor_data_loc_->insert({op.new_block, data_pos}); + } + data_pos += op.data_length; + } // :TODO: sequence buffer & resume buffer follow // Once we implement labels, we'll have to discard unused ops and adjust // the header as needed. CHECK(!label); ops_->shrink_to_fit(); + return true; } diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h index 5760cf2ba..e2663cc59 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h +++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h @@ -45,13 +45,10 @@ class CowParserV3 final : public CowParserBase { bool Parse(android::base::borrowed_fd fd, const CowHeaderV3& header, std::optional label = {}) override; bool Translate(TranslatedCowOps* out) override; - std::shared_ptr> data_loc() const override { - return nullptr; - }; private: bool ParseOps(android::base::borrowed_fd fd, std::optional label); - + off_t GetDataOffset() const; CowHeaderV3 header_ = {}; std::shared_ptr> ops_; }; diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp index c5d7a025f..3277c990e 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp @@ -218,7 +218,7 @@ TEST_F(CowTestV3, CopyOp) { CowReader reader; ASSERT_TRUE(reader.Parse(cow_->fd)); - const auto& header = reader.GetHeader(); + const auto& header = reader.header_v3(); ASSERT_EQ(header.prefix.magic, kCowMagicNumber); ASSERT_EQ(header.prefix.major_version, 3); ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor); @@ -242,5 +242,84 @@ TEST_F(CowTestV3, CopyOp) { ASSERT_EQ(i, 100); } +TEST_F(CowTestV3, XorOp) { + CowOptions options; + options.op_count_max = 100; + auto writer = CreateCowWriter(3, options, GetCowFd()); + + std::string data = "This is test data-1. Testing xor"; + data.resize(options.block_size, '\0'); + ASSERT_TRUE(writer->AddXorBlocks(50, data.data(), data.size(), 24, 10)); + ASSERT_TRUE(writer->Finalize()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + CowReader reader; + + ASSERT_TRUE(reader.Parse(cow_->fd)); + + const auto& header = reader.header_v3(); + ASSERT_EQ(header.op_count, 1); + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + ASSERT_FALSE(iter->AtEnd()); + auto op = iter->Get(); + std::string sink(data.size(), '\0'); + + ASSERT_EQ(op->type, kCowXorOp); + ASSERT_EQ(op->data_length, 4096); + ASSERT_EQ(op->new_block, 50); + ASSERT_EQ(GetCowOpSourceInfoData(*op), 98314); // 4096 * 24 + 10 + ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size())); + ASSERT_EQ(sink, data); +} + +TEST_F(CowTestV3, ConsecutiveXorOp) { + CowOptions options; + options.op_count_max = 100; + auto writer = CreateCowWriter(3, options, GetCowFd()); + + std::string data; + data.resize(options.block_size * 5); + for (int i = 0; i < data.size(); i++) { + data[i] = char(rand() % 256); + } + + ASSERT_TRUE(writer->AddXorBlocks(50, data.data(), data.size(), 24, 10)); + ASSERT_TRUE(writer->Finalize()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + CowReader reader; + + ASSERT_TRUE(reader.Parse(cow_->fd)); + + const auto& header = reader.header_v3(); + ASSERT_EQ(header.op_count, 5); + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + ASSERT_FALSE(iter->AtEnd()); + + std::string sink(data.size(), '\0'); + size_t i = 0; + + while (!iter->AtEnd()) { + auto op = iter->Get(); + ASSERT_EQ(op->type, kCowXorOp); + ASSERT_EQ(op->data_length, 4096); + ASSERT_EQ(op->new_block, 50 + i); + ASSERT_EQ(GetCowOpSourceInfoData(*op), 98314 + (i * options.block_size)); // 4096 * 24 + 10 + ASSERT_TRUE( + ReadData(reader, op, sink.data() + (i * options.block_size), options.block_size)); + iter->Next(); + i++; + } + ASSERT_EQ(sink, data); + + ASSERT_EQ(i, 5); +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp index aeb088d2e..13b615782 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp @@ -163,7 +163,6 @@ bool CowWriterV3::OpenForWrite() { } next_data_pos_ = sizeof(CowHeaderV3) + header_.buffer_size + header_.op_count_max * sizeof(CowOperation); - return true; } @@ -187,19 +186,12 @@ bool CowWriterV3::EmitRawBlocks(uint64_t new_block_start, const void* data, size bool CowWriterV3::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, uint16_t offset) { - LOG(ERROR) << __LINE__ << " " << __FILE__ << " <- function here should never be called"; - if (new_block_start || old_block || offset || data || size) return false; - return false; + return EmitBlocks(new_block_start, data, size, old_block, offset, kCowXorOp); } bool CowWriterV3::EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block, uint16_t offset, uint8_t type) { const uint8_t* iter = reinterpret_cast(data); - - // Placing here until we support XOR ops - CHECK_EQ(old_block, 0); - CHECK_EQ(offset, 0); - const size_t num_blocks = (size / header_.block_size); for (size_t i = 0; i < num_blocks; i++) { @@ -207,12 +199,18 @@ bool CowWriterV3::EmitBlocks(uint64_t new_block_start, const void* data, size_t op.new_block = new_block_start + i; op.type = type; - op.source_info = next_data_pos_; op.data_length = static_cast(header_.block_size); + + if (type == kCowXorOp) { + op.source_info = (old_block + i) * header_.block_size + offset; + } else { + op.source_info = next_data_pos_; + } if (!WriteOperation(op, iter, header_.block_size)) { LOG(ERROR) << "AddRawBlocks: write failed"; return false; } + iter += header_.block_size; }