tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge changes I07031e89,I1ba276e1

Browse source 
* changes:
  libsnapshot: Remove direct accesses of CowOperation::source and compression.
  libsnapshot: Add helpers for accessing CowOperation offsets.
This commit is contained in:
David Anderson 2023-06-26 18:39:57 +00:00 committed by Gerrit Code Review
commit b41cd681aa
11 changed files with 105 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
View file

@ -166,6 +166,13 @@ static constexpr uint8_t kCowReadAheadNotStarted = 0;
static constexpr uint8_t kCowReadAheadInProgress = 1;
static constexpr uint8_t kCowReadAheadDone = 2;
static inline uint64_t GetCowOpSourceInfoData(const CowOperation* op) {
return op->source;
}
static inline bool GetCowOpSourceInfoCompression(const CowOperation* op) {
return op->compression != kCowCompressNone;
}
struct CowFooter {
CowFooterOperation op;
uint8_t unused[64];

14
fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
View file

@ -73,8 +73,20 @@ class ICowReader {
// The operation pointer must derive from ICowOpIter::Get().
virtual ssize_t ReadData(const CowOperation* op, void* buffer, size_t buffer_size,
size_t ignore_bytes = 0) = 0;
// Get the absolute source offset, in bytes, of a CowOperation. Returns
// false if the operation does not read from source partitions.
virtual bool GetSourceOffset(const CowOperation* op, uint64_t* source_offset) = 0;
};
static constexpr uint64_t GetBlockFromOffset(const CowHeader& header, uint64_t offset) {
return offset / header.block_size;
}
static constexpr uint64_t GetBlockRelativeOffset(const CowHeader& header, uint64_t offset) {
return offset % header.block_size;
}
// Iterate over a sequence of COW operations. The iterator is bidirectional.
class ICowOpIter {
public:
@ -119,6 +131,7 @@ class CowReader final : public ICowReader {
bool VerifyMergeOps() override;
bool GetFooter(CowFooter* footer) override;
bool GetLastLabel(uint64_t* label) override;
bool GetSourceOffset(const CowOperation* op, uint64_t* source_offset) override;
// Create a CowOpIter object which contains footer_.num_ops
// CowOperation objects. Get() returns a unique CowOperation object
@ -155,6 +168,7 @@ class CowReader final : public ICowReader {
bool ParseOps(std::optional<uint64_t> label);
bool PrepMergeOps();
uint64_t FindNumCopyops();
uint8_t GetCompressionType(const CowOperation* op);
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;

41
fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp
View file

@ -312,19 +312,15 @@ bool CowReader::VerifyMergeOps() {
std::unordered_map<uint64_t, const CowOperation*> overwritten_blocks;
while (!itr->AtEnd()) {
const auto& op = itr->Get();
uint64_t block;
bool offset;
if (op->type == kCowCopyOp) {
block = op->source;
offset = false;
} else if (op->type == kCowXorOp) {
block = op->source / header_.block_size;
offset = (op->source % header_.block_size) != 0;
} else {
uint64_t offset;
if (!GetSourceOffset(op, &offset)) {
itr->Next();
continue;
}
uint64_t block = GetBlockFromOffset(header_, offset);
bool misaligned = (GetBlockRelativeOffset(header_, offset) != 0);
const CowOperation* overwrite = nullptr;
if (overwritten_blocks.count(block)) {
overwrite = overwritten_blocks[block];
@ -332,7 +328,7 @@ bool CowReader::VerifyMergeOps() {
<< op << "\noverwritten by previously merged op:\n"
<< *overwrite;
}
if (offset && overwritten_blocks.count(block + 1)) {
if (misaligned && overwritten_blocks.count(block + 1)) {
overwrite = overwritten_blocks[block + 1];
LOG(ERROR) << "Invalid Sequence! Block needed for op:\n"
<< op << "\noverwritten by previously merged op:\n"
@ -521,7 +517,7 @@ bool CowReader::GetRawBytes(const CowOperation* op, void* buffer, size_t len, si
case kCowSequenceOp:
case kCowReplaceOp:
case kCowXorOp:
return GetRawBytes(op->source, buffer, len, read);
return GetRawBytes(GetCowOpSourceInfoData(op), buffer, len, read);
default:
LOG(ERROR) << "Cannot get raw bytes of non-data op: " << *op;
return false;
@ -578,10 +574,14 @@ class CowDataStream final : public IByteStream {
size_t remaining_;
};
uint8_t CowReader::GetCompressionType(const CowOperation* op) {
return op->compression;
}
ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_size,
size_t ignore_bytes) {
std::unique_ptr<IDecompressor> decompressor;
switch (op->compression) {
switch (GetCompressionType(op)) {
case kCowCompressNone:
break;
case kCowCompressGz:
@ -601,7 +601,7 @@ ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_
}
break;
default:
LOG(ERROR) << "Unknown compression type: " << op->compression;
LOG(ERROR) << "Unknown compression type: " << GetCompressionType(op);
return -1;
}
@ -609,7 +609,7 @@ ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_
if (op->type == kCowXorOp) {
offset = data_loc_->at(op->new_block);
} else {
offset = op->source;
offset = GetCowOpSourceInfoData(op);
}
if (!decompressor) {
@ -622,5 +622,18 @@ ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_
return decompressor->Decompress(buffer, buffer_size, header_.block_size, ignore_bytes);
}
bool CowReader::GetSourceOffset(const CowOperation* op, uint64_t* source_offset) {
switch (op->type) {
case kCowCopyOp:
*source_offset = GetCowOpSourceInfoData(op) * header_.block_size;
return true;
case kCowXorOp:
*source_offset = GetCowOpSourceInfoData(op);
return true;
default:
return false;
}
}
} // namespace snapshot
} // namespace android

14
fs_mgr/libsnapshot/libsnapshot_cow/snapshot_reader.cpp
View file

@ -155,7 +155,12 @@ ssize_t CompressedSnapshotReader::ReadBlock(uint64_t chunk, size_t start_offset,
}
if (op) {
chunk = op->source;
uint64_t source_offset;
if (!cow_->GetSourceOffset(op, &source_offset)) {
LOG(ERROR) << "GetSourceOffset failed in CompressedSnapshotReader for op: " << *op;
return false;
}
chunk = GetBlockFromOffset(cow_->GetHeader(), source_offset);
}
off64_t offset = (chunk * block_size_) + start_offset;
@ -179,7 +184,12 @@ ssize_t CompressedSnapshotReader::ReadBlock(uint64_t chunk, size_t start_offset,
return -1;
}
off64_t offset = op->source + start_offset;
uint64_t source_offset;
if (!cow_->GetSourceOffset(op, &source_offset)) {
LOG(ERROR) << "GetSourceOffset failed in CompressedSnapshotReader for op: " << *op;
return false;
}
off64_t offset = source_offset + start_offset;
std::string data(bytes_to_read, '\0');
if (!android::base::ReadFullyAtOffset(fd, data.data(), data.size(), offset)) {

16
fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
View file

@ -145,7 +145,7 @@ TEST_F(CowTest, ReadWrite) {
op = iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_FALSE(GetCowOpSourceInfoCompression(op));
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50);
ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
@ -224,10 +224,10 @@ TEST_F(CowTest, ReadWriteXor) {
op = iter->Get();
ASSERT_EQ(op->type, kCowXorOp);
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_FALSE(GetCowOpSourceInfoCompression(op));
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 98314); // 4096 * 24 + 10
ASSERT_EQ(GetCowOpSourceInfoData(op), 98314); // 4096 * 24 + 10
ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
ASSERT_EQ(sink, data);
@ -283,7 +283,7 @@ TEST_F(CowTest, CompressGz) {
std::string sink(data.size(), '\0');
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_TRUE(GetCowOpSourceInfoCompression(op));
ASSERT_EQ(op->data_length, 56); // compressed!
ASSERT_EQ(op->new_block, 50);
ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
@ -339,7 +339,7 @@ TEST_P(CompressionTest, ThreadedBatchWrites) {
total_blocks += 1;
std::string sink(xor_data.size(), '\0');
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 98314); // 4096 * 24 + 10
ASSERT_EQ(GetCowOpSourceInfoData(op), 98314); // 4096 * 24 + 10
ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
ASSERT_EQ(sink, xor_data);
}
@ -528,7 +528,7 @@ TEST_F(CowTest, ClusterCompressGz) {
std::string sink(data.size(), '\0');
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_TRUE(GetCowOpSourceInfoCompression(op));
ASSERT_EQ(op->data_length, 56); // compressed!
ASSERT_EQ(op->new_block, 50);
ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
@ -546,7 +546,7 @@ TEST_F(CowTest, ClusterCompressGz) {
sink = {};
sink.resize(data2.size(), '\0');
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_TRUE(GetCowOpSourceInfoCompression(op));
ASSERT_EQ(op->data_length, 41); // compressed!
ASSERT_EQ(op->new_block, 51);
ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
@ -591,7 +591,7 @@ TEST_F(CowTest, CompressTwoBlocks) {
auto op = iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_TRUE(GetCowOpSourceInfoCompression(op));
ASSERT_EQ(op->new_block, 51);
ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
}

2
fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/snapuserd.cpp
View file

@ -508,7 +508,7 @@ bool Snapuserd::ReadMetadata() {
// the merge of operations are done based on the ops present
// in the file.
//===========================================================
uint64_t block_source = cow_op->source;
uint64_t block_source = GetCowOpSourceInfoData(cow_op);
if (prev_id.has_value()) {
if (dest_blocks.count(cow_op->new_block) || source_blocks.count(block_source)) {
break;

6
fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/snapuserd_readahead.cpp
View file

@ -172,7 +172,7 @@ ReadAheadThread::ReadAheadThread(const std::string& cow_device, const std::strin
}
void ReadAheadThread::CheckOverlap(const CowOperation* cow_op) {
uint64_t source_block = cow_op->source;
uint64_t source_block = GetCowOpSourceInfoData(cow_op);
if (dest_blocks_.count(cow_op->new_block) || source_blocks_.count(source_block)) {
overlap_ = true;
}
@ -191,7 +191,7 @@ void ReadAheadThread::PrepareReadAhead(uint64_t* source_offset, int* pending_ops
// Get the first block with offset
const CowOperation* cow_op = GetRAOpIter();
CHECK_NE(cow_op, nullptr);
*source_offset = cow_op->source;
*source_offset = GetCowOpSourceInfoData(cow_op);
if (cow_op->type == kCowCopyOp) {
*source_offset *= BLOCK_SZ;
}
@ -210,7 +210,7 @@ void ReadAheadThread::PrepareReadAhead(uint64_t* source_offset, int* pending_ops
while (!RAIterDone() && num_ops) {
const CowOperation* op = GetRAOpIter();
CHECK_NE(op, nullptr);
uint64_t next_offset = op->source;
uint64_t next_offset = GetCowOpSourceInfoData(op);
if (op->type == kCowCopyOp) {
next_offset *= BLOCK_SZ;
}

13
fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/snapuserd_worker.cpp
View file

@ -115,12 +115,13 @@ bool WorkerThread::ReadFromBaseDevice(const CowOperation* cow_op) {
SNAP_LOG(ERROR) << "ReadFromBaseDevice: Failed to get payload buffer";
return false;
}
SNAP_LOG(DEBUG) << " ReadFromBaseDevice...: new-block: " << cow_op->new_block
<< " Source: " << cow_op->source;
uint64_t offset = cow_op->source;
if (cow_op->type == kCowCopyOp) {
offset *= BLOCK_SZ;
uint64_t offset;
if (!reader_->GetSourceOffset(cow_op, &offset)) {
SNAP_LOG(ERROR) << "ReadFromBaseDevice: Failed to get source offset";
return false;
}
SNAP_LOG(DEBUG) << " ReadFromBaseDevice...: new-block: " << cow_op->new_block
<< " Source: " << offset;
if (!android::base::ReadFullyAtOffset(backing_store_fd_, buffer, BLOCK_SZ, offset)) {
SNAP_PLOG(ERROR) << "Copy op failed. Read from backing store: " << backing_store_device_
<< "at block :" << offset / BLOCK_SZ << " offset:" << offset % BLOCK_SZ;
@ -508,7 +509,7 @@ int WorkerThread::GetNumberOfMergedOps(void* merged_buffer, void* unmerged_buffe
if (read_ahead_buffer_map.find(cow_op->new_block) == read_ahead_buffer_map.end()) {
SNAP_LOG(ERROR)
<< " Block: " << cow_op->new_block << " not found in read-ahead cache"
<< " Source: " << cow_op->source;
<< " Op: " << *cow_op;
return -1;
}
// If this is a final block merged in the read-ahead buffer

11
fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_dm_user.cpp
View file

@ -94,12 +94,13 @@ bool Worker::ReadFromSourceDevice(const CowOperation* cow_op) {
SNAP_LOG(ERROR) << "ReadFromBaseDevice: Failed to get payload buffer";
return false;
}
SNAP_LOG(DEBUG) << " ReadFromBaseDevice...: new-block: " << cow_op->new_block
<< " Source: " << cow_op->source;
uint64_t offset = cow_op->source;
if (cow_op->type == kCowCopyOp) {
offset *= BLOCK_SZ;
uint64_t offset;
if (!reader_->GetSourceOffset(cow_op, &offset)) {
SNAP_LOG(ERROR) << "ReadFromSourceDevice: Failed to get source offset";
return false;
}
SNAP_LOG(DEBUG) << " ReadFromBaseDevice...: new-block: " << cow_op->new_block
<< " Op: " << *cow_op;
if (!android::base::ReadFullyAtOffset(backing_store_fd_, buffer, BLOCK_SZ, offset)) {
std::string op;
if (cow_op->type == kCowCopyOp)

33
fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
View file

@ -34,14 +34,17 @@ ReadAhead::ReadAhead(const std::string& cow_device, const std::string& backing_d
}
void ReadAhead::CheckOverlap(const CowOperation* cow_op) {
uint64_t source_block = cow_op->source;
uint64_t source_offset = 0;
if (cow_op->type == kCowXorOp) {
source_block /= BLOCK_SZ;
source_offset = cow_op->source % BLOCK_SZ;
uint64_t source_offset;
if (!reader_->GetSourceOffset(cow_op, &source_offset)) {
SNAP_LOG(ERROR) << "ReadAhead operation has no source offset: " << *cow_op;
return;
}
uint64_t source_block = GetBlockFromOffset(header_, source_offset);
bool misaligned = (GetBlockRelativeOffset(header_, source_offset) != 0);
if (dest_blocks_.count(cow_op->new_block) || source_blocks_.count(source_block) ||
(source_offset > 0 && source_blocks_.count(source_block + 1))) {
(misaligned && source_blocks_.count(source_block + 1))) {
overlap_ = true;
}
@ -66,11 +69,12 @@ int ReadAhead::PrepareNextReadAhead(uint64_t* source_offset, int* pending_ops,
// Get the first block with offset
const CowOperation* cow_op = GetRAOpIter();
*source_offset = cow_op->source;
if (cow_op->type == kCowCopyOp) {
*source_offset *= BLOCK_SZ;
} else if (cow_op->type == kCowXorOp) {
if (!reader_->GetSourceOffset(cow_op, source_offset)) {
SNAP_LOG(ERROR) << "PrepareNextReadAhead operation has no source offset: " << *cow_op;
return nr_consecutive;
}
if (cow_op->type == kCowXorOp) {
xor_op_vec.push_back(cow_op);
}
@ -88,10 +92,10 @@ int ReadAhead::PrepareNextReadAhead(uint64_t* source_offset, int* pending_ops,
*/
while (!RAIterDone() && num_ops) {
const CowOperation* op = GetRAOpIter();
uint64_t next_offset = op->source;
if (cow_op->type == kCowCopyOp) {
next_offset *= BLOCK_SZ;
uint64_t next_offset;
if (!reader_->GetSourceOffset(op, &next_offset)) {
SNAP_LOG(ERROR) << "PrepareNextReadAhead operation has no source offset: " << *cow_op;
break;
}
// Check for consecutive blocks
@ -803,6 +807,7 @@ bool ReadAhead::InitReader() {
if (!reader_->InitForMerge(std::move(cow_fd_))) {
return false;
}
header_ = reader_->GetHeader();
return true;
}

1
fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.h
View file

@ -85,6 +85,7 @@ class ReadAhead {
std::shared_ptr<SnapshotHandler> snapuserd_;
std::unique_ptr<CowReader> reader_;
CowHeader header_;
std::unordered_set<uint64_t> dest_blocks_;
std::unordered_set<uint64_t> source_blocks_;