tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

libsnapshot: Refactor COW reading/writing of footers.

Browse source 
This simplifies the footer and label resume system for COW files.
Previously, a footer could be missing, and we'd try to recover the file.
Any ops we could recover were handed off to the CowWriter, which then
searched up to the correct resume label.

In the new model, a reader is opened with an optional label (resume
mode), or without a label.

When resuming to a label, a footer is optional. The read is terminated
by EOF, discovery of the requested label, or the presence of a footer.
However the requested label must be found, and parsing fails if it is
not found.

When opening a COW without a label, the footer must be present, as this
signals the file was successfully flushed. Parsing fails if no footer is
found.

Bug: N/A
Test: cow_api_test
      full OTA with VABC
Change-Id: Ie79ab5259f532dd16c72f0e42da7568c5c5c4623
This commit is contained in:
David Anderson 2020-11-09 20:40:06 -08:00
parent ce69f2985b
commit ea79136740
4 changed files with 79 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
fs_mgr/libsnapshot/cow_api_test.cpp
View file

@ -264,10 +264,7 @@ TEST_F(CowTest, GetSize) {
ASSERT_EQ(size_before, size_after);
struct stat buf;
if (fstat(cow_->fd, &buf) < 0) {
perror("Fails to determine size of cow image written");
FAIL();
}
ASSERT_GE(fstat(cow_->fd, &buf), 0) << strerror(errno);
ASSERT_EQ(buf.st_size, writer.GetCowSize());
}
@ -408,7 +405,7 @@ TEST_F(CowTest, AppendExtendedCorrupted) {
// Get the last known good label
CowReader label_reader;
uint64_t label;
ASSERT_TRUE(label_reader.Parse(cow_->fd));
ASSERT_TRUE(label_reader.Parse(cow_->fd, {5}));
ASSERT_TRUE(label_reader.GetLastLabel(&label));
ASSERT_EQ(label, 5);

137
fs_mgr/libsnapshot/cow_reader.cpp
View file

@ -31,14 +31,7 @@
namespace android {
namespace snapshot {
CowReader::CowReader()
: fd_(-1),
header_(),
footer_(),
fd_size_(0),
has_footer_(false),
last_label_(0),
has_last_label_(false) {}
CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {}
static void SHA256(const void*, size_t, uint8_t[]) {
#if 0
@ -49,12 +42,12 @@ static void SHA256(const void*, size_t, uint8_t[]) {
#endif
}
bool CowReader::Parse(android::base::unique_fd&& fd) {
bool CowReader::Parse(android::base::unique_fd&& fd, std::optional<uint64_t> label) {
owned_fd_ = std::move(fd);
return Parse(android::base::borrowed_fd{owned_fd_});
return Parse(android::base::borrowed_fd{owned_fd_}, label);
}
bool CowReader::Parse(android::base::borrowed_fd fd) {
bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label) {
fd_ = fd;
auto pos = lseek(fd_.get(), 0, SEEK_END);
@ -99,105 +92,107 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
return false;
}
auto footer_pos = lseek(fd_.get(), -header_.footer_size, SEEK_END);
if (footer_pos != fd_size_ - header_.footer_size) {
LOG(ERROR) << "Failed to read full footer!";
return false;
}
if (!android::base::ReadFully(fd_, &footer_, sizeof(footer_))) {
PLOG(ERROR) << "read footer failed";
return false;
}
has_footer_ = (footer_.op.type == kCowFooterOp);
return ParseOps();
return ParseOps(label);
}
bool CowReader::ParseOps() {
bool CowReader::ParseOps(std::optional<uint64_t> label) {
uint64_t pos = lseek(fd_.get(), sizeof(header_), SEEK_SET);
if (pos != sizeof(header_)) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
std::optional<uint64_t> next_last_label;
auto ops_buffer = std::make_shared<std::vector<CowOperation>>();
if (has_footer_) ops_buffer->reserve(footer_.op.num_ops);
uint64_t current_op_num = 0;
// Look until we reach the last possible non-footer position.
uint64_t last_pos = fd_size_ - (has_footer_ ? sizeof(footer_) : sizeof(CowOperation));
// Alternating op and data
while (pos < last_pos) {
ops_buffer->resize(current_op_num + 1);
if (!android::base::ReadFully(fd_, ops_buffer->data() + current_op_num,
sizeof(CowOperation))) {
while (true) {
ops_buffer->emplace_back();
if (!android::base::ReadFully(fd_, &ops_buffer->back(), sizeof(CowOperation))) {
PLOG(ERROR) << "read op failed";
return false;
}
auto& current_op = ops_buffer->data()[current_op_num];
pos = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR);
if (pos == uint64_t(-1)) {
auto& current_op = ops_buffer->back();
off_t offs = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR);
if (offs < 0) {
PLOG(ERROR) << "lseek next op failed";
return false;
}
current_op_num++;
if (next_last_label) {
last_label_ = next_last_label.value();
has_last_label_ = true;
}
pos = static_cast<uint64_t>(offs);
if (current_op.type == kCowLabelOp) {
// If we don't have a footer, the last label may be incomplete.
// If we see any operation after it, we can infer the flush finished.
if (has_footer_) {
has_last_label_ = true;
last_label_ = current_op.source;
} else {
next_last_label = {current_op.source};
last_label_ = {current_op.source};
// If we reach the requested label, stop reading.
if (label && label.value() == current_op.source) {
break;
}
} else if (current_op.type == kCowFooterOp) {
memcpy(&footer_.op, &current_op, sizeof(footer_.op));
// we don't consider this an operation for the checksum
current_op_num--;
if (android::base::ReadFully(fd_, &footer_.data, sizeof(footer_.data))) {
has_footer_ = true;
if (next_last_label) {
last_label_ = next_last_label.value();
has_last_label_ = true;
}
footer_.emplace();
CowFooter* footer = &footer_.value();
memcpy(&footer_->op, &current_op, sizeof(footer->op));
if (!android::base::ReadFully(fd_, &footer->data, sizeof(footer->data))) {
LOG(ERROR) << "Could not read COW footer";
return false;
}
// Drop the footer from the op stream.
ops_buffer->pop_back();
break;
}
}
// To successfully parse a COW file, we need either:
// (1) a label to read up to, and for that label to be found, or
// (2) a valid footer.
if (label) {
if (!last_label_) {
LOG(ERROR) << "Did not find label " << label.value()
<< " while reading COW (no labels found)";
return false;
}
if (last_label_.value() != label.value()) {
LOG(ERROR) << "Did not find label " << label.value()
<< ", last label=" << last_label_.value();
return false;
}
} else if (!footer_) {
LOG(ERROR) << "No COW footer found";
return false;
}
uint8_t csum[32];
memset(csum, 0, sizeof(uint8_t) * 32);
if (has_footer_) {
if (ops_buffer->size() != footer_.op.num_ops) {
if (footer_) {
if (ops_buffer->size() != footer_->op.num_ops) {
LOG(ERROR) << "num ops does not match";
return false;
}
if (ops_buffer->size() * sizeof(CowOperation) != footer_.op.ops_size) {
if (ops_buffer->size() * sizeof(CowOperation) != footer_->op.ops_size) {
LOG(ERROR) << "ops size does not match ";
return false;
}
SHA256(&footer_.op, sizeof(footer_.op), footer_.data.footer_checksum);
if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) {
SHA256(&footer_->op, sizeof(footer_->op), footer_->data.footer_checksum);
if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return false;
}
SHA256(ops_buffer.get()->data(), footer_.op.ops_size, csum);
if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) {
SHA256(ops_buffer.get()->data(), footer_->op.ops_size, csum);
if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return false;
}
} else {
LOG(INFO) << "No Footer, recovered data";
LOG(INFO) << "No COW Footer, recovered data";
}
if (header_.num_merge_ops > 0) {
uint64_t merge_ops = header_.num_merge_ops;
uint64_t metadata_ops = 0;
current_op_num = 0;
uint64_t current_op_num = 0;
CHECK(ops_buffer->size() >= merge_ops);
while (merge_ops) {
@ -223,14 +218,14 @@ bool CowReader::GetHeader(CowHeader* header) {
}
bool CowReader::GetFooter(CowFooter* footer) {
if (!has_footer_) return false;
*footer = footer_;
if (!footer_) return false;
*footer = footer_.value();
return true;
}
bool CowReader::GetLastLabel(uint64_t* label) {
if (!has_last_label_) return false;
*label = last_label_;
if (!last_label_) return false;
*label = last_label_.value();
return true;
}
@ -308,8 +303,8 @@ std::unique_ptr<ICowOpReverseIter> CowReader::GetRevOpIter() {
bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {
// Validate the offset, taking care to acknowledge possible overflow of offset+len.
if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(footer_) || len >= fd_size_ ||
offset + len > fd_size_ - sizeof(footer_)) {
if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(CowFooter) || len >= fd_size_ ||
offset + len > fd_size_ - sizeof(CowFooter)) {
LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";
return false;
}

18
fs_mgr/libsnapshot/cow_writer.cpp
View file

@ -191,12 +191,10 @@ bool CowWriter::OpenForWrite() {
bool CowWriter::OpenForAppend(uint64_t label) {
auto reader = std::make_unique<CowReader>();
std::queue<CowOperation> toAdd;
bool found_label = false;
if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {
if (!reader->Parse(fd_, {label}) || !reader->GetHeader(&header_)) {
return false;
}
reader->GetFooter(&footer_);
options_.block_size = header_.block_size;
@ -206,21 +204,11 @@ bool CowWriter::OpenForAppend(uint64_t label) {
ops_.resize(0);
auto iter = reader->GetOpIter();
while (!iter->Done() && !found_label) {
const CowOperation& op = iter->Get();
if (op.type == kCowFooterOp) break;
if (op.type == kCowLabelOp && op.source == label) found_label = true;
AddOperation(op);
while (!iter->Done()) {
AddOperation(iter->Get());
iter->Next();
}
if (!found_label) {
LOG(ERROR) << "Failed to find last label";
return false;
}
// Free reader so we own the descriptor position again.
reader = nullptr;

15
fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
View file

@ -18,6 +18,7 @@
#include <functional>
#include <memory>
#include <optional>
#include <android-base/unique_fd.h>
#include <libsnapshot/cow_format.h>
@ -116,8 +117,10 @@ class CowReader : public ICowReader {
public:
CowReader();
bool Parse(android::base::unique_fd&& fd);
bool Parse(android::base::borrowed_fd fd);
// Parse the COW, optionally, up to the given label. If no label is
// specified, the COW must have an intact footer.
bool Parse(android::base::unique_fd&& fd, std::optional<uint64_t> label = {});
bool Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label = {});
bool GetHeader(CowHeader* header) override;
bool GetFooter(CowFooter* footer) override;
@ -138,16 +141,14 @@ class CowReader : public ICowReader {
void UpdateMergeProgress(uint64_t merge_ops) { header_.num_merge_ops += merge_ops; }
private:
bool ParseOps();
bool ParseOps(std::optional<uint64_t> label);
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
CowHeader header_;
CowFooter footer_;
std::optional<CowFooter> footer_;
uint64_t fd_size_;
bool has_footer_;
uint64_t last_label_;
bool has_last_label_;
std::optional<uint64_t> last_label_;
std::shared_ptr<std::vector<CowOperation>> ops_;
};