tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "libsnapshot: Retrieve base sector for dm-user device creation"

Browse source
This commit is contained in:
Akilesh Kailash 2020-11-12 00:45:21 +00:00 committed by Gerrit Code Review
commit 70d27113d0
8 changed files with 155 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
fs_mgr/libsnapshot/cow_snapuserd_test.cpp
View file

@ -108,6 +108,8 @@ class SnapuserdTest : public ::testing::Test {
std::unique_ptr<uint8_t[]> product_buffer_;
void Init();
void InitCowDevices();
void InitDaemon();
void CreateCowDevice(std::unique_ptr<TemporaryFile>& cow);
void CreateSystemDmUser(std::unique_ptr<TemporaryFile>& cow);
void CreateProductDmUser(std::unique_ptr<TemporaryFile>& cow);
@ -238,12 +240,6 @@ void SnapuserdTest::CreateSystemDmUser(std::unique_ptr<TemporaryFile>& cow) {
system_device_name_.clear();
system_device_ctrl_name_.clear();
// Create a COW device. Number of sectors is chosen random which can
// hold at least 400MB of data
int err = ioctl(sys_fd_.get(), BLKGETSIZE, &system_blksize_);
ASSERT_GE(err, 0);
std::string str(cow->path);
std::size_t found = str.find_last_of("/\\");
ASSERT_NE(found, std::string::npos);
@ -280,12 +276,6 @@ void SnapuserdTest::CreateProductDmUser(std::unique_ptr<TemporaryFile>& cow) {
product_device_name_.clear();
product_device_ctrl_name_.clear();
// Create a COW device. Number of sectors is chosen random which can
// hold at least 400MB of data
int err = ioctl(product_fd_.get(), BLKGETSIZE, &product_blksize_);
ASSERT_GE(err, 0);
std::string str(cow->path);
std::size_t found = str.find_last_of("/\\");
ASSERT_NE(found, std::string::npos);
@ -297,12 +287,15 @@ void SnapuserdTest::CreateProductDmUser(std::unique_ptr<TemporaryFile>& cow) {
system(cmd.c_str());
}
void SnapuserdTest::StartSnapuserdDaemon() {
ASSERT_TRUE(EnsureSnapuserdStarted());
void SnapuserdTest::InitCowDevices() {
system_blksize_ = client_->InitDmUserCow(cow_system_->path);
ASSERT_NE(system_blksize_, 0);
client_ = SnapuserdClient::Connect(kSnapuserdSocket, 5s);
ASSERT_NE(client_, nullptr);
product_blksize_ = client_->InitDmUserCow(cow_product_->path);
ASSERT_NE(product_blksize_, 0);
}
void SnapuserdTest::InitDaemon() {
bool ok = client_->InitializeSnapuserd(cow_system_->path, system_a_loop_->device(),
GetSystemControlPath());
ASSERT_TRUE(ok);
@ -312,6 +305,13 @@ void SnapuserdTest::StartSnapuserdDaemon() {
ASSERT_TRUE(ok);
}
void SnapuserdTest::StartSnapuserdDaemon() {
ASSERT_TRUE(EnsureSnapuserdStarted());
client_ = SnapuserdClient::Connect(kSnapuserdSocket, 5s);
ASSERT_NE(client_, nullptr);
}
void SnapuserdTest::CreateSnapshotDevices() {
std::string cmd;
@ -435,10 +435,13 @@ TEST_F(SnapuserdTest, ReadWrite) {
CreateCowDevice(cow_system_);
CreateCowDevice(cow_product_);
StartSnapuserdDaemon();
InitCowDevices();
CreateSystemDmUser(cow_system_);
CreateProductDmUser(cow_product_);
StartSnapuserdDaemon();
InitDaemon();
CreateSnapshotDevices();

19
fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h
View file

@ -61,23 +61,19 @@ class BufferSink : public IByteSink {
class Snapuserd final {
public:
Snapuserd(const std::string& in_cow_device, const std::string& in_backing_store_device,
const std::string& in_control_device)
: cow_device_(in_cow_device),
backing_store_device_(in_backing_store_device),
control_device_(in_control_device),
metadata_read_done_(false) {}
bool Init();
bool InitBackingAndControlDevice(std::string& backing_device, std::string& control_device);
bool InitCowDevice(std::string& cow_device);
int Run();
const std::string& GetControlDevicePath() { return control_device_; }
const std::string& GetCowDevice() { return cow_device_; }
uint64_t GetNumSectors() { return num_sectors_; }
private:
int ReadDmUserHeader();
bool ReadDmUserPayload(void* buffer, size_t size);
int WriteDmUserPayload(size_t size);
int ConstructKernelCowHeader();
int ReadMetadata();
bool ReadMetadata();
int ZerofillDiskExceptions(size_t read_size);
int ReadDiskExceptions(chunk_t chunk, size_t size);
int ReadData(chunk_t chunk, size_t size);
@ -94,6 +90,8 @@ class Snapuserd final {
int unmerged_exceptions);
bool AdvanceMergedOps(int merged_ops_cur_iter);
bool ProcessMergeComplete(chunk_t chunk, void* buffer);
sector_t ChunkToSector(chunk_t chunk) { return chunk << CHUNK_SHIFT; }
chunk_t SectorToChunk(sector_t sector) { return sector >> CHUNK_SHIFT; }
std::string cow_device_;
std::string backing_store_device_;
@ -104,6 +102,7 @@ class Snapuserd final {
unique_fd ctrl_fd_;
uint32_t exceptions_per_area_;
uint64_t num_sectors_;
std::unique_ptr<ICowOpIter> cowop_iter_;
std::unique_ptr<ICowOpReverseIter> cowop_riter_;
@ -118,7 +117,7 @@ class Snapuserd final {
// Value - cow operation
std::unordered_map<chunk_t, const CowOperation*> chunk_map_;
bool metadata_read_done_;
bool metadata_read_done_ = false;
BufferSink bufsink_;
};

2
fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h
View file

@ -58,7 +58,7 @@ class SnapuserdClient {
std::chrono::milliseconds timeout_ms);
bool StopSnapuserd();
int RestartSnapuserd(std::vector<std::vector<std::string>>& vec);
uint64_t InitDmUserCow(const std::string& cow_device);
bool InitializeSnapuserd(const std::string& cow_device, const std::string& backing_device,
const std::string& control_device);

1
fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h
View file

@ -36,6 +36,7 @@ namespace snapshot {
static constexpr uint32_t MAX_PACKET_SIZE = 512;
enum class DaemonOperations {
INIT,
START,
QUERY,
STOP,

41
fs_mgr/libsnapshot/snapshot.cpp
View file

@ -386,24 +386,6 @@ bool SnapshotManager::MapDmUserCow(LockedFile* lock, const std::string& name,
auto& dm = DeviceMapper::Instance();
// Use the size of the base device for the COW device. It doesn't really
// matter, it just needs to look similar enough so the kernel doesn't complain
// about alignment or being too small.
uint64_t base_sectors = 0;
{
unique_fd fd(open(base_device.c_str(), O_RDONLY | O_CLOEXEC));
if (fd < 0) {
PLOG(ERROR) << "open failed: " << base_device;
return false;
}
auto dev_size = get_block_device_size(fd);
if (!dev_size) {
PLOG(ERROR) << "Could not determine block device size: " << base_device;
return false;
}
base_sectors = dev_size / kSectorSize;
}
// Use an extra decoration for first-stage init, so we can transition
// to a new table entry in second-stage.
std::string misc_name = name;
@ -411,13 +393,19 @@ bool SnapshotManager::MapDmUserCow(LockedFile* lock, const std::string& name,
misc_name += "-init";
}
DmTable table;
table.Emplace<DmTargetUser>(0, base_sectors, misc_name);
if (!dm.CreateDevice(name, table, path, timeout_ms)) {
if (!EnsureSnapuserdConnected()) {
return false;
}
if (!EnsureSnapuserdConnected()) {
uint64_t base_sectors = snapuserd_client_->InitDmUserCow(cow_file);
if (base_sectors == 0) {
LOG(ERROR) << "Failed to retrieve base_sectors from Snapuserd";
return false;
}
DmTable table;
table.Emplace<DmTargetUser>(0, base_sectors, misc_name);
if (!dm.CreateDevice(name, table, path, timeout_ms)) {
return false;
}
@ -1366,6 +1354,15 @@ bool SnapshotManager::PerformSecondStageTransition() {
continue;
}
uint64_t base_sectors = snapuserd_client_->InitDmUserCow(cow_device);
if (base_sectors == 0) {
// Unrecoverable as metadata reads from cow device failed
LOG(FATAL) << "Failed to retrieve base_sectors from Snapuserd";
return false;
}
CHECK(base_sectors == target.spec.length);
if (!snapuserd_client_->InitializeSnapuserd(cow_device, backing_device, control_device)) {
// This error is unrecoverable. We cannot proceed because reads to
// the underlying device will fail.

70
fs_mgr/libsnapshot/snapuserd.cpp
View file

@ -505,7 +505,7 @@ chunk_t Snapuserd::GetNextAllocatableChunkId(chunk_t chunk) {
* exceptions_per_area_
* 12: Kernel will stop issuing metadata IO request when new-chunk ID is 0.
*/
int Snapuserd::ReadMetadata() {
bool Snapuserd::ReadMetadata() {
reader_ = std::make_unique<CowReader>();
CowHeader header;
CowOptions options;
@ -516,12 +516,12 @@ int Snapuserd::ReadMetadata() {
if (!reader_->Parse(cow_fd_)) {
LOG(ERROR) << "Failed to parse";
return 1;
return false;
}
if (!reader_->GetHeader(&header)) {
LOG(ERROR) << "Failed to get header";
return 1;
return false;
}
CHECK(header.block_size == BLOCK_SIZE);
@ -563,7 +563,7 @@ int Snapuserd::ReadMetadata() {
if (!(cow_op->type == kCowReplaceOp || cow_op->type == kCowZeroOp ||
cow_op->type == kCowCopyOp)) {
LOG(ERROR) << "Unknown operation-type found: " << cow_op->type;
return 1;
return false;
}
metadata_found = true;
@ -609,16 +609,24 @@ int Snapuserd::ReadMetadata() {
}
// Partially filled area or there is no metadata
// If there is no metadata, fill with zero so that kernel
// is aware that merge is completed.
if (num_ops || !metadata_found) {
vec_.push_back(std::move(de_ptr));
LOG(DEBUG) << "ReadMetadata() completed. Partially filled area num_ops: " << num_ops
<< "Areas : " << vec_.size();
}
LOG(DEBUG) << "ReadMetadata() completed. chunk_id: " << next_free
<< "Num Sector: " << ChunkToSector(next_free);
// Initialize the iterator for merging
cowop_iter_ = reader_->GetOpIter();
return 0;
// Total number of sectors required for creating dm-user device
num_sectors_ = ChunkToSector(next_free);
metadata_read_done_ = true;
return true;
}
void MyLogger(android::base::LogId, android::base::LogSeverity severity, const char*, const char*,
@ -664,12 +672,8 @@ bool Snapuserd::ReadDmUserPayload(void* buffer, size_t size) {
return true;
}
bool Snapuserd::Init() {
backing_store_fd_.reset(open(backing_store_device_.c_str(), O_RDONLY));
if (backing_store_fd_ < 0) {
PLOG(ERROR) << "Open Failed: " << backing_store_device_;
return false;
}
bool Snapuserd::InitCowDevice(std::string& cow_device) {
cow_device_ = cow_device;
cow_fd_.reset(open(cow_device_.c_str(), O_RDWR));
if (cow_fd_ < 0) {
@ -677,16 +681,6 @@ bool Snapuserd::Init() {
return false;
}
std::string control_path = GetControlDevicePath();
LOG(DEBUG) << "Opening control device " << control_path;
ctrl_fd_.reset(open(control_path.c_str(), O_RDWR));
if (ctrl_fd_ < 0) {
PLOG(ERROR) << "Unable to open " << control_path;
return false;
}
// Allocate the buffer which is used to communicate between
// daemon and dm-user. The buffer comprises of header and a fixed payload.
// If the dm-user requests a big IO, the IO will be broken into chunks
@ -694,6 +688,26 @@ bool Snapuserd::Init() {
size_t buf_size = sizeof(struct dm_user_header) + PAYLOAD_SIZE;
bufsink_.Initialize(buf_size);
return ReadMetadata();
}
bool Snapuserd::InitBackingAndControlDevice(std::string& backing_device,
std::string& control_device) {
backing_store_device_ = backing_device;
control_device_ = control_device;
backing_store_fd_.reset(open(backing_store_device_.c_str(), O_RDONLY));
if (backing_store_fd_ < 0) {
PLOG(ERROR) << "Open Failed: " << backing_store_device_;
return false;
}
ctrl_fd_.reset(open(control_device_.c_str(), O_RDWR));
if (ctrl_fd_ < 0) {
PLOG(ERROR) << "Unable to open " << control_device_;
return false;
}
return true;
}
@ -729,15 +743,7 @@ int Snapuserd::Run() {
// never see multiple IO requests. Additionally this IO
// will always be a single 4k.
if (header->sector == 0) {
// Read the metadata from internal COW device
// and build the in-memory data structures
// for all the operations in the internal COW.
if (!metadata_read_done_ && ReadMetadata()) {
LOG(ERROR) << "Metadata read failed";
return 1;
}
metadata_read_done_ = true;
CHECK(metadata_read_done_ == true);
CHECK(read_size == BLOCK_SIZE);
ret = ConstructKernelCowHeader();
if (ret < 0) return ret;
@ -747,7 +753,7 @@ int Snapuserd::Run() {
// Check if the chunk ID represents a metadata
// page. If the chunk ID is not found in the
// vector, then it points to a metadata page.
chunk_t chunk = (header->sector >> CHUNK_SHIFT);
chunk_t chunk = SectorToChunk(header->sector);
if (chunk_map_.find(chunk) == chunk_map_.end()) {
ret = ReadDiskExceptions(chunk, read_size);
@ -789,7 +795,7 @@ int Snapuserd::Run() {
size_t read_size = std::min(PAYLOAD_SIZE, remaining_size);
CHECK(read_size == BLOCK_SIZE);
CHECK(header->sector > 0);
chunk_t chunk = (header->sector >> CHUNK_SHIFT);
chunk_t chunk = SectorToChunk(header->sector);
CHECK(chunk_map_.find(chunk) == chunk_map_.end());
void* buffer = bufsink_.GetPayloadBuffer(read_size);

81
fs_mgr/libsnapshot/snapuserd_client.cpp
View file

@ -27,9 +27,12 @@
#include <unistd.h>
#include <chrono>
#include <sstream>
#include <android-base/logging.h>
#include <android-base/parseint.h>
#include <android-base/properties.h>
#include <android-base/strings.h>
#include <libsnapshot/snapuserd_client.h>
namespace android {
@ -114,7 +117,7 @@ bool SnapuserdClient::ValidateConnection() {
std::string str = Receivemsg();
// If the daemon is passive then fallback to secondary active daemon. Daemon
// is passive during transition phase. Please see RestartSnapuserd()
// is passive during transition phase.
if (str.find("passive") != std::string::npos) {
LOG(ERROR) << "Snapuserd is terminating";
return false;
@ -199,77 +202,31 @@ bool SnapuserdClient::InitializeSnapuserd(const std::string& cow_device,
return true;
}
/*
* Transition from first stage snapuserd daemon to second stage daemon involves
* series of steps viz:
*
* 1: Create new dm-user devices - This is done by libsnapshot
*
* 2: Spawn the new snapuserd daemon - This is the second stage daemon which
* will start the server but the dm-user misc devices is not binded yet.
*
* 3: Vector to this function contains pair of cow_device and source device.
* Ex: {{system_cow,system_a}, {product_cow, product_a}, {vendor_cow,
* vendor_a}}. This vector will be populated by the libsnapshot.
*
* 4: Initialize the Second stage daemon passing the information from the
* vector. This will bind the daemon with dm-user misc device and will be ready
* to serve the IO. Up until this point, first stage daemon is still active.
* However, client library will mark the first stage daemon as passive and hence
* all the control message from hereon will be sent to active second stage
* daemon.
*
* 5: Create new dm-snapshot table. This is done by libsnapshot. When new table
* is created, kernel will issue metadata read once again which will be served
* by second stage daemon. However, any active IO will still be served by first
* stage daemon.
*
* 6: Swap the snapshot table atomically - This is done by libsnapshot. Once
* the swapping is done, all the IO will be served by second stage daemon.
*
* 7: Stop the first stage daemon. After this point second stage daemon is
* completely active to serve the IO and merging process.
*
*/
int SnapuserdClient::RestartSnapuserd(std::vector<std::vector<std::string>>& vec) {
std::string msg = "terminate-request";
uint64_t SnapuserdClient::InitDmUserCow(const std::string& cow_device) {
std::string msg = "init," + cow_device;
if (!Sendmsg(msg)) {
LOG(ERROR) << "Failed to send message " << msg << " to snapuserd daemon";
return -1;
return 0;
}
std::string str = Receivemsg();
if (str.find("fail") != std::string::npos) {
LOG(ERROR) << "Failed to receive ack for " << msg << " from snapuserd daemon";
return -1;
std::vector<std::string> input = android::base::Split(str, ",");
if (input[0] != "success") {
LOG(ERROR) << "Failed to receive number of sectors for " << msg << " from snapuserd daemon";
return 0;
}
CHECK(str.find("success") != std::string::npos);
LOG(DEBUG) << "Snapuserd daemon COW device initialized: " << cow_device
<< " Num-sectors: " << input[1];
// Start the new daemon
if (!EnsureSnapuserdStarted()) {
LOG(ERROR) << "Failed to start new daemon";
return -1;
uint64_t num_sectors = 0;
if (!android::base::ParseUint(input[1], &num_sectors)) {
LOG(ERROR) << "Failed to parse input string to sectors";
return 0;
}
LOG(DEBUG) << "Second stage Snapuserd daemon created successfully";
// Vector contains all the device information to be passed to the new
// daemon. Note that the caller can choose to initialize separately
// by calling InitializeSnapuserd() API as well. In that case, vector
// should be empty
for (int i = 0; i < vec.size(); i++) {
std::string& cow_device = vec[i][0];
std::string& base_device = vec[i][1];
std::string& control_device = vec[i][2];
InitializeSnapuserd(cow_device, base_device, control_device);
LOG(DEBUG) << "Daemon initialized with " << cow_device << ", " << base_device << " and "
<< control_device;
}
return 0;
return num_sectors;
}
} // namespace snapshot

61
fs_mgr/libsnapshot/snapuserd_server.cpp
View file

@ -33,6 +33,7 @@ namespace android {
namespace snapshot {
DaemonOperations SnapuserdServer::Resolveop(std::string& input) {
if (input == "init") return DaemonOperations::INIT;
if (input == "start") return DaemonOperations::START;
if (input == "stop") return DaemonOperations::STOP;
if (input == "query") return DaemonOperations::QUERY;
@ -123,6 +124,32 @@ bool SnapuserdServer::Receivemsg(android::base::borrowed_fd fd, const std::strin
DaemonOperations op = Resolveop(out[0]);
switch (op) {
case DaemonOperations::INIT: {
// Message format:
// init,<cow_device_path>
//
// Reads the metadata and send the number of sectors
if (out.size() != 2) {
LOG(ERROR) << "Malformed init message, " << out.size() << " parts";
return Sendmsg(fd, "fail");
}
auto snapuserd = std::make_unique<Snapuserd>();
if (!snapuserd->InitCowDevice(out[1])) {
LOG(ERROR) << "Failed to initialize Snapuserd";
return Sendmsg(fd, "fail");
}
std::string retval = "success," + std::to_string(snapuserd->GetNumSectors());
auto handler = std::make_unique<DmUserHandler>(std::move(snapuserd));
{
std::lock_guard<std::mutex> lock(lock_);
dm_users_.push_back(std::move(handler));
}
return Sendmsg(fd, retval);
}
case DaemonOperations::START: {
// Message format:
// start,<cow_device_path>,<source_device_path>,<control_device>
@ -133,21 +160,29 @@ bool SnapuserdServer::Receivemsg(android::base::borrowed_fd fd, const std::strin
return Sendmsg(fd, "fail");
}
auto snapuserd = std::make_unique<Snapuserd>(out[1], out[2], out[3]);
if (!snapuserd->Init()) {
LOG(ERROR) << "Failed to initialize Snapuserd";
return Sendmsg(fd, "fail");
}
auto handler = std::make_unique<DmUserHandler>(std::move(snapuserd));
bool found = false;
{
std::lock_guard<std::mutex> lock(lock_);
handler->thread() =
std::thread(std::bind(&SnapuserdServer::RunThread, this, handler.get()));
dm_users_.push_back(std::move(handler));
auto iter = dm_users_.begin();
while (iter != dm_users_.end()) {
if ((*iter)->snapuserd()->GetCowDevice() == out[1]) {
if (!((*iter)->snapuserd()->InitBackingAndControlDevice(out[2], out[3]))) {
LOG(ERROR) << "Failed to initialize control device: " << out[3];
break;
}
(*iter)->thread() = std::thread(
std::bind(&SnapuserdServer::RunThread, this, (*iter).get()));
found = true;
break;
}
iter++;
}
}
if (found) {
return Sendmsg(fd, "success");
} else {
return Sendmsg(fd, "fail");
}
return Sendmsg(fd, "success");
}
case DaemonOperations::STOP: {
// Message format: stop
@ -172,7 +207,7 @@ bool SnapuserdServer::Receivemsg(android::base::borrowed_fd fd, const std::strin
}
case DaemonOperations::DELETE: {
// Message format:
// delete,<cow_device_path>
// delete,<control_device_path>
if (out.size() != 2) {
LOG(ERROR) << "Malformed delete message, " << out.size() << " parts";
return Sendmsg(fd, "fail");