tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "libfiemap: Create/Open returns FiemapStatus"

Browse source
This commit is contained in:
Yifan Hong 2020-01-08 01:00:43 +00:00 committed by Gerrit Code Review
commit 0ed21b604c
10 changed files with 220 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
fs_mgr/libfiemap/Android.bp
View file

@ -24,6 +24,7 @@ filegroup {
name: "libfiemap_srcs",
srcs: [
"fiemap_writer.cpp",
"fiemap_status.cpp",
"image_manager.cpp",
"metadata.cpp",
"split_fiemap_writer.cpp",

42
fs_mgr/libfiemap/fiemap_status.cpp Normal file
View file

@ -0,0 +1,42 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <libfiemap/fiemap_status.h>
namespace android::fiemap {
// FiemapStatus -> string
std::string FiemapStatus::string() const {
if (error_code() == ErrorCode::ERROR) {
return "Error";
}
return strerror(-static_cast<int>(error_code()));
}
// -errno -> known ErrorCode
// unknown ErrorCode -> known ErrorCode
FiemapStatus::ErrorCode FiemapStatus::CastErrorCode(int error_code) {
switch (error_code) {
case static_cast<int32_t>(ErrorCode::SUCCESS):
case static_cast<int32_t>(ErrorCode::NO_SPACE):
return static_cast<ErrorCode>(error_code);
case static_cast<int32_t>(ErrorCode::ERROR):
default:
return ErrorCode::ERROR;
}
}
} // namespace android::fiemap

105
fs_mgr/libfiemap/fiemap_writer.cpp
View file

@ -262,9 +262,9 @@ static bool PerformFileChecks(const std::string& file_path, uint64_t* blocksz, u
return true;
}
static bool FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_size,
const std::string& file_path,
const std::function<bool(uint64_t, uint64_t)>& on_progress) {
static FiemapStatus FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_size,
const std::string& file_path,
const std::function<bool(uint64_t, uint64_t)>& on_progress) {
// Even though this is much faster than writing zeroes, it is still slow
// enough that we need to fire the progress callback periodically. To
// easily achieve this, we seek in chunks. We use 1000 chunks since
@ -280,22 +280,22 @@ static bool FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_si
auto rv = TEMP_FAILURE_RETRY(lseek(file_fd, cursor - 1, SEEK_SET));
if (rv < 0) {
PLOG(ERROR) << "Failed to lseek " << file_path;
return false;
return FiemapStatus::FromErrno(errno);
}
if (rv != cursor - 1) {
LOG(ERROR) << "Seek returned wrong offset " << rv << " for file " << file_path;
return false;
return FiemapStatus::Error();
}
char buffer[] = {0};
if (!android::base::WriteFully(file_fd, buffer, 1)) {
PLOG(ERROR) << "Write failed: " << file_path;
return false;
return FiemapStatus::FromErrno(errno);
}
if (on_progress && !on_progress(cursor, file_size)) {
return false;
return FiemapStatus::Error();
}
}
return true;
return FiemapStatus::Ok();
}
// F2FS-specific ioctl
@ -382,19 +382,19 @@ static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type)
// write zeroes in 'blocksz' byte increments until we reach file_size to make sure the data
// blocks are actually written to by the file system and thus getting rid of the holes in the
// file.
static bool WriteZeroes(int file_fd, const std::string& file_path, size_t blocksz,
uint64_t file_size,
const std::function<bool(uint64_t, uint64_t)>& on_progress) {
static FiemapStatus WriteZeroes(int file_fd, const std::string& file_path, size_t blocksz,
uint64_t file_size,
const std::function<bool(uint64_t, uint64_t)>& on_progress) {
auto buffer = std::unique_ptr<void, decltype(&free)>(calloc(1, blocksz), free);
if (buffer == nullptr) {
LOG(ERROR) << "failed to allocate memory for writing file";
return false;
return FiemapStatus::Error();
}
off64_t offset = lseek64(file_fd, 0, SEEK_SET);
if (offset < 0) {
PLOG(ERROR) << "Failed to seek at the beginning of : " << file_path;
return false;
return FiemapStatus::FromErrno(errno);
}
int permille = -1;
@ -402,7 +402,7 @@ static bool WriteZeroes(int file_fd, const std::string& file_path, size_t blocks
if (!::android::base::WriteFully(file_fd, buffer.get(), blocksz)) {
PLOG(ERROR) << "Failed to write" << blocksz << " bytes at offset" << offset
<< " in file " << file_path;
return false;
return FiemapStatus::FromErrno(errno);
}
offset += blocksz;
@ -412,7 +412,7 @@ static bool WriteZeroes(int file_fd, const std::string& file_path, size_t blocks
int new_permille = (static_cast<uint64_t>(offset) * 1000) / file_size;
if (new_permille != permille && static_cast<uint64_t>(offset) != file_size) {
if (on_progress && !on_progress(offset, file_size)) {
return false;
return FiemapStatus::Error();
}
permille = new_permille;
}
@ -420,18 +420,18 @@ static bool WriteZeroes(int file_fd, const std::string& file_path, size_t blocks
if (lseek64(file_fd, 0, SEEK_SET) < 0) {
PLOG(ERROR) << "Failed to reset offset at the beginning of : " << file_path;
return false;
return FiemapStatus::FromErrno(errno);
}
return true;
return FiemapStatus::Ok();
}
// Reserve space for the file on the file system and write it out to make sure the extents
// don't come back unwritten. Return from this function with the kernel file offset set to 0.
// If the filesystem is f2fs, then we also PIN the file on disk to make sure the blocks
// aren't moved around.
static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz,
uint64_t file_size, unsigned int fs_type,
std::function<bool(uint64_t, uint64_t)> on_progress) {
static FiemapStatus AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz,
uint64_t file_size, unsigned int fs_type,
std::function<bool(uint64_t, uint64_t)> on_progress) {
bool need_explicit_writes = true;
switch (fs_type) {
case EXT4_SUPER_MAGIC:
@ -439,11 +439,11 @@ static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blo
case F2FS_SUPER_MAGIC: {
bool supported;
if (!F2fsPinBeforeAllocate(file_fd, &supported)) {
return false;
return FiemapStatus::Error();
}
if (supported) {
if (!PinFile(file_fd, file_path, fs_type)) {
return false;
return FiemapStatus::Error();
}
need_explicit_writes = false;
}
@ -455,29 +455,32 @@ static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blo
return FallocateFallback(file_fd, blocksz, file_size, file_path, on_progress);
default:
LOG(ERROR) << "Missing fallocate() support for file system " << fs_type;
return false;
return FiemapStatus::Error();
}
if (fallocate(file_fd, 0, 0, file_size)) {
PLOG(ERROR) << "Failed to allocate space for file: " << file_path << " size: " << file_size;
return false;
return FiemapStatus::FromErrno(errno);
}
if (need_explicit_writes && !WriteZeroes(file_fd, file_path, blocksz, file_size, on_progress)) {
return false;
if (need_explicit_writes) {
auto status = WriteZeroes(file_fd, file_path, blocksz, file_size, on_progress);
if (!status.is_ok()) {
return status;
}
}
// flush all writes here ..
if (fsync(file_fd)) {
PLOG(ERROR) << "Failed to synchronize written file:" << file_path;
return false;
return FiemapStatus::FromErrno(errno);
}
// Send one last progress notification.
if (on_progress && !on_progress(file_size, file_size)) {
return false;
return FiemapStatus::Error();
}
return true;
return FiemapStatus::Ok();
}
bool FiemapWriter::HasPinnedExtents(const std::string& file_path) {
@ -671,6 +674,18 @@ static bool ReadFibmap(int file_fd, const std::string& file_path,
FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_size, bool create,
std::function<bool(uint64_t, uint64_t)> progress) {
FiemapUniquePtr ret;
if (!Open(file_path, file_size, &ret, create, progress).is_ok()) {
return nullptr;
}
return ret;
}
FiemapStatus FiemapWriter::Open(const std::string& file_path, uint64_t file_size,
FiemapUniquePtr* out, bool create,
std::function<bool(uint64_t, uint64_t)> progress) {
out->reset();
// if 'create' is false, open an existing file and do not truncate.
int open_flags = O_RDWR | O_CLOEXEC;
if (create) {
@ -683,43 +698,46 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s
TEMP_FAILURE_RETRY(open(file_path.c_str(), open_flags, S_IRUSR | S_IWUSR)));
if (file_fd < 0) {
PLOG(ERROR) << "Failed to create file at: " << file_path;
return nullptr;
return FiemapStatus::FromErrno(errno);
}
std::string abs_path;
if (!::android::base::Realpath(file_path, &abs_path)) {
int saved_errno = errno;
PLOG(ERROR) << "Invalid file path: " << file_path;
cleanup(file_path, create);
return nullptr;
return FiemapStatus::FromErrno(saved_errno);
}
std::string bdev_path;
if (!GetBlockDeviceForFile(abs_path, &bdev_path)) {
LOG(ERROR) << "Failed to get block dev path for file: " << file_path;
cleanup(abs_path, create);
return nullptr;
return FiemapStatus::Error();
}
::android::base::unique_fd bdev_fd(
TEMP_FAILURE_RETRY(open(bdev_path.c_str(), O_RDONLY | O_CLOEXEC)));
if (bdev_fd < 0) {
int saved_errno = errno;
PLOG(ERROR) << "Failed to open block device: " << bdev_path;
cleanup(file_path, create);
return nullptr;
return FiemapStatus::FromErrno(saved_errno);
}
uint64_t bdevsz;
if (!GetBlockDeviceSize(bdev_fd, bdev_path, &bdevsz)) {
int saved_errno = errno;
LOG(ERROR) << "Failed to get block device size for : " << bdev_path;
cleanup(file_path, create);
return nullptr;
return FiemapStatus::FromErrno(saved_errno);
}
if (!create) {
file_size = GetFileSize(abs_path);
if (file_size == 0) {
LOG(ERROR) << "Invalid file size of zero bytes for file: " << abs_path;
return nullptr;
return FiemapStatus::FromErrno(errno);
}
}
@ -728,7 +746,7 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s
if (!PerformFileChecks(abs_path, &blocksz, &fs_type)) {
LOG(ERROR) << "Failed to validate file or file system for file:" << abs_path;
cleanup(abs_path, create);
return nullptr;
return FiemapStatus::Error();
}
// Align up to the nearest block size.
@ -737,11 +755,13 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s
}
if (create) {
if (!AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress))) {
auto status =
AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress));
if (!status.is_ok()) {
LOG(ERROR) << "Failed to allocate file: " << abs_path << " of size: " << file_size
<< " bytes";
cleanup(abs_path, create);
return nullptr;
return status;
}
}
@ -749,7 +769,7 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s
if (!PinFile(file_fd, abs_path, fs_type)) {
cleanup(abs_path, create);
LOG(ERROR) << "Failed to pin the file in storage";
return nullptr;
return FiemapStatus::Error();
}
// now allocate the FiemapWriter and start setting it up
@ -760,14 +780,14 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s
if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) {
LOG(ERROR) << "Failed to read fiemap of file: " << abs_path;
cleanup(abs_path, create);
return nullptr;
return FiemapStatus::Error();
}
break;
case MSDOS_SUPER_MAGIC:
if (!ReadFibmap(file_fd, abs_path, &fmap->extents_)) {
LOG(ERROR) << "Failed to read fibmap of file: " << abs_path;
cleanup(abs_path, create);
return nullptr;
return FiemapStatus::Error();
}
break;
}
@ -781,7 +801,8 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s
LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device "
<< bdev_path;
return fmap;
*out = std::move(fmap);
return FiemapStatus::Ok();
}
} // namespace fiemap

4
fs_mgr/libfiemap/fiemap_writer_test.cpp
View file

@ -193,7 +193,9 @@ TEST_F(FiemapWriterTest, FileDeletedOnError) {
}
TEST_F(FiemapWriterTest, MaxBlockSize) {
ASSERT_GT(DetermineMaximumFileSize(testfile), 0);
uint64_t max_piece_size = 0;
ASSERT_TRUE(DetermineMaximumFileSize(testfile, &max_piece_size));
ASSERT_GT(max_piece_size, 0);
}
TEST_F(FiemapWriterTest, FibmapBlockAddressing) {

60
fs_mgr/libfiemap/include/libfiemap/fiemap_status.h Normal file
View file

@ -0,0 +1,60 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <errno.h>
#include <stdint.h>
#include <string>
namespace android::fiemap {
// Represent error status of libfiemap classes.
class FiemapStatus {
public:
enum class ErrorCode : int32_t {
SUCCESS = 0,
// Generic non-recoverable failure.
ERROR = INT32_MIN,
// Not enough space
NO_SPACE = -ENOSPC,
};
// Create from a given errno (specified in errno,h)
static FiemapStatus FromErrno(int error_num) { return FiemapStatus(CastErrorCode(-error_num)); }
// Generic error.
static FiemapStatus Error() { return FiemapStatus(ErrorCode::ERROR); }
// Success.
static FiemapStatus Ok() { return FiemapStatus(ErrorCode::SUCCESS); }
ErrorCode error_code() const { return error_code_; }
bool is_ok() const { return error_code() == ErrorCode::SUCCESS; }
operator bool() const { return is_ok(); }
// For logging and debugging only.
std::string string() const;
private:
ErrorCode error_code_;
FiemapStatus(ErrorCode code) : error_code_(code) {}
static ErrorCode CastErrorCode(int error);
};
} // namespace android::fiemap

5
fs_mgr/libfiemap/include/libfiemap/fiemap_writer.h
View file

@ -27,6 +27,8 @@
#include <android-base/unique_fd.h>
#include <libfiemap/fiemap_status.h>
namespace android {
namespace fiemap {
@ -47,6 +49,9 @@ class FiemapWriter final {
static FiemapUniquePtr Open(const std::string& file_path, uint64_t file_size,
bool create = true,
std::function<bool(uint64_t, uint64_t)> progress = {});
static FiemapStatus Open(const std::string& file_path, uint64_t file_size, FiemapUniquePtr* out,
bool create = true,
std::function<bool(uint64_t, uint64_t)> progress = {});
// Check that a file still has the same extents since it was last opened with FiemapWriter,
// assuming the file was not resized outside of FiemapWriter. Returns false either on error

6
fs_mgr/libfiemap/include/libfiemap/split_fiemap_writer.h
View file

@ -25,7 +25,8 @@
#include <android-base/unique_fd.h>
#include "fiemap_writer.h"
#include <libfiemap/fiemap_status.h>
#include <libfiemap/fiemap_writer.h>
namespace android {
namespace fiemap {
@ -43,6 +44,9 @@ class SplitFiemap final {
static std::unique_ptr<SplitFiemap> Create(const std::string& file_path, uint64_t file_size,
uint64_t max_piece_size,
ProgressCallback progress = {});
static FiemapStatus Create(const std::string& file_path, uint64_t file_size,
uint64_t max_piece_size, std::unique_ptr<SplitFiemap>* out_val,
ProgressCallback progress = {});
// Open an existing split fiemap file.
static std::unique_ptr<SplitFiemap> Open(const std::string& file_path);

41
fs_mgr/libfiemap/split_fiemap_writer.cpp
View file

@ -45,16 +45,28 @@ static const size_t kMaxFilePieces = 500;
std::unique_ptr<SplitFiemap> SplitFiemap::Create(const std::string& file_path, uint64_t file_size,
uint64_t max_piece_size,
ProgressCallback progress) {
std::unique_ptr<SplitFiemap> ret;
if (!Create(file_path, file_size, max_piece_size, &ret, progress).is_ok()) {
return nullptr;
}
return ret;
}
FiemapStatus SplitFiemap::Create(const std::string& file_path, uint64_t file_size,
uint64_t max_piece_size, std::unique_ptr<SplitFiemap>* out_val,
ProgressCallback progress) {
out_val->reset();
if (!file_size) {
LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path;
return nullptr;
return FiemapStatus::Error();
}
if (!max_piece_size) {
max_piece_size = DetermineMaximumFileSize(file_path);
if (!max_piece_size) {
auto status = DetermineMaximumFileSize(file_path, &max_piece_size);
if (!status.is_ok()) {
LOG(ERROR) << "Could not determine maximum file size for " << file_path;
return nullptr;
return status;
}
}
@ -75,7 +87,6 @@ std::unique_ptr<SplitFiemap> SplitFiemap::Create(const std::string& file_path, u
}
return true;
};
std::unique_ptr<SplitFiemap> out(new SplitFiemap());
out->creating_ = true;
out->list_file_ = file_path;
@ -85,14 +96,17 @@ std::unique_ptr<SplitFiemap> SplitFiemap::Create(const std::string& file_path, u
while (remaining_bytes) {
if (out->files_.size() >= kMaxFilePieces) {
LOG(ERROR) << "Requested size " << file_size << " created too many split files";
return nullptr;
out.reset();
return FiemapStatus::Error();
}
std::string chunk_path =
android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size());
uint64_t chunk_size = std::min(max_piece_size, remaining_bytes);
auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress);
if (!writer) {
return nullptr;
FiemapUniquePtr writer;
auto status = FiemapWriter::Open(chunk_path, chunk_size, &writer, true, on_progress);
if (!status.is_ok()) {
out.reset();
return status;
}
// To make sure the alignment doesn't create too much inconsistency, we
@ -110,20 +124,23 @@ std::unique_ptr<SplitFiemap> SplitFiemap::Create(const std::string& file_path, u
unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660));
if (fd < 0) {
PLOG(ERROR) << "Failed to open " << file_path;
return nullptr;
out.reset();
return FiemapStatus::FromErrno(errno);
}
for (const auto& writer : out->files_) {
std::string line = android::base::Basename(writer->file_path()) + "\n";
if (!android::base::WriteFully(fd, line.data(), line.size())) {
PLOG(ERROR) << "Write failed " << file_path;
return nullptr;
out.reset();
return FiemapStatus::FromErrno(errno);
}
}
// Unset this bit, so we don't unlink on destruction.
out->creating_ = false;
return out;
*out_val = std::move(out);
return FiemapStatus::Ok();
}
std::unique_ptr<SplitFiemap> SplitFiemap::Open(const std::string& file_path) {

21
fs_mgr/libfiemap/utility.cpp
View file

@ -37,29 +37,30 @@ using android::base::unique_fd;
static constexpr char kUserdataDevice[] = "/dev/block/by-name/userdata";
uint64_t DetermineMaximumFileSize(const std::string& file_path) {
FiemapStatus DetermineMaximumFileSize(const std::string& file_path, uint64_t* result) {
// Create the smallest file possible (one block).
auto writer = FiemapWriter::Open(file_path, 1);
if (!writer) {
return 0;
FiemapUniquePtr writer;
auto status = FiemapWriter::Open(file_path, 1, &writer);
if (!status.is_ok()) {
return status;
}
uint64_t result = 0;
*result = 0;
switch (writer->fs_type()) {
case EXT4_SUPER_MAGIC:
// The minimum is 16GiB, so just report that. If we wanted we could parse the
// superblock and figure out if 64-bit support is enabled.
result = 17179869184ULL;
*result = 17179869184ULL;
break;
case F2FS_SUPER_MAGIC:
// Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt
// 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB.
result = 4329690886144ULL;
*result = 4329690886144ULL;
break;
case MSDOS_SUPER_MAGIC:
// 4GB-1, which we want aligned to the block size.
result = 4294967295;
result -= (result % writer->block_size());
*result = 4294967295;
*result -= (*result % writer->block_size());
break;
default:
LOG(ERROR) << "Unknown file system type: " << writer->fs_type();
@ -70,7 +71,7 @@ uint64_t DetermineMaximumFileSize(const std::string& file_path) {
writer = nullptr;
unlink(file_path.c_str());
return result;
return FiemapStatus::Ok();
}
// Given a SplitFiemap, this returns a device path that will work during first-

2
fs_mgr/libfiemap/utility.h
View file

@ -28,7 +28,7 @@ namespace fiemap {
// Given a file that will be created, determine the maximum size its containing
// filesystem allows. Note this is a theoretical maximum size; free space is
// ignored entirely.
uint64_t DetermineMaximumFileSize(const std::string& file_path);
FiemapStatus DetermineMaximumFileSize(const std::string& file_path, uint64_t* result);
// Given a SplitFiemap, this returns a device path that will work during first-
// stage init (i.e., its path can be found by InitRequiredDevices).