tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "fs_mgr: Move libfiemap back to fs_mgr from system/gsid."

Browse source
This commit is contained in:
David Anderson 2019-12-17 17:32:09 +00:00 committed by Gerrit Code Review
commit 1526a46e92
20 changed files with 3902 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
fs_mgr/libfiemap/Android.bp Normal file
View file

@ -0,0 +1,135 @@
//
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
cc_library_headers {
name: "libfiemap_headers",
recovery_available: true,
export_include_dirs: ["include"],
}
cc_defaults {
name: "libfiemap_defaults",
defaults: ["fs_mgr_defaults"],
cflags: [
"-D_FILE_OFFSET_BITS=64",
"-Wall",
"-Werror",
],
srcs: [
"fiemap_writer.cpp",
"image_manager.cpp",
"metadata.cpp",
"split_fiemap_writer.cpp",
"utility.cpp",
],
static_libs: [
"libdm",
"libext2_uuid",
"libext4_utils",
"liblp",
"libfs_mgr",
],
shared_libs: [
"libbase",
],
header_libs: [
"libfiemap_headers",
"liblog_headers",
],
export_shared_lib_headers: [
"libbase",
],
export_header_lib_headers: [
"libfiemap_headers",
],
}
// Open up a binder IImageManager interface.
cc_library_static {
name: "libfiemap_binder",
defaults: ["libfiemap_defaults"],
srcs: [
"binder.cpp",
],
whole_static_libs: [
"gsi_aidl_interface-cpp",
"libgsi",
],
shared_libs: [
"libbinder",
],
}
// Open up a passthrough IImageManager interface. Use libfiemap_binder whenever
// possible. This should only be used when binder is not available.
cc_library_static {
name: "libfiemap_passthrough",
defaults: ["libfiemap_defaults"],
recovery_available: true,
srcs: [
"passthrough.cpp",
],
}
cc_test {
name: "fiemap_writer_test",
defaults: ["libfiemap_defaults"],
static_libs: [
"libbase",
"libdm",
"liblog",
],
data: [
"testdata/unaligned_file",
"testdata/file_4k",
"testdata/file_32k",
],
srcs: [
"fiemap_writer_test.cpp",
],
}
cc_test {
name: "fiemap_image_test",
defaults: ["libfiemap_defaults"],
static_libs: [
"libdm",
"libext4_utils",
"libfs_mgr",
"liblp",
],
shared_libs: [
"libcrypto",
"libcrypto_utils",
"libcutils",
"liblog",
],
srcs: [
"image_test.cpp",
],
}
vts_config {
name: "VtsFiemapWriterTest",
}

30
fs_mgr/libfiemap/AndroidTest.xml Normal file
View file

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2019 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration description="Config for VTS VtsFiemapWriterTest">
<option name="config-descriptor:metadata" key="plan" value="vts-kernel" />
<target_preparer class="com.android.compatibility.common.tradefed.targetprep.VtsFilePusher">
<option name="abort-on-push-failure" value="false"/>
<option name="push-group" value="HostDrivenTest.push"/>
</target_preparer>
<test class="com.android.tradefed.testtype.VtsMultiDeviceTest">
<option name="test-module-name" value="VtsFiemapWriterTest"/>
<option name="binary-test-source" value="_32bit::DATA/nativetest/fiemap_writer_test/fiemap_writer_test" />
<option name="binary-test-source" value="_64bit::DATA/nativetest64/fiemap_writer_test/fiemap_writer_test" />
<option name="binary-test-type" value="gtest"/>
<option name="precondition-first-api-level" value="29" />
<option name="test-timeout" value="1m"/>
</test>
</configuration>

75
fs_mgr/libfiemap/README.md Normal file
View file

@ -0,0 +1,75 @@
libfiemap
=============
`libfiemap` is a library for creating block-devices that are backed by
storage in read-write partitions. It exists primary for gsid. Generally, the
library works by using `libfiemap_writer` to allocate large files within
filesystem, and then tracks their extents.
There are three main uses for `libfiemap`:
- Creating images that will act as block devices. For example, gsid needs to
create a `system_gsi` image to store Dynamic System Updates.
- Mapping the image as a block device while /data is mounted. This is fairly
tricky and is described in more detail below.
- Mapping the image as a block device during first-stage init. This is simple
because it uses the same logic from dynamic partitions.
Image creation is done through `SplitFiemap`. Depending on the file system,
a large image may have to be split into multiple files. On Ext4 the limit is
16GiB and on FAT32 it's 4GiB. Images are saved into `/data/gsi/<name>/`
where `<name>` is chosen by the process requesting the image.
At the same time, a file called `/metadata/gsi/<name>/lp_metadata` is created.
This is a super partition header that allows first-stage init to create dynamic
partitions from the image files. It also tracks the canonical size of the image,
since the file size may be larger due to alignment.
Mapping
-------
It is easy to make block devices out of blocks on `/data` when it is not
mounted, so first-stage init has no issues mapping dynamic partitions from
images. After `/data` is mounted however, there are two problems:
- `/data` is encrypted.
- `/dev/block/by-name/data` may be marked as in-use.
We break the problem down into three scenarios.
### FDE and Metadata Encrypted Devices
When FDE or metadata encryption is used, `/data` is not mounted from
`/dev/block/by-name/data`. Instead, it is mounted from an intermediate
`dm-crypt` or `dm-default-key` device. This means the underlying device is
not marked in use, and we can create new dm-linear devices on top of it.
On these devices, a block device for an image will consist of a single
device-mapper device with a `dm-linear` table entry for each extent in the
backing file.
### Unencrypted and FBE-encrypted Devices
When a device is unencrypted, or is encrypted with FBE but not metadata
encryption, we instead use a loop device with `LOOP_SET_DIRECT_IO` enabled.
Since `/data/gsi` has encryption disabled, this means the raw blocks will be
unencrypted as well.
### Split Images
If an image was too large to store a single file on the underlying filesystem,
on an FBE/unencrypted device we will have multiple loop devices. In this case,
we create a device-mapper device as well. For each loop device it will have one
`dm-linear` table entry spanning the length of the device.
State Tracking
--------------
It's important that we know whether or not an image is currently in-use by a
block device. It could be catastrophic to write to a dm-linear device if the
underlying blocks are no longer owned by the original file. Thus, when mapping
an image, we create a property called `gsid.mapped_image.<name>` and set it to
the path of the block device.
Additionally, we create a `/metadata/gsi/<subdir>/<name>.status` file. Each
line in this file denotes a dependency on either a device-mapper node or a loop
device. When deleting a block device, this file is used to release all
resources.

224
fs_mgr/libfiemap/binder.cpp Normal file
View file

@ -0,0 +1,224 @@
//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#if !defined(__ANDROID_RECOVERY__)
#include <android-base/logging.h>
#include <android-base/properties.h>
#include <android/gsi/IGsiService.h>
#include <android/gsi/IGsid.h>
#include <binder/IServiceManager.h>
#include <libfiemap/image_manager.h>
#include <libgsi/libgsi.h>
namespace android {
namespace fiemap {
using namespace android::gsi;
using namespace std::chrono_literals;
class ImageManagerBinder final : public IImageManager {
public:
ImageManagerBinder(android::sp<IGsiService>&& service, android::sp<IImageService>&& manager);
bool CreateBackingImage(const std::string& name, uint64_t size, int flags) override;
bool DeleteBackingImage(const std::string& name) override;
bool MapImageDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms,
std::string* path) override;
bool UnmapImageDevice(const std::string& name) override;
bool BackingImageExists(const std::string& name) override;
bool IsImageMapped(const std::string& name) override;
bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name,
std::string* dev) override;
bool ZeroFillNewImage(const std::string& name, uint64_t bytes) override;
bool RemoveAllImages() override;
std::vector<std::string> GetAllBackingImages() override;
private:
android::sp<IGsiService> service_;
android::sp<IImageService> manager_;
};
ImageManagerBinder::ImageManagerBinder(android::sp<IGsiService>&& service,
android::sp<IImageService>&& manager)
: service_(std::move(service)), manager_(std::move(manager)) {}
bool ImageManagerBinder::CreateBackingImage(const std::string& name, uint64_t size, int flags) {
auto status = manager_->createBackingImage(name, size, flags);
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
return false;
}
return true;
}
bool ImageManagerBinder::DeleteBackingImage(const std::string& name) {
auto status = manager_->deleteBackingImage(name);
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
return false;
}
return true;
}
bool ImageManagerBinder::MapImageDevice(const std::string& name,
const std::chrono::milliseconds& timeout_ms,
std::string* path) {
int32_t timeout_ms_count =
static_cast<int32_t>(std::clamp<typename std::chrono::milliseconds::rep>(
timeout_ms.count(), INT32_MIN, INT32_MAX));
MappedImage map;
auto status = manager_->mapImageDevice(name, timeout_ms_count, &map);
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
return false;
}
*path = map.path;
return true;
}
bool ImageManagerBinder::UnmapImageDevice(const std::string& name) {
auto status = manager_->unmapImageDevice(name);
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
return false;
}
return true;
}
bool ImageManagerBinder::BackingImageExists(const std::string& name) {
bool retval;
auto status = manager_->backingImageExists(name, &retval);
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
return false;
}
return retval;
}
bool ImageManagerBinder::IsImageMapped(const std::string& name) {
bool retval;
auto status = manager_->isImageMapped(name, &retval);
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
return false;
}
return retval;
}
bool ImageManagerBinder::MapImageWithDeviceMapper(const IPartitionOpener& opener,
const std::string& name, std::string* dev) {
(void)opener;
(void)name;
(void)dev;
LOG(ERROR) << "MapImageWithDeviceMapper is not available over binder.";
return false;
}
std::vector<std::string> ImageManagerBinder::GetAllBackingImages() {
std::vector<std::string> retval;
auto status = manager_->getAllBackingImages(&retval);
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
}
return retval;
}
bool ImageManagerBinder::ZeroFillNewImage(const std::string& name, uint64_t bytes) {
auto status = manager_->zeroFillNewImage(name, bytes);
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
return false;
}
return true;
}
bool ImageManagerBinder::RemoveAllImages() {
auto status = manager_->removeAllImages();
if (!status.isOk()) {
LOG(ERROR) << __PRETTY_FUNCTION__
<< " binder returned: " << status.exceptionMessage().string();
return false;
}
return true;
}
static android::sp<IGsid> AcquireIGsid(const std::chrono::milliseconds& timeout_ms) {
if (android::base::GetProperty("init.svc.gsid", "") != "running") {
if (!android::base::SetProperty("ctl.start", "gsid") ||
!android::base::WaitForProperty("init.svc.gsid", "running", timeout_ms)) {
LOG(ERROR) << "Could not start the gsid service";
return nullptr;
}
// Sleep for 250ms to give the service time to register.
usleep(250 * 1000);
}
auto sm = android::defaultServiceManager();
auto name = android::String16(kGsiServiceName);
auto service = sm->checkService(name);
return android::interface_cast<IGsid>(service);
}
static android::sp<IGsid> GetGsiService(const std::chrono::milliseconds& timeout_ms) {
auto start_time = std::chrono::steady_clock::now();
std::chrono::milliseconds elapsed = std::chrono::milliseconds::zero();
do {
if (auto gsid = AcquireIGsid(timeout_ms - elapsed); gsid != nullptr) {
return gsid;
}
auto now = std::chrono::steady_clock::now();
elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - start_time);
} while (elapsed <= timeout_ms);
LOG(ERROR) << "Timed out trying to acquire IGsid interface";
return nullptr;
}
std::unique_ptr<IImageManager> IImageManager::Open(const std::string& dir,
const std::chrono::milliseconds& timeout_ms) {
auto gsid = GetGsiService(timeout_ms);
if (!gsid) {
return nullptr;
}
android::sp<IGsiService> service;
auto status = gsid->getClient(&service);
if (!status.isOk() || !service) {
LOG(ERROR) << "Could not acquire IGsiService";
return nullptr;
}
android::sp<IImageService> manager;
status = service->openImageService(dir, &manager);
if (!status.isOk() || !manager) {
LOG(ERROR) << "Could not acquire IImageManager: " << status.exceptionMessage().string();
return nullptr;
}
return std::make_unique<ImageManagerBinder>(std::move(service), std::move(manager));
}
} // namespace fiemap
} // namespace android
#endif // __ANDROID_RECOVERY__

782
fs_mgr/libfiemap/fiemap_writer.cpp Normal file
View file

@ -0,0 +1,782 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <libfiemap/fiemap_writer.h>
#include <dirent.h>
#include <fcntl.h>
#include <linux/fs.h>
#include <stdio.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/sysmacros.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>
#include <limits>
#include <string>
#include <utility>
#include <vector>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <libdm/dm.h>
#include "utility.h"
namespace android {
namespace fiemap {
using namespace android::dm;
// We cap the maximum number of extents as a sanity measure.
static constexpr uint32_t kMaxExtents = 50000;
// TODO: Fallback to using fibmap if FIEMAP_EXTENT_MERGED is set.
static constexpr const uint32_t kUnsupportedExtentFlags =
FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_DELALLOC |
FIEMAP_EXTENT_NOT_ALIGNED | FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_DATA_TAIL |
FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_SHARED | FIEMAP_EXTENT_MERGED;
// Large file support must be enabled.
static_assert(sizeof(off_t) == sizeof(uint64_t));
static inline void cleanup(const std::string& file_path, bool created) {
if (created) {
unlink(file_path.c_str());
}
}
static bool ValidateDmTarget(const DeviceMapper::TargetInfo& target) {
const auto& entry = target.spec;
if (entry.sector_start != 0) {
LOG(INFO) << "Stopping at target with non-zero starting sector";
return false;
}
auto target_type = DeviceMapper::GetTargetType(entry);
if (target_type == "bow" || target_type == "default-key" || target_type == "crypt") {
return true;
}
if (target_type == "linear") {
auto pieces = android::base::Split(target.data, " ");
if (pieces[1] != "0") {
LOG(INFO) << "Stopping at complex linear target with non-zero starting sector: "
<< pieces[1];
return false;
}
return true;
}
LOG(INFO) << "Stopping at complex target type " << target_type;
return false;
}
static bool DeviceMapperStackPop(const std::string& bdev, std::string* bdev_raw) {
*bdev_raw = bdev;
if (!::android::base::StartsWith(bdev, "dm-")) {
// We are at the bottom of the device mapper stack.
return true;
}
// Get the device name.
auto dm_name_file = "/sys/block/" + bdev + "/dm/name";
std::string dm_name;
if (!android::base::ReadFileToString(dm_name_file, &dm_name)) {
PLOG(ERROR) << "Could not read file: " << dm_name_file;
return false;
}
dm_name = android::base::Trim(dm_name);
auto& dm = DeviceMapper::Instance();
std::vector<DeviceMapper::TargetInfo> table;
if (!dm.GetTableInfo(dm_name, &table)) {
LOG(ERROR) << "Could not read device-mapper table for " << dm_name << " at " << bdev;
return false;
}
// The purpose of libfiemap is to provide an extent-based view into
// a file. This is difficult if devices are not layered in a 1:1 manner;
// we would have to translate and break up extents based on the actual
// block mapping. Since this is too complex, we simply stop processing
// the device-mapper stack if we encounter a complex case.
//
// It is up to the caller to decide whether stopping at a virtual block
// device is allowable. In most cases it is not, because we want either
// "userdata" or an external volume. It is useful for tests however.
// Callers can check by comparing the device number to that of userdata,
// or by checking whether is a device-mapper node.
if (table.size() > 1) {
LOG(INFO) << "Stopping at complex table for " << dm_name << " at " << bdev;
return true;
}
if (!ValidateDmTarget(table[0])) {
return true;
}
auto dm_leaf_dir = "/sys/block/" + bdev + "/slaves";
auto d = std::unique_ptr<DIR, decltype(&closedir)>(opendir(dm_leaf_dir.c_str()), closedir);
if (d == nullptr) {
PLOG(ERROR) << "Failed to open: " << dm_leaf_dir;
return false;
}
struct dirent* de;
uint32_t num_leaves = 0;
std::string bdev_next = "";
while ((de = readdir(d.get())) != nullptr) {
if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) {
continue;
}
// We set the first name we find here
if (bdev_next.empty()) {
bdev_next = de->d_name;
}
num_leaves++;
}
// if we have more than one leaves, we return immediately. We can't continue to create the
// file since we don't know how to write it out using fiemap, so it will be readable via the
// underlying block devices later. The reader will also have to construct the same device mapper
// target in order read the file out.
if (num_leaves > 1) {
LOG(ERROR) << "Found " << num_leaves << " leaf block devices under device mapper device "
<< bdev;
return false;
}
// recursively call with the block device we found in order to pop the device mapper stack.
return DeviceMapperStackPop(bdev_next, bdev_raw);
}
bool FiemapWriter::GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path,
bool* uses_dm) {
struct stat sb;
if (stat(file_path.c_str(), &sb)) {
PLOG(ERROR) << "Failed to get stat for: " << file_path;
return false;
}
std::string bdev;
if (!BlockDeviceToName(major(sb.st_dev), minor(sb.st_dev), &bdev)) {
LOG(ERROR) << "Failed to get block device name for " << major(sb.st_dev) << ":"
<< minor(sb.st_dev);
return false;
}
std::string bdev_raw;
if (!DeviceMapperStackPop(bdev, &bdev_raw)) {
LOG(ERROR) << "Failed to get the bottom of the device mapper stack for device: " << bdev;
return false;
}
if (uses_dm) {
*uses_dm = (bdev_raw != bdev);
}
LOG(DEBUG) << "Popped device (" << bdev_raw << ") from device mapper stack starting with ("
<< bdev << ")";
*bdev_path = ::android::base::StringPrintf("/dev/block/%s", bdev_raw.c_str());
// Make sure we are talking to a block device before calling it a success.
if (stat(bdev_path->c_str(), &sb)) {
PLOG(ERROR) << "Failed to get stat for block device: " << *bdev_path;
return false;
}
if ((sb.st_mode & S_IFMT) != S_IFBLK) {
PLOG(ERROR) << "File: " << *bdev_path << " is not a block device";
return false;
}
return true;
}
static bool GetBlockDeviceSize(int bdev_fd, const std::string& bdev_path, uint64_t* bdev_size) {
uint64_t size_in_bytes = 0;
if (ioctl(bdev_fd, BLKGETSIZE64, &size_in_bytes)) {
PLOG(ERROR) << "Failed to get total size for: " << bdev_path;
return false;
}
*bdev_size = size_in_bytes;
return true;
}
static uint64_t GetFileSize(const std::string& file_path) {
struct stat sb;
if (stat(file_path.c_str(), &sb)) {
PLOG(ERROR) << "Failed to get size for file: " << file_path;
return 0;
}
return sb.st_size;
}
static bool PerformFileChecks(const std::string& file_path, uint64_t file_size, uint64_t* blocksz,
uint32_t* fs_type) {
struct statfs64 sfs;
if (statfs64(file_path.c_str(), &sfs)) {
PLOG(ERROR) << "Failed to read file system status at: " << file_path;
return false;
}
if (!sfs.f_bsize) {
LOG(ERROR) << "Unsupported block size: " << sfs.f_bsize;
return false;
}
// Check if the filesystem is of supported types.
// Only ext4, f2fs, and vfat are tested and supported.
switch (sfs.f_type) {
case EXT4_SUPER_MAGIC:
case F2FS_SUPER_MAGIC:
case MSDOS_SUPER_MAGIC:
break;
default:
LOG(ERROR) << "Unsupported file system type: 0x" << std::hex << sfs.f_type;
return false;
}
uint64_t available_bytes = sfs.f_bsize * sfs.f_bavail;
if (access(file_path.c_str(), F_OK) != 0 && available_bytes <= file_size) {
LOG(ERROR) << "Not enough free space in file system to create file of size : " << file_size;
return false;
}
*blocksz = sfs.f_bsize;
*fs_type = sfs.f_type;
return true;
}
static bool FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_size,
const std::string& file_path,
const std::function<bool(uint64_t, uint64_t)>& on_progress) {
// Even though this is much faster than writing zeroes, it is still slow
// enough that we need to fire the progress callback periodically. To
// easily achieve this, we seek in chunks. We use 1000 chunks since
// normally we only fire the callback on 1/1000th increments.
uint64_t bytes_per_chunk = std::max(file_size / 1000, block_size);
// Seek just to the end of each chunk and write a single byte, causing
// the filesystem to allocate blocks.
off_t cursor = 0;
off_t end = static_cast<off_t>(file_size);
while (cursor < end) {
cursor = std::min(static_cast<off_t>(cursor + bytes_per_chunk), end);
auto rv = TEMP_FAILURE_RETRY(lseek(file_fd, cursor - 1, SEEK_SET));
if (rv < 0) {
PLOG(ERROR) << "Failed to lseek " << file_path;
return false;
}
if (rv != cursor - 1) {
LOG(ERROR) << "Seek returned wrong offset " << rv << " for file " << file_path;
return false;
}
char buffer[] = {0};
if (!android::base::WriteFully(file_fd, buffer, 1)) {
PLOG(ERROR) << "Write failed: " << file_path;
return false;
}
if (on_progress && !on_progress(cursor, file_size)) {
return false;
}
}
return true;
}
// F2FS-specific ioctl
// It requires the below kernel commit merged in v4.16-rc1.
// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file")
// In android-4.4,
// 56ee1e817908 ("f2fs: updates on v4.16-rc1")
// In android-4.9,
// 2f17e34672a8 ("f2fs: updates on v4.16-rc1")
// In android-4.14,
// ce767d9a55bc ("f2fs: updates on v4.16-rc1")
#ifndef F2FS_IOC_SET_PIN_FILE
#ifndef F2FS_IOCTL_MAGIC
#define F2FS_IOCTL_MAGIC 0xf5
#endif
#define F2FS_IOC_GET_PIN_FILE _IOR(F2FS_IOCTL_MAGIC, 14, __u32)
#define F2FS_IOC_SET_PIN_FILE _IOW(F2FS_IOCTL_MAGIC, 13, __u32)
#endif
static bool IsFilePinned(int file_fd, const std::string& file_path, uint32_t fs_type) {
if (fs_type != F2FS_SUPER_MAGIC) {
// No pinning necessary for ext4 or vfat. The blocks, once allocated,
// are expected to be fixed.
return true;
}
// f2fs: export FS_NOCOW_FL flag to user
uint32_t flags;
int error = ioctl(file_fd, FS_IOC_GETFLAGS, &flags);
if (error < 0) {
if ((errno == ENOTTY) || (errno == ENOTSUP)) {
PLOG(ERROR) << "Failed to get flags, not supported by kernel: " << file_path;
} else {
PLOG(ERROR) << "Failed to get flags: " << file_path;
}
return false;
}
if (!(flags & FS_NOCOW_FL)) {
return false;
}
// F2FS_IOC_GET_PIN_FILE returns the number of blocks moved.
uint32_t moved_blocks_nr;
error = ioctl(file_fd, F2FS_IOC_GET_PIN_FILE, &moved_blocks_nr);
if (error < 0) {
if ((errno == ENOTTY) || (errno == ENOTSUP)) {
PLOG(ERROR) << "Failed to get file pin status, not supported by kernel: " << file_path;
} else {
PLOG(ERROR) << "Failed to get file pin status: " << file_path;
}
return false;
}
if (moved_blocks_nr) {
LOG(WARNING) << moved_blocks_nr << " blocks moved in file " << file_path;
}
return moved_blocks_nr == 0;
}
static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type) {
if (IsFilePinned(file_fd, file_path, fs_type)) {
return true;
}
if (fs_type != F2FS_SUPER_MAGIC) {
// No pinning necessary for ext4/msdos. The blocks, once allocated, are
// expected to be fixed.
return true;
}
uint32_t pin_status = 1;
int error = ioctl(file_fd, F2FS_IOC_SET_PIN_FILE, &pin_status);
if (error < 0) {
if ((errno == ENOTTY) || (errno == ENOTSUP)) {
PLOG(ERROR) << "Failed to pin file, not supported by kernel: " << file_path;
} else {
PLOG(ERROR) << "Failed to pin file: " << file_path;
}
return false;
}
return true;
}
// Reserve space for the file on the file system and write it out to make sure the extents
// don't come back unwritten. Return from this function with the kernel file offset set to 0.
// If the filesystem is f2fs, then we also PIN the file on disk to make sure the blocks
// aren't moved around.
static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz,
uint64_t file_size, unsigned int fs_type,
std::function<bool(uint64_t, uint64_t)> on_progress) {
switch (fs_type) {
case EXT4_SUPER_MAGIC:
break;
case F2FS_SUPER_MAGIC: {
bool supported;
if (!F2fsPinBeforeAllocate(file_fd, &supported)) {
return false;
}
if (supported && !PinFile(file_fd, file_path, fs_type)) {
return false;
}
break;
}
case MSDOS_SUPER_MAGIC:
// fallocate() is not supported, and not needed, since VFAT does not support holes.
// Instead we can perform a much faster allocation.
return FallocateFallback(file_fd, blocksz, file_size, file_path, on_progress);
default:
LOG(ERROR) << "Missing fallocate() support for file system " << fs_type;
return false;
}
if (fallocate(file_fd, FALLOC_FL_ZERO_RANGE, 0, file_size)) {
PLOG(ERROR) << "Failed to allocate space for file: " << file_path << " size: " << file_size;
return false;
}
// write zeroes in 'blocksz' byte increments until we reach file_size to make sure the data
// blocks are actually written to by the file system and thus getting rid of the holes in the
// file.
auto buffer = std::unique_ptr<void, decltype(&free)>(calloc(1, blocksz), free);
if (buffer == nullptr) {
LOG(ERROR) << "failed to allocate memory for writing file";
return false;
}
off64_t offset = lseek64(file_fd, 0, SEEK_SET);
if (offset < 0) {
PLOG(ERROR) << "Failed to seek at the beginning of : " << file_path;
return false;
}
int permille = -1;
while (offset < file_size) {
if (!::android::base::WriteFully(file_fd, buffer.get(), blocksz)) {
PLOG(ERROR) << "Failed to write" << blocksz << " bytes at offset" << offset
<< " in file " << file_path;
return false;
}
offset += blocksz;
// Don't invoke the callback every iteration - wait until a significant
// chunk (here, 1/1000th) of the data has been processed.
int new_permille = (static_cast<uint64_t>(offset) * 1000) / file_size;
if (new_permille != permille && static_cast<uint64_t>(offset) != file_size) {
if (on_progress && !on_progress(offset, file_size)) {
return false;
}
permille = new_permille;
}
}
if (lseek64(file_fd, 0, SEEK_SET) < 0) {
PLOG(ERROR) << "Failed to reset offset at the beginning of : " << file_path;
return false;
}
// flush all writes here ..
if (fsync(file_fd)) {
PLOG(ERROR) << "Failed to synchronize written file:" << file_path;
return false;
}
// Send one last progress notification.
if (on_progress && !on_progress(file_size, file_size)) {
return false;
}
return true;
}
bool FiemapWriter::HasPinnedExtents(const std::string& file_path) {
android::base::unique_fd fd(open(file_path.c_str(), O_NOFOLLOW | O_CLOEXEC | O_RDONLY));
if (fd < 0) {
PLOG(ERROR) << "open: " << file_path;
return false;
}
struct statfs64 sfs;
if (fstatfs64(fd, &sfs)) {
PLOG(ERROR) << "fstatfs64: " << file_path;
return false;
}
return IsFilePinned(fd, file_path, sfs.f_type);
}
static bool CountFiemapExtents(int file_fd, const std::string& file_path, uint32_t* num_extents) {
struct fiemap fiemap = {};
fiemap.fm_start = 0;
fiemap.fm_length = UINT64_MAX;
fiemap.fm_flags = FIEMAP_FLAG_SYNC;
fiemap.fm_extent_count = 0;
if (ioctl(file_fd, FS_IOC_FIEMAP, &fiemap)) {
PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path;
return false;
}
if (num_extents) {
*num_extents = fiemap.fm_mapped_extents;
}
return true;
}
static bool IsValidExtent(const fiemap_extent* extent, std::string_view file_path) {
if (extent->fe_flags & kUnsupportedExtentFlags) {
LOG(ERROR) << "Extent at location " << extent->fe_logical << " of file " << file_path
<< " has unsupported flags";
return false;
}
return true;
}
static bool IsLastExtent(const fiemap_extent* extent) {
if (!(extent->fe_flags & FIEMAP_EXTENT_LAST)) {
LOG(ERROR) << "Extents are being received out-of-order";
return false;
}
return true;
}
static bool FiemapToExtents(struct fiemap* fiemap, std::vector<struct fiemap_extent>* extents,
uint32_t num_extents, std::string_view file_path) {
if (num_extents == 0) return false;
const struct fiemap_extent* last_extent = &fiemap->fm_extents[num_extents - 1];
if (!IsLastExtent(last_extent)) {
LOG(ERROR) << "FIEMAP did not return a final extent for file: " << file_path;
return false;
}
// Iterate through each extent, read and make sure its valid before adding it to the vector
// merging contiguous extents.
fiemap_extent* prev = &fiemap->fm_extents[0];
if (!IsValidExtent(prev, file_path)) return false;
for (uint32_t i = 1; i < num_extents; i++) {
fiemap_extent* next = &fiemap->fm_extents[i];
// Make sure extents are returned in order
if (next != last_extent && IsLastExtent(next)) return false;
// Check if extent's flags are valid
if (!IsValidExtent(next, file_path)) return false;
// Check if the current extent is contiguous with the previous one.
// An extent can be combined with its predecessor only if:
// 1. There is no physical space between the previous and the current
// extent, and
// 2. The physical distance between the previous and current extent
// corresponds to their logical distance (contiguous mapping).
if (prev->fe_physical + prev->fe_length == next->fe_physical &&
next->fe_physical - prev->fe_physical == next->fe_logical - prev->fe_logical) {
prev->fe_length += next->fe_length;
} else {
extents->emplace_back(*prev);
prev = next;
}
}
extents->emplace_back(*prev);
return true;
}
static bool ReadFiemap(int file_fd, const std::string& file_path,
std::vector<struct fiemap_extent>* extents) {
uint32_t num_extents;
if (!CountFiemapExtents(file_fd, file_path, &num_extents)) {
return false;
}
if (num_extents == 0) {
LOG(ERROR) << "File " << file_path << " has zero extents";
return false;
}
if (num_extents > kMaxExtents) {
LOG(ERROR) << "File has " << num_extents << ", maximum is " << kMaxExtents << ": "
<< file_path;
return false;
}
uint64_t fiemap_size =
sizeof(struct fiemap_extent) + num_extents * sizeof(struct fiemap_extent);
auto buffer = std::unique_ptr<void, decltype(&free)>(calloc(1, fiemap_size), free);
if (buffer == nullptr) {
LOG(ERROR) << "Failed to allocate memory for fiemap";
return false;
}
struct fiemap* fiemap = reinterpret_cast<struct fiemap*>(buffer.get());
fiemap->fm_start = 0;
fiemap->fm_length = UINT64_MAX;
// make sure file is synced to disk before we read the fiemap
fiemap->fm_flags = FIEMAP_FLAG_SYNC;
fiemap->fm_extent_count = num_extents;
if (ioctl(file_fd, FS_IOC_FIEMAP, fiemap)) {
PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path;
return false;
}
if (fiemap->fm_mapped_extents != num_extents) {
LOG(ERROR) << "FIEMAP returned unexpected extent count (" << num_extents
<< " expected, got " << fiemap->fm_mapped_extents << ") for file: " << file_path;
return false;
}
return FiemapToExtents(fiemap, extents, num_extents, file_path);
}
static bool ReadFibmap(int file_fd, const std::string& file_path,
std::vector<struct fiemap_extent>* extents) {
struct stat s;
if (fstat(file_fd, &s)) {
PLOG(ERROR) << "Failed to stat " << file_path;
return false;
}
unsigned int blksize;
if (ioctl(file_fd, FIGETBSZ, &blksize) < 0) {
PLOG(ERROR) << "Failed to get FIGETBSZ for " << file_path;
return false;
}
if (!blksize) {
LOG(ERROR) << "Invalid filesystem block size: " << blksize;
return false;
}
uint64_t num_blocks = (s.st_size + blksize - 1) / blksize;
if (num_blocks > std::numeric_limits<uint32_t>::max()) {
LOG(ERROR) << "Too many blocks for FIBMAP (" << num_blocks << ")";
return false;
}
for (uint32_t last_block, block_number = 0; block_number < num_blocks; block_number++) {
uint32_t block = block_number;
if (ioctl(file_fd, FIBMAP, &block)) {
PLOG(ERROR) << "Failed to get FIBMAP for file " << file_path;
return false;
}
if (!block) {
LOG(ERROR) << "Logical block " << block_number << " is a hole, which is not supported";
return false;
}
if (!extents->empty() && block == last_block + 1) {
extents->back().fe_length += blksize;
} else {
extents->push_back(fiemap_extent{.fe_logical = block_number,
.fe_physical = static_cast<uint64_t>(block) * blksize,
.fe_length = static_cast<uint64_t>(blksize),
.fe_flags = 0});
if (extents->size() > kMaxExtents) {
LOG(ERROR) << "File has more than " << kMaxExtents << "extents: " << file_path;
return false;
}
}
last_block = block;
}
return true;
}
FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_size, bool create,
std::function<bool(uint64_t, uint64_t)> progress) {
// if 'create' is false, open an existing file and do not truncate.
int open_flags = O_RDWR | O_CLOEXEC;
if (create) {
if (access(file_path.c_str(), F_OK) == 0) {
LOG(WARNING) << "File " << file_path << " already exists, truncating";
}
open_flags |= O_CREAT | O_TRUNC;
}
::android::base::unique_fd file_fd(
TEMP_FAILURE_RETRY(open(file_path.c_str(), open_flags, S_IRUSR | S_IWUSR)));
if (file_fd < 0) {
PLOG(ERROR) << "Failed to create file at: " << file_path;
return nullptr;
}
std::string abs_path;
if (!::android::base::Realpath(file_path, &abs_path)) {
PLOG(ERROR) << "Invalid file path: " << file_path;
cleanup(file_path, create);
return nullptr;
}
std::string bdev_path;
if (!GetBlockDeviceForFile(abs_path, &bdev_path)) {
LOG(ERROR) << "Failed to get block dev path for file: " << file_path;
cleanup(abs_path, create);
return nullptr;
}
::android::base::unique_fd bdev_fd(
TEMP_FAILURE_RETRY(open(bdev_path.c_str(), O_RDONLY | O_CLOEXEC)));
if (bdev_fd < 0) {
PLOG(ERROR) << "Failed to open block device: " << bdev_path;
cleanup(file_path, create);
return nullptr;
}
uint64_t bdevsz;
if (!GetBlockDeviceSize(bdev_fd, bdev_path, &bdevsz)) {
LOG(ERROR) << "Failed to get block device size for : " << bdev_path;
cleanup(file_path, create);
return nullptr;
}
if (!create) {
file_size = GetFileSize(abs_path);
if (file_size == 0) {
LOG(ERROR) << "Invalid file size of zero bytes for file: " << abs_path;
return nullptr;
}
}
uint64_t blocksz;
uint32_t fs_type;
if (!PerformFileChecks(abs_path, file_size, &blocksz, &fs_type)) {
LOG(ERROR) << "Failed to validate file or file system for file:" << abs_path;
cleanup(abs_path, create);
return nullptr;
}
// Align up to the nearest block size.
if (file_size % blocksz) {
file_size += blocksz - (file_size % blocksz);
}
if (create) {
if (!AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress))) {
LOG(ERROR) << "Failed to allocate file: " << abs_path << " of size: " << file_size
<< " bytes";
cleanup(abs_path, create);
return nullptr;
}
}
// f2fs may move the file blocks around.
if (!PinFile(file_fd, abs_path, fs_type)) {
cleanup(abs_path, create);
LOG(ERROR) << "Failed to pin the file in storage";
return nullptr;
}
// now allocate the FiemapWriter and start setting it up
FiemapUniquePtr fmap(new FiemapWriter());
switch (fs_type) {
case EXT4_SUPER_MAGIC:
case F2FS_SUPER_MAGIC:
if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) {
LOG(ERROR) << "Failed to read fiemap of file: " << abs_path;
cleanup(abs_path, create);
return nullptr;
}
break;
case MSDOS_SUPER_MAGIC:
if (!ReadFibmap(file_fd, abs_path, &fmap->extents_)) {
LOG(ERROR) << "Failed to read fibmap of file: " << abs_path;
cleanup(abs_path, create);
return nullptr;
}
break;
}
fmap->file_path_ = abs_path;
fmap->bdev_path_ = bdev_path;
fmap->file_size_ = file_size;
fmap->bdev_size_ = bdevsz;
fmap->fs_type_ = fs_type;
fmap->block_size_ = blocksz;
LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device "
<< bdev_path;
return fmap;
}
} // namespace fiemap
} // namespace android

541
fs_mgr/libfiemap/fiemap_writer_test.cpp Normal file
View file

@ -0,0 +1,541 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fcntl.h>
#include <inttypes.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>
#include <string>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/unique_fd.h>
#include <gtest/gtest.h>
#include <libdm/loop_control.h>
#include <libfiemap/fiemap_writer.h>
#include <libfiemap/split_fiemap_writer.h>
#include "utility.h"
namespace android {
namespace fiemap {
using namespace std;
using namespace std::string_literals;
using namespace android::fiemap;
using unique_fd = android::base::unique_fd;
using LoopDevice = android::dm::LoopDevice;
std::string gTestDir;
uint64_t testfile_size = 536870912; // default of 512MiB
size_t gBlockSize = 0;
class FiemapWriterTest : public ::testing::Test {
protected:
void SetUp() override {
const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
testfile = gTestDir + "/"s + tinfo->name();
}
void TearDown() override { unlink(testfile.c_str()); }
// name of the file we use for testing
std::string testfile;
};
class SplitFiemapTest : public ::testing::Test {
protected:
void SetUp() override {
const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
testfile = gTestDir + "/"s + tinfo->name();
}
void TearDown() override {
std::string message;
if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) {
cerr << "Could not remove all split files: " << message;
}
}
// name of the file we use for testing
std::string testfile;
};
TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) {
// Try creating a file of size ~100TB but aligned to
// 512 byte to make sure block alignment tests don't
// fail.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184);
EXPECT_EQ(fptr, nullptr);
EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
EXPECT_EQ(errno, ENOENT);
}
TEST_F(FiemapWriterTest, CreateUnalignedFile) {
// Try creating a file of size 4097 bytes which is guaranteed
// to be unaligned to all known block sizes.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1);
ASSERT_NE(fptr, nullptr);
ASSERT_EQ(fptr->size(), gBlockSize * 2);
}
TEST_F(FiemapWriterTest, CheckFilePath) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
ASSERT_NE(fptr, nullptr);
EXPECT_EQ(fptr->size(), gBlockSize);
EXPECT_EQ(fptr->file_path(), testfile);
EXPECT_EQ(access(testfile.c_str(), F_OK), 0);
}
TEST_F(FiemapWriterTest, CheckFileSize) {
// Create a large-ish file and test that the expected size matches.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16);
ASSERT_NE(fptr, nullptr);
struct stat s;
ASSERT_EQ(stat(testfile.c_str(), &s), 0);
EXPECT_EQ(static_cast<uint64_t>(s.st_size), fptr->size());
}
TEST_F(FiemapWriterTest, CheckProgress) {
std::vector<uint64_t> expected;
size_t invocations = 0;
auto callback = [&](uint64_t done, uint64_t total) -> bool {
if (invocations >= expected.size()) {
return false;
}
EXPECT_EQ(done, expected[invocations]);
EXPECT_EQ(total, gBlockSize);
invocations++;
return true;
};
expected.push_back(gBlockSize);
auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
EXPECT_NE(ptr, nullptr);
EXPECT_EQ(invocations, expected.size());
}
TEST_F(FiemapWriterTest, CheckPinning) {
auto ptr = FiemapWriter::Open(testfile, 4096);
ASSERT_NE(ptr, nullptr);
EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile));
}
TEST_F(FiemapWriterTest, CheckBlockDevicePath) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
EXPECT_EQ(fptr->size(), gBlockSize);
EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0));
EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos);
}
TEST_F(FiemapWriterTest, CheckFileCreated) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768);
ASSERT_NE(fptr, nullptr);
unique_fd fd(open(testfile.c_str(), O_RDONLY));
EXPECT_GT(fd, -1);
}
TEST_F(FiemapWriterTest, CheckFileSizeActual) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
ASSERT_NE(fptr, nullptr);
struct stat sb;
ASSERT_EQ(stat(testfile.c_str(), &sb), 0);
EXPECT_GE(sb.st_size, testfile_size);
}
TEST_F(FiemapWriterTest, CheckFileExtents) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
ASSERT_NE(fptr, nullptr);
EXPECT_GT(fptr->extents().size(), 0);
}
TEST_F(FiemapWriterTest, ExistingFile) {
// Create the file.
{ ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); }
// Test that we can still open it.
{
auto ptr = FiemapWriter::Open(testfile, 0, false);
ASSERT_NE(ptr, nullptr);
EXPECT_GT(ptr->extents().size(), 0);
}
}
TEST_F(FiemapWriterTest, FileDeletedOnError) {
auto callback = [](uint64_t, uint64_t) -> bool { return false; };
auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
EXPECT_EQ(ptr, nullptr);
EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
EXPECT_EQ(errno, ENOENT);
}
TEST_F(FiemapWriterTest, MaxBlockSize) {
ASSERT_GT(DetermineMaximumFileSize(testfile), 0);
}
TEST_F(FiemapWriterTest, FibmapBlockAddressing) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
ASSERT_NE(fptr, nullptr);
switch (fptr->fs_type()) {
case F2FS_SUPER_MAGIC:
case EXT4_SUPER_MAGIC:
// Skip the test for FIEMAP supported filesystems. This is really
// because f2fs/ext4 have caches that seem to defeat reading back
// directly from the block device, and writing directly is too
// dangerous.
std::cout << "Skipping test, filesystem does not use FIBMAP\n";
return;
}
bool uses_dm;
std::string bdev_path;
ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm));
if (uses_dm) {
// We could use a device-mapper wrapper here to bypass encryption, but
// really this test is for FIBMAP correctness on VFAT (where encryption
// is never used), so we don't bother.
std::cout << "Skipping test, block device is metadata encrypted\n";
return;
}
std::string data(fptr->size(), '\0');
for (size_t i = 0; i < data.size(); i++) {
data[i] = 'A' + static_cast<char>(data.size() % 26);
}
{
unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC));
ASSERT_GE(fd, 0);
ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size()));
ASSERT_EQ(fsync(fd), 0);
}
ASSERT_FALSE(fptr->extents().empty());
const auto& first_extent = fptr->extents()[0];
unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC));
ASSERT_GE(bdev, 0);
off_t where = first_extent.fe_physical;
ASSERT_EQ(lseek(bdev, where, SEEK_SET), where);
// Note: this will fail on encrypted folders.
std::string actual(data.size(), '\0');
ASSERT_GE(first_extent.fe_length, data.size());
ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size()));
EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0);
}
TEST_F(SplitFiemapTest, Create) {
auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
ASSERT_NE(ptr, nullptr);
auto extents = ptr->extents();
// Destroy the fiemap, closing file handles. This should not delete them.
ptr = nullptr;
std::vector<std::string> files;
ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files));
for (const auto& path : files) {
EXPECT_EQ(access(path.c_str(), F_OK), 0);
}
ASSERT_GE(extents.size(), files.size());
}
TEST_F(SplitFiemapTest, Open) {
{
auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
ASSERT_NE(ptr, nullptr);
}
auto ptr = SplitFiemap::Open(testfile);
ASSERT_NE(ptr, nullptr);
auto extents = ptr->extents();
ASSERT_GE(extents.size(), 24);
}
TEST_F(SplitFiemapTest, DeleteOnFail) {
auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1);
ASSERT_EQ(ptr, nullptr);
std::string first_file = testfile + ".0001";
ASSERT_NE(access(first_file.c_str(), F_OK), 0);
ASSERT_EQ(errno, ENOENT);
ASSERT_NE(access(testfile.c_str(), F_OK), 0);
ASSERT_EQ(errno, ENOENT);
}
static string ReadSplitFiles(const std::string& base_path, size_t num_files) {
std::string result;
for (int i = 0; i < num_files; i++) {
std::string path = base_path + android::base::StringPrintf(".%04d", i);
std::string data;
if (!android::base::ReadFileToString(path, &data)) {
return {};
}
result += data;
}
return result;
}
TEST_F(SplitFiemapTest, WriteWholeFile) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WriteFileInChunks1) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
// Write in chunks of 1000 (so some writes straddle the boundary of two
// files).
size_t bytes_written = 0;
while (bytes_written < kSize) {
size_t to_write = std::min(kSize - bytes_written, (size_t)1000);
char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
ASSERT_TRUE(ptr->Write(data, to_write));
bytes_written += to_write;
}
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WriteFileInChunks2) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
// Write in chunks of 32KiB so every write is exactly at the end of the
// current file.
size_t bytes_written = 0;
while (bytes_written < kSize) {
size_t to_write = std::min(kSize - bytes_written, kChunkSize);
char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
ASSERT_TRUE(ptr->Write(data, to_write));
bytes_written += to_write;
}
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WritePastEnd) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
ASSERT_FALSE(ptr->Write(buffer.get(), kSize));
}
class VerifyBlockWritesExt4 : public ::testing::Test {
// 2GB Filesystem and 4k block size by default
static constexpr uint64_t block_size = 4096;
static constexpr uint64_t fs_size = 2147483648;
protected:
void SetUp() override {
fs_path = std::string(getenv("TMPDIR")) + "/ext4_2G.img";
uint64_t count = fs_size / block_size;
std::string dd_cmd =
::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
" count=%" PRIu64 " > /dev/null 2>&1",
fs_path.c_str(), block_size, count);
std::string mkfs_cmd =
::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path.c_str());
// create mount point
mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt";
ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0);
// create file for the file system
int ret = system(dd_cmd.c_str());
ASSERT_EQ(ret, 0);
// Get and attach a loop device to the filesystem we created
LoopDevice loop_dev(fs_path, 10s);
ASSERT_TRUE(loop_dev.valid());
// create file system
ret = system(mkfs_cmd.c_str());
ASSERT_EQ(ret, 0);
// mount the file system
ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "ext4", 0, nullptr), 0);
}
void TearDown() override {
umount(mntpoint.c_str());
rmdir(mntpoint.c_str());
unlink(fs_path.c_str());
}
std::string mntpoint;
std::string fs_path;
};
class VerifyBlockWritesF2fs : public ::testing::Test {
// 2GB Filesystem and 4k block size by default
static constexpr uint64_t block_size = 4096;
static constexpr uint64_t fs_size = 2147483648;
protected:
void SetUp() override {
fs_path = std::string(getenv("TMPDIR")) + "/f2fs_2G.img";
uint64_t count = fs_size / block_size;
std::string dd_cmd =
::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
" count=%" PRIu64 " > /dev/null 2>&1",
fs_path.c_str(), block_size, count);
std::string mkfs_cmd =
::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path.c_str());
// create mount point
mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt";
ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0);
// create file for the file system
int ret = system(dd_cmd.c_str());
ASSERT_EQ(ret, 0);
// Get and attach a loop device to the filesystem we created
LoopDevice loop_dev(fs_path, 10s);
ASSERT_TRUE(loop_dev.valid());
// create file system
ret = system(mkfs_cmd.c_str());
ASSERT_EQ(ret, 0);
// mount the file system
ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "f2fs", 0, nullptr), 0);
}
void TearDown() override {
umount(mntpoint.c_str());
rmdir(mntpoint.c_str());
unlink(fs_path.c_str());
}
std::string mntpoint;
std::string fs_path;
};
bool DetermineBlockSize() {
struct statfs s;
if (statfs(gTestDir.c_str(), &s)) {
std::cerr << "Could not call statfs: " << strerror(errno) << "\n";
return false;
}
if (!s.f_bsize) {
std::cerr << "Invalid block size: " << s.f_bsize << "\n";
return false;
}
gBlockSize = s.f_bsize;
return true;
}
} // namespace fiemap
} // namespace android
using namespace android::fiemap;
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
if (argc > 1 && argv[1] == "-h"s) {
cerr << "Usage: [test_dir] [file_size]\n";
cerr << "\n";
cerr << "Note: test_dir must be a writable, unencrypted directory.\n";
exit(EXIT_FAILURE);
}
::android::base::InitLogging(argv, ::android::base::StderrLogger);
std::string root_dir = "/data/local/unencrypted";
if (access(root_dir.c_str(), F_OK)) {
root_dir = "/data";
}
std::string tempdir = root_dir + "/XXXXXX"s;
if (!mkdtemp(tempdir.data())) {
cerr << "unable to create tempdir on " << root_dir << "\n";
exit(EXIT_FAILURE);
}
if (!android::base::Realpath(tempdir, &gTestDir)) {
cerr << "unable to find realpath for " << tempdir;
exit(EXIT_FAILURE);
}
if (argc > 2) {
testfile_size = strtoull(argv[2], NULL, 0);
if (testfile_size == ULLONG_MAX) {
testfile_size = 512 * 1024 * 1024;
}
}
if (!DetermineBlockSize()) {
exit(EXIT_FAILURE);
}
auto result = RUN_ALL_TESTS();
std::string cmd = "rm -rf " + gTestDir;
system(cmd.c_str());
return result;
}

674
fs_mgr/libfiemap/image_manager.cpp Normal file
View file

@ -0,0 +1,674 @@
//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <libfiemap/image_manager.h>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/properties.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <ext4_utils/ext4_utils.h>
#include <fs_mgr/file_wait.h>
#include <fs_mgr_dm_linear.h>
#include <libdm/loop_control.h>
#include <libfiemap/split_fiemap_writer.h>
#include "metadata.h"
#include "utility.h"
namespace android {
namespace fiemap {
using namespace std::literals;
using android::base::unique_fd;
using android::dm::DeviceMapper;
using android::dm::DmDeviceState;
using android::dm::DmTable;
using android::dm::DmTargetLinear;
using android::dm::LoopControl;
using android::fs_mgr::CreateLogicalPartition;
using android::fs_mgr::CreateLogicalPartitionParams;
using android::fs_mgr::DestroyLogicalPartition;
using android::fs_mgr::GetPartitionName;
static constexpr char kTestImageMetadataDir[] = "/metadata/gsi/test";
std::unique_ptr<ImageManager> ImageManager::Open(const std::string& dir_prefix) {
auto metadata_dir = "/metadata/gsi/" + dir_prefix;
auto data_dir = "/data/gsi/" + dir_prefix;
return Open(metadata_dir, data_dir);
}
std::unique_ptr<ImageManager> ImageManager::Open(const std::string& metadata_dir,
const std::string& data_dir) {
return std::unique_ptr<ImageManager>(new ImageManager(metadata_dir, data_dir));
}
ImageManager::ImageManager(const std::string& metadata_dir, const std::string& data_dir)
: metadata_dir_(metadata_dir), data_dir_(data_dir) {
partition_opener_ = std::make_unique<android::fs_mgr::PartitionOpener>();
}
std::string ImageManager::GetImageHeaderPath(const std::string& name) {
return JoinPaths(data_dir_, name) + ".img";
}
// The status file has one entry per line, with each entry formatted as one of:
// dm:<name>
// loop:<path>
//
// This simplifies the process of tearing down a mapping, since we can simply
// unmap each entry in the order it appears.
std::string ImageManager::GetStatusFilePath(const std::string& image_name) {
return JoinPaths(metadata_dir_, image_name) + ".status";
}
static std::string GetStatusPropertyName(const std::string& image_name) {
// Note: we don't prefix |image_name|, because CreateLogicalPartition won't
// prefix the name either. There are no plans to change this at the moment,
// consumers of the image API must take care to use globally-unique image
// names.
return "gsid.mapped_image." + image_name;
}
void ImageManager::set_partition_opener(std::unique_ptr<IPartitionOpener>&& opener) {
partition_opener_ = std::move(opener);
}
bool ImageManager::IsImageMapped(const std::string& image_name) {
auto prop_name = GetStatusPropertyName(image_name);
if (android::base::GetProperty(prop_name, "").empty()) {
// If mapped in first-stage init, the dm-device will exist but not the
// property.
auto& dm = DeviceMapper::Instance();
return dm.GetState(image_name) != DmDeviceState::INVALID;
}
return true;
}
std::vector<std::string> ImageManager::GetAllBackingImages() {
std::vector<std::string> images;
if (!MetadataExists(metadata_dir_)) {
return images;
}
auto metadata = OpenMetadata(metadata_dir_);
if (metadata) {
for (auto&& partition : metadata->partitions) {
images.push_back(partition.name);
}
}
return images;
}
bool ImageManager::PartitionExists(const std::string& name) {
if (!MetadataExists(metadata_dir_)) {
return false;
}
auto metadata = OpenMetadata(metadata_dir_);
if (!metadata) {
return false;
}
return !!FindPartition(*metadata.get(), name);
}
bool ImageManager::BackingImageExists(const std::string& name) {
auto header_file = GetImageHeaderPath(name);
return access(header_file.c_str(), F_OK) == 0;
}
bool ImageManager::CreateBackingImage(const std::string& name, uint64_t size, int flags) {
return CreateBackingImage(name, size, flags, nullptr);
}
static bool IsUnreliablePinningAllowed(const std::string& path) {
return android::base::StartsWith(path, "/data/gsi/dsu/") ||
android::base::StartsWith(path, "/data/gsi/test/") ||
android::base::StartsWith(path, "/data/gsi/ota/test/");
}
bool ImageManager::CreateBackingImage(const std::string& name, uint64_t size, int flags,
std::function<bool(uint64_t, uint64_t)>&& on_progress) {
auto data_path = GetImageHeaderPath(name);
auto fw = SplitFiemap::Create(data_path, size, 0, on_progress);
if (!fw) {
return false;
}
bool reliable_pinning;
if (!FilesystemHasReliablePinning(data_path, &reliable_pinning)) {
return false;
}
if (!reliable_pinning && !IsUnreliablePinningAllowed(data_path)) {
// For historical reasons, we allow unreliable pinning for certain use
// cases (DSUs, testing) because the ultimate use case is either
// developer-oriented or ephemeral (the intent is to boot immediately
// into DSUs). For everything else - such as snapshots/OTAs or adb
// remount, we have a higher bar, and require the filesystem to support
// proper pinning.
LOG(ERROR) << "File system does not have reliable block pinning";
SplitFiemap::RemoveSplitFiles(data_path);
return false;
}
// Except for testing, we do not allow persisting metadata that references
// device-mapper devices. It just doesn't make sense, because the device
// numbering may change on reboot. We allow it for testing since the images
// are not meant to survive reboot. Outside of tests, this can only happen
// if device-mapper is stacked in some complex way not supported by
// FiemapWriter.
auto device_path = GetDevicePathForFile(fw.get());
if (android::base::StartsWith(device_path, "/dev/block/dm-") &&
!android::base::StartsWith(metadata_dir_, kTestImageMetadataDir)) {
LOG(ERROR) << "Cannot persist images against device-mapper device: " << device_path;
fw = {};
SplitFiemap::RemoveSplitFiles(data_path);
return false;
}
bool readonly = !!(flags & CREATE_IMAGE_READONLY);
if (!UpdateMetadata(metadata_dir_, name, fw.get(), size, readonly)) {
return false;
}
if (flags & CREATE_IMAGE_ZERO_FILL) {
if (!ZeroFillNewImage(name, 0)) {
DeleteBackingImage(name);
return false;
}
}
return true;
}
bool ImageManager::ZeroFillNewImage(const std::string& name, uint64_t bytes) {
auto data_path = GetImageHeaderPath(name);
// See the comment in MapImageDevice() about how this works.
std::string block_device;
bool can_use_devicemapper;
if (!FiemapWriter::GetBlockDeviceForFile(data_path, &block_device, &can_use_devicemapper)) {
LOG(ERROR) << "Could not determine block device for " << data_path;
return false;
}
if (!can_use_devicemapper) {
// We've backed with loop devices, and since we store files in an
// unencrypted folder, the initial zeroes we wrote will suffice.
return true;
}
// data is dm-crypt, or FBE + dm-default-key. This means the zeroes written
// by libfiemap were encrypted, so we need to map the image in and correct
// this.
auto device = MappedDevice::Open(this, 10s, name);
if (!device) {
return false;
}
static constexpr size_t kChunkSize = 4096;
std::string zeroes(kChunkSize, '\0');
uint64_t remaining;
if (bytes) {
remaining = bytes;
} else {
remaining = get_block_device_size(device->fd());
if (!remaining) {
PLOG(ERROR) << "Could not get block device size for " << device->path();
return false;
}
}
while (remaining) {
uint64_t to_write = std::min(static_cast<uint64_t>(zeroes.size()), remaining);
if (!android::base::WriteFully(device->fd(), zeroes.data(),
static_cast<size_t>(to_write))) {
PLOG(ERROR) << "write failed: " << device->path();
return false;
}
remaining -= to_write;
}
return true;
}
bool ImageManager::DeleteBackingImage(const std::string& name) {
// For dm-linear devices sitting on top of /data, we cannot risk deleting
// the file. The underlying blocks could be reallocated by the filesystem.
if (IsImageMapped(name)) {
LOG(ERROR) << "Backing image " << name << " is currently mapped to a block device";
return false;
}
std::string message;
auto header_file = GetImageHeaderPath(name);
if (!SplitFiemap::RemoveSplitFiles(header_file, &message)) {
// This is fatal, because we don't want to leave these files dangling.
LOG(ERROR) << "Error removing image " << name << ": " << message;
return false;
}
auto status_file = GetStatusFilePath(name);
if (!android::base::RemoveFileIfExists(status_file)) {
LOG(ERROR) << "Error removing " << status_file << ": " << message;
}
return RemoveImageMetadata(metadata_dir_, name);
}
// Create a block device for an image file, using its extents in its
// lp_metadata.
bool ImageManager::MapWithDmLinear(const IPartitionOpener& opener, const std::string& name,
const std::chrono::milliseconds& timeout_ms, std::string* path) {
// :TODO: refresh extents in metadata file until f2fs is fixed.
auto metadata = OpenMetadata(metadata_dir_);
if (!metadata) {
return false;
}
auto super = android::fs_mgr::GetMetadataSuperBlockDevice(*metadata.get());
auto block_device = android::fs_mgr::GetBlockDevicePartitionName(*super);
CreateLogicalPartitionParams params = {
.block_device = block_device,
.metadata = metadata.get(),
.partition_name = name,
.force_writable = true,
.timeout_ms = timeout_ms,
.partition_opener = &opener,
};
if (!CreateLogicalPartition(params, path)) {
LOG(ERROR) << "Error creating device-mapper node for image " << name;
return false;
}
auto status_string = "dm:" + name;
auto status_file = GetStatusFilePath(name);
if (!android::base::WriteStringToFile(status_string, status_file)) {
PLOG(ERROR) << "Could not write status file: " << status_file;
DestroyLogicalPartition(name);
return false;
}
return true;
}
// Helper to create a loop device for a file.
static bool CreateLoopDevice(LoopControl& control, const std::string& file,
const std::chrono::milliseconds& timeout_ms, std::string* path) {
static constexpr int kOpenFlags = O_RDWR | O_NOFOLLOW | O_CLOEXEC;
android::base::unique_fd file_fd(open(file.c_str(), kOpenFlags));
if (file_fd < 0) {
PLOG(ERROR) << "Could not open file: " << file;
return false;
}
if (!control.Attach(file_fd, timeout_ms, path)) {
LOG(ERROR) << "Could not create loop device for: " << file;
return false;
}
LOG(INFO) << "Created loop device " << *path << " for file " << file;
return true;
}
class AutoDetachLoopDevices final {
public:
AutoDetachLoopDevices(LoopControl& control, const std::vector<std::string>& devices)
: control_(control), devices_(devices), commit_(false) {}
~AutoDetachLoopDevices() {
if (commit_) return;
for (const auto& device : devices_) {
control_.Detach(device);
}
}
void Commit() { commit_ = true; }
private:
LoopControl& control_;
const std::vector<std::string>& devices_;
bool commit_;
};
// If an image is stored across multiple files, this takes a list of loop
// devices and joins them together using device-mapper.
bool ImageManager::MapWithLoopDeviceList(const std::vector<std::string>& device_list,
const std::string& name,
const std::chrono::milliseconds& timeout_ms,
std::string* path) {
auto metadata = OpenMetadata(metadata_dir_);
if (!metadata) {
return false;
}
auto partition = FindPartition(*metadata.get(), name);
if (!partition) {
LOG(ERROR) << "Could not find image in metadata: " << name;
return false;
}
// Since extent lengths are in sector units, the size should be a multiple
// of the sector size.
uint64_t partition_size = GetPartitionSize(*metadata.get(), *partition);
if (partition_size % LP_SECTOR_SIZE != 0) {
LOG(ERROR) << "Partition size not sector aligned: " << name << ", " << partition_size
<< " bytes";
return false;
}
DmTable table;
uint64_t start_sector = 0;
uint64_t sectors_needed = partition_size / LP_SECTOR_SIZE;
for (const auto& block_device : device_list) {
// The final block device must be == partition_size, otherwise we
// can't find the AVB footer on verified partitions.
static constexpr int kOpenFlags = O_RDWR | O_NOFOLLOW | O_CLOEXEC;
unique_fd fd(open(block_device.c_str(), kOpenFlags));
if (fd < 0) {
PLOG(ERROR) << "Open failed: " << block_device;
return false;
}
uint64_t file_size = get_block_device_size(fd);
uint64_t file_sectors = file_size / LP_SECTOR_SIZE;
uint64_t segment_size = std::min(file_sectors, sectors_needed);
table.Emplace<DmTargetLinear>(start_sector, segment_size, block_device, 0);
start_sector += segment_size;
sectors_needed -= segment_size;
if (sectors_needed == 0) {
break;
}
}
auto& dm = DeviceMapper::Instance();
if (!dm.CreateDevice(name, table, path, timeout_ms)) {
LOG(ERROR) << "Could not create device-mapper device over loop set";
return false;
}
// Build the status file.
std::vector<std::string> lines;
lines.emplace_back("dm:" + name);
for (const auto& block_device : device_list) {
lines.emplace_back("loop:" + block_device);
}
auto status_message = android::base::Join(lines, "\n");
auto status_file = GetStatusFilePath(name);
if (!android::base::WriteStringToFile(status_message, status_file)) {
PLOG(ERROR) << "Write failed: " << status_file;
dm.DeleteDevice(name);
return false;
}
return true;
}
static bool OptimizeLoopDevices(const std::vector<std::string>& device_list) {
for (const auto& device : device_list) {
unique_fd fd(open(device.c_str(), O_RDWR | O_CLOEXEC | O_NOFOLLOW));
if (fd < 0) {
PLOG(ERROR) << "Open failed: " << device;
return false;
}
if (!LoopControl::EnableDirectIo(fd)) {
return false;
}
}
return true;
}
// Helper to use one or more loop devices around image files.
bool ImageManager::MapWithLoopDevice(const std::string& name,
const std::chrono::milliseconds& timeout_ms,
std::string* path) {
auto image_header = GetImageHeaderPath(name);
std::vector<std::string> file_list;
if (!SplitFiemap::GetSplitFileList(image_header, &file_list)) {
LOG(ERROR) << "Could not get image file list";
return false;
}
// Map each image file as a loopback device.
LoopControl control;
std::vector<std::string> loop_devices;
AutoDetachLoopDevices auto_detach(control, loop_devices);
auto start_time = std::chrono::steady_clock::now();
for (const auto& file : file_list) {
auto now = std::chrono::steady_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - start_time);
std::string loop_device;
if (!CreateLoopDevice(control, file, timeout_ms - elapsed, &loop_device)) {
break;
}
loop_devices.emplace_back(loop_device);
}
if (loop_devices.size() != file_list.size()) {
// The number of devices will mismatch if CreateLoopDevice() failed.
return false;
}
// If OptimizeLoopDevices fails, we'd use double the memory.
if (!OptimizeLoopDevices(loop_devices)) {
return false;
}
// If there's only one loop device (by far the most common case, splits
// will normally only happen on sdcards with FAT32), then just return that
// as the block device. Otherwise, we need to use dm-linear to stitch
// together all the loop devices we just created.
if (loop_devices.size() > 1) {
if (!MapWithLoopDeviceList(loop_devices, name, timeout_ms, path)) {
return false;
}
}
auto status_message = "loop:" + loop_devices.back();
auto status_file = GetStatusFilePath(name);
if (!android::base::WriteStringToFile(status_message, status_file)) {
PLOG(ERROR) << "Write failed: " << status_file;
return false;
}
auto_detach.Commit();
*path = loop_devices.back();
return true;
}
bool ImageManager::MapImageDevice(const std::string& name,
const std::chrono::milliseconds& timeout_ms, std::string* path) {
if (IsImageMapped(name)) {
LOG(ERROR) << "Backing image " << name << " is already mapped";
return false;
}
auto image_header = GetImageHeaderPath(name);
// If there is a device-mapper node wrapping the block device, then we're
// able to create another node around it; the dm layer does not carry the
// exclusion lock down the stack when a mount occurs.
//
// If there is no intermediate device-mapper node, then partitions cannot be
// opened writable due to sepolicy and exclusivity of having a mounted
// filesystem. This should only happen on devices with no encryption, or
// devices with FBE and no metadata encryption. For these cases it suffices
// to perform normal file writes to /data/gsi (which is unencrypted).
std::string block_device;
bool can_use_devicemapper;
if (!FiemapWriter::GetBlockDeviceForFile(image_header, &block_device, &can_use_devicemapper)) {
LOG(ERROR) << "Could not determine block device for " << image_header;
return false;
}
if (can_use_devicemapper) {
if (!MapWithDmLinear(*partition_opener_.get(), name, timeout_ms, path)) {
return false;
}
} else if (!MapWithLoopDevice(name, timeout_ms, path)) {
return false;
}
// Set a property so we remember this is mapped.
auto prop_name = GetStatusPropertyName(name);
if (!android::base::SetProperty(prop_name, *path)) {
UnmapImageDevice(name, true);
return false;
}
return true;
}
bool ImageManager::MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name,
std::string* dev) {
std::string ignore_path;
if (!MapWithDmLinear(opener, name, {}, &ignore_path)) {
return false;
}
auto& dm = DeviceMapper::Instance();
if (!dm.GetDeviceString(name, dev)) {
return false;
}
return true;
}
bool ImageManager::UnmapImageDevice(const std::string& name) {
return UnmapImageDevice(name, false);
}
bool ImageManager::UnmapImageDevice(const std::string& name, bool force) {
if (!force && !IsImageMapped(name)) {
LOG(ERROR) << "Backing image " << name << " is not mapped";
return false;
}
auto& dm = DeviceMapper::Instance();
LoopControl loop;
std::string status;
auto status_file = GetStatusFilePath(name);
if (!android::base::ReadFileToString(status_file, &status)) {
PLOG(ERROR) << "Read failed: " << status_file;
return false;
}
auto lines = android::base::Split(status, "\n");
for (const auto& line : lines) {
auto pieces = android::base::Split(line, ":");
if (pieces.size() != 2) {
LOG(ERROR) << "Unknown status line";
continue;
}
if (pieces[0] == "dm") {
// Failure to remove a dm node is fatal, since we can't safely
// remove the file or loop devices.
const auto& name = pieces[1];
if (!dm.DeleteDeviceIfExists(name)) {
return false;
}
} else if (pieces[0] == "loop") {
// Failure to remove a loop device is not fatal, since we can still
// remove the backing file if we want.
loop.Detach(pieces[1]);
} else {
LOG(ERROR) << "Unknown status: " << pieces[0];
}
}
std::string message;
if (!android::base::RemoveFileIfExists(status_file, &message)) {
LOG(ERROR) << "Could not remove " << status_file << ": " << message;
}
auto status_prop = GetStatusPropertyName(name);
android::base::SetProperty(status_prop, "");
return true;
}
bool ImageManager::RemoveAllImages() {
if (!MetadataExists(metadata_dir_)) {
return true;
}
auto metadata = OpenMetadata(metadata_dir_);
if (!metadata) {
return RemoveAllMetadata(metadata_dir_);
}
bool ok = true;
for (const auto& partition : metadata->partitions) {
auto partition_name = GetPartitionName(partition);
ok &= DeleteBackingImage(partition_name);
}
return ok && RemoveAllMetadata(metadata_dir_);
}
bool ImageManager::Validate() {
auto metadata = OpenMetadata(metadata_dir_);
if (!metadata) {
return false;
}
for (const auto& partition : metadata->partitions) {
auto name = GetPartitionName(partition);
auto image_path = GetImageHeaderPath(name);
auto fiemap = SplitFiemap::Open(image_path);
if (!fiemap || !fiemap->HasPinnedExtents()) {
LOG(ERROR) << "Image is missing or was moved: " << image_path;
return false;
}
}
return true;
}
std::unique_ptr<MappedDevice> MappedDevice::Open(IImageManager* manager,
const std::chrono::milliseconds& timeout_ms,
const std::string& name) {
std::string path;
if (!manager->MapImageDevice(name, timeout_ms, &path)) {
return nullptr;
}
auto device = std::unique_ptr<MappedDevice>(new MappedDevice(manager, name, path));
if (device->fd() < 0) {
return nullptr;
}
return device;
}
MappedDevice::MappedDevice(IImageManager* manager, const std::string& name, const std::string& path)
: manager_(manager), name_(name), path_(path) {
// The device is already mapped; try and open it.
fd_.reset(open(path.c_str(), O_RDWR | O_CLOEXEC));
}
MappedDevice::~MappedDevice() {
fd_ = {};
manager_->UnmapImageDevice(name_);
}
bool IImageManager::UnmapImageIfExists(const std::string& name) {
// No lock is needed even though this seems to be vulnerable to TOCTOU. If process A
// calls MapImageDevice() while process B calls UnmapImageIfExists(), and MapImageDevice()
// happens after process B checks IsImageMapped(), it would be as if MapImageDevice() is called
// after process B finishes calling UnmapImageIfExists(), resulting the image to be mapped,
// which is a reasonable sequence.
if (!IsImageMapped(name)) {
return true;
}
return UnmapImageDevice(name);
}
} // namespace fiemap
} // namespace android

251
fs_mgr/libfiemap/image_test.cpp Normal file
View file

@ -0,0 +1,251 @@
//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <chrono>
#include <iostream>
#include <thread>
#include <android-base/file.h>
#include <android-base/properties.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <ext4_utils/ext4_utils.h>
#include <fs_mgr/file_wait.h>
#include <gtest/gtest.h>
#include <libdm/dm.h>
#include <libfiemap/image_manager.h>
using namespace android::dm;
using namespace std::literals;
using android::base::unique_fd;
using android::fiemap::ImageManager;
using android::fs_mgr::BlockDeviceInfo;
using android::fs_mgr::PartitionOpener;
using android::fs_mgr::WaitForFile;
static std::string gDataPath;
static std::string gDataMountPath;
static constexpr char kMetadataPath[] = "/metadata/gsi/test";
static constexpr uint64_t kTestImageSize = 1024 * 1024;
class TestPartitionOpener final : public PartitionOpener {
public:
android::base::unique_fd Open(const std::string& partition_name, int flags) const override {
return PartitionOpener::Open(GetPathForBlockDeviceName(partition_name), flags);
}
bool GetInfo(const std::string& partition_name, BlockDeviceInfo* info) const override {
return PartitionOpener::GetInfo(GetPathForBlockDeviceName(partition_name), info);
}
std::string GetDeviceString(const std::string& partition_name) const override {
return PartitionOpener::GetDeviceString(GetPathForBlockDeviceName(partition_name));
}
private:
static std::string GetPathForBlockDeviceName(const std::string& name) {
if (android::base::StartsWith(name, "loop") || android::base::StartsWith(name, "dm-")) {
return "/dev/block/"s + name;
}
return name;
}
};
// This fixture is for tests against the device's native configuration.
class NativeTest : public ::testing::Test {
protected:
void SetUp() override {
manager_ = ImageManager::Open(kMetadataPath, gDataPath);
ASSERT_NE(manager_, nullptr);
manager_->set_partition_opener(std::make_unique<TestPartitionOpener>());
const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
base_name_ = tinfo->name();
}
void TearDown() override {
manager_->UnmapImageDevice(base_name_);
manager_->DeleteBackingImage(base_name_);
}
std::string PropertyName() { return "gsid.mapped_image." + base_name_; }
std::unique_ptr<ImageManager> manager_;
std::string base_name_;
};
TEST_F(NativeTest, CreateAndMap) {
ASSERT_TRUE(manager_->CreateBackingImage(base_name_, kTestImageSize, false, nullptr));
std::string path;
ASSERT_TRUE(manager_->MapImageDevice(base_name_, 5s, &path));
ASSERT_TRUE(manager_->IsImageMapped(base_name_));
ASSERT_EQ(android::base::GetProperty(PropertyName(), ""), path);
{
unique_fd fd(open(path.c_str(), O_RDWR | O_NOFOLLOW | O_CLOEXEC));
ASSERT_GE(fd, 0);
ASSERT_EQ(get_block_device_size(fd), kTestImageSize);
}
ASSERT_TRUE(manager_->UnmapImageDevice(base_name_));
ASSERT_FALSE(manager_->IsImageMapped(base_name_));
ASSERT_EQ(android::base::GetProperty(PropertyName(), ""), "");
}
// This fixture is for tests against a simulated device environment. Rather
// than use /data, we create an image and then layer a new filesystem within
// it. Each test then decides how to mount and create layered images. This
// allows us to test FBE vs FDE configurations.
class ImageTest : public ::testing::Test {
public:
ImageTest() : dm_(DeviceMapper::Instance()) {}
void SetUp() override {
manager_ = ImageManager::Open(kMetadataPath, gDataPath);
ASSERT_NE(manager_, nullptr);
manager_->set_partition_opener(std::make_unique<TestPartitionOpener>());
submanager_ = ImageManager::Open(kMetadataPath + "/mnt"s, gDataPath + "/mnt"s);
ASSERT_NE(submanager_, nullptr);
submanager_->set_partition_opener(std::make_unique<TestPartitionOpener>());
// Ensure that metadata is cleared in between runs.
submanager_->RemoveAllImages();
manager_->RemoveAllImages();
const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
base_name_ = tinfo->name();
test_image_name_ = base_name_ + "-base";
wrapper_device_name_ = base_name_ + "-wrapper";
ASSERT_TRUE(manager_->CreateBackingImage(base_name_, kTestImageSize * 16, false, nullptr));
ASSERT_TRUE(manager_->MapImageDevice(base_name_, 5s, &base_device_));
}
void TearDown() override {
submanager_->UnmapImageDevice(test_image_name_);
umount(gDataMountPath.c_str());
dm_.DeleteDeviceIfExists(wrapper_device_name_);
manager_->UnmapImageDevice(base_name_);
manager_->DeleteBackingImage(base_name_);
}
protected:
bool DoFormat(const std::string& device) {
// clang-format off
std::vector<std::string> mkfs_args = {
"/system/bin/mke2fs",
"-F",
"-b 4096",
"-t ext4",
"-m 0",
"-O has_journal",
device,
">/dev/null",
"2>/dev/null",
"</dev/null",
};
// clang-format on
auto command = android::base::Join(mkfs_args, " ");
return system(command.c_str()) == 0;
}
std::unique_ptr<ImageManager> manager_;
std::unique_ptr<ImageManager> submanager_;
DeviceMapper& dm_;
std::string base_name_;
std::string base_device_;
std::string test_image_name_;
std::string wrapper_device_name_;
};
TEST_F(ImageTest, DirectMount) {
ASSERT_TRUE(DoFormat(base_device_));
ASSERT_EQ(mount(base_device_.c_str(), gDataMountPath.c_str(), "ext4", 0, nullptr), 0);
ASSERT_TRUE(submanager_->CreateBackingImage(test_image_name_, kTestImageSize, false, nullptr));
std::string path;
ASSERT_TRUE(submanager_->MapImageDevice(test_image_name_, 5s, &path));
ASSERT_TRUE(android::base::StartsWith(path, "/dev/block/loop"));
}
TEST_F(ImageTest, IndirectMount) {
// Create a simple wrapper around the base device that we'll mount from
// instead. This will simulate the code paths for dm-crypt/default-key/bow
// and force us to use device-mapper rather than loop devices.
uint64_t device_size = 0;
{
unique_fd fd(open(base_device_.c_str(), O_RDWR | O_CLOEXEC));
ASSERT_GE(fd, 0);
device_size = get_block_device_size(fd);
ASSERT_EQ(device_size, kTestImageSize * 16);
}
uint64_t num_sectors = device_size / 512;
auto& dm = DeviceMapper::Instance();
DmTable table;
table.Emplace<DmTargetLinear>(0, num_sectors, base_device_, 0);
ASSERT_TRUE(dm.CreateDevice(wrapper_device_name_, table));
// Format and mount.
std::string wrapper_device;
ASSERT_TRUE(dm.GetDmDevicePathByName(wrapper_device_name_, &wrapper_device));
ASSERT_TRUE(WaitForFile(wrapper_device, 5s));
ASSERT_TRUE(DoFormat(wrapper_device));
ASSERT_EQ(mount(wrapper_device.c_str(), gDataMountPath.c_str(), "ext4", 0, nullptr), 0);
ASSERT_TRUE(submanager_->CreateBackingImage(test_image_name_, kTestImageSize, false, nullptr));
std::string path;
ASSERT_TRUE(submanager_->MapImageDevice(test_image_name_, 5s, &path));
ASSERT_TRUE(android::base::StartsWith(path, "/dev/block/dm-"));
}
bool Mkdir(const std::string& path) {
if (mkdir(path.c_str(), 0700) && errno != EEXIST) {
std::cerr << "Could not mkdir " << path << ": " << strerror(errno) << std::endl;
return false;
}
return true;
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
if (argc >= 2) {
gDataPath = argv[1];
} else {
gDataPath = "/data/gsi/test";
}
gDataMountPath = gDataPath + "/mnt"s;
if (!Mkdir(gDataPath) || !Mkdir(kMetadataPath) || !Mkdir(gDataMountPath) ||
!Mkdir(kMetadataPath + "/mnt"s)) {
return 1;
}
return RUN_ALL_TESTS();
}

118
fs_mgr/libfiemap/include/libfiemap/fiemap_writer.h Normal file
View file

@ -0,0 +1,118 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <linux/fiemap.h>
#include <stdint.h>
#include <sys/types.h>
#include <unistd.h>
#include <functional>
#include <string>
#include <vector>
#include <android-base/unique_fd.h>
namespace android {
namespace fiemap {
class FiemapWriter;
using FiemapUniquePtr = std::unique_ptr<FiemapWriter>;
class FiemapWriter final {
public:
// Factory method for FiemapWriter.
// The method returns FiemapUniquePtr that contains all the data necessary to be able to write
// to the given file directly using raw block i/o. The optional progress callback will be
// invoked, if create is true, while the file is being initialized. It receives the bytes
// written and the number of total bytes. If the callback returns false, the operation will
// fail.
//
// Note: when create is true, the file size will be aligned up to the nearest file system
// block.
static FiemapUniquePtr Open(const std::string& file_path, uint64_t file_size,
bool create = true,
std::function<bool(uint64_t, uint64_t)> progress = {});
// Check that a file still has the same extents since it was last opened with FiemapWriter,
// assuming the file was not resized outside of FiemapWriter. Returns false either on error
// or if the file was not pinned.
//
// This will always return true on Ext4. On F2FS, it will return true if either of the
// following cases are true:
// - The file was never pinned.
// - The file is pinned and has not been moved by the GC.
// Thus, this method should only be called for pinned files (such as those returned by
// FiemapWriter::Open).
static bool HasPinnedExtents(const std::string& file_path);
// Returns the underlying block device of a file. This will look past device-mapper layers
// as long as each layer would not change block mappings (i.e., dm-crypt, dm-bow, and dm-
// default-key tables are okay; dm-linear is not). If a mapping such as dm-linear is found,
// it will be returned in place of any physical block device.
//
// It is the caller's responsibility to check whether the returned block device is acceptable.
// Gsid, for example, will only accept /dev/block/by-name/userdata as the bottom device.
// Callers can check the device name (dm- or loop prefix), inspect sysfs, or compare the major
// number against a boot device.
//
// If device-mapper nodes were encountered, then |uses_dm| will be set to true.
static bool GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path,
bool* uses_dm = nullptr);
~FiemapWriter() = default;
const std::string& file_path() const { return file_path_; };
uint64_t size() const { return file_size_; };
const std::string& bdev_path() const { return bdev_path_; };
uint64_t block_size() const { return block_size_; };
const std::vector<struct fiemap_extent>& extents() { return extents_; };
uint32_t fs_type() const { return fs_type_; }
// Non-copyable & Non-movable
FiemapWriter(const FiemapWriter&) = delete;
FiemapWriter& operator=(const FiemapWriter&) = delete;
FiemapWriter& operator=(FiemapWriter&&) = delete;
FiemapWriter(FiemapWriter&&) = delete;
private:
// Name of the file managed by this class.
std::string file_path_;
// Block device on which we have created the file.
std::string bdev_path_;
// Size in bytes of the file this class is writing
uint64_t file_size_;
// total size in bytes of the block device
uint64_t bdev_size_;
// Filesystem type where the file is being created.
// See: <uapi/linux/magic.h> for filesystem magic numbers
uint32_t fs_type_;
// block size as reported by the kernel of the underlying block device;
uint64_t block_size_;
// This file's fiemap
std::vector<struct fiemap_extent> extents_;
FiemapWriter() = default;
};
} // namespace fiemap
} // namespace android

185
fs_mgr/libfiemap/include/libfiemap/image_manager.h Normal file
View file

@ -0,0 +1,185 @@
//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#pragma once
#include <stdint.h>
#include <chrono>
#include <functional>
#include <memory>
#include <string>
#include <android-base/unique_fd.h>
#include <liblp/partition_opener.h>
namespace android {
namespace fiemap {
class IImageManager {
public:
using IPartitionOpener = android::fs_mgr::IPartitionOpener;
virtual ~IImageManager() {}
// When linking to libfiemap_binder, the Open() call will use binder.
// Otherwise, the Open() call will use the ImageManager implementation
// below.
static std::unique_ptr<IImageManager> Open(const std::string& dir_prefix,
const std::chrono::milliseconds& timeout_ms);
// Flags for CreateBackingImage().
static constexpr int CREATE_IMAGE_DEFAULT = 0x0;
static constexpr int CREATE_IMAGE_READONLY = 0x1;
static constexpr int CREATE_IMAGE_ZERO_FILL = 0x2;
// Create an image that can be mapped as a block-device. If |force_zero_fill|
// is true, the image will be zero-filled. Otherwise, the initial content
// of the image is undefined. If zero-fill is requested, and the operation
// cannot be completed, the image will be deleted and this function will
// return false.
virtual bool CreateBackingImage(const std::string& name, uint64_t size, int flags) = 0;
// Delete an image created with CreateBackingImage. Its entry will be
// removed from the associated lp_metadata file.
virtual bool DeleteBackingImage(const std::string& name) = 0;
// Create a block device for an image previously created with
// CreateBackingImage. This will wait for at most |timeout_ms| milliseconds
// for |path| to be available, and will return false if not available in
// the requested time. If |timeout_ms| is zero, this is NOT guaranteed to
// return true. A timeout of 10s is recommended.
//
// Note that snapshots created with a readonly flag are always mapped
// writable. The flag is persisted in the lp_metadata file however, so if
// fs_mgr::CreateLogicalPartition(s) is used, the flag will be respected.
virtual bool MapImageDevice(const std::string& name,
const std::chrono::milliseconds& timeout_ms, std::string* path) = 0;
// Unmap a block device previously mapped with mapBackingImage.
virtual bool UnmapImageDevice(const std::string& name) = 0;
// Returns true whether the named backing image exists.
virtual bool BackingImageExists(const std::string& name) = 0;
// Returns true if the specified image is mapped to a device.
virtual bool IsImageMapped(const std::string& name) = 0;
// Map an image using device-mapper. This is not available over binder, and
// is intended only for first-stage init. The returned device is a major:minor
// device string.
virtual bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name,
std::string* dev) = 0;
// Get all backing image names.
virtual std::vector<std::string> GetAllBackingImages() = 0;
// Writes |bytes| zeros to |name| file. If |bytes| is 0, then the
// whole file if filled with zeros.
virtual bool ZeroFillNewImage(const std::string& name, uint64_t bytes) = 0;
// Find and remove all images and metadata for this manager.
virtual bool RemoveAllImages() = 0;
virtual bool UnmapImageIfExists(const std::string& name);
};
class ImageManager final : public IImageManager {
public:
// Return an ImageManager for the given metadata and data directories. Both
// directories must already exist.
static std::unique_ptr<ImageManager> Open(const std::string& metadata_dir,
const std::string& data_dir);
// Helper function that derives the metadata and data dirs given a single
// prefix.
static std::unique_ptr<ImageManager> Open(const std::string& dir_prefix);
// Methods that must be implemented from IImageManager.
bool CreateBackingImage(const std::string& name, uint64_t size, int flags) override;
bool DeleteBackingImage(const std::string& name) override;
bool MapImageDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms,
std::string* path) override;
bool UnmapImageDevice(const std::string& name) override;
bool BackingImageExists(const std::string& name) override;
bool IsImageMapped(const std::string& name) override;
bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name,
std::string* dev) override;
bool RemoveAllImages() override;
std::vector<std::string> GetAllBackingImages();
// Same as CreateBackingImage, but provides a progress notification.
bool CreateBackingImage(const std::string& name, uint64_t size, int flags,
std::function<bool(uint64_t, uint64_t)>&& on_progress);
// Returns true if the named partition exists. This does not check the
// consistency of the backing image/data file.
bool PartitionExists(const std::string& name);
// Validates that all images still have pinned extents. This will be removed
// once b/134588268 is fixed.
bool Validate();
void set_partition_opener(std::unique_ptr<IPartitionOpener>&& opener);
// Writes |bytes| zeros at the beginning of the passed image
bool ZeroFillNewImage(const std::string& name, uint64_t bytes);
private:
ImageManager(const std::string& metadata_dir, const std::string& data_dir);
std::string GetImageHeaderPath(const std::string& name);
std::string GetStatusFilePath(const std::string& image_name);
bool MapWithLoopDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms,
std::string* path);
bool MapWithLoopDeviceList(const std::vector<std::string>& device_list, const std::string& name,
const std::chrono::milliseconds& timeout_ms, std::string* path);
bool MapWithDmLinear(const IPartitionOpener& opener, const std::string& name,
const std::chrono::milliseconds& timeout_ms, std::string* path);
bool UnmapImageDevice(const std::string& name, bool force);
ImageManager(const ImageManager&) = delete;
ImageManager& operator=(const ImageManager&) = delete;
ImageManager& operator=(ImageManager&&) = delete;
ImageManager(ImageManager&&) = delete;
std::string metadata_dir_;
std::string data_dir_;
std::unique_ptr<IPartitionOpener> partition_opener_;
};
// RAII helper class for mapping and opening devices with an ImageManager.
class MappedDevice final {
public:
static std::unique_ptr<MappedDevice> Open(IImageManager* manager,
const std::chrono::milliseconds& timeout_ms,
const std::string& name);
~MappedDevice();
int fd() const { return fd_; }
const std::string& path() const { return path_; }
protected:
MappedDevice(IImageManager* manager, const std::string& name, const std::string& path);
IImageManager* manager_;
std::string name_;
std::string path_;
android::base::unique_fd fd_;
};
} // namespace fiemap
} // namespace android

97
fs_mgr/libfiemap/include/libfiemap/split_fiemap_writer.h Normal file
View file

@ -0,0 +1,97 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <stdint.h>
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include <android-base/unique_fd.h>
#include "fiemap_writer.h"
namespace android {
namespace fiemap {
// Wrapper around FiemapWriter that is able to split images across files if
// necessary.
class SplitFiemap final {
public:
using ProgressCallback = std::function<bool(uint64_t, uint64_t)>;
// Create a new split fiemap file. If |max_piece_size| is 0, the number of
// pieces will be determined automatically by detecting the filesystem.
// Otherwise, the file will be split evenly (with the remainder in the
// final file).
static std::unique_ptr<SplitFiemap> Create(const std::string& file_path, uint64_t file_size,
uint64_t max_piece_size,
ProgressCallback progress = {});
// Open an existing split fiemap file.
static std::unique_ptr<SplitFiemap> Open(const std::string& file_path);
~SplitFiemap();
// Return a list of all files created for a split file.
static bool GetSplitFileList(const std::string& file_path, std::vector<std::string>* list);
// Destroy all components of a split file. If the root file does not exist,
// this returns true and does not report an error.
static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr);
// Return whether all components of a split file still have pinned extents.
bool HasPinnedExtents() const;
// Helper method for writing data that spans files. Note there is no seek
// method (yet); this starts at 0 and increments the position by |bytes|.
bool Write(const void* data, uint64_t bytes);
// Flush all writes to all split files.
bool Flush();
const std::vector<struct fiemap_extent>& extents();
uint32_t block_size() const;
uint64_t size() const { return total_size_; }
const std::string& bdev_path() const;
// Non-copyable & Non-movable
SplitFiemap(const SplitFiemap&) = delete;
SplitFiemap& operator=(const SplitFiemap&) = delete;
SplitFiemap& operator=(SplitFiemap&&) = delete;
SplitFiemap(SplitFiemap&&) = delete;
private:
SplitFiemap() = default;
void AddFile(FiemapUniquePtr&& file);
bool creating_ = false;
std::string list_file_;
std::vector<FiemapUniquePtr> files_;
std::vector<struct fiemap_extent> extents_;
uint64_t total_size_ = 0;
// Most recently open file and position for Write().
size_t cursor_index_ = 0;
uint64_t cursor_file_pos_ = 0;
android::base::unique_fd cursor_fd_;
};
} // namespace fiemap
} // namespace android

196
fs_mgr/libfiemap/metadata.cpp Normal file
View file

@ -0,0 +1,196 @@
//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include "metadata.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <liblp/builder.h>
#include "utility.h"
namespace android {
namespace fiemap {
using namespace android::fs_mgr;
static constexpr uint32_t kMaxMetadataSize = 256 * 1024;
std::string GetMetadataFile(const std::string& metadata_dir) {
return JoinPaths(metadata_dir, "lp_metadata");
}
bool MetadataExists(const std::string& metadata_dir) {
auto metadata_file = GetMetadataFile(metadata_dir);
return access(metadata_file.c_str(), F_OK) == 0;
}
std::unique_ptr<LpMetadata> OpenMetadata(const std::string& metadata_dir) {
auto metadata_file = GetMetadataFile(metadata_dir);
auto metadata = ReadFromImageFile(metadata_file);
if (!metadata) {
LOG(ERROR) << "Could not read metadata file " << metadata_file;
return nullptr;
}
return metadata;
}
// :TODO: overwrite on create if open fails
std::unique_ptr<MetadataBuilder> OpenOrCreateMetadata(const std::string& metadata_dir,
SplitFiemap* file) {
auto metadata_file = GetMetadataFile(metadata_dir);
PartitionOpener opener;
std::unique_ptr<MetadataBuilder> builder;
if (access(metadata_file.c_str(), R_OK)) {
if (errno != ENOENT) {
PLOG(ERROR) << "access " << metadata_file << " failed:";
return nullptr;
}
auto data_device = GetDevicePathForFile(file);
BlockDeviceInfo device_info;
if (!opener.GetInfo(data_device, &device_info)) {
LOG(ERROR) << "Could not read partition: " << data_device;
return nullptr;
}
std::vector<BlockDeviceInfo> block_devices = {device_info};
auto super_name = android::base::Basename(data_device);
builder = MetadataBuilder::New(block_devices, super_name, kMaxMetadataSize, 1);
} else {
auto metadata = OpenMetadata(metadata_dir);
if (!metadata) {
return nullptr;
}
builder = MetadataBuilder::New(*metadata.get(), &opener);
}
if (!builder) {
LOG(ERROR) << "Could not create metadata builder";
return nullptr;
}
return builder;
}
bool SaveMetadata(MetadataBuilder* builder, const std::string& metadata_dir) {
auto exported = builder->Export();
if (!exported) {
LOG(ERROR) << "Unable to export new metadata";
return false;
}
// If there are no more partitions in the metadata, just delete the file.
auto metadata_file = GetMetadataFile(metadata_dir);
if (exported->partitions.empty() && android::base::RemoveFileIfExists(metadata_file)) {
return true;
}
if (!WriteToImageFile(metadata_file, *exported.get())) {
LOG(ERROR) << "Unable to save new metadata";
return false;
}
return true;
}
bool RemoveAllMetadata(const std::string& dir) {
auto metadata_file = GetMetadataFile(dir);
return android::base::RemoveFileIfExists(metadata_file);
}
bool FillPartitionExtents(MetadataBuilder* builder, Partition* partition, SplitFiemap* file,
uint64_t partition_size) {
auto block_device = android::base::Basename(GetDevicePathForFile(file));
uint64_t sectors_needed = partition_size / LP_SECTOR_SIZE;
for (const auto& extent : file->extents()) {
if (extent.fe_length % LP_SECTOR_SIZE != 0) {
LOG(ERROR) << "Extent is not sector-aligned: " << extent.fe_length;
return false;
}
if (extent.fe_physical % LP_SECTOR_SIZE != 0) {
LOG(ERROR) << "Extent physical sector is not sector-aligned: " << extent.fe_physical;
return false;
}
uint64_t num_sectors =
std::min(static_cast<uint64_t>(extent.fe_length / LP_SECTOR_SIZE), sectors_needed);
if (!num_sectors || !sectors_needed) {
// This should never happen, but we include it just in case. It would
// indicate that the last filesystem block had multiple extents.
LOG(WARNING) << "FiemapWriter allocated extra blocks";
break;
}
uint64_t physical_sector = extent.fe_physical / LP_SECTOR_SIZE;
if (!builder->AddLinearExtent(partition, block_device, num_sectors, physical_sector)) {
LOG(ERROR) << "Could not add extent to lp metadata";
return false;
}
sectors_needed -= num_sectors;
}
return true;
}
bool RemoveImageMetadata(const std::string& metadata_dir, const std::string& partition_name) {
if (!MetadataExists(metadata_dir)) {
return true;
}
auto metadata = OpenMetadata(metadata_dir);
if (!metadata) {
return false;
}
PartitionOpener opener;
auto builder = MetadataBuilder::New(*metadata.get(), &opener);
if (!builder) {
return false;
}
builder->RemovePartition(partition_name);
return SaveMetadata(builder.get(), metadata_dir);
}
bool UpdateMetadata(const std::string& metadata_dir, const std::string& partition_name,
SplitFiemap* file, uint64_t partition_size, bool readonly) {
auto builder = OpenOrCreateMetadata(metadata_dir, file);
if (!builder) {
return false;
}
auto partition = builder->FindPartition(partition_name);
if (!partition) {
int attrs = 0;
if (readonly) attrs |= LP_PARTITION_ATTR_READONLY;
if ((partition = builder->AddPartition(partition_name, attrs)) == nullptr) {
LOG(ERROR) << "Could not add partition " << partition_name << " to metadata";
return false;
}
}
partition->RemoveExtents();
if (!FillPartitionExtents(builder.get(), partition, file, partition_size)) {
return false;
}
return SaveMetadata(builder.get(), metadata_dir);
}
} // namespace fiemap
} // namespace android

36
fs_mgr/libfiemap/metadata.h Normal file
View file

@ -0,0 +1,36 @@
//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <stdint.h>
#include <memory>
#include <string>
#include <libfiemap/split_fiemap_writer.h>
#include <liblp/liblp.h>
namespace android {
namespace fiemap {
bool MetadataExists(const std::string& metadata_dir);
std::unique_ptr<android::fs_mgr::LpMetadata> OpenMetadata(const std::string& metadata_dir);
bool UpdateMetadata(const std::string& metadata_dir, const std::string& partition_name,
SplitFiemap* file, uint64_t partition_size, bool readonly);
bool RemoveImageMetadata(const std::string& metadata_dir, const std::string& partition_name);
bool RemoveAllMetadata(const std::string& dir);
} // namespace fiemap
} // namespace android

29
fs_mgr/libfiemap/passthrough.cpp Normal file
View file

@ -0,0 +1,29 @@
//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <libfiemap/image_manager.h>
namespace android {
namespace fiemap {
std::unique_ptr<IImageManager> IImageManager::Open(const std::string& dir_prefix,
const std::chrono::milliseconds& timeout_ms) {
(void)timeout_ms;
return ImageManager::Open(dir_prefix);
}
} // namespace fiemap
} // namespace android

303
fs_mgr/libfiemap/split_fiemap_writer.cpp Normal file
View file

@ -0,0 +1,303 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <libfiemap/split_fiemap_writer.h>
#include <fcntl.h>
#include <stdint.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <memory>
#include <string>
#include <vector>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include "utility.h"
namespace android {
namespace fiemap {
using android::base::unique_fd;
// We use a four-digit suffix at the end of filenames.
static const size_t kMaxFilePieces = 500;
std::unique_ptr<SplitFiemap> SplitFiemap::Create(const std::string& file_path, uint64_t file_size,
uint64_t max_piece_size,
ProgressCallback progress) {
if (!file_size) {
LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path;
return nullptr;
}
if (!max_piece_size) {
max_piece_size = DetermineMaximumFileSize(file_path);
if (!max_piece_size) {
LOG(ERROR) << "Could not determine maximum file size for " << file_path;
return nullptr;
}
}
// Remove any existing file.
RemoveSplitFiles(file_path);
// Call |progress| only when the total percentage would significantly change.
int permille = -1;
uint64_t total_bytes_written = 0;
auto on_progress = [&](uint64_t written, uint64_t) -> bool {
uint64_t actual_written = total_bytes_written + written;
int new_permille = (actual_written * 1000) / file_size;
if (new_permille != permille && actual_written < file_size) {
if (progress && !progress(actual_written, file_size)) {
return false;
}
permille = new_permille;
}
return true;
};
std::unique_ptr<SplitFiemap> out(new SplitFiemap());
out->creating_ = true;
out->list_file_ = file_path;
// Create the split files.
uint64_t remaining_bytes = file_size;
while (remaining_bytes) {
if (out->files_.size() >= kMaxFilePieces) {
LOG(ERROR) << "Requested size " << file_size << " created too many split files";
return nullptr;
}
std::string chunk_path =
android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size());
uint64_t chunk_size = std::min(max_piece_size, remaining_bytes);
auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress);
if (!writer) {
return nullptr;
}
// To make sure the alignment doesn't create too much inconsistency, we
// account the *actual* size, not the requested size.
total_bytes_written += writer->size();
// writer->size() is block size aligned and could be bigger than remaining_bytes
// If remaining_bytes is bigger, set remaining_bytes to 0 to avoid underflow error.
remaining_bytes = remaining_bytes > writer->size() ? (remaining_bytes - writer->size()) : 0;
out->AddFile(std::move(writer));
}
// Create the split file list.
unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660));
if (fd < 0) {
PLOG(ERROR) << "Failed to open " << file_path;
return nullptr;
}
for (const auto& writer : out->files_) {
std::string line = android::base::Basename(writer->file_path()) + "\n";
if (!android::base::WriteFully(fd, line.data(), line.size())) {
PLOG(ERROR) << "Write failed " << file_path;
return nullptr;
}
}
// Unset this bit, so we don't unlink on destruction.
out->creating_ = false;
return out;
}
std::unique_ptr<SplitFiemap> SplitFiemap::Open(const std::string& file_path) {
std::vector<std::string> files;
if (!GetSplitFileList(file_path, &files)) {
return nullptr;
}
std::unique_ptr<SplitFiemap> out(new SplitFiemap());
out->list_file_ = file_path;
for (const auto& file : files) {
auto writer = FiemapWriter::Open(file, 0, false);
if (!writer) {
// Error was logged in Open().
return nullptr;
}
out->AddFile(std::move(writer));
}
return out;
}
bool SplitFiemap::GetSplitFileList(const std::string& file_path, std::vector<std::string>* list) {
// This is not the most efficient thing, but it is simple and recovering
// the fiemap/fibmap is much more expensive.
std::string contents;
if (!android::base::ReadFileToString(file_path, &contents, true)) {
PLOG(ERROR) << "Error reading file: " << file_path;
return false;
}
std::vector<std::string> names = android::base::Split(contents, "\n");
std::string dir = android::base::Dirname(file_path);
for (const auto& name : names) {
if (!name.empty()) {
list->emplace_back(dir + "/" + name);
}
}
return true;
}
bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* message) {
// Early exit if this does not exist, and do not report an error.
if (access(file_path.c_str(), F_OK) && errno == ENOENT) {
return true;
}
bool ok = true;
std::vector<std::string> files;
if (GetSplitFileList(file_path, &files)) {
for (const auto& file : files) {
ok &= android::base::RemoveFileIfExists(file, message);
}
}
ok &= android::base::RemoveFileIfExists(file_path, message);
return ok;
}
bool SplitFiemap::HasPinnedExtents() const {
for (const auto& file : files_) {
if (!FiemapWriter::HasPinnedExtents(file->file_path())) {
return false;
}
}
return true;
}
const std::vector<struct fiemap_extent>& SplitFiemap::extents() {
if (extents_.empty()) {
for (const auto& file : files_) {
const auto& extents = file->extents();
extents_.insert(extents_.end(), extents.begin(), extents.end());
}
}
return extents_;
}
bool SplitFiemap::Write(const void* data, uint64_t bytes) {
// Open the current file.
FiemapWriter* file = files_[cursor_index_].get();
const uint8_t* data_ptr = reinterpret_cast<const uint8_t*>(data);
uint64_t bytes_remaining = bytes;
while (bytes_remaining) {
// How many bytes can we write into the current file?
uint64_t file_bytes_left = file->size() - cursor_file_pos_;
if (!file_bytes_left) {
if (cursor_index_ == files_.size() - 1) {
LOG(ERROR) << "write past end of file requested";
return false;
}
// No space left in the current file, but we have more files to
// use, so prep the next one.
cursor_fd_ = {};
cursor_file_pos_ = 0;
file = files_[++cursor_index_].get();
file_bytes_left = file->size();
}
// Open the current file if it's not open.
if (cursor_fd_ < 0) {
cursor_fd_.reset(open(file->file_path().c_str(), O_CLOEXEC | O_WRONLY));
if (cursor_fd_ < 0) {
PLOG(ERROR) << "open failed: " << file->file_path();
return false;
}
CHECK(cursor_file_pos_ == 0);
}
if (!FiemapWriter::HasPinnedExtents(file->file_path())) {
LOG(ERROR) << "file is no longer pinned: " << file->file_path();
return false;
}
uint64_t bytes_to_write = std::min(file_bytes_left, bytes_remaining);
if (!android::base::WriteFully(cursor_fd_, data_ptr, bytes_to_write)) {
PLOG(ERROR) << "write failed: " << file->file_path();
return false;
}
data_ptr += bytes_to_write;
bytes_remaining -= bytes_to_write;
cursor_file_pos_ += bytes_to_write;
}
// If we've reached the end of the current file, close it for sanity.
if (cursor_file_pos_ == file->size()) {
cursor_fd_ = {};
}
return true;
}
bool SplitFiemap::Flush() {
for (const auto& file : files_) {
unique_fd fd(open(file->file_path().c_str(), O_RDONLY | O_CLOEXEC));
if (fd < 0) {
PLOG(ERROR) << "open failed: " << file->file_path();
return false;
}
if (fsync(fd)) {
PLOG(ERROR) << "fsync failed: " << file->file_path();
return false;
}
}
return true;
}
SplitFiemap::~SplitFiemap() {
if (!creating_) {
return;
}
// We failed to finish creating, so unlink everything.
unlink(list_file_.c_str());
for (auto&& file : files_) {
std::string path = file->file_path();
file = nullptr;
unlink(path.c_str());
}
}
void SplitFiemap::AddFile(FiemapUniquePtr&& file) {
total_size_ += file->size();
files_.emplace_back(std::move(file));
}
uint32_t SplitFiemap::block_size() const {
return files_[0]->block_size();
}
const std::string& SplitFiemap::bdev_path() const {
return files_[0]->bdev_path();
}
} // namespace fiemap
} // namespace android

BIN
fs_mgr/libfiemap/testdata/file_32k vendored Normal file
View file

Binary file not shown.

BIN
fs_mgr/libfiemap/testdata/file_4k vendored Normal file
View file

Binary file not shown.

BIN
fs_mgr/libfiemap/testdata/unaligned_file vendored Normal file
View file

Binary file not shown.

171
fs_mgr/libfiemap/utility.cpp Normal file
View file

@ -0,0 +1,171 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "utility.h"
#include <stdint.h>
#include <sys/stat.h>
#include <sys/sysmacros.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <libfiemap/fiemap_writer.h>
namespace android {
namespace fiemap {
using namespace std::string_literals;
using android::base::unique_fd;
static constexpr char kUserdataDevice[] = "/dev/block/by-name/userdata";
uint64_t DetermineMaximumFileSize(const std::string& file_path) {
// Create the smallest file possible (one block).
auto writer = FiemapWriter::Open(file_path, 1);
if (!writer) {
return 0;
}
uint64_t result = 0;
switch (writer->fs_type()) {
case EXT4_SUPER_MAGIC:
// The minimum is 16GiB, so just report that. If we wanted we could parse the
// superblock and figure out if 64-bit support is enabled.
result = 17179869184ULL;
break;
case F2FS_SUPER_MAGIC:
// Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt
// 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB.
result = 4329690886144ULL;
break;
case MSDOS_SUPER_MAGIC:
// 4GB-1, which we want aligned to the block size.
result = 4294967295;
result -= (result % writer->block_size());
break;
default:
LOG(ERROR) << "Unknown file system type: " << writer->fs_type();
break;
}
// Close and delete the temporary file.
writer = nullptr;
unlink(file_path.c_str());
return result;
}
// Given a SplitFiemap, this returns a device path that will work during first-
// stage init (i.e., its path can be found by InitRequiredDevices).
std::string GetDevicePathForFile(SplitFiemap* file) {
auto bdev_path = file->bdev_path();
struct stat userdata, given;
if (!stat(bdev_path.c_str(), &given) && !stat(kUserdataDevice, &userdata)) {
if (S_ISBLK(given.st_mode) && S_ISBLK(userdata.st_mode) &&
given.st_rdev == userdata.st_rdev) {
return kUserdataDevice;
}
}
return bdev_path;
}
std::string JoinPaths(const std::string& dir, const std::string& file) {
if (android::base::EndsWith(dir, "/")) {
return dir + file;
}
return dir + "/" + file;
}
bool F2fsPinBeforeAllocate(int file_fd, bool* supported) {
struct stat st;
if (fstat(file_fd, &st) < 0) {
PLOG(ERROR) << "stat failed";
return false;
}
std::string bdev;
if (!BlockDeviceToName(major(st.st_dev), minor(st.st_dev), &bdev)) {
LOG(ERROR) << "Failed to get block device name for " << major(st.st_dev) << ":"
<< minor(st.st_dev);
return false;
}
std::string contents;
std::string feature_file = "/sys/fs/f2fs/" + bdev + "/features";
if (!android::base::ReadFileToString(feature_file, &contents)) {
PLOG(ERROR) << "read failed: " << feature_file;
return false;
}
contents = android::base::Trim(contents);
auto features = android::base::Split(contents, ", ");
auto iter = std::find(features.begin(), features.end(), "pin_file"s);
*supported = (iter != features.end());
return true;
}
bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name) {
// The symlinks in /sys/dev/block point to the block device node under /sys/device/..
// The directory name in the target corresponds to the name of the block device. We use
// that to extract the block device name.
// e.g for block device name 'ram0', there exists a symlink named '1:0' in /sys/dev/block as
// follows.
// 1:0 -> ../../devices/virtual/block/ram0
std::string sysfs_path = ::android::base::StringPrintf("/sys/dev/block/%u:%u", major, minor);
std::string sysfs_bdev;
if (!::android::base::Readlink(sysfs_path, &sysfs_bdev)) {
PLOG(ERROR) << "Failed to read link at: " << sysfs_path;
return false;
}
*bdev_name = ::android::base::Basename(sysfs_bdev);
// Paranoid sanity check to make sure we just didn't get the
// input in return as-is.
if (sysfs_bdev == *bdev_name) {
LOG(ERROR) << "Malformed symlink for block device: " << sysfs_bdev;
return false;
}
return true;
}
bool FilesystemHasReliablePinning(const std::string& file, bool* supported) {
struct statfs64 sfs;
if (statfs64(file.c_str(), &sfs)) {
PLOG(ERROR) << "statfs failed: " << file;
return false;
}
if (sfs.f_type != F2FS_SUPER_MAGIC) {
*supported = true;
return true;
}
unique_fd fd(open(file.c_str(), O_RDONLY | O_CLOEXEC));
if (fd < 0) {
PLOG(ERROR) << "open failed: " << file;
return false;
}
return F2fsPinBeforeAllocate(fd, supported);
}
} // namespace fiemap
} // namespace android

55
fs_mgr/libfiemap/utility.h Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <stdint.h>
#include <string>
#include <libfiemap/split_fiemap_writer.h>
namespace android {
namespace fiemap {
// Given a file that will be created, determine the maximum size its containing
// filesystem allows. Note this is a theoretical maximum size; free space is
// ignored entirely.
uint64_t DetermineMaximumFileSize(const std::string& file_path);
// Given a SplitFiemap, this returns a device path that will work during first-
// stage init (i.e., its path can be found by InitRequiredDevices).
std::string GetDevicePathForFile(android::fiemap::SplitFiemap* file);
// Combine two path components into a single path.
std::string JoinPaths(const std::string& dir, const std::string& file);
// Given a file within an F2FS filesystem, return whether or not the filesystem
// supports the "pin_file" feature, which requires pinning before fallocation.
bool F2fsPinBeforeAllocate(int file_fd, bool* supported);
// Given a major/minor device number, return its canonical name such that
// /dev/block/<name> resolves to the device.
bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name);
// This is the same as F2fsPinBeforeAllocate, however, it will return true
// (and supported = true) for non-f2fs filesystems. It is intended to be used
// in conjunction with ImageManager to reject image requests for reliable use
// cases (such as snapshots or adb remount).
bool FilesystemHasReliablePinning(const std::string& file, bool* supported);
} // namespace fiemap
} // namespace android