Implement FuseBlockDataProvider

Adds a fuse data provider that parses the metadata from a block map,
reads the data from the given ranges of the block device; and provides
the data to the fuse.

Bug: 127071893
Test: unit tests pass, install a package from block map
Change-Id: Ie9925ee9144e98642505b3f5e1a4a186d2b21ed0

fuse_sideload/Android.bp

@@ -34,6 +34,10 @@ cc_library {
         "include",
     ],
 
+    static_libs: [
+        "libotautil",
+    ],
+
     shared_libs: [
         "libbase",
         "libcrypto",

fuse_sideload/fuse_provider.cpp

@@ -27,8 +27,11 @@
 #include <functional>
 
 #include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/strings.h>
 
 #include "fuse_sideload.h"
+#include "otautil/sysutil.h"
 
 FuseFileDataProvider::FuseFileDataProvider(const std::string& path, uint32_t block_size) {
   struct stat sb;
@@ -69,3 +72,79 @@ bool FuseFileDataProvider::ReadBlockAlignedData(uint8_t* buffer, uint32_t fetch_
 void FuseFileDataProvider::Close() {
   fd_.reset();
 }
+
+FuseBlockDataProvider::FuseBlockDataProvider(uint64_t file_size, uint32_t fuse_block_size,
+                                             android::base::unique_fd&& fd,
+                                             uint32_t source_block_size, RangeSet ranges)
+    : FuseDataProvider(file_size, fuse_block_size),
+      fd_(std::move(fd)),
+      source_block_size_(source_block_size),
+      ranges_(std::move(ranges)) {
+  // Make sure the offset is also aligned with the blocks on the block device when we call
+  // ReadBlockAlignedData().
+  CHECK_EQ(0, fuse_block_size_ % source_block_size_);
+}
+
+bool FuseBlockDataProvider::ReadBlockAlignedData(uint8_t* buffer, uint32_t fetch_size,
+                                                 uint32_t start_block) const {
+  uint64_t offset = static_cast<uint64_t>(start_block) * fuse_block_size_;
+  if (fetch_size > file_size_ || offset > file_size_ - fetch_size) {
+    LOG(ERROR) << "Out of bound read, offset: " << offset << ", fetch size: " << fetch_size
+               << ", file size " << file_size_;
+    return false;
+  }
+
+  auto read_ranges =
+      ranges_.GetSubRanges(offset / source_block_size_, fetch_size / source_block_size_);
+  if (!read_ranges) {
+    return false;
+  }
+
+  uint8_t* next_out = buffer;
+  for (const auto& [range_start, range_end] : read_ranges.value()) {
+    uint64_t bytes_start = static_cast<uint64_t>(range_start) * source_block_size_;
+    uint64_t bytes_to_read = static_cast<uint64_t>(range_end - range_start) * source_block_size_;
+    if (!android::base::ReadFullyAtOffset(fd_, next_out, bytes_to_read, bytes_start)) {
+      PLOG(ERROR) << "Failed to read " << bytes_to_read << " bytes at offset " << bytes_start;
+      return false;
+    }
+
+    next_out += bytes_to_read;
+  }
+
+  if (uint64_t tailing_bytes = fetch_size % source_block_size_; tailing_bytes != 0) {
+    // Calculate the offset to last partial block.
+    uint64_t tailing_offset =
+        read_ranges.value()
+            ? static_cast<uint64_t>((read_ranges->cend() - 1)->second) * source_block_size_
+            : static_cast<uint64_t>(start_block) * source_block_size_;
+    if (!android::base::ReadFullyAtOffset(fd_, next_out, tailing_bytes, tailing_offset)) {
+      PLOG(ERROR) << "Failed to read tailing " << tailing_bytes << " bytes at offset "
+                  << tailing_offset;
+      return false;
+    }
+  }
+  return true;
+}
+
+std::unique_ptr<FuseBlockDataProvider> FuseBlockDataProvider::CreateFromBlockMap(
+    const std::string& block_map_path, uint32_t fuse_block_size) {
+  auto block_map = BlockMapData::ParseBlockMapFile(block_map_path);
+  if (!block_map) {
+    return nullptr;
+  }
+
+  android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(block_map.path().c_str(), O_RDONLY)));
+  if (fd == -1) {
+    PLOG(ERROR) << "Failed to open " << block_map.path();
+    return nullptr;
+  }
+
+  return std::unique_ptr<FuseBlockDataProvider>(
+      new FuseBlockDataProvider(block_map.file_size(), fuse_block_size, std::move(fd),
+                                block_map.block_size(), block_map.block_ranges()));
+}
+
+void FuseBlockDataProvider::Close() {
+  fd_.reset();
+}

fuse_sideload/include/fuse_provider.h

@@ -18,10 +18,13 @@
 
 #include <stdint.h>
 
+#include <memory>
 #include <string>
 
 #include <android-base/unique_fd.h>
 
+#include "otautil/rangeset.h"
+
 // This is the base class to read data from source and provide the data to FUSE.
 class FuseDataProvider {
  public:
@@ -70,3 +73,28 @@ class FuseFileDataProvider : public FuseDataProvider {
   // The underlying source to read data from.
   android::base::unique_fd fd_;
 };
+
+// This class parses a block map and reads data from the underlying block device.
+class FuseBlockDataProvider : public FuseDataProvider {
+ public:
+  // Constructs the fuse provider from the block map.
+  static std::unique_ptr<FuseBlockDataProvider> CreateFromBlockMap(
+      const std::string& block_map_path, uint32_t fuse_block_size);
+
+  RangeSet ranges() const {
+    return ranges_;
+  }
+  bool ReadBlockAlignedData(uint8_t* buffer, uint32_t fetch_size,
+                            uint32_t start_block) const override;
+  void Close() override;
+
+ private:
+  FuseBlockDataProvider(uint64_t file_size, uint32_t fuse_block_size, android::base::unique_fd&& fd,
+                        uint32_t source_block_size, RangeSet ranges);
+  // The underlying block device to read data from.
+  android::base::unique_fd fd_;
+  // The block size of the source block device.
+  uint32_t source_block_size_;
+  // The block ranges from the source block device that consist of the file
+  RangeSet ranges_;
+};

minadbd/Android.bp

@@ -43,6 +43,10 @@ cc_library {
         "minadbd_services.cpp",
     ],
 
+    static_libs: [
+        "libotautil",
+    ],
+
     shared_libs: [
         "libadbd",
         "libbase",
@@ -96,6 +100,7 @@ cc_test {
     static_libs: [
         "libminadbd_services",
         "libfusesideload",
+        "libotautil",
         "libadbd",
         "libcrypto",
     ],

otautil/include/otautil/rangeset.h

@@ -18,6 +18,7 @@
 
 #include <stddef.h>
 
+#include <optional>
 #include <string>
 #include <utility>
 #include <vector>
@@ -49,6 +50,12 @@ class RangeSet {
   // bounds. For example, "3,5" contains blocks 3 and 4. So "3,5" and "5,7" are not overlapped.
   bool Overlaps(const RangeSet& other) const;
 
+  // Returns a subset of ranges starting from |start_index| with respect to the original range. The
+  // output range will have |num_of_blocks| blocks in size. Returns std::nullopt if the input is
+  // invalid. e.g. RangeSet({{0, 5}, {10, 15}}).GetSubRanges(1, 5) returns
+  // RangeSet({{1, 5}, {10, 11}}).
+  std::optional<RangeSet> GetSubRanges(size_t start_index, size_t num_of_blocks) const;
+
   // Returns a vector of RangeSets that contain the same set of blocks represented by the current
   // RangeSet. The RangeSets in the vector contain similar number of blocks, with a maximum delta
   // of 1-block between any two of them. For example, 14 blocks would be split into 4 + 4 + 3 + 3,

otautil/rangeset.cpp

@@ -184,6 +184,58 @@ bool RangeSet::Overlaps(const RangeSet& other) const {
   return false;
 }
 
+std::optional<RangeSet> RangeSet::GetSubRanges(size_t start_index, size_t num_of_blocks) const {
+  size_t end_index = start_index + num_of_blocks;  // The index of final block to read plus one
+  if (start_index > end_index || end_index > blocks_) {
+    LOG(ERROR) << "Failed to get the sub ranges for start_index " << start_index
+               << " num_of_blocks " << num_of_blocks
+               << " total number of blocks the range contains is " << blocks_;
+    return std::nullopt;
+  }
+
+  if (num_of_blocks == 0) {
+    LOG(WARNING) << "num_of_blocks is zero when calling GetSubRanges()";
+    return RangeSet();
+  }
+
+  RangeSet result;
+  size_t current_index = 0;
+  for (const auto& [range_start, range_end] : ranges_) {
+    CHECK_LT(range_start, range_end);
+    size_t blocks_in_range = range_end - range_start;
+    // Linear search to skip the ranges until we reach start_block.
+    if (current_index + blocks_in_range <= start_index) {
+      current_index += blocks_in_range;
+      continue;
+    }
+
+    size_t trimmed_range_start = range_start;
+    // We have found the first block range to read, trim the heading blocks.
+    if (current_index < start_index) {
+      trimmed_range_start += start_index - current_index;
+    }
+    // Trim the trailing blocks if the last range has more blocks than desired; also return the
+    // result.
+    if (current_index + blocks_in_range >= end_index) {
+      size_t trimmed_range_end = range_end - (current_index + blocks_in_range - end_index);
+      if (!result.PushBack({ trimmed_range_start, trimmed_range_end })) {
+        return std::nullopt;
+      }
+
+      return result;
+    }
+
+    if (!result.PushBack({ trimmed_range_start, range_end })) {
+      return std::nullopt;
+    }
+    current_index += blocks_in_range;
+  }
+
+  LOG(ERROR) << "Failed to construct byte ranges to read, start_block: " << start_index
+             << ", num_of_blocks: " << num_of_blocks << " total number of blocks: " << blocks_;
+  return std::nullopt;
+}
+
 // Ranges in the the set should be mutually exclusive; and they're sorted by the start block.
 SortedRangeSet::SortedRangeSet(std::vector<Range>&& pairs) : RangeSet(std::move(pairs)) {
   std::sort(ranges_.begin(), ranges_.end());

otautil/sysutil.cpp

@@ -94,6 +94,11 @@ BlockMapData BlockMapData::ParseBlockMapFile(const std::string& block_map_path)
     remaining_blocks -= range_blocks;
   }
 
+  if (remaining_blocks != 0) {
+    LOG(ERROR) << "Invalid ranges: remaining blocks " << remaining_blocks;
+    return {};
+  }
+
   return BlockMapData(block_dev, file_size, blksize, std::move(ranges));
 }
 

tests/Android.bp

@@ -118,6 +118,7 @@ cc_test {
 
     static_libs: libapplypatch_static_libs + librecovery_static_libs + [
         "librecovery_ui",
+        "libfusesideload",
         "libminui",
         "libotautil",
         "libupdater",

tests/unit/fuse_provider_test.cpp

@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+#include <unistd.h>
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/strings.h>
+#include <android-base/unique_fd.h>
+#include <gtest/gtest.h>
+
+#include "fuse_provider.h"
+#include "fuse_sideload.h"
+#include "install/install.h"
+
+TEST(FuseBlockMapTest, CreateFromBlockMap_smoke) {
+  TemporaryFile fake_block_device;
+  std::vector<std::string> lines = {
+    fake_block_device.path, "10000 4096", "3", "10 11", "20 21", "22 23",
+  };
+
+  TemporaryFile temp_file;
+  android::base::WriteStringToFile(android::base::Join(lines, '\n'), temp_file.path);
+  auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(temp_file.path, 4096);
+
+  ASSERT_TRUE(block_map_data);
+  ASSERT_EQ(10000, block_map_data->file_size());
+  ASSERT_EQ(4096, block_map_data->fuse_block_size());
+  ASSERT_EQ(RangeSet({ { 10, 11 }, { 20, 21 }, { 22, 23 } }), block_map_data->ranges());
+}
+
+TEST(FuseBlockMapTest, ReadBlockAlignedData_smoke) {
+  std::string content;
+  content.reserve(40960);
+  for (char c = 0; c < 10; c++) {
+    content += std::string(4096, c);
+  }
+  TemporaryFile fake_block_device;
+  ASSERT_TRUE(android::base::WriteStringToFile(content, fake_block_device.path));
+
+  std::vector<std::string> lines = {
+    fake_block_device.path,
+    "20000 4096",
+    "1",
+    "0 5",
+  };
+  TemporaryFile temp_file;
+  android::base::WriteStringToFile(android::base::Join(lines, '\n'), temp_file.path);
+  auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(temp_file.path, 4096);
+
+  std::vector<uint8_t> result(2000);
+  ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 2000, 1));
+  ASSERT_EQ(std::vector<uint8_t>(content.begin() + 4096, content.begin() + 6096), result);
+
+  result.resize(20000);
+  ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 0));
+  ASSERT_EQ(std::vector<uint8_t>(content.begin(), content.begin() + 20000), result);
+}
+
+TEST(FuseBlockMapTest, ReadBlockAlignedData_large_fuse_block) {
+  std::string content;
+  for (char c = 0; c < 10; c++) {
+    content += std::string(4096, c);
+  }
+
+  TemporaryFile temp_file;
+  ASSERT_TRUE(android::base::WriteStringToFile(content, temp_file.path));
+
+  std::vector<std::string> lines = {
+    temp_file.path, "36384 4096", "2", "0 5", "6 10",
+  };
+  TemporaryFile block_map;
+  ASSERT_TRUE(android::base::WriteStringToFile(android::base::Join(lines, '\n'), block_map.path));
+
+  auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(block_map.path, 16384);
+  ASSERT_TRUE(block_map_data);
+
+  std::vector<uint8_t> result(20000);
+  // Out of bound read
+  ASSERT_FALSE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 2));
+  ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 1));
+  // expected source block contains: 4, 6-9
+  std::string expected = content.substr(16384, 4096) + content.substr(24576, 15904);
+  ASSERT_EQ(std::vector<uint8_t>(expected.begin(), expected.end()), result);
+}

tests/unit/rangeset_test.cpp

@@ -18,6 +18,7 @@
 #include <sys/types.h>
 
 #include <limits>
+#include <optional>
 #include <vector>
 
 #include <gtest/gtest.h>
@@ -248,6 +249,29 @@ TEST(RangeSetTest, ToString) {
   ASSERT_EQ("6,1,3,4,6,15,22", RangeSet::Parse("6,1,3,4,6,15,22").ToString());
 }
 
+TEST(RangeSetTest, GetSubRanges_invalid) {
+  RangeSet range0({ { 1, 11 }, { 20, 30 } });
+  ASSERT_FALSE(range0.GetSubRanges(0, 21));  // too many blocks
+  ASSERT_FALSE(range0.GetSubRanges(21, 1));  // start block OOB
+}
+
+TEST(RangeSetTest, GetSubRanges_empty) {
+  RangeSet range0({ { 1, 11 }, { 20, 30 } });
+  ASSERT_EQ(RangeSet{}, range0.GetSubRanges(1, 0));  // empty num_of_blocks
+}
+
+TEST(RangeSetTest, GetSubRanges_smoke) {
+  RangeSet range0({ { 10, 11 } });
+  ASSERT_EQ(RangeSet({ { 10, 11 } }), range0.GetSubRanges(0, 1));
+
+  RangeSet range1({ { 10, 11 }, { 20, 21 }, { 30, 31 } });
+  ASSERT_EQ(range1, range1.GetSubRanges(0, 3));
+  ASSERT_EQ(RangeSet({ { 20, 21 } }), range1.GetSubRanges(1, 1));
+
+  RangeSet range2({ { 1, 11 }, { 20, 25 }, { 30, 35 } });
+  ASSERT_EQ(RangeSet({ { 10, 11 }, { 20, 25 }, { 30, 31 } }), range2.GetSubRanges(9, 7));
+}
+
 TEST(SortedRangeSetTest, Insert) {
   SortedRangeSet rs({ { 2, 3 }, { 4, 6 }, { 8, 14 } });
   rs.Insert({ 1, 2 });

tests/unit/sysutil_test.cpp

@@ -67,7 +67,7 @@ TEST(SysUtilTest, ParseBlockMapFile_invalid_size) {
     "/dev/abc",
     "42949672950 4294967295",
     "1",
-    "0 9",
+    "0 10",
   };
 
   TemporaryFile temp_file;