Implement the TargetFile and BuildInfo

The TargetFile class parses a target-file and provides functions to read
its contents. And the BuildInfo tries to simulate the device with files
on host. Some work it does includes parsing the build properties,
and extracting the image files for partitions specified in the fstab.

Bug: 131911365
Test: unit tests pass, run simulator with cuttlefish, wear devices and from extracted TF.
Change-Id: Iefe4a96d619d2e4b3d038e31480f11a0f9a70afa

updater/Android.bp

@@ -128,6 +128,7 @@ cc_library_host_static {
     ],
 
     srcs: [
+        "build_info.cpp",
         "simulator_runtime.cpp",
         "target_files.cpp",
     ],
@@ -135,6 +136,7 @@ cc_library_host_static {
     static_libs: [
         "libupdater_core",
         "libfstab",
+        "libc++fs",
     ],
 
     target: {

updater/Android.mk

@@ -136,7 +136,8 @@ LOCAL_STATIC_LIBRARIES := \
     $(TARGET_RECOVERY_UPDATER_HOST_LIBS) \
     $(TARGET_RECOVERY_UPDATER_HOST_EXTRA_LIBS) \
     $(updater_common_static_libraries) \
-    libfstab
+    libfstab \
+    libc++fs
 
 LOCAL_MODULE_CLASS := EXECUTABLES
 inc := $(call local-generated-sources-dir)/register.inc

updater/build_info.cpp

@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "updater/build_info.h"
+
+#include <set>
+#include <vector>
+
+#include <android-base/logging.h>
+#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
+
+#include "updater/target_files.h"
+
+bool BuildInfo::ParseTargetFile(const std::string_view target_file_path, bool extracted_input) {
+  TargetFile target_file(std::string(target_file_path), extracted_input);
+  if (!target_file.Open()) {
+    return false;
+  }
+
+  if (!target_file.GetBuildProps(&build_props_)) {
+    return false;
+  }
+
+  std::vector<FstabInfo> fstab_info_list;
+  if (!target_file.ParseFstabInfo(&fstab_info_list)) {
+    return false;
+  }
+
+  for (const auto& fstab_info : fstab_info_list) {
+    for (const auto& directory : { "IMAGES", "RADIO" }) {
+      std::string entry_name = directory + fstab_info.mount_point + ".img";
+      if (!target_file.EntryExists(entry_name)) {
+        LOG(WARNING) << "Failed to find the image entry in the target file: " << entry_name;
+        continue;
+      }
+
+      temp_files_.emplace_back(work_dir_);
+      auto& image_file = temp_files_.back();
+      if (!target_file.ExtractImage(entry_name, fstab_info, work_dir_, &image_file)) {
+        LOG(ERROR) << "Failed to set up source image files.";
+        return false;
+      }
+
+      LOG(INFO) << "Mounted " << fstab_info.mount_point << "\nMapping: " << fstab_info.blockdev_name
+                << " to " << image_file.path;
+
+      blockdev_map_.emplace(
+          fstab_info.blockdev_name,
+          FakeBlockDevice(fstab_info.blockdev_name, fstab_info.mount_point, image_file.path));
+      break;
+    }
+  }
+
+  return true;
+}
+
+std::string BuildInfo::GetProperty(const std::string_view key,
+                                   const std::string_view default_value) const {
+  // The logic to parse the ro.product properties should be in line with the generation script.
+  // More details in common.py BuildInfo.GetBuildProp.
+  // TODO(xunchang) handle the oem property and the source order defined in
+  // ro.product.property_source_order
+  const std::set<std::string, std::less<>> ro_product_props = {
+    "ro.product.brand", "ro.product.device", "ro.product.manufacturer", "ro.product.model",
+    "ro.product.name"
+  };
+  const std::vector<std::string> source_order = {
+    "product", "product_services", "odm", "vendor", "system",
+  };
+  if (ro_product_props.find(key) != ro_product_props.end()) {
+    std::string_view key_suffix(key);
+    CHECK(android::base::ConsumePrefix(&key_suffix, "ro.product"));
+    for (const auto& source : source_order) {
+      std::string resolved_key = "ro.product." + source + std::string(key_suffix);
+      if (auto entry = build_props_.find(resolved_key); entry != build_props_.end()) {
+        return entry->second;
+      }
+    }
+    LOG(WARNING) << "Failed to find property: " << key;
+    return std::string(default_value);
+  } else if (key == "ro.build.fingerprint") {
+    // clang-format off
+    return android::base::StringPrintf("%s/%s/%s:%s/%s/%s:%s/%s",
+        GetProperty("ro.product.brand", "").c_str(),
+        GetProperty("ro.product.name", "").c_str(),
+        GetProperty("ro.product.device", "").c_str(),
+        GetProperty("ro.build.version.release", "").c_str(),
+        GetProperty("ro.build.id", "").c_str(),
+        GetProperty("ro.build.version.incremental", "").c_str(),
+        GetProperty("ro.build.type", "").c_str(),
+        GetProperty("ro.build.tags", "").c_str());
+    // clang-format on
+  }
+
+  auto entry = build_props_.find(key);
+  if (entry == build_props_.end()) {
+    LOG(WARNING) << "Failed to find property: " << key;
+    return std::string(default_value);
+  }
+
+  return entry->second;
+}
+
+std::string BuildInfo::FindBlockDeviceName(const std::string_view name) const {
+  auto entry = blockdev_map_.find(name);
+  if (entry == blockdev_map_.end()) {
+    LOG(WARNING) << "Failed to find path to block device " << name;
+    return "";
+  }
+
+  return entry->second.mounted_file_path;
+}

updater/include/updater/build_info.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <list>
+#include <map>
+#include <string>
+#include <string_view>
+
+#include <android-base/file.h>
+
+// This class serves as the aggregation of the fake block device information during update
+// simulation on host. In specific, it has the name of the block device, its mount point, and the
+// path to the temporary file that fakes this block device.
+class FakeBlockDevice {
+ public:
+  FakeBlockDevice(std::string block_device, std::string mount_point, std::string temp_file_path)
+      : blockdev_name(std::move(block_device)),
+        mount_point(std::move(mount_point)),
+        mounted_file_path(std::move(temp_file_path)) {}
+
+  std::string blockdev_name;
+  std::string mount_point;
+  std::string mounted_file_path;  // path to the temp file that mocks the block device
+};
+
+// This class stores the information of the source build. For example, it creates and maintains
+// the temporary files to simulate the block devices on host. Therefore, the simulator runtime can
+// query the information and run the update on host.
+class BuildInfo {
+ public:
+  explicit BuildInfo(const std::string_view work_dir) : work_dir_(work_dir) {}
+  // Returns the value of the build properties.
+  std::string GetProperty(const std::string_view key, const std::string_view default_value) const;
+  // Returns the path to the mock block device.
+  std::string FindBlockDeviceName(const std::string_view name) const;
+  // Parses the given target-file, initializes the build properties and extracts the images.
+  bool ParseTargetFile(const std::string_view target_file_path, bool extracted_input);
+
+ private:
+  // A map to store the system properties during simulation.
+  std::map<std::string, std::string, std::less<>> build_props_;
+  // A map from the blockdev_name to the FakeBlockDevice object, which contains the path to the
+  // temporary file.
+  std::map<std::string, FakeBlockDevice, std::less<>> blockdev_map_;
+
+  std::list<TemporaryFile> temp_files_;
+  std::string work_dir_;  // A temporary directory to store the extracted image files
+};

updater/include/updater/simulator_runtime.h

@@ -24,11 +24,11 @@
 #include <vector>
 
 #include "edify/updater_runtime_interface.h"
-#include "updater/target_files.h"
+#include "updater/build_info.h"
 
 class SimulatorRuntime : public UpdaterRuntimeInterface {
  public:
-  explicit SimulatorRuntime(TargetFiles* source) : source_(source) {}
+  explicit SimulatorRuntime(BuildInfo* source) : source_(source) {}
 
   bool IsSimulator() const override {
     return true;
@@ -53,6 +53,6 @@ class SimulatorRuntime : public UpdaterRuntimeInterface {
  private:
   std::string FindBlockDeviceName(const std::string_view name) const override;
 
-  TargetFiles* source_;
+  BuildInfo* source_;
   std::map<std::string, std::string, std::less<>> mounted_partitions_;
 };

updater/include/updater/target_files.h

@@ -16,21 +16,56 @@
 
 #pragma once
 
+#include <map>
 #include <string>
+#include <string_view>
+#include <vector>
 
-// This class parses a given target file for the build properties and image files. Then it creates
+#include <android-base/file.h>
-// and maintains the temporary files to simulate the block devices on host.
+#include <ziparchive/zip_archive.h>
-class TargetFiles {
+
+// This class represents the mount information for each line in a fstab file.
+class FstabInfo {
  public:
-  TargetFiles(std::string path, std::string work_dir)
+  FstabInfo(std::string blockdev_name, std::string mount_point, std::string fs_type)
-      : path_(std::move(path)), work_dir_(std::move(work_dir)) {}
+      : blockdev_name(std::move(blockdev_name)),
+        mount_point(std::move(mount_point)),
+        fs_type(std::move(fs_type)) {}
 
-  std::string GetProperty(const std::string_view key, const std::string_view default_value) const;
+  std::string blockdev_name;
+  std::string mount_point;
+  std::string fs_type;
+};
 
-  std::string FindBlockDeviceName(const std::string_view name) const;
+// This class parses a target file from a zip file or an extracted directory. It also provides the
+// function to read the its content for simulation.
+class TargetFile {
+ public:
+  TargetFile(std::string path, bool extracted_input)
+      : path_(std::move(path)), extracted_input_(extracted_input) {}
+
+  // Opens the input target file (or extracted directory) and parses the misc_info.txt.
+  bool Open();
+  // Parses the build properties in all possible locations and save them in |props_map|
+  bool GetBuildProps(std::map<std::string, std::string, std::less<>>* props_map) const;
+  // Parses the fstab and save the information about each partition to mount into |fstab_info_list|.
+  bool ParseFstabInfo(std::vector<FstabInfo>* fstab_info_list) const;
+  // Returns true if the given entry exists in the target file.
+  bool EntryExists(const std::string_view name) const;
+  // Extracts the image file |entry_name|. Returns true on success.
+  bool ExtractImage(const std::string_view entry_name, const FstabInfo& fstab_info,
+                    const std::string_view work_dir, TemporaryFile* image_file) const;
 
  private:
-  std::string path_;  // Path to the target file.
+  // Wrapper functions to read the entry from either the zipped target-file, or the extracted input
+  // directory.
+  bool ReadEntryToString(const std::string_view name, std::string* content) const;
+  bool ExtractEntryToTempFile(const std::string_view name, TemporaryFile* temp_file) const;
 
-  std::string work_dir_;  // A temporary directory to store the extracted image files
+  std::string path_;      // Path to the zipped target-file or an extracted directory.
+  bool extracted_input_;  // True if the target-file has been extracted.
+  ZipArchiveHandle handle_{ nullptr };
+
+  // The properties under META/misc_info.txt
+  std::map<std::string, std::string, std::less<>> misc_info_;
 };

updater/target_files.cpp

@@ -16,11 +16,267 @@
 
 #include "updater/target_files.h"
 
-std::string TargetFiles::GetProperty(const std::string_view /*key*/,
+#include <unistd.h>
-                                     const std::string_view default_value) const {
+
-  return std::string(default_value);
+#include <algorithm>
+#include <filesystem>
+#include <memory>
+
+#include <android-base/logging.h>
+#include <android-base/strings.h>
+#include <sparse/sparse.h>
+
+static bool SimgToImg(int input_fd, int output_fd) {
+  if (lseek64(input_fd, 0, SEEK_SET) == -1) {
+    PLOG(ERROR) << "Failed to lseek64 on the input sparse image";
+    return false;
+  }
+
+  if (lseek64(output_fd, 0, SEEK_SET) == -1) {
+    PLOG(ERROR) << "Failed to lseek64 on the output raw image";
+    return false;
+  }
+
+  std::unique_ptr<sparse_file, decltype(&sparse_file_destroy)> s_file(
+      sparse_file_import(input_fd, true, false), sparse_file_destroy);
+  if (!s_file) {
+    LOG(ERROR) << "Failed to import the sparse image.";
+    return false;
+  }
+
+  if (sparse_file_write(s_file.get(), output_fd, false, false, false) < 0) {
+    PLOG(ERROR) << "Failed to output the raw image file.";
+    return false;
+  }
+
+  return true;
 }
 
-std::string TargetFiles::FindBlockDeviceName(const std::string_view name) const {
+static bool ParsePropertyFile(const std::string_view prop_content,
-  return std::string(name);
+                              std::map<std::string, std::string, std::less<>>* props_map) {
+  LOG(INFO) << "Start parsing build property\n";
+  std::vector<std::string> lines = android::base::Split(std::string(prop_content), "\n");
+  for (const auto& line : lines) {
+    if (line.empty() || line[0] == '#') continue;
+    auto pos = line.find('=');
+    if (pos == std::string::npos) continue;
+    std::string key = line.substr(0, pos);
+    std::string value = line.substr(pos + 1);
+    LOG(INFO) << key << ": " << value;
+    props_map->emplace(key, value);
+  }
+
+  return true;
+}
+
+static bool ParseFstab(const std::string_view fstab, std::vector<FstabInfo>* fstab_info_list) {
+  LOG(INFO) << "parsing fstab\n";
+  std::vector<std::string> lines = android::base::Split(std::string(fstab), "\n");
+  for (const auto& line : lines) {
+    if (line.empty() || line[0] == '#') continue;
+
+    // <block_device>  <mount_point>  <fs_type>  <mount_flags>  optional:<fs_mgr_flags>
+    std::vector<std::string> tokens = android::base::Split(line, " ");
+    tokens.erase(std::remove(tokens.begin(), tokens.end(), ""), tokens.end());
+    if (tokens.size() != 4 && tokens.size() != 5) {
+      LOG(ERROR) << "Unexpected token size: " << tokens.size() << std::endl
+                 << "Error parsing fstab line: " << line;
+      return false;
+    }
+
+    const auto& blockdev = tokens[0];
+    const auto& mount_point = tokens[1];
+    const auto& fs_type = tokens[2];
+    if (!android::base::StartsWith(mount_point, "/")) {
+      LOG(WARNING) << "mount point '" << mount_point << "' does not start with '/'";
+      continue;
+    }
+
+    // The simulator only supports ext4 and emmc for now.
+    if (fs_type != "ext4" && fs_type != "emmc") {
+      LOG(WARNING) << "Unsupported fs_type in " << line;
+      continue;
+    }
+
+    fstab_info_list->emplace_back(blockdev, mount_point, fs_type);
+  }
+
+  return true;
+}
+
+bool TargetFile::EntryExists(const std::string_view name) const {
+  if (extracted_input_) {
+    std::string entry_path = path_ + "/" + std::string(name);
+    if (access(entry_path.c_str(), O_RDONLY) != 0) {
+      PLOG(WARNING) << "Failed to access " << entry_path;
+      return false;
+    }
+    return true;
+  }
+
+  CHECK(handle_);
+  ZipEntry img_entry;
+  return FindEntry(handle_, name, &img_entry) == 0;
+}
+
+bool TargetFile::ReadEntryToString(const std::string_view name, std::string* content) const {
+  if (extracted_input_) {
+    std::string entry_path = path_ + "/" + std::string(name);
+    return android::base::ReadFileToString(entry_path, content);
+  }
+
+  CHECK(handle_);
+  ZipEntry entry;
+  if (auto find_err = FindEntry(handle_, name, &entry); find_err != 0) {
+    LOG(ERROR) << "failed to find " << name << " in the package: " << ErrorCodeString(find_err);
+    return false;
+  }
+
+  content->resize(entry.uncompressed_length);
+  if (auto extract_err = ExtractToMemory(
+          handle_, &entry, reinterpret_cast<uint8_t*>(&content->at(0)), entry.uncompressed_length);
+      extract_err != 0) {
+    LOG(ERROR) << "failed to read " << name << " from package: " << ErrorCodeString(extract_err);
+    return false;
+  }
+
+  return true;
+}
+
+bool TargetFile::ExtractEntryToTempFile(const std::string_view name,
+                                        TemporaryFile* temp_file) const {
+  if (extracted_input_) {
+    std::string entry_path = path_ + "/" + std::string(name);
+    return std::filesystem::copy_file(entry_path, temp_file->path,
+                                      std::filesystem::copy_options::overwrite_existing);
+  }
+
+  CHECK(handle_);
+  ZipEntry entry;
+  if (auto find_err = FindEntry(handle_, name, &entry); find_err != 0) {
+    LOG(ERROR) << "failed to find " << name << " in the package: " << ErrorCodeString(find_err);
+    return false;
+  }
+
+  if (auto status = ExtractEntryToFile(handle_, &entry, temp_file->fd); status != 0) {
+    LOG(ERROR) << "Failed to extract zip entry " << name << " : " << ErrorCodeString(status);
+    return false;
+  }
+  return true;
+}
+
+bool TargetFile::Open() {
+  if (!extracted_input_) {
+    if (auto ret = OpenArchive(path_.c_str(), &handle_); ret != 0) {
+      LOG(ERROR) << "failed to open source target file " << path_ << ": " << ErrorCodeString(ret);
+      return false;
+    }
+  }
+
+  // Parse the misc info.
+  std::string misc_info_content;
+  if (!ReadEntryToString("META/misc_info.txt", &misc_info_content)) {
+    return false;
+  }
+  if (!ParsePropertyFile(misc_info_content, &misc_info_)) {
+    return false;
+  }
+
+  return true;
+}
+
+bool TargetFile::GetBuildProps(std::map<std::string, std::string, std::less<>>* props_map) const {
+  props_map->clear();
+  // Parse the source zip to mock the system props and block devices. We try all the possible
+  // locations for build props.
+  constexpr std::string_view kPropLocations[] = {
+    "SYSTEM/build.prop",
+    "VENDOR/build.prop",
+    "PRODUCT/build.prop",
+    "PRODUCT_SERVICES/build.prop",
+    "SYSTEM/vendor/build.prop",
+    "SYSTEM/product/build.prop",
+    "SYSTEM/product_services/build.prop",
+    "ODM/build.prop",  // legacy
+    "ODM/etc/build.prop",
+    "VENDOR/odm/build.prop",  // legacy
+    "VENDOR/odm/etc/build.prop",
+  };
+  for (const auto& name : kPropLocations) {
+    std::string build_prop_content;
+    if (!ReadEntryToString(name, &build_prop_content)) {
+      continue;
+    }
+    std::map<std::string, std::string, std::less<>> props;
+    if (!ParsePropertyFile(build_prop_content, &props)) {
+      LOG(ERROR) << "Failed to parse build prop in " << name;
+      return false;
+    }
+    for (const auto& [key, value] : props) {
+      if (auto it = props_map->find(key); it != props_map->end() && it->second != value) {
+        LOG(WARNING) << "Property " << key << " has different values in property files, we got "
+                     << it->second << " and " << value;
+      }
+      props_map->emplace(key, value);
+    }
+  }
+
+  return true;
+}
+
+bool TargetFile::ExtractImage(const std::string_view entry_name, const FstabInfo& fstab_info,
+                              const std::string_view work_dir, TemporaryFile* image_file) const {
+  if (!EntryExists(entry_name)) {
+    return false;
+  }
+
+  // We don't need extra work for 'emmc'; use the image file as the block device.
+  if (fstab_info.fs_type == "emmc" || misc_info_.find("extfs_sparse_flag") == misc_info_.end()) {
+    if (!ExtractEntryToTempFile(entry_name, image_file)) {
+      return false;
+    }
+  } else {  // treated as ext4 sparse image
+    TemporaryFile sparse_image{ std::string(work_dir) };
+    if (!ExtractEntryToTempFile(entry_name, &sparse_image)) {
+      return false;
+    }
+
+    // Convert the sparse image to raw.
+    if (!SimgToImg(sparse_image.fd, image_file->fd)) {
+      LOG(ERROR) << "Failed to convert " << fstab_info.mount_point << " to raw.";
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool TargetFile::ParseFstabInfo(std::vector<FstabInfo>* fstab_info_list) const {
+  // Parse the fstab file and extract the image files. The location of the fstab actually depends
+  // on some flags e.g. "no_recovery", "recovery_as_boot". Here we just try all possibilities.
+  constexpr std::string_view kRecoveryFstabLocations[] = {
+    "RECOVERY/RAMDISK/system/etc/recovery.fstab",
+    "RECOVERY/RAMDISK/etc/recovery.fstab",
+    "BOOT/RAMDISK/system/etc/recovery.fstab",
+    "BOOT/RAMDISK/etc/recovery.fstab",
+  };
+  std::string fstab_content;
+  for (const auto& name : kRecoveryFstabLocations) {
+    if (std::string content; ReadEntryToString(name, &content)) {
+      fstab_content = std::move(content);
+      break;
+    }
+  }
+  if (fstab_content.empty()) {
+    LOG(ERROR) << "Failed to parse the recovery fstab file";
+    return false;
+  }
+
+  // Extract the images and convert them to raw.
+  if (!ParseFstab(fstab_content, fstab_info_list)) {
+    LOG(ERROR) << "Failed to mount the block devices for source build.";
+    return false;
+  }
+
+  return true;
 }

updater/update_simulator_main.cpp

@@ -22,9 +22,10 @@
 #include "otautil/error_code.h"
 #include "otautil/paths.h"
 #include "updater/blockimg.h"
+#include "updater/build_info.h"
+#include "updater/dynamic_partitions.h"
 #include "updater/install.h"
 #include "updater/simulator_runtime.h"
-#include "updater/target_files.h"
 #include "updater/updater.h"
 
 int main(int argc, char** argv) {
@@ -34,7 +35,7 @@ int main(int argc, char** argv) {
   if (argc != 3 && argc != 4) {
     LOG(ERROR) << "unexpected number of arguments: " << argc << std::endl
                << "Usage: " << argv[0] << " <source_target-file> <ota_package>";
-    return 1;
+    return EXIT_FAILURE;
   }
 
   // TODO(xunchang) implement a commandline parser, e.g. it can take an oem property so that the
@@ -57,17 +58,21 @@ int main(int argc, char** argv) {
   Paths::Get().set_stash_directory_base(temp_stash_base.path);
 
   TemporaryFile cmd_pipe;
-
   TemporaryDir source_temp_dir;
-  TargetFiles source(source_target_file, source_temp_dir.path);
 
-  Updater updater(std::make_unique<SimulatorRuntime>(&source));
+  BuildInfo source_build_info(source_temp_dir.path);
+  if (!source_build_info.ParseTargetFile(source_target_file, false)) {
+    LOG(ERROR) << "Failed to parse the target file " << source_target_file;
+    return EXIT_FAILURE;
+  }
+
+  Updater updater(std::make_unique<SimulatorRuntime>(&source_build_info));
   if (!updater.Init(cmd_pipe.release(), package_name, false)) {
-    return 1;
+    return EXIT_FAILURE;
   }
 
   if (!updater.RunUpdate()) {
-    return 1;
+    return EXIT_FAILURE;
   }
 
   LOG(INFO) << "\nscript succeeded, result: " << updater.GetResult();