Merge "Implement the TargetFile and BuildInfo"

am: 9b76970e9a

Change-Id: I2d188eae7305bb82c7407bcbb04282b65768dde1

updater/Android.bp

@@ -128,6 +128,7 @@ cc_library_host_static {
     ],
 
     srcs: [
+        "build_info.cpp",
         "simulator_runtime.cpp",
         "target_files.cpp",
     ],
@@ -135,6 +136,7 @@ cc_library_host_static {
     static_libs: [
         "libupdater_core",
         "libfstab",
+        "libc++fs",
     ],
 
     target: {

updater/Android.mk

@@ -136,7 +136,8 @@ LOCAL_STATIC_LIBRARIES := \
     $(TARGET_RECOVERY_UPDATER_HOST_LIBS) \
     $(TARGET_RECOVERY_UPDATER_HOST_EXTRA_LIBS) \
     $(updater_common_static_libraries) \
-    libfstab
+    libfstab \
+    libc++fs
 
 LOCAL_MODULE_CLASS := EXECUTABLES
 inc := $(call local-generated-sources-dir)/register.inc

updater/build_info.cpp

@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "updater/build_info.h"
+
+#include <set>
+#include <vector>
+
+#include <android-base/logging.h>
+#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
+
+#include "updater/target_files.h"
+
+bool BuildInfo::ParseTargetFile(const std::string_view target_file_path, bool extracted_input) {
+  TargetFile target_file(std::string(target_file_path), extracted_input);
+  if (!target_file.Open()) {
+    return false;
+  }
+
+  if (!target_file.GetBuildProps(&build_props_)) {
+    return false;
+  }
+
+  std::vector<FstabInfo> fstab_info_list;
+  if (!target_file.ParseFstabInfo(&fstab_info_list)) {
+    return false;
+  }
+
+  for (const auto& fstab_info : fstab_info_list) {
+    for (const auto& directory : { "IMAGES", "RADIO" }) {
+      std::string entry_name = directory + fstab_info.mount_point + ".img";
+      if (!target_file.EntryExists(entry_name)) {
+        LOG(WARNING) << "Failed to find the image entry in the target file: " << entry_name;
+        continue;
+      }
+
+      temp_files_.emplace_back(work_dir_);
+      auto& image_file = temp_files_.back();
+      if (!target_file.ExtractImage(entry_name, fstab_info, work_dir_, &image_file)) {
+        LOG(ERROR) << "Failed to set up source image files.";
+        return false;
+      }
+
+      LOG(INFO) << "Mounted " << fstab_info.mount_point << "\nMapping: " << fstab_info.blockdev_name
+                << " to " << image_file.path;
+
+      blockdev_map_.emplace(
+          fstab_info.blockdev_name,
+          FakeBlockDevice(fstab_info.blockdev_name, fstab_info.mount_point, image_file.path));
+      break;
+    }
+  }
+
+  return true;
+}
+
+std::string BuildInfo::GetProperty(const std::string_view key,
+                                   const std::string_view default_value) const {
+  // The logic to parse the ro.product properties should be in line with the generation script.
+  // More details in common.py BuildInfo.GetBuildProp.
+  // TODO(xunchang) handle the oem property and the source order defined in
+  // ro.product.property_source_order
+  const std::set<std::string, std::less<>> ro_product_props = {
+    "ro.product.brand", "ro.product.device", "ro.product.manufacturer", "ro.product.model",
+    "ro.product.name"
+  };
+  const std::vector<std::string> source_order = {
+    "product", "product_services", "odm", "vendor", "system",
+  };
+  if (ro_product_props.find(key) != ro_product_props.end()) {
+    std::string_view key_suffix(key);
+    CHECK(android::base::ConsumePrefix(&key_suffix, "ro.product"));
+    for (const auto& source : source_order) {
+      std::string resolved_key = "ro.product." + source + std::string(key_suffix);
+      if (auto entry = build_props_.find(resolved_key); entry != build_props_.end()) {
+        return entry->second;
+      }
+    }
+    LOG(WARNING) << "Failed to find property: " << key;
+    return std::string(default_value);
+  } else if (key == "ro.build.fingerprint") {
+    // clang-format off
+    return android::base::StringPrintf("%s/%s/%s:%s/%s/%s:%s/%s",
+        GetProperty("ro.product.brand", "").c_str(),
+        GetProperty("ro.product.name", "").c_str(),
+        GetProperty("ro.product.device", "").c_str(),
+        GetProperty("ro.build.version.release", "").c_str(),
+        GetProperty("ro.build.id", "").c_str(),
+        GetProperty("ro.build.version.incremental", "").c_str(),
+        GetProperty("ro.build.type", "").c_str(),
+        GetProperty("ro.build.tags", "").c_str());
+    // clang-format on
+  }
+
+  auto entry = build_props_.find(key);
+  if (entry == build_props_.end()) {
+    LOG(WARNING) << "Failed to find property: " << key;
+    return std::string(default_value);
+  }
+
+  return entry->second;
+}
+
+std::string BuildInfo::FindBlockDeviceName(const std::string_view name) const {
+  auto entry = blockdev_map_.find(name);
+  if (entry == blockdev_map_.end()) {
+    LOG(WARNING) << "Failed to find path to block device " << name;
+    return "";
+  }
+
+  return entry->second.mounted_file_path;
+}

updater/include/updater/build_info.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <list>
+#include <map>
+#include <string>
+#include <string_view>
+
+#include <android-base/file.h>
+
+// This class serves as the aggregation of the fake block device information during update
+// simulation on host. In specific, it has the name of the block device, its mount point, and the
+// path to the temporary file that fakes this block device.
+class FakeBlockDevice {
+ public:
+  FakeBlockDevice(std::string block_device, std::string mount_point, std::string temp_file_path)
+      : blockdev_name(std::move(block_device)),
+        mount_point(std::move(mount_point)),
+        mounted_file_path(std::move(temp_file_path)) {}
+
+  std::string blockdev_name;
+  std::string mount_point;
+  std::string mounted_file_path;  // path to the temp file that mocks the block device
+};
+
+// This class stores the information of the source build. For example, it creates and maintains
+// the temporary files to simulate the block devices on host. Therefore, the simulator runtime can
+// query the information and run the update on host.
+class BuildInfo {
+ public:
+  explicit BuildInfo(const std::string_view work_dir) : work_dir_(work_dir) {}
+  // Returns the value of the build properties.
+  std::string GetProperty(const std::string_view key, const std::string_view default_value) const;
+  // Returns the path to the mock block device.
+  std::string FindBlockDeviceName(const std::string_view name) const;
+  // Parses the given target-file, initializes the build properties and extracts the images.
+  bool ParseTargetFile(const std::string_view target_file_path, bool extracted_input);
+
+ private:
+  // A map to store the system properties during simulation.
+  std::map<std::string, std::string, std::less<>> build_props_;
+  // A map from the blockdev_name to the FakeBlockDevice object, which contains the path to the
+  // temporary file.
+  std::map<std::string, FakeBlockDevice, std::less<>> blockdev_map_;
+
+  std::list<TemporaryFile> temp_files_;
+  std::string work_dir_;  // A temporary directory to store the extracted image files
+};

updater/include/updater/simulator_runtime.h

@@ -24,11 +24,11 @@
 #include <vector>
 
 #include "edify/updater_runtime_interface.h"
-#include "updater/target_files.h"
+#include "updater/build_info.h"
 
 class SimulatorRuntime : public UpdaterRuntimeInterface {
  public:
-  explicit SimulatorRuntime(TargetFiles* source) : source_(source) {}
+  explicit SimulatorRuntime(BuildInfo* source) : source_(source) {}
 
   bool IsSimulator() const override {
     return true;
@@ -53,6 +53,6 @@ class SimulatorRuntime : public UpdaterRuntimeInterface {
  private:
   std::string FindBlockDeviceName(const std::string_view name) const override;
 
-  TargetFiles* source_;
+  BuildInfo* source_;
   std::map<std::string, std::string, std::less<>> mounted_partitions_;
 };

updater/include/updater/target_files.h

@@ -16,21 +16,56 @@
 
 #pragma once
 
+#include <map>
 #include <string>
+#include <string_view>
+#include <vector>
 
-// This class parses a given target file for the build properties and image files. Then it creates
+#include <android-base/file.h>
-// and maintains the temporary files to simulate the block devices on host.
+#include <ziparchive/zip_archive.h>
-class TargetFiles {
+
+// This class represents the mount information for each line in a fstab file.
+class FstabInfo {
  public:
-  TargetFiles(std::string path, std::string work_dir)
+  FstabInfo(std::string blockdev_name, std::string mount_point, std::string fs_type)
-      : path_(std::move(path)), work_dir_(std::move(work_dir)) {}
+      : blockdev_name(std::move(blockdev_name)),
+        mount_point(std::move(mount_point)),
+        fs_type(std::move(fs_type)) {}
 
-  std::string GetProperty(const std::string_view key, const std::string_view default_value) const;
+  std::string blockdev_name;
+  std::string mount_point;
+  std::string fs_type;
+};
 
-  std::string FindBlockDeviceName(const std::string_view name) const;
+// This class parses a target file from a zip file or an extracted directory. It also provides the
+// function to read the its content for simulation.
+class TargetFile {
+ public:
+  TargetFile(std::string path, bool extracted_input)
+      : path_(std::move(path)), extracted_input_(extracted_input) {}
+
+  // Opens the input target file (or extracted directory) and parses the misc_info.txt.
+  bool Open();
+  // Parses the build properties in all possible locations and save them in |props_map|
+  bool GetBuildProps(std::map<std::string, std::string, std::less<>>* props_map) const;
+  // Parses the fstab and save the information about each partition to mount into |fstab_info_list|.
+  bool ParseFstabInfo(std::vector<FstabInfo>* fstab_info_list) const;
+  // Returns true if the given entry exists in the target file.
+  bool EntryExists(const std::string_view name) const;
+  // Extracts the image file |entry_name|. Returns true on success.
+  bool ExtractImage(const std::string_view entry_name, const FstabInfo& fstab_info,
+                    const std::string_view work_dir, TemporaryFile* image_file) const;
 
  private:
-  std::string path_;  // Path to the target file.
+  // Wrapper functions to read the entry from either the zipped target-file, or the extracted input
+  // directory.
+  bool ReadEntryToString(const std::string_view name, std::string* content) const;
+  bool ExtractEntryToTempFile(const std::string_view name, TemporaryFile* temp_file) const;
 
-  std::string work_dir_;  // A temporary directory to store the extracted image files
+  std::string path_;      // Path to the zipped target-file or an extracted directory.
+  bool extracted_input_;  // True if the target-file has been extracted.
+  ZipArchiveHandle handle_{ nullptr };
+
+  // The properties under META/misc_info.txt
+  std::map<std::string, std::string, std::less<>> misc_info_;
 };

updater/target_files.cpp

@@ -16,11 +16,267 @@
 
 #include "updater/target_files.h"
 
-std::string TargetFiles::GetProperty(const std::string_view /*key*/,
+#include <unistd.h>
-                                     const std::string_view default_value) const {
+
-  return std::string(default_value);
+#include <algorithm>
+#include <filesystem>
+#include <memory>
+
+#include <android-base/logging.h>
+#include <android-base/strings.h>
+#include <sparse/sparse.h>
+
+static bool SimgToImg(int input_fd, int output_fd) {
+  if (lseek64(input_fd, 0, SEEK_SET) == -1) {
+    PLOG(ERROR) << "Failed to lseek64 on the input sparse image";
+    return false;
+  }
+
+  if (lseek64(output_fd, 0, SEEK_SET) == -1) {
+    PLOG(ERROR) << "Failed to lseek64 on the output raw image";
+    return false;
+  }
+
+  std::unique_ptr<sparse_file, decltype(&sparse_file_destroy)> s_file(
+      sparse_file_import(input_fd, true, false), sparse_file_destroy);
+  if (!s_file) {
+    LOG(ERROR) << "Failed to import the sparse image.";
+    return false;
+  }
+
+  if (sparse_file_write(s_file.get(), output_fd, false, false, false) < 0) {
+    PLOG(ERROR) << "Failed to output the raw image file.";
+    return false;
+  }
+
+  return true;
 }
 
-std::string TargetFiles::FindBlockDeviceName(const std::string_view name) const {
+static bool ParsePropertyFile(const std::string_view prop_content,
-  return std::string(name);
+                              std::map<std::string, std::string, std::less<>>* props_map) {
+  LOG(INFO) << "Start parsing build property\n";
+  std::vector<std::string> lines = android::base::Split(std::string(prop_content), "\n");
+  for (const auto& line : lines) {
+    if (line.empty() || line[0] == '#') continue;
+    auto pos = line.find('=');
+    if (pos == std::string::npos) continue;
+    std::string key = line.substr(0, pos);
+    std::string value = line.substr(pos + 1);
+    LOG(INFO) << key << ": " << value;
+    props_map->emplace(key, value);
+  }
+
+  return true;
+}
+
+static bool ParseFstab(const std::string_view fstab, std::vector<FstabInfo>* fstab_info_list) {
+  LOG(INFO) << "parsing fstab\n";
+  std::vector<std::string> lines = android::base::Split(std::string(fstab), "\n");
+  for (const auto& line : lines) {
+    if (line.empty() || line[0] == '#') continue;
+
+    // <block_device>  <mount_point>  <fs_type>  <mount_flags>  optional:<fs_mgr_flags>
+    std::vector<std::string> tokens = android::base::Split(line, " ");
+    tokens.erase(std::remove(tokens.begin(), tokens.end(), ""), tokens.end());
+    if (tokens.size() != 4 && tokens.size() != 5) {
+      LOG(ERROR) << "Unexpected token size: " << tokens.size() << std::endl
+                 << "Error parsing fstab line: " << line;
+      return false;
+    }
+
+    const auto& blockdev = tokens[0];
+    const auto& mount_point = tokens[1];
+    const auto& fs_type = tokens[2];
+    if (!android::base::StartsWith(mount_point, "/")) {
+      LOG(WARNING) << "mount point '" << mount_point << "' does not start with '/'";
+      continue;
+    }
+
+    // The simulator only supports ext4 and emmc for now.
+    if (fs_type != "ext4" && fs_type != "emmc") {
+      LOG(WARNING) << "Unsupported fs_type in " << line;
+      continue;
+    }
+
+    fstab_info_list->emplace_back(blockdev, mount_point, fs_type);
+  }
+
+  return true;
+}
+
+bool TargetFile::EntryExists(const std::string_view name) const {
+  if (extracted_input_) {
+    std::string entry_path = path_ + "/" + std::string(name);
+    if (access(entry_path.c_str(), O_RDONLY) != 0) {
+      PLOG(WARNING) << "Failed to access " << entry_path;
+      return false;
+    }
+    return true;
+  }
+
+  CHECK(handle_);
+  ZipEntry img_entry;
+  return FindEntry(handle_, name, &img_entry) == 0;
+}
+
+bool TargetFile::ReadEntryToString(const std::string_view name, std::string* content) const {
+  if (extracted_input_) {
+    std::string entry_path = path_ + "/" + std::string(name);
+    return android::base::ReadFileToString(entry_path, content);
+  }
+
+  CHECK(handle_);
+  ZipEntry entry;
+  if (auto find_err = FindEntry(handle_, name, &entry); find_err != 0) {
+    LOG(ERROR) << "failed to find " << name << " in the package: " << ErrorCodeString(find_err);
+    return false;
+  }
+
+  content->resize(entry.uncompressed_length);
+  if (auto extract_err = ExtractToMemory(
+          handle_, &entry, reinterpret_cast<uint8_t*>(&content->at(0)), entry.uncompressed_length);
+      extract_err != 0) {
+    LOG(ERROR) << "failed to read " << name << " from package: " << ErrorCodeString(extract_err);
+    return false;
+  }
+
+  return true;
+}
+
+bool TargetFile::ExtractEntryToTempFile(const std::string_view name,
+                                        TemporaryFile* temp_file) const {
+  if (extracted_input_) {
+    std::string entry_path = path_ + "/" + std::string(name);
+    return std::filesystem::copy_file(entry_path, temp_file->path,
+                                      std::filesystem::copy_options::overwrite_existing);
+  }
+
+  CHECK(handle_);
+  ZipEntry entry;
+  if (auto find_err = FindEntry(handle_, name, &entry); find_err != 0) {
+    LOG(ERROR) << "failed to find " << name << " in the package: " << ErrorCodeString(find_err);
+    return false;
+  }
+
+  if (auto status = ExtractEntryToFile(handle_, &entry, temp_file->fd); status != 0) {
+    LOG(ERROR) << "Failed to extract zip entry " << name << " : " << ErrorCodeString(status);
+    return false;
+  }
+  return true;
+}
+
+bool TargetFile::Open() {
+  if (!extracted_input_) {
+    if (auto ret = OpenArchive(path_.c_str(), &handle_); ret != 0) {
+      LOG(ERROR) << "failed to open source target file " << path_ << ": " << ErrorCodeString(ret);
+      return false;
+    }
+  }
+
+  // Parse the misc info.
+  std::string misc_info_content;
+  if (!ReadEntryToString("META/misc_info.txt", &misc_info_content)) {
+    return false;
+  }
+  if (!ParsePropertyFile(misc_info_content, &misc_info_)) {
+    return false;
+  }
+
+  return true;
+}
+
+bool TargetFile::GetBuildProps(std::map<std::string, std::string, std::less<>>* props_map) const {
+  props_map->clear();
+  // Parse the source zip to mock the system props and block devices. We try all the possible
+  // locations for build props.
+  constexpr std::string_view kPropLocations[] = {
+    "SYSTEM/build.prop",
+    "VENDOR/build.prop",
+    "PRODUCT/build.prop",
+    "PRODUCT_SERVICES/build.prop",
+    "SYSTEM/vendor/build.prop",
+    "SYSTEM/product/build.prop",
+    "SYSTEM/product_services/build.prop",
+    "ODM/build.prop",  // legacy
+    "ODM/etc/build.prop",
+    "VENDOR/odm/build.prop",  // legacy
+    "VENDOR/odm/etc/build.prop",
+  };
+  for (const auto& name : kPropLocations) {
+    std::string build_prop_content;
+    if (!ReadEntryToString(name, &build_prop_content)) {
+      continue;
+    }
+    std::map<std::string, std::string, std::less<>> props;
+    if (!ParsePropertyFile(build_prop_content, &props)) {
+      LOG(ERROR) << "Failed to parse build prop in " << name;
+      return false;
+    }
+    for (const auto& [key, value] : props) {
+      if (auto it = props_map->find(key); it != props_map->end() && it->second != value) {
+        LOG(WARNING) << "Property " << key << " has different values in property files, we got "
+                     << it->second << " and " << value;
+      }
+      props_map->emplace(key, value);
+    }
+  }
+
+  return true;
+}
+
+bool TargetFile::ExtractImage(const std::string_view entry_name, const FstabInfo& fstab_info,
+                              const std::string_view work_dir, TemporaryFile* image_file) const {
+  if (!EntryExists(entry_name)) {
+    return false;
+  }
+
+  // We don't need extra work for 'emmc'; use the image file as the block device.
+  if (fstab_info.fs_type == "emmc" || misc_info_.find("extfs_sparse_flag") == misc_info_.end()) {
+    if (!ExtractEntryToTempFile(entry_name, image_file)) {
+      return false;
+    }
+  } else {  // treated as ext4 sparse image
+    TemporaryFile sparse_image{ std::string(work_dir) };
+    if (!ExtractEntryToTempFile(entry_name, &sparse_image)) {
+      return false;
+    }
+
+    // Convert the sparse image to raw.
+    if (!SimgToImg(sparse_image.fd, image_file->fd)) {
+      LOG(ERROR) << "Failed to convert " << fstab_info.mount_point << " to raw.";
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool TargetFile::ParseFstabInfo(std::vector<FstabInfo>* fstab_info_list) const {
+  // Parse the fstab file and extract the image files. The location of the fstab actually depends
+  // on some flags e.g. "no_recovery", "recovery_as_boot". Here we just try all possibilities.
+  constexpr std::string_view kRecoveryFstabLocations[] = {
+    "RECOVERY/RAMDISK/system/etc/recovery.fstab",
+    "RECOVERY/RAMDISK/etc/recovery.fstab",
+    "BOOT/RAMDISK/system/etc/recovery.fstab",
+    "BOOT/RAMDISK/etc/recovery.fstab",
+  };
+  std::string fstab_content;
+  for (const auto& name : kRecoveryFstabLocations) {
+    if (std::string content; ReadEntryToString(name, &content)) {
+      fstab_content = std::move(content);
+      break;
+    }
+  }
+  if (fstab_content.empty()) {
+    LOG(ERROR) << "Failed to parse the recovery fstab file";
+    return false;
+  }
+
+  // Extract the images and convert them to raw.
+  if (!ParseFstab(fstab_content, fstab_info_list)) {
+    LOG(ERROR) << "Failed to mount the block devices for source build.";
+    return false;
+  }
+
+  return true;
 }

updater/update_simulator_main.cpp

@@ -22,9 +22,10 @@
 #include "otautil/error_code.h"
 #include "otautil/paths.h"
 #include "updater/blockimg.h"
+#include "updater/build_info.h"
+#include "updater/dynamic_partitions.h"
 #include "updater/install.h"
 #include "updater/simulator_runtime.h"
-#include "updater/target_files.h"
 #include "updater/updater.h"
 
 int main(int argc, char** argv) {
@@ -34,7 +35,7 @@ int main(int argc, char** argv) {
   if (argc != 3 && argc != 4) {
     LOG(ERROR) << "unexpected number of arguments: " << argc << std::endl
                << "Usage: " << argv[0] << " <source_target-file> <ota_package>";
-    return 1;
+    return EXIT_FAILURE;
   }
 
   // TODO(xunchang) implement a commandline parser, e.g. it can take an oem property so that the
@@ -57,17 +58,21 @@ int main(int argc, char** argv) {
   Paths::Get().set_stash_directory_base(temp_stash_base.path);
 
   TemporaryFile cmd_pipe;
-
   TemporaryDir source_temp_dir;
-  TargetFiles source(source_target_file, source_temp_dir.path);
 
-  Updater updater(std::make_unique<SimulatorRuntime>(&source));
+  BuildInfo source_build_info(source_temp_dir.path);
+  if (!source_build_info.ParseTargetFile(source_target_file, false)) {
+    LOG(ERROR) << "Failed to parse the target file " << source_target_file;
+    return EXIT_FAILURE;
+  }
+
+  Updater updater(std::make_unique<SimulatorRuntime>(&source_build_info));
   if (!updater.Init(cmd_pipe.release(), package_name, false)) {
-    return 1;
+    return EXIT_FAILURE;
   }
 
   if (!updater.RunUpdate()) {
-    return 1;
+    return EXIT_FAILURE;
   }
 
   LOG(INFO) << "\nscript succeeded, result: " << updater.GetResult();