Merge Android R (rvc-dev-plus-aosp-without-vendor@6692709)

Bug: 166295507
Merged-In: Id417587a550b0f4abf5a6a3e4b4535011b21f627
Change-Id: Ibb5e8cf5f36dad408cf047dd0498aba24249b695

Android.bp

@@ -28,6 +28,7 @@ cc_defaults {
     name: "vold_default_libs",
 
     static_libs: [
+        "libasync_safe",
         "libavb",
         "libbootloader_message",
         "libdm",
@@ -156,6 +157,7 @@ cc_library_static {
     ],
     whole_static_libs: [
         "com.android.sysprop.apex",
+        "libc++fs"
     ],
 }
 
@@ -221,6 +223,7 @@ cc_binary {
         "libhidlbase",
         "libkeymaster4support",
         "libkeymaster4_1support",
+        "libutils",
     ],
 }
 
@@ -257,6 +260,7 @@ filegroup {
     srcs: [
         "binder/android/os/IVold.aidl",
         "binder/android/os/IVoldListener.aidl",
+        "binder/android/os/IVoldMountCallback.aidl",
         "binder/android/os/IVoldTaskListener.aidl",
     ],
     path: "binder",

FsCrypt.cpp

@@ -40,6 +40,7 @@
 #include <unistd.h>
 
 #include <private/android_filesystem_config.h>
+#include <private/android_projectid_config.h>
 
 #include "android/os/IVold.h"
 
@@ -66,11 +67,14 @@ using android::base::StartsWith;
 using android::base::StringPrintf;
 using android::fs_mgr::GetEntryForMountPoint;
 using android::vold::BuildDataPath;
+using android::vold::IsFilesystemSupported;
 using android::vold::kEmptyAuthentication;
 using android::vold::KeyBuffer;
 using android::vold::KeyGeneration;
 using android::vold::retrieveKey;
 using android::vold::retrieveOrGenerateKey;
+using android::vold::SetQuotaInherit;
+using android::vold::SetQuotaProjectId;
 using android::vold::writeStringToFile;
 using namespace android::fscrypt;
 using namespace android::dm;
@@ -856,6 +860,7 @@ bool fscrypt_prepare_user_storage(const std::string& volume_uuid, userid_t user_
             if (!prepare_dir(vendor_ce_path, 0771, AID_ROOT, AID_ROOT)) return false;
         }
         if (!prepare_dir(media_ce_path, 0770, AID_MEDIA_RW, AID_MEDIA_RW)) return false;
+
         if (!prepare_dir(user_ce_path, 0771, AID_SYSTEM, AID_SYSTEM)) return false;
 
         if (fscrypt_is_native()) {

Keymaster.h

@@ -133,7 +133,7 @@ class Keymaster {
     static void earlyBootEnded();
 
   private:
-    std::unique_ptr<KmDevice> mDevice;
+    sp<KmDevice> mDevice;
     DISALLOW_COPY_AND_ASSIGN(Keymaster);
     static bool hmacKeyGenerated;
 };

Process.cpp

@@ -29,6 +29,7 @@
 #include <unistd.h>
 
 #include <fstream>
+#include <mntent.h>
 #include <unordered_set>
 
 #include <android-base/file.h>
@@ -81,6 +82,51 @@ static bool checkSymlink(const std::string& path, const std::string& prefix) {
     return false;
 }
 
+// TODO: Refactor the code with KillProcessesWithOpenFiles().
+int KillProcessesWithMounts(const std::string& prefix, int signal) {
+    std::unordered_set<pid_t> pids;
+
+    auto proc_d = std::unique_ptr<DIR, int (*)(DIR*)>(opendir("/proc"), closedir);
+    if (!proc_d) {
+        PLOG(ERROR) << "Failed to open proc";
+        return -1;
+    }
+
+    struct dirent* proc_de;
+    while ((proc_de = readdir(proc_d.get())) != nullptr) {
+        // We only care about valid PIDs
+        pid_t pid;
+        if (proc_de->d_type != DT_DIR) continue;
+        if (!android::base::ParseInt(proc_de->d_name, &pid)) continue;
+
+        // Look for references to prefix
+        std::string mounts_file(StringPrintf("/proc/%d/mounts", pid));
+        auto fp = std::unique_ptr<FILE, int (*)(FILE*)>(
+                setmntent(mounts_file.c_str(), "r"), endmntent);
+        if (!fp) {
+            PLOG(WARNING) << "Failed to open " << mounts_file;
+            continue;
+        }
+
+        // Check if obb directory is mounted, and get all packages of mounted app data directory.
+        mntent* mentry;
+        while ((mentry = getmntent(fp.get())) != nullptr) {
+            if (android::base::StartsWith(mentry->mnt_dir, prefix)) {
+                pids.insert(pid);
+                break;
+            }
+        }
+    }
+    if (signal != 0) {
+        for (const auto& pid : pids) {
+            LOG(WARNING) << "Killing pid "<< pid << " with signal " << strsignal(signal) <<
+                    " because it has a mount with prefix " << prefix;
+            kill(pid, signal);
+        }
+    }
+    return pids.size();
+}
+
 int KillProcessesWithOpenFiles(const std::string& prefix, int signal) {
     std::unordered_set<pid_t> pids;
 

Process.h

@@ -21,6 +21,7 @@ namespace android {
 namespace vold {
 
 int KillProcessesWithOpenFiles(const std::string& path, int signal);
+int KillProcessesWithMounts(const std::string& path, int signal);
 
 }  // namespace vold
 }  // namespace android

TEST_MAPPING

@@ -0,0 +1,10 @@
+{
+  "presubmit": [
+    {
+      "name": "CtsScopedStorageHostTest"
+    },
+    {
+      "name": "AdoptableHostTest"
+    }
+  ]
+}

Utils.cpp

@@ -29,24 +29,29 @@
 #include <cutils/fs.h>
 #include <logwrap/logwrap.h>
 #include <private/android_filesystem_config.h>
+#include <private/android_projectid_config.h>
 
 #include <dirent.h>
 #include <fcntl.h>
 #include <linux/fs.h>
+#include <linux/posix_acl.h>
+#include <linux/posix_acl_xattr.h>
 #include <mntent.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <sys/mount.h>
 #include <sys/stat.h>
 #include <sys/statvfs.h>
 #include <sys/sysmacros.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <sys/xattr.h>
 #include <unistd.h>
 
+#include <filesystem>
 #include <list>
 #include <mutex>
+#include <regex>
 #include <thread>
 
 #ifndef UMOUNT_NOFOLLOW
@@ -54,8 +59,11 @@
 #endif
 
 using namespace std::chrono_literals;
+using android::base::EndsWith;
 using android::base::ReadFileToString;
+using android::base::StartsWith;
 using android::base::StringPrintf;
+using android::base::unique_fd;
 
 namespace android {
 namespace vold {
@@ -73,10 +81,22 @@ static const char* kKeyPath = "/data/misc/vold";
 static const char* kProcDevices = "/proc/devices";
 static const char* kProcFilesystems = "/proc/filesystems";
 
+static const char* kAndroidDir = "/Android/";
+static const char* kAppDataDir = "/Android/data/";
+static const char* kAppMediaDir = "/Android/media/";
+static const char* kAppObbDir = "/Android/obb/";
+
+static const char* kMediaProviderCtx = "u:r:mediaprovider:";
+static const char* kMediaProviderAppCtx = "u:r:mediaprovider_app:";
+
 // Lock used to protect process-level SELinux changes from racing with each
 // other between multiple threads.
 static std::mutex kSecurityLock;
 
+std::string GetFuseMountPathForUser(userid_t user_id, const std::string& relative_upper_path) {
+    return StringPrintf("/mnt/user/%d/%s", user_id, relative_upper_path.c_str());
+}
+
 status_t CreateDeviceNode(const std::string& path, dev_t dev) {
     std::lock_guard<std::mutex> lock(kSecurityLock);
     const char* cpath = path.c_str();
@@ -115,6 +135,287 @@ status_t DestroyDeviceNode(const std::string& path) {
     }
 }
 
+// Sets a default ACL on the directory.
+int SetDefaultAcl(const std::string& path, mode_t mode, uid_t uid, gid_t gid,
+                  std::vector<gid_t> additionalGids) {
+    if (IsSdcardfsUsed()) {
+        // sdcardfs magically takes care of this
+        return OK;
+    }
+
+    size_t num_entries = 3 + (additionalGids.size() > 0 ? additionalGids.size() + 1 : 0);
+    size_t size = sizeof(posix_acl_xattr_header) + num_entries * sizeof(posix_acl_xattr_entry);
+    auto buf = std::make_unique<uint8_t[]>(size);
+
+    posix_acl_xattr_header* acl_header = reinterpret_cast<posix_acl_xattr_header*>(buf.get());
+    acl_header->a_version = POSIX_ACL_XATTR_VERSION;
+
+    posix_acl_xattr_entry* entry =
+            reinterpret_cast<posix_acl_xattr_entry*>(buf.get() + sizeof(posix_acl_xattr_header));
+
+    int tag_index = 0;
+
+    entry[tag_index].e_tag = ACL_USER_OBJ;
+    // The existing mode_t mask has the ACL in the lower 9 bits:
+    // the lowest 3 for "other", the next 3 the group, the next 3 for the owner
+    // Use the mode_t masks to get these bits out, and shift them to get the
+    // correct value per entity.
+    //
+    // Eg if mode_t = 0700, rwx for the owner, then & S_IRWXU >> 6 results in 7
+    entry[tag_index].e_perm = (mode & S_IRWXU) >> 6;
+    entry[tag_index].e_id = uid;
+    tag_index++;
+
+    entry[tag_index].e_tag = ACL_GROUP_OBJ;
+    entry[tag_index].e_perm = (mode & S_IRWXG) >> 3;
+    entry[tag_index].e_id = gid;
+    tag_index++;
+
+    if (additionalGids.size() > 0) {
+        for (gid_t additional_gid : additionalGids) {
+            entry[tag_index].e_tag = ACL_GROUP;
+            entry[tag_index].e_perm = (mode & S_IRWXG) >> 3;
+            entry[tag_index].e_id = additional_gid;
+            tag_index++;
+        }
+
+        entry[tag_index].e_tag = ACL_MASK;
+        entry[tag_index].e_perm = (mode & S_IRWXG) >> 3;
+        entry[tag_index].e_id = 0;
+        tag_index++;
+    }
+
+    entry[tag_index].e_tag = ACL_OTHER;
+    entry[tag_index].e_perm = mode & S_IRWXO;
+    entry[tag_index].e_id = 0;
+
+    int ret = setxattr(path.c_str(), XATTR_NAME_POSIX_ACL_DEFAULT, acl_header, size, 0);
+
+    if (ret != 0) {
+        PLOG(ERROR) << "Failed to set default ACL on " << path;
+    }
+
+    return ret;
+}
+
+int SetQuotaInherit(const std::string& path) {
+    unsigned long flags;
+
+    android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
+    if (fd == -1) {
+        PLOG(ERROR) << "Failed to open " << path << " to set project id inheritance.";
+        return -1;
+    }
+
+    int ret = ioctl(fd, FS_IOC_GETFLAGS, &flags);
+    if (ret == -1) {
+        PLOG(ERROR) << "Failed to get flags for " << path << " to set project id inheritance.";
+        return ret;
+    }
+
+    flags |= FS_PROJINHERIT_FL;
+
+    ret = ioctl(fd, FS_IOC_SETFLAGS, &flags);
+    if (ret == -1) {
+        PLOG(ERROR) << "Failed to set flags for " << path << " to set project id inheritance.";
+        return ret;
+    }
+
+    return 0;
+}
+
+int SetQuotaProjectId(const std::string& path, long projectId) {
+    struct fsxattr fsx;
+
+    android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
+    if (fd == -1) {
+        PLOG(ERROR) << "Failed to open " << path << " to set project id.";
+        return -1;
+    }
+
+    int ret = ioctl(fd, FS_IOC_FSGETXATTR, &fsx);
+    if (ret == -1) {
+        PLOG(ERROR) << "Failed to get extended attributes for " << path << " to get project id.";
+        return ret;
+    }
+
+    fsx.fsx_projid = projectId;
+    return ioctl(fd, FS_IOC_FSSETXATTR, &fsx);
+}
+
+int PrepareDirWithProjectId(const std::string& path, mode_t mode, uid_t uid, gid_t gid,
+                            long projectId) {
+    int ret = fs_prepare_dir(path.c_str(), mode, uid, gid);
+
+    if (ret != 0) {
+        return ret;
+    }
+
+    if (!IsSdcardfsUsed()) {
+        ret = SetQuotaProjectId(path, projectId);
+    }
+
+    return ret;
+}
+
+static int FixupAppDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid, long projectId) {
+    namespace fs = std::filesystem;
+
+    // Setup the directory itself correctly
+    int ret = PrepareDirWithProjectId(path, mode, uid, gid, projectId);
+    if (ret != OK) {
+        return ret;
+    }
+
+    // Fixup all of its file entries
+    for (const auto& itEntry : fs::directory_iterator(path)) {
+        ret = lchown(itEntry.path().c_str(), uid, gid);
+        if (ret != 0) {
+            return ret;
+        }
+
+        ret = chmod(itEntry.path().c_str(), mode);
+        if (ret != 0) {
+            return ret;
+        }
+
+        if (!IsSdcardfsUsed()) {
+            ret = SetQuotaProjectId(itEntry.path(), projectId);
+            if (ret != 0) {
+                return ret;
+            }
+        }
+    }
+
+    return OK;
+}
+
+int PrepareAppDirFromRoot(const std::string& path, const std::string& root, int appUid,
+                          bool fixupExisting) {
+    long projectId;
+    size_t pos;
+    int ret = 0;
+    bool sdcardfsSupport = IsSdcardfsUsed();
+
+    // Make sure the Android/ directories exist and are setup correctly
+    ret = PrepareAndroidDirs(root);
+    if (ret != 0) {
+        LOG(ERROR) << "Failed to prepare Android/ directories.";
+        return ret;
+    }
+
+    // Now create the application-specific subdir(s)
+    // path is something like /data/media/0/Android/data/com.foo/files
+    // First, chop off the volume root, eg /data/media/0
+    std::string pathFromRoot = path.substr(root.length());
+
+    uid_t uid = appUid;
+    gid_t gid = AID_MEDIA_RW;
+    std::vector<gid_t> additionalGids;
+    std::string appDir;
+
+    // Check that the next part matches one of the allowed Android/ dirs
+    if (StartsWith(pathFromRoot, kAppDataDir)) {
+        appDir = kAppDataDir;
+        if (!sdcardfsSupport) {
+            gid = AID_EXT_DATA_RW;
+            // Also add the app's own UID as a group; since apps belong to a group
+            // that matches their UID, this ensures that they will always have access to
+            // the files created in these dirs, even if they are created by other processes
+            additionalGids.push_back(uid);
+        }
+    } else if (StartsWith(pathFromRoot, kAppMediaDir)) {
+        appDir = kAppMediaDir;
+        if (!sdcardfsSupport) {
+            gid = AID_MEDIA_RW;
+        }
+    } else if (StartsWith(pathFromRoot, kAppObbDir)) {
+        appDir = kAppObbDir;
+        if (!sdcardfsSupport) {
+            gid = AID_EXT_OBB_RW;
+            // See comments for kAppDataDir above
+            additionalGids.push_back(uid);
+        }
+    } else {
+        LOG(ERROR) << "Invalid application directory: " << path;
+        return -EINVAL;
+    }
+
+    // mode = 770, plus sticky bit on directory to inherit GID when apps
+    // create subdirs
+    mode_t mode = S_IRWXU | S_IRWXG | S_ISGID;
+    // the project ID for application-specific directories is directly
+    // derived from their uid
+
+    // Chop off the generic application-specific part, eg /Android/data/
+    // this leaves us with something like com.foo/files/
+    std::string leftToCreate = pathFromRoot.substr(appDir.length());
+    if (!EndsWith(leftToCreate, "/")) {
+        leftToCreate += "/";
+    }
+    std::string pathToCreate = root + appDir;
+    int depth = 0;
+    // Derive initial project ID
+    if (appDir == kAppDataDir || appDir == kAppMediaDir) {
+        projectId = uid - AID_APP_START + PROJECT_ID_EXT_DATA_START;
+    } else if (appDir == kAppObbDir) {
+        projectId = uid - AID_APP_START + PROJECT_ID_EXT_OBB_START;
+    }
+
+    while ((pos = leftToCreate.find('/')) != std::string::npos) {
+        std::string component = leftToCreate.substr(0, pos + 1);
+        leftToCreate = leftToCreate.erase(0, pos + 1);
+        pathToCreate = pathToCreate + component;
+
+        if (appDir == kAppDataDir && depth == 1 && component == "cache/") {
+            // All dirs use the "app" project ID, except for the cache dirs in
+            // Android/data, eg Android/data/com.foo/cache
+            // Note that this "sticks" - eg subdirs of this dir need the same
+            // project ID.
+            projectId = uid - AID_APP_START + PROJECT_ID_EXT_CACHE_START;
+        }
+
+        if (fixupExisting && access(pathToCreate.c_str(), F_OK) == 0) {
+            // Fixup all files in this existing directory with the correct UID/GID
+            // and project ID.
+            ret = FixupAppDir(pathToCreate, mode, uid, gid, projectId);
+        } else {
+            ret = PrepareDirWithProjectId(pathToCreate, mode, uid, gid, projectId);
+        }
+
+        if (ret != 0) {
+            return ret;
+        }
+
+        if (depth == 0) {
+            // Set the default ACL on the top-level application-specific directories,
+            // to ensure that even if applications run with a umask of 0077,
+            // new directories within these directories will allow the GID
+            // specified here to write; this is necessary for apps like
+            // installers and MTP, that require access here.
+            //
+            // See man (5) acl for more details.
+            ret = SetDefaultAcl(pathToCreate, mode, uid, gid, additionalGids);
+            if (ret != 0) {
+                return ret;
+            }
+
+            if (!sdcardfsSupport) {
+                // Set project ID inheritance, so that future subdirectories inherit the
+                // same project ID
+                ret = SetQuotaInherit(pathToCreate);
+                if (ret != 0) {
+                    return ret;
+                }
+            }
+        }
+
+        depth++;
+    }
+
+    return OK;
+}
+
 status_t PrepareDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid) {
     std::lock_guard<std::mutex> lock(kSecurityLock);
     const char* cpath = path.c_str();
@@ -166,10 +467,35 @@ status_t ForceUnmount(const std::string& path) {
     if (!umount2(cpath, UMOUNT_NOFOLLOW) || errno == EINVAL || errno == ENOENT) {
         return OK;
     }
-
+    PLOG(INFO) << "ForceUnmount failed";
     return -errno;
 }
 
+status_t KillProcessesWithMountPrefix(const std::string& path) {
+    if (KillProcessesWithMounts(path, SIGINT) == 0) {
+        return OK;
+    }
+    if (sSleepOnUnmount) sleep(5);
+
+    if (KillProcessesWithMounts(path, SIGTERM) == 0) {
+        return OK;
+    }
+    if (sSleepOnUnmount) sleep(5);
+
+    if (KillProcessesWithMounts(path, SIGKILL) == 0) {
+        return OK;
+    }
+    if (sSleepOnUnmount) sleep(5);
+
+    // Send SIGKILL a second time to determine if we've
+    // actually killed everyone mount
+    if (KillProcessesWithMounts(path, SIGKILL) == 0) {
+        return OK;
+    }
+    PLOG(ERROR) << "Failed to kill processes using " << path;
+    return -EBUSY;
+}
+
 status_t KillProcessesUsingPath(const std::string& path) {
     if (KillProcessesWithOpenFiles(path, SIGINT) == 0) {
         return OK;
@@ -627,6 +953,19 @@ uint64_t GetTreeBytes(const std::string& path) {
     }
 }
 
+// TODO: Use a better way to determine if it's media provider app.
+bool IsFuseDaemon(const pid_t pid) {
+    auto path = StringPrintf("/proc/%d/mounts", pid);
+    char* tmp;
+    if (lgetfilecon(path.c_str(), &tmp) < 0) {
+        return false;
+    }
+    bool result = android::base::StartsWith(tmp, kMediaProviderAppCtx)
+            || android::base::StartsWith(tmp, kMediaProviderCtx);
+    freecon(tmp);
+    return result;
+}
+
 bool IsFilesystemSupported(const std::string& fsType) {
     std::string supported;
     if (!ReadFileToString(kProcFilesystems, &supported)) {
@@ -636,6 +975,11 @@ bool IsFilesystemSupported(const std::string& fsType) {
     return supported.find(fsType + "\n") != std::string::npos;
 }
 
+bool IsSdcardfsUsed() {
+    return IsFilesystemSupported("sdcardfs") &&
+           base::GetBoolProperty(kExternalStorageSdcardfs, true);
+}
+
 status_t WipeBlockDevice(const std::string& path) {
     status_t res = -1;
     const char* c_path = path.c_str();
@@ -1017,5 +1361,265 @@ bool writeStringToFile(const std::string& payload, const std::string& filename)
     return true;
 }
 
+status_t AbortFuseConnections() {
+    namespace fs = std::filesystem;
+
+    for (const auto& itEntry : fs::directory_iterator("/sys/fs/fuse/connections")) {
+        std::string abortPath = itEntry.path().string() + "/abort";
+        LOG(DEBUG) << "Aborting fuse connection entry " << abortPath;
+        bool ret = writeStringToFile("1", abortPath);
+        if (!ret) {
+            LOG(WARNING) << "Failed to write to " << abortPath;
+        }
+    }
+
+    return OK;
+}
+
+status_t EnsureDirExists(const std::string& path, mode_t mode, uid_t uid, gid_t gid) {
+    if (access(path.c_str(), F_OK) != 0) {
+        PLOG(WARNING) << "Dir does not exist: " << path;
+        if (fs_prepare_dir(path.c_str(), mode, uid, gid) != 0) {
+            return -errno;
+        }
+    }
+    return OK;
+}
+
+// Gets the sysfs path for parameters of the backing device info (bdi)
+static std::string getBdiPathForMount(const std::string& mount) {
+    // First figure out MAJOR:MINOR of mount. Simplest way is to stat the path.
+    struct stat info;
+    if (stat(mount.c_str(), &info) != 0) {
+        PLOG(ERROR) << "Failed to stat " << mount;
+        return "";
+    }
+    unsigned int maj = major(info.st_dev);
+    unsigned int min = minor(info.st_dev);
+
+    return StringPrintf("/sys/class/bdi/%u:%u", maj, min);
+}
+
+// Configures max_ratio for the FUSE filesystem.
+void ConfigureMaxDirtyRatioForFuse(const std::string& fuse_mount, unsigned int max_ratio) {
+    LOG(INFO) << "Configuring max_ratio of " << fuse_mount << " fuse filesystem to " << max_ratio;
+    if (max_ratio > 100) {
+        LOG(ERROR) << "Invalid max_ratio: " << max_ratio;
+        return;
+    }
+    std::string fuseBdiPath = getBdiPathForMount(fuse_mount);
+    if (fuseBdiPath == "") {
+        return;
+    }
+    std::string max_ratio_file = StringPrintf("%s/max_ratio", fuseBdiPath.c_str());
+    unique_fd fd(TEMP_FAILURE_RETRY(open(max_ratio_file.c_str(), O_WRONLY | O_CLOEXEC)));
+    if (fd.get() == -1) {
+        PLOG(ERROR) << "Failed to open " << max_ratio_file;
+        return;
+    }
+    LOG(INFO) << "Writing " << max_ratio << " to " << max_ratio_file;
+    if (!WriteStringToFd(std::to_string(max_ratio), fd)) {
+        PLOG(ERROR) << "Failed to write to " << max_ratio_file;
+    }
+}
+
+// Configures read ahead property of the fuse filesystem with the mount point |fuse_mount| by
+// writing |read_ahead_kb| to the /sys/class/bdi/MAJOR:MINOR/read_ahead_kb.
+void ConfigureReadAheadForFuse(const std::string& fuse_mount, size_t read_ahead_kb) {
+    LOG(INFO) << "Configuring read_ahead of " << fuse_mount << " fuse filesystem to "
+              << read_ahead_kb << "kb";
+    std::string fuseBdiPath = getBdiPathForMount(fuse_mount);
+    if (fuseBdiPath == "") {
+        return;
+    }
+    // We found the bdi path for our filesystem, time to configure read ahead!
+    std::string read_ahead_file = StringPrintf("%s/read_ahead_kb", fuseBdiPath.c_str());
+    unique_fd fd(TEMP_FAILURE_RETRY(open(read_ahead_file.c_str(), O_WRONLY | O_CLOEXEC)));
+    if (fd.get() == -1) {
+        PLOG(ERROR) << "Failed to open " << read_ahead_file;
+        return;
+    }
+    LOG(INFO) << "Writing " << read_ahead_kb << " to " << read_ahead_file;
+    if (!WriteStringToFd(std::to_string(read_ahead_kb), fd)) {
+        PLOG(ERROR) << "Failed to write to " << read_ahead_file;
+    }
+}
+
+status_t MountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                       const std::string& relative_upper_path, android::base::unique_fd* fuse_fd) {
+    std::string pre_fuse_path(StringPrintf("/mnt/user/%d", user_id));
+    std::string fuse_path(
+            StringPrintf("%s/%s", pre_fuse_path.c_str(), relative_upper_path.c_str()));
+
+    std::string pre_pass_through_path(StringPrintf("/mnt/pass_through/%d", user_id));
+    std::string pass_through_path(
+            StringPrintf("%s/%s", pre_pass_through_path.c_str(), relative_upper_path.c_str()));
+
+    // Ensure that /mnt/user is 0700. With FUSE, apps don't need access to /mnt/user paths directly.
+    // Without FUSE however, apps need /mnt/user access so /mnt/user in init.rc is 0755 until here
+    auto result = PrepareDir("/mnt/user", 0750, AID_ROOT, AID_MEDIA_RW);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory /mnt/user";
+        return -1;
+    }
+
+    // Shell is neither AID_ROOT nor AID_EVERYBODY. Since it equally needs 'execute' access to
+    // /mnt/user/0 to 'adb shell ls /sdcard' for instance, we set the uid bit of /mnt/user/0 to
+    // AID_SHELL. This gives shell access along with apps running as group everybody (user 0 apps)
+    // These bits should be consistent with what is set in zygote in
+    // com_android_internal_os_Zygote#MountEmulatedStorage on volume bind mount during app fork
+    result = PrepareDir(pre_fuse_path, 0710, user_id ? AID_ROOT : AID_SHELL,
+                             multiuser_get_uid(user_id, AID_EVERYBODY));
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << pre_fuse_path;
+        return -1;
+    }
+
+    result = PrepareDir(fuse_path, 0700, AID_ROOT, AID_ROOT);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << fuse_path;
+        return -1;
+    }
+
+    result = PrepareDir(pre_pass_through_path, 0710, AID_ROOT, AID_MEDIA_RW);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << pre_pass_through_path;
+        return -1;
+    }
+
+    result = PrepareDir(pass_through_path, 0710, AID_ROOT, AID_MEDIA_RW);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << pass_through_path;
+        return -1;
+    }
+
+    if (relative_upper_path == "emulated") {
+        std::string linkpath(StringPrintf("/mnt/user/%d/self", user_id));
+        result = PrepareDir(linkpath, 0755, AID_ROOT, AID_ROOT);
+        if (result != android::OK) {
+            PLOG(ERROR) << "Failed to prepare directory " << linkpath;
+            return -1;
+        }
+        linkpath += "/primary";
+        Symlink("/storage/emulated/" + std::to_string(user_id), linkpath);
+
+        std::string pass_through_linkpath(StringPrintf("/mnt/pass_through/%d/self", user_id));
+        result = PrepareDir(pass_through_linkpath, 0710, AID_ROOT, AID_MEDIA_RW);
+        if (result != android::OK) {
+            PLOG(ERROR) << "Failed to prepare directory " << pass_through_linkpath;
+            return -1;
+        }
+        pass_through_linkpath += "/primary";
+        Symlink("/storage/emulated/" + std::to_string(user_id), pass_through_linkpath);
+    }
+
+    // Open fuse fd.
+    fuse_fd->reset(open("/dev/fuse", O_RDWR | O_CLOEXEC));
+    if (fuse_fd->get() == -1) {
+        PLOG(ERROR) << "Failed to open /dev/fuse";
+        return -1;
+    }
+
+    // Note: leaving out default_permissions since we don't want kernel to do lower filesystem
+    // permission checks before routing to FUSE daemon.
+    const auto opts = StringPrintf(
+        "fd=%i,"
+        "rootmode=40000,"
+        "allow_other,"
+        "user_id=0,group_id=0,",
+        fuse_fd->get());
+
+    result = TEMP_FAILURE_RETRY(mount("/dev/fuse", fuse_path.c_str(), "fuse",
+                                      MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_NOATIME | MS_LAZYTIME,
+                                      opts.c_str()));
+    if (result != 0) {
+        PLOG(ERROR) << "Failed to mount " << fuse_path;
+        return -errno;
+    }
+
+    if (IsSdcardfsUsed()) {
+        std::string sdcardfs_path(
+                StringPrintf("/mnt/runtime/full/%s", relative_upper_path.c_str()));
+
+        LOG(INFO) << "Bind mounting " << sdcardfs_path << " to " << pass_through_path;
+        return BindMount(sdcardfs_path, pass_through_path);
+    } else {
+        LOG(INFO) << "Bind mounting " << absolute_lower_path << " to " << pass_through_path;
+        return BindMount(absolute_lower_path, pass_through_path);
+    }
+}
+
+status_t UnmountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                         const std::string& relative_upper_path) {
+    std::string fuse_path(StringPrintf("/mnt/user/%d/%s", user_id, relative_upper_path.c_str()));
+    std::string pass_through_path(
+            StringPrintf("/mnt/pass_through/%d/%s", user_id, relative_upper_path.c_str()));
+
+    // Best effort unmount pass_through path
+    sSleepOnUnmount = false;
+    LOG(INFO) << "Unmounting pass_through_path " << pass_through_path;
+    auto status = ForceUnmount(pass_through_path);
+    if (status != android::OK) {
+        LOG(ERROR) << "Failed to unmount " << pass_through_path;
+    }
+    rmdir(pass_through_path.c_str());
+
+    LOG(INFO) << "Unmounting fuse path " << fuse_path;
+    android::status_t result = ForceUnmount(fuse_path);
+    sSleepOnUnmount = true;
+    if (result != android::OK) {
+        // TODO(b/135341433): MNT_DETACH is needed for fuse because umount2 can fail with EBUSY.
+        // Figure out why we get EBUSY and remove this special casing if possible.
+        PLOG(ERROR) << "Failed to unmount. Trying MNT_DETACH " << fuse_path << " ...";
+        if (umount2(fuse_path.c_str(), UMOUNT_NOFOLLOW | MNT_DETACH) && errno != EINVAL &&
+            errno != ENOENT) {
+            PLOG(ERROR) << "Failed to unmount with MNT_DETACH " << fuse_path;
+            return -errno;
+        }
+        result = android::OK;
+    }
+    rmdir(fuse_path.c_str());
+
+    return result;
+}
+
+status_t PrepareAndroidDirs(const std::string& volumeRoot) {
+    std::string androidDir = volumeRoot + kAndroidDir;
+    std::string androidDataDir = volumeRoot + kAppDataDir;
+    std::string androidObbDir = volumeRoot + kAppObbDir;
+    std::string androidMediaDir = volumeRoot + kAppMediaDir;
+
+    bool useSdcardFs = IsSdcardfsUsed();
+
+    // mode 0771 + sticky bit for inheriting GIDs
+    mode_t mode = S_IRWXU | S_IRWXG | S_IXOTH | S_ISGID;
+    if (fs_prepare_dir(androidDir.c_str(), mode, AID_MEDIA_RW, AID_MEDIA_RW) != 0) {
+        PLOG(ERROR) << "Failed to create " << androidDir;
+        return -errno;
+    }
+
+    gid_t dataGid = useSdcardFs ? AID_MEDIA_RW : AID_EXT_DATA_RW;
+    if (fs_prepare_dir(androidDataDir.c_str(), mode, AID_MEDIA_RW, dataGid) != 0) {
+        PLOG(ERROR) << "Failed to create " << androidDataDir;
+        return -errno;
+    }
+
+    gid_t obbGid = useSdcardFs ? AID_MEDIA_RW : AID_EXT_OBB_RW;
+    if (fs_prepare_dir(androidObbDir.c_str(), mode, AID_MEDIA_RW, obbGid) != 0) {
+        PLOG(ERROR) << "Failed to create " << androidObbDir;
+        return -errno;
+    }
+    // Some other apps, like installers, have write access to the OBB directory
+    // to pre-download them. To make sure newly created folders in this directory
+    // have the right permissions, set a default ACL.
+    SetDefaultAcl(androidObbDir, mode, AID_MEDIA_RW, obbGid, {});
+
+    if (fs_prepare_dir(androidMediaDir.c_str(), mode, AID_MEDIA_RW, AID_MEDIA_RW) != 0) {
+        PLOG(ERROR) << "Failed to create " << androidMediaDir;
+        return -errno;
+    }
+
+    return OK;
+}
 }  // namespace vold
 }  // namespace android

Utils.h

@@ -20,6 +20,7 @@
 #include "KeyBuffer.h"
 
 #include <android-base/macros.h>
+#include <android-base/unique_fd.h>
 #include <cutils/multiuser.h>
 #include <selinux/selinux.h>
 #include <utils/Errors.h>
@@ -33,6 +34,10 @@ struct DIR;
 namespace android {
 namespace vold {
 
+static const char* kPropFuse = "persist.sys.fuse";
+static const char* kVoldAppDataIsolationEnabled = "persist.sys.vold_app_data_isolation_enabled";
+static const char* kExternalStorageSdcardfs = "external_storage.sdcardfs.enabled";
+
 /* SELinux contexts used depending on the block device type */
 extern security_context_t sBlkidContext;
 extern security_context_t sBlkidUntrustedContext;
@@ -42,9 +47,25 @@ extern security_context_t sFsckUntrustedContext;
 // TODO remove this with better solution, b/64143519
 extern bool sSleepOnUnmount;
 
+std::string GetFuseMountPathForUser(userid_t user_id, const std::string& relative_upper_path);
+
 status_t CreateDeviceNode(const std::string& path, dev_t dev);
 status_t DestroyDeviceNode(const std::string& path);
 
+status_t AbortFuseConnections();
+
+int SetQuotaInherit(const std::string& path);
+int SetQuotaProjectId(const std::string& path, long projectId);
+/*
+ * Creates and sets up an application-specific path on external
+ * storage with the correct ACL and project ID (if needed).
+ *
+ * ONLY for use with app-specific data directories on external storage!
+ * (eg, /Android/data/com.foo, /Android/obb/com.foo, etc.)
+ */
+int PrepareAppDirFromRoot(const std::string& path, const std::string& root, int appUid,
+                          bool fixupExisting);
+
 /* fs_prepare_dir wrapper that creates with SELinux context */
 status_t PrepareDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid);
 
@@ -54,6 +75,9 @@ status_t ForceUnmount(const std::string& path);
 /* Kills any processes using given path */
 status_t KillProcessesUsingPath(const std::string& path);
 
+/* Kills any processes using given mount prifix */
+status_t KillProcessesWithMountPrefix(const std::string& path);
+
 /* Creates bind mount from source to target */
 status_t BindMount(const std::string& source, const std::string& target);
 
@@ -105,6 +129,8 @@ uint64_t GetFreeBytes(const std::string& path);
 uint64_t GetTreeBytes(const std::string& path);
 
 bool IsFilesystemSupported(const std::string& fsType);
+bool IsSdcardfsUsed();
+bool IsFuseDaemon(const pid_t pid);
 
 /* Wipes contents of block device at given path */
 status_t WipeBlockDevice(const std::string& path);
@@ -128,6 +154,8 @@ std::string BuildDataUserDePath(const std::string& volumeUuid, userid_t userid);
 
 dev_t GetDevice(const std::string& path);
 
+status_t EnsureDirExists(const std::string& path, mode_t mode, uid_t uid, gid_t gid);
+
 status_t RestoreconRecursive(const std::string& path);
 
 // TODO: promote to android::base
@@ -147,6 +175,18 @@ status_t WaitForFile(const char* filename, std::chrono::nanoseconds timeout);
 bool FsyncDirectory(const std::string& dirname);
 
 bool writeStringToFile(const std::string& payload, const std::string& filename);
+
+void ConfigureMaxDirtyRatioForFuse(const std::string& fuse_mount, unsigned int max_ratio);
+
+void ConfigureReadAheadForFuse(const std::string& fuse_mount, size_t read_ahead_kb);
+
+status_t MountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                       const std::string& relative_upper_path, android::base::unique_fd* fuse_fd);
+
+status_t UnmountUserFuse(userid_t userId, const std::string& absolute_lower_path,
+                         const std::string& relative_upper_path);
+
+status_t PrepareAndroidDirs(const std::string& volumeRoot);
 }  // namespace vold
 }  // namespace android
 

VoldNativeService.cpp

@@ -18,6 +18,17 @@
 
 #include "VoldNativeService.h"
 
+#include <android-base/logging.h>
+#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
+#include <fs_mgr.h>
+#include <fscrypt/fscrypt.h>
+#include <private/android_filesystem_config.h>
+#include <utils/Trace.h>
+
+#include <fstream>
+#include <thread>
+
 #include "Benchmark.h"
 #include "Checkpoint.h"
 #include "FsCrypt.h"
@@ -29,19 +40,7 @@
 #include "VoldUtil.h"
 #include "VolumeManager.h"
 #include "cryptfs.h"
-
+#include "incfs.h"
-#include "incfs_ndk.h"
-
-#include <fstream>
-#include <thread>
-
-#include <android-base/logging.h>
-#include <android-base/stringprintf.h>
-#include <android-base/strings.h>
-#include <fs_mgr.h>
-#include <fscrypt/fscrypt.h>
-#include <private/android_filesystem_config.h>
-#include <utils/Trace.h>
 
 using android::base::StringPrintf;
 using std::endl;
@@ -143,7 +142,7 @@ status_t VoldNativeService::dump(int fd, const Vector<String16>& /* args */) {
 }
 
 binder::Status VoldNativeService::setListener(
-    const android::sp<android::os::IVoldListener>& listener) {
+        const android::sp<android::os::IVoldListener>& listener) {
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_LOCK;
 
@@ -175,6 +174,13 @@ binder::Status VoldNativeService::shutdown() {
     return translate(VolumeManager::Instance()->shutdown());
 }
 
+binder::Status VoldNativeService::abortFuse() {
+    ENFORCE_SYSTEM_OR_ROOT;
+    ACQUIRE_LOCK;
+
+    return translate(VolumeManager::Instance()->abortFuse());
+}
+
 binder::Status VoldNativeService::onUserAdded(int32_t userId, int32_t userSerial) {
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_LOCK;
@@ -252,8 +258,9 @@ binder::Status VoldNativeService::forgetPartition(const std::string& partGuid,
     return translate(VolumeManager::Instance()->forgetPartition(partGuid, fsUuid));
 }
 
-binder::Status VoldNativeService::mount(const std::string& volId, int32_t mountFlags,
+binder::Status VoldNativeService::mount(
-                                        int32_t mountUserId) {
+        const std::string& volId, int32_t mountFlags, int32_t mountUserId,
+        const android::sp<android::os::IVoldMountCallback>& callback) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_ID(volId);
     ACQUIRE_LOCK;
@@ -266,10 +273,14 @@ binder::Status VoldNativeService::mount(const std::string& volId, int32_t mountF
     vol->setMountFlags(mountFlags);
     vol->setMountUserId(mountUserId);
 
+    vol->setMountCallback(callback);
     int res = vol->mount();
+    vol->setMountCallback(nullptr);
+
     if (res != OK) {
         return translate(res);
     }
+
     if ((mountFlags & MOUNT_FLAG_PRIMARY) != 0) {
         res = VolumeManager::Instance()->setPrimary(vol);
         if (res != OK) {
@@ -326,7 +337,7 @@ static binder::Status pathForVolId(const std::string& volId, std::string* path)
 }
 
 binder::Status VoldNativeService::benchmark(
-    const std::string& volId, const android::sp<android::os::IVoldTaskListener>& listener) {
+        const std::string& volId, const android::sp<android::os::IVoldTaskListener>& listener) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_ID(volId);
     ACQUIRE_LOCK;
@@ -340,8 +351,8 @@ binder::Status VoldNativeService::benchmark(
 }
 
 binder::Status VoldNativeService::moveStorage(
-    const std::string& fromVolId, const std::string& toVolId,
+        const std::string& fromVolId, const std::string& toVolId,
-    const android::sp<android::os::IVoldTaskListener>& listener) {
+        const android::sp<android::os::IVoldTaskListener>& listener) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_ID(fromVolId);
     CHECK_ARGUMENT_ID(toVolId);
@@ -366,12 +377,28 @@ binder::Status VoldNativeService::remountUid(int32_t uid, int32_t remountMode) {
     return translate(VolumeManager::Instance()->remountUid(uid, remountMode));
 }
 
-binder::Status VoldNativeService::mkdirs(const std::string& path) {
+binder::Status VoldNativeService::remountAppStorageDirs(int uid, int pid,
+        const std::vector<std::string>& packageNames) {
+    ENFORCE_SYSTEM_OR_ROOT;
+    ACQUIRE_LOCK;
+
+    return translate(VolumeManager::Instance()->remountAppStorageDirs(uid, pid, packageNames));
+}
+
+binder::Status VoldNativeService::setupAppDir(const std::string& path, int32_t appUid) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_PATH(path);
     ACQUIRE_LOCK;
 
-    return translate(VolumeManager::Instance()->mkdirs(path));
+    return translate(VolumeManager::Instance()->setupAppDir(path, appUid));
+}
+
+binder::Status VoldNativeService::fixupAppDir(const std::string& path, int32_t appUid) {
+    ENFORCE_SYSTEM_OR_ROOT;
+    CHECK_ARGUMENT_PATH(path);
+    ACQUIRE_LOCK;
+
+    return translate(VolumeManager::Instance()->fixupAppDir(path, appUid));
 }
 
 binder::Status VoldNativeService::createObb(const std::string& sourcePath,
@@ -383,7 +410,7 @@ binder::Status VoldNativeService::createObb(const std::string& sourcePath,
     ACQUIRE_LOCK;
 
     return translate(
-        VolumeManager::Instance()->createObb(sourcePath, sourceKey, ownerGid, _aidl_return));
+            VolumeManager::Instance()->createObb(sourcePath, sourceKey, ownerGid, _aidl_return));
 }
 
 binder::Status VoldNativeService::destroyObb(const std::string& volId) {
@@ -394,9 +421,12 @@ binder::Status VoldNativeService::destroyObb(const std::string& volId) {
     return translate(VolumeManager::Instance()->destroyObb(volId));
 }
 
-binder::Status VoldNativeService::createStubVolume(
+binder::Status VoldNativeService::createStubVolume(const std::string& sourcePath,
-    const std::string& sourcePath, const std::string& mountPath, const std::string& fsType,
+                                                   const std::string& mountPath,
-    const std::string& fsUuid, const std::string& fsLabel, std::string* _aidl_return) {
+                                                   const std::string& fsType,
+                                                   const std::string& fsUuid,
+                                                   const std::string& fsLabel, int32_t flags,
+                                                   std::string* _aidl_return) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_PATH(sourcePath);
     CHECK_ARGUMENT_PATH(mountPath);
@@ -405,8 +435,8 @@ binder::Status VoldNativeService::createStubVolume(
     // is quite meaningless.
     ACQUIRE_LOCK;
 
-    return translate(VolumeManager::Instance()->createStubVolume(sourcePath, mountPath, fsType,
+    return translate(VolumeManager::Instance()->createStubVolume(
-                                                                 fsUuid, fsLabel, _aidl_return));
+            sourcePath, mountPath, fsType, fsUuid, fsLabel, flags, _aidl_return));
 }
 
 binder::Status VoldNativeService::destroyStubVolume(const std::string& volId) {
@@ -418,7 +448,7 @@ binder::Status VoldNativeService::destroyStubVolume(const std::string& volId) {
 }
 
 binder::Status VoldNativeService::fstrim(
-    int32_t fstrimFlags, const android::sp<android::os::IVoldTaskListener>& listener) {
+        int32_t fstrimFlags, const android::sp<android::os::IVoldTaskListener>& listener) {
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_LOCK;
 
@@ -427,7 +457,7 @@ binder::Status VoldNativeService::fstrim(
 }
 
 binder::Status VoldNativeService::runIdleMaint(
-    const android::sp<android::os::IVoldTaskListener>& listener) {
+        const android::sp<android::os::IVoldTaskListener>& listener) {
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_LOCK;
 
@@ -436,7 +466,7 @@ binder::Status VoldNativeService::runIdleMaint(
 }
 
 binder::Status VoldNativeService::abortIdleMaint(
-    const android::sp<android::os::IVoldTaskListener>& listener) {
+        const android::sp<android::os::IVoldTaskListener>& listener) {
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_LOCK;
 
@@ -653,7 +683,8 @@ binder::Status VoldNativeService::encryptFstab(const std::string& blkDevice,
     return translateBool(fscrypt_mount_metadata_encrypted(blkDevice, mountPoint, true));
 }
 
-binder::Status VoldNativeService::createUserKey(int32_t userId, int32_t userSerial, bool ephemeral) {
+binder::Status VoldNativeService::createUserKey(int32_t userId, int32_t userSerial,
+                                                bool ephemeral) {
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_CRYPT_LOCK;
 
@@ -851,7 +882,7 @@ binder::Status VoldNativeService::resetCheckpoint() {
 binder::Status VoldNativeService::incFsEnabled(bool* _aidl_return) {
     ENFORCE_SYSTEM_OR_ROOT;
 
-    *_aidl_return = IncFs_IsEnabled();
+    *_aidl_return = incfs::enabled();
     return Ok();
 }
 
@@ -862,21 +893,19 @@ binder::Status VoldNativeService::mountIncFs(
     CHECK_ARGUMENT_PATH(backingPath);
     CHECK_ARGUMENT_PATH(targetDir);
 
-    auto control = IncFs_Mount(backingPath.c_str(), targetDir.c_str(),
+    auto control = incfs::mount(backingPath, targetDir,
-                               {.flags = IncFsMountFlags(flags),
+                                {.flags = IncFsMountFlags(flags),
-                                .defaultReadTimeoutMs = INCFS_DEFAULT_READ_TIMEOUT_MS,
+                                 .defaultReadTimeoutMs = INCFS_DEFAULT_READ_TIMEOUT_MS,
-                                .readLogBufferPages = 4});
+                                 // Mount with read logs disabled.
-    if (control == nullptr) {
+                                 .readLogBufferPages = 0});
-        return translate(-1);
+    if (!control) {
+        return translate(-errno);
     }
-    using unique_fd = ::android::base::unique_fd;
+    auto fds = control.releaseFds();
-    _aidl_return->cmd.emplace(unique_fd(dup(IncFs_GetControlFd(control, CMD))));
+    using android::base::unique_fd;
-    _aidl_return->pendingReads.emplace(unique_fd(dup(IncFs_GetControlFd(control, PENDING_READS))));
+    _aidl_return->cmd.reset(unique_fd(fds[CMD].release()));
-    auto logsFd = IncFs_GetControlFd(control, LOGS);
+    _aidl_return->pendingReads.reset(unique_fd(fds[PENDING_READS].release()));
-    if (logsFd >= 0) {
+    _aidl_return->log.reset(unique_fd(fds[LOGS].release()));
-        _aidl_return->log.emplace(unique_fd(dup(logsFd)));
-    }
-    IncFs_DeleteControl(control);
     return Ok();
 }
 
@@ -884,7 +913,32 @@ binder::Status VoldNativeService::unmountIncFs(const std::string& dir) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_PATH(dir);
 
-    return translate(IncFs_Unmount(dir.c_str()));
+    return translate(incfs::unmount(dir));
+}
+
+binder::Status VoldNativeService::setIncFsMountOptions(
+        const ::android::os::incremental::IncrementalFileSystemControlParcel& control,
+        bool enableReadLogs) {
+    ENFORCE_SYSTEM_OR_ROOT;
+
+    auto incfsControl =
+            incfs::createControl(control.cmd.get(), control.pendingReads.get(), control.log.get());
+    auto cleanupFunc = [](auto incfsControl) {
+        for (auto& fd : incfsControl->releaseFds()) {
+            (void)fd.release();
+        }
+    };
+    auto cleanup =
+            std::unique_ptr<incfs::Control, decltype(cleanupFunc)>(&incfsControl, cleanupFunc);
+    if (auto error = incfs::setOptions(
+                incfsControl,
+                {.defaultReadTimeoutMs = INCFS_DEFAULT_READ_TIMEOUT_MS,
+                 .readLogBufferPages = enableReadLogs ? INCFS_DEFAULT_PAGE_READ_BUFFER_PAGES : 0});
+        error < 0) {
+        return binder::Status::fromServiceSpecificError(error);
+    }
+
+    return Ok();
 }
 
 binder::Status VoldNativeService::bindMount(const std::string& sourceDir,
@@ -893,7 +947,7 @@ binder::Status VoldNativeService::bindMount(const std::string& sourceDir,
     CHECK_ARGUMENT_PATH(sourceDir);
     CHECK_ARGUMENT_PATH(targetDir);
 
-    return translate(IncFs_BindMount(sourceDir.c_str(), targetDir.c_str()));
+    return translate(incfs::bindMount(sourceDir, targetDir));
 }
 
 }  // namespace vold

VoldNativeService.h

@@ -36,6 +36,7 @@ class VoldNativeService : public BinderService<VoldNativeService>, public os::Bn
     binder::Status monitor();
     binder::Status reset();
     binder::Status shutdown();
+    binder::Status abortFuse();
 
     binder::Status onUserAdded(int32_t userId, int32_t userSerial);
     binder::Status onUserRemoved(int32_t userId);
@@ -52,7 +53,8 @@ class VoldNativeService : public BinderService<VoldNativeService>, public os::Bn
     binder::Status partition(const std::string& diskId, int32_t partitionType, int32_t ratio);
     binder::Status forgetPartition(const std::string& partGuid, const std::string& fsUuid);
 
-    binder::Status mount(const std::string& volId, int32_t mountFlags, int32_t mountUserId);
+    binder::Status mount(const std::string& volId, int32_t mountFlags, int32_t mountUserId,
+                         const android::sp<android::os::IVoldMountCallback>& callback);
     binder::Status unmount(const std::string& volId);
     binder::Status format(const std::string& volId, const std::string& fsType);
     binder::Status benchmark(const std::string& volId,
@@ -62,8 +64,11 @@ class VoldNativeService : public BinderService<VoldNativeService>, public os::Bn
                                const android::sp<android::os::IVoldTaskListener>& listener);
 
     binder::Status remountUid(int32_t uid, int32_t remountMode);
+    binder::Status remountAppStorageDirs(int uid, int pid,
+                               const std::vector<std::string>& packageNames);
 
-    binder::Status mkdirs(const std::string& path);
+    binder::Status setupAppDir(const std::string& path, int32_t appUid);
+    binder::Status fixupAppDir(const std::string& path, int32_t appUid);
 
     binder::Status createObb(const std::string& sourcePath, const std::string& sourceKey,
                              int32_t ownerGid, std::string* _aidl_return);
@@ -71,7 +76,8 @@ class VoldNativeService : public BinderService<VoldNativeService>, public os::Bn
 
     binder::Status createStubVolume(const std::string& sourcePath, const std::string& mountPath,
                                     const std::string& fsType, const std::string& fsUuid,
-                                    const std::string& fsLabel, std::string* _aidl_return);
+                                    const std::string& fsLabel, int32_t flags,
+                                    std::string* _aidl_return);
     binder::Status destroyStubVolume(const std::string& volId);
 
     binder::Status fstrim(int32_t fstrimFlags,
@@ -149,6 +155,9 @@ class VoldNativeService : public BinderService<VoldNativeService>, public os::Bn
             const std::string& backingPath, const std::string& targetDir, int32_t flags,
             ::android::os::incremental::IncrementalFileSystemControlParcel* _aidl_return) override;
     binder::Status unmountIncFs(const std::string& dir) override;
+    binder::Status setIncFsMountOptions(
+            const ::android::os::incremental::IncrementalFileSystemControlParcel& control,
+            bool enableReadLogs) override;
     binder::Status bindMount(const std::string& sourceDir, const std::string& targetDir) override;
 };
 

VoldNativeServiceValidation.cpp

@@ -39,11 +39,10 @@ binder::Status Exception(uint32_t code, const std::string& msg) {
 }
 
 binder::Status CheckPermission(const char* permission) {
-    pid_t pid;
+    int32_t pid;
-    uid_t uid;
+    int32_t uid;
 
-    if (checkCallingPermission(String16(permission), reinterpret_cast<int32_t*>(&pid),
+    if (checkCallingPermission(String16(permission), &pid, &uid)) {
-                               reinterpret_cast<int32_t*>(&uid))) {
         return Ok();
     } else {
         return Exception(binder::Status::EX_SECURITY,
@@ -66,7 +65,7 @@ binder::Status CheckArgumentId(const std::string& id) {
         return Exception(binder::Status::EX_ILLEGAL_ARGUMENT, "Missing ID");
     }
     for (const char& c : id) {
-        if (!std::isalnum(c) && c != ':' && c != ',') {
+        if (!std::isalnum(c) && c != ':' && c != ',' && c != ';') {
             return Exception(binder::Status::EX_ILLEGAL_ARGUMENT,
                              StringPrintf("ID %s is malformed", id.c_str()));
         }

VolumeManager.cpp

@@ -19,6 +19,7 @@
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <limits.h>
 #include <mntent.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -40,6 +41,7 @@
 #include <android-base/properties.h>
 #include <android-base/stringprintf.h>
 #include <android-base/strings.h>
+#include <async_safe/log.h>
 
 #include <cutils/fs.h>
 #include <utils/Trace.h>
@@ -66,6 +68,7 @@
 #include "fs/Vfat.h"
 #include "model/EmulatedVolume.h"
 #include "model/ObbVolume.h"
+#include "model/PrivateVolume.h"
 #include "model/StubVolume.h"
 
 using android::OK;
@@ -78,11 +81,18 @@ using android::vold::BindMount;
 using android::vold::CreateDir;
 using android::vold::DeleteDirContents;
 using android::vold::DeleteDirContentsAndDir;
+using android::vold::EnsureDirExists;
+using android::vold::IsFilesystemSupported;
+using android::vold::IsSdcardfsUsed;
 using android::vold::IsVirtioBlkDevice;
+using android::vold::PrepareAndroidDirs;
+using android::vold::PrepareAppDirFromRoot;
+using android::vold::PrivateVolume;
 using android::vold::Symlink;
 using android::vold::Unlink;
 using android::vold::UnmountTree;
 using android::vold::VoldNativeService;
+using android::vold::VolumeBase;
 
 static const char* kPathUserMount = "/mnt/user";
 static const char* kPathVirtualDisk = "/data/misc/vold/virtual_disk";
@@ -96,6 +106,8 @@ static const unsigned int kSizeVirtualDisk = 536870912;
 
 static const unsigned int kMajorBlockMmc = 179;
 
+using ScanProcCallback = bool(*)(uid_t uid, pid_t pid, int nsFd, const char* name, void* params);
+
 VolumeManager* VolumeManager::sInstance = NULL;
 
 VolumeManager* VolumeManager::Instance() {
@@ -106,7 +118,7 @@ VolumeManager* VolumeManager::Instance() {
 VolumeManager::VolumeManager() {
     mDebug = false;
     mNextObbId = 0;
-    mNextStubVolumeId = 0;
+    mNextStubId = 0;
     // For security reasons, assume that a secure keyguard is
     // showing until we hear otherwise
     mSecureKeyguardShowing = true;
@@ -172,10 +184,13 @@ int VolumeManager::start() {
 
     // Assume that we always have an emulated volume on internal
     // storage; the framework will decide if it should be mounted.
-    CHECK(mInternalEmulated == nullptr);
+    CHECK(mInternalEmulatedVolumes.empty());
-    mInternalEmulated = std::shared_ptr<android::vold::VolumeBase>(
+
-        new android::vold::EmulatedVolume("/data/media"));
+    auto vol = std::shared_ptr<android::vold::VolumeBase>(
-    mInternalEmulated->create();
+            new android::vold::EmulatedVolume("/data/media", 0));
+    vol->setMountUserId(0);
+    vol->create();
+    mInternalEmulatedVolumes.push_back(vol);
 
     // Consider creating a virtual disk
     updateVirtualDisk();
@@ -184,9 +199,12 @@ int VolumeManager::start() {
 }
 
 int VolumeManager::stop() {
-    CHECK(mInternalEmulated != nullptr);
+    CHECK(!mInternalEmulatedVolumes.empty());
-    mInternalEmulated->destroy();
+    for (const auto& vol : mInternalEmulatedVolumes) {
-    mInternalEmulated = nullptr;
+        vol->destroy();
+    }
+    mInternalEmulatedVolumes.clear();
+
     return 0;
 }
 
@@ -249,10 +267,17 @@ void VolumeManager::handleBlockEvent(NetlinkEvent* evt) {
 void VolumeManager::handleDiskAdded(const std::shared_ptr<android::vold::Disk>& disk) {
     // For security reasons, if secure keyguard is showing, wait
     // until the user unlocks the device to actually touch it
+    // Additionally, wait until user 0 is actually started, since we need
+    // the user to be up before we can mount a FUSE daemon to handle the disk.
+    bool userZeroStarted = mStartedUsers.find(0) != mStartedUsers.end();
     if (mSecureKeyguardShowing) {
         LOG(INFO) << "Found disk at " << disk->getEventPath()
                   << " but delaying scan due to secure keyguard";
         mPendingDisks.push_back(disk);
+    } else if (!userZeroStarted) {
+        LOG(INFO) << "Found disk at " << disk->getEventPath()
+                  << " but delaying scan due to user zero not having started";
+        mPendingDisks.push_back(disk);
     } else {
         disk->create();
         mDisks.push_back(disk);
@@ -306,11 +331,10 @@ std::shared_ptr<android::vold::Disk> VolumeManager::findDisk(const std::string&
 }
 
 std::shared_ptr<android::vold::VolumeBase> VolumeManager::findVolume(const std::string& id) {
-    // Vold could receive "mount" after "shutdown" command in the extreme case.
+    for (const auto& vol : mInternalEmulatedVolumes) {
-    // If this happens, mInternalEmulated will equal nullptr and
+        if (vol->getId() == id) {
-    // we need to deal with it in order to avoid null pointer crash.
+            return vol;
-    if (mInternalEmulated != nullptr && mInternalEmulated->getId() == id) {
+        }
-        return mInternalEmulated;
     }
     for (const auto& disk : mDisks) {
         auto vol = disk->findVolume(id);
@@ -318,11 +342,6 @@ std::shared_ptr<android::vold::VolumeBase> VolumeManager::findVolume(const std::
             return vol;
         }
     }
-    for (const auto& vol : mStubVolumes) {
-        if (vol->getId() == id) {
-            return vol;
-        }
-    }
     for (const auto& vol : mObbVolumes) {
         if (vol->getId() == id) {
             return vol;
@@ -361,60 +380,145 @@ int VolumeManager::forgetPartition(const std::string& partGuid, const std::strin
 }
 
 int VolumeManager::linkPrimary(userid_t userId) {
-    std::string source(mPrimary->getPath());
+    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
-    if (mPrimary->isEmulated()) {
+        std::string source(mPrimary->getPath());
-        source = StringPrintf("%s/%d", source.c_str(), userId);
+        if (mPrimary->isEmulated()) {
-        fs_prepare_dir(source.c_str(), 0755, AID_ROOT, AID_ROOT);
+            source = StringPrintf("%s/%d", source.c_str(), userId);
-    }
+            fs_prepare_dir(source.c_str(), 0755, AID_ROOT, AID_ROOT);
+        }
 
-    std::string target(StringPrintf("/mnt/user/%d/primary", userId));
+        std::string target(StringPrintf("/mnt/user/%d/primary", userId));
-    LOG(DEBUG) << "Linking " << source << " to " << target;
+        LOG(DEBUG) << "Linking " << source << " to " << target;
-    Symlink(source, target);
+        Symlink(source, target);
+    }
     return 0;
 }
 
+void VolumeManager::destroyEmulatedVolumesForUser(userid_t userId) {
+    // Destroy and remove all unstacked EmulatedVolumes for the user
+    auto i = mInternalEmulatedVolumes.begin();
+    while (i != mInternalEmulatedVolumes.end()) {
+        auto vol = *i;
+        if (vol->getMountUserId() == userId) {
+            vol->destroy();
+            i = mInternalEmulatedVolumes.erase(i);
+        } else {
+            i++;
+        }
+    }
+
+    // Destroy and remove all stacked EmulatedVolumes for the user on each mounted private volume
+    std::list<std::string> private_vols;
+    listVolumes(VolumeBase::Type::kPrivate, private_vols);
+    for (const std::string& id : private_vols) {
+        PrivateVolume* pvol = static_cast<PrivateVolume*>(findVolume(id).get());
+        std::list<std::shared_ptr<VolumeBase>> vols_to_remove;
+        if (pvol->getState() == VolumeBase::State::kMounted) {
+            for (const auto& vol : pvol->getVolumes()) {
+                if (vol->getMountUserId() == userId) {
+                    vols_to_remove.push_back(vol);
+                }
+            }
+            for (const auto& vol : vols_to_remove) {
+                vol->destroy();
+                pvol->removeVolume(vol);
+            }
+        }  // else EmulatedVolumes will be destroyed on VolumeBase#unmount
+    }
+}
+
+void VolumeManager::createEmulatedVolumesForUser(userid_t userId) {
+    // Create unstacked EmulatedVolumes for the user
+    auto vol = std::shared_ptr<android::vold::VolumeBase>(
+            new android::vold::EmulatedVolume("/data/media", userId));
+    vol->setMountUserId(userId);
+    mInternalEmulatedVolumes.push_back(vol);
+    vol->create();
+
+    // Create stacked EmulatedVolumes for the user on each PrivateVolume
+    std::list<std::string> private_vols;
+    listVolumes(VolumeBase::Type::kPrivate, private_vols);
+    for (const std::string& id : private_vols) {
+        PrivateVolume* pvol = static_cast<PrivateVolume*>(findVolume(id).get());
+        if (pvol->getState() == VolumeBase::State::kMounted) {
+            auto evol =
+                    std::shared_ptr<android::vold::VolumeBase>(new android::vold::EmulatedVolume(
+                            pvol->getPath() + "/media", pvol->getRawDevice(), pvol->getFsUuid(),
+                            userId));
+            evol->setMountUserId(userId);
+            pvol->addVolume(evol);
+            evol->create();
+        }  // else EmulatedVolumes will be created per user when on PrivateVolume#doMount
+    }
+}
+
 int VolumeManager::onUserAdded(userid_t userId, int userSerialNumber) {
+    LOG(INFO) << "onUserAdded: " << userId;
+
     mAddedUsers[userId] = userSerialNumber;
     return 0;
 }
 
 int VolumeManager::onUserRemoved(userid_t userId) {
+    LOG(INFO) << "onUserRemoved: " << userId;
+
+    onUserStopped(userId);
     mAddedUsers.erase(userId);
     return 0;
 }
 
 int VolumeManager::onUserStarted(userid_t userId) {
-    LOG(VERBOSE) << "onUserStarted: " << userId;
+    LOG(INFO) << "onUserStarted: " << userId;
-    // Note that sometimes the system will spin up processes from Zygote
+
-    // before actually starting the user, so we're okay if Zygote
+    if (mStartedUsers.find(userId) == mStartedUsers.end()) {
-    // already created this directory.
+        createEmulatedVolumesForUser(userId);
-    std::string path(StringPrintf("%s/%d", kPathUserMount, userId));
+    }
-    fs_prepare_dir(path.c_str(), 0755, AID_ROOT, AID_ROOT);
+
+    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
+        // Note that sometimes the system will spin up processes from Zygote
+        // before actually starting the user, so we're okay if Zygote
+        // already created this directory.
+        std::string path(StringPrintf("%s/%d", kPathUserMount, userId));
+        fs_prepare_dir(path.c_str(), 0755, AID_ROOT, AID_ROOT);
+
+        if (mPrimary) {
+            linkPrimary(userId);
+        }
+    }
 
     mStartedUsers.insert(userId);
-    if (mPrimary) {
+
-        linkPrimary(userId);
+    createPendingDisksIfNeeded();
-    }
     return 0;
 }
 
 int VolumeManager::onUserStopped(userid_t userId) {
     LOG(VERBOSE) << "onUserStopped: " << userId;
+
+    if (mStartedUsers.find(userId) != mStartedUsers.end()) {
+        destroyEmulatedVolumesForUser(userId);
+    }
+
     mStartedUsers.erase(userId);
     return 0;
 }
 
-int VolumeManager::onSecureKeyguardStateChanged(bool isShowing) {
+void VolumeManager::createPendingDisksIfNeeded() {
-    mSecureKeyguardShowing = isShowing;
+    bool userZeroStarted = mStartedUsers.find(0) != mStartedUsers.end();
-    if (!mSecureKeyguardShowing) {
+    if (!mSecureKeyguardShowing && userZeroStarted) {
-        // Now that secure keyguard has been dismissed, process
+        // Now that secure keyguard has been dismissed and user 0 has
-        // any pending disks
+        // started, process any pending disks
         for (const auto& disk : mPendingDisks) {
             disk->create();
             mDisks.push_back(disk);
         }
         mPendingDisks.clear();
     }
+}
+
+int VolumeManager::onSecureKeyguardStateChanged(bool isShowing) {
+    mSecureKeyguardShowing = isShowing;
+    createPendingDisksIfNeeded();
     return 0;
 }
 
@@ -426,32 +530,107 @@ int VolumeManager::setPrimary(const std::shared_ptr<android::vold::VolumeBase>&
     return 0;
 }
 
-int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
+// This code is executed after a fork so it's very important that the set of
-    std::string mode;
+// methods we call here is strictly limited.
+//
+// TODO: Get rid of this guesswork altogether and instead exec a process
+// immediately after fork to do our bindding for us.
+static bool childProcess(const char* storageSource, const char* userSource, int nsFd,
+                         const char* name) {
+    if (setns(nsFd, CLONE_NEWNS) != 0) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to setns for %s :%s", name,
+                              strerror(errno));
+        return false;
+    }
+
+    // NOTE: Inlined from vold::UnmountTree here to avoid using PLOG methods and
+    // to also protect against future changes that may cause issues across a
+    // fork.
+    if (TEMP_FAILURE_RETRY(umount2("/storage/", MNT_DETACH)) < 0 && errno != EINVAL &&
+        errno != ENOENT) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to unmount /storage/ :%s",
+                              strerror(errno));
+        return false;
+    }
+
+    if (TEMP_FAILURE_RETRY(mount(storageSource, "/storage", NULL, MS_BIND | MS_REC, NULL)) == -1) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s for %s :%s",
+                              storageSource, name, strerror(errno));
+        return false;
+    }
+
+    if (TEMP_FAILURE_RETRY(mount(NULL, "/storage", NULL, MS_REC | MS_SLAVE, NULL)) == -1) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold",
+                              "Failed to set MS_SLAVE to /storage for %s :%s", name,
+                              strerror(errno));
+        return false;
+    }
+
+    if (TEMP_FAILURE_RETRY(mount(userSource, "/storage/self", NULL, MS_BIND, NULL)) == -1) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s for %s :%s",
+                              userSource, name, strerror(errno));
+        return false;
+    }
+
+    return true;
+}
+
+// Fork the process and remount storage
+bool forkAndRemountChild(uid_t uid, pid_t pid, int nsFd, const char* name, void* params) {
+    int32_t mountMode = *static_cast<int32_t*>(params);
+    std::string userSource;
+    std::string storageSource;
+    pid_t child;
+    // Need to fix these paths to account for when sdcardfs is gone
     switch (mountMode) {
         case VoldNativeService::REMOUNT_MODE_NONE:
-            mode = "none";
+            return true;
-            break;
         case VoldNativeService::REMOUNT_MODE_DEFAULT:
-            mode = "default";
+            storageSource = "/mnt/runtime/default";
             break;
         case VoldNativeService::REMOUNT_MODE_READ:
-            mode = "read";
+            storageSource = "/mnt/runtime/read";
             break;
         case VoldNativeService::REMOUNT_MODE_WRITE:
         case VoldNativeService::REMOUNT_MODE_LEGACY:
         case VoldNativeService::REMOUNT_MODE_INSTALLER:
-            mode = "write";
+            storageSource = "/mnt/runtime/write";
             break;
         case VoldNativeService::REMOUNT_MODE_FULL:
-            mode = "full";
+            storageSource = "/mnt/runtime/full";
             break;
+        case VoldNativeService::REMOUNT_MODE_PASS_THROUGH:
+            return true;
         default:
             PLOG(ERROR) << "Unknown mode " << std::to_string(mountMode);
-            return -1;
+            return false;
     }
-    LOG(DEBUG) << "Remounting " << uid << " as mode " << mode;
+    LOG(DEBUG) << "Remounting " << uid << " as " << storageSource;
 
+    // Fork a child to mount user-specific symlink helper into place
+    userSource = StringPrintf("/mnt/user/%d", multiuser_get_user_id(uid));
+    if (!(child = fork())) {
+        if (childProcess(storageSource.c_str(), userSource.c_str(), nsFd, name)) {
+            _exit(0);
+        } else {
+            _exit(1);
+        }
+    }
+
+    if (child == -1) {
+        PLOG(ERROR) << "Failed to fork";
+        return false;
+    } else {
+        TEMP_FAILURE_RETRY(waitpid(child, nullptr, 0));
+    }
+    return true;
+}
+
+// Helper function to scan all processes in /proc and call the callback if:
+// 1). pid belongs to an app process
+// 2). If input uid is 0 or it matches the process uid
+// 3). If userId is not -1 or userId matches the process userId
+bool scanProcProcesses(uid_t uid, userid_t userId, ScanProcCallback callback, void* params) {
     DIR* dir;
     struct dirent* de;
     std::string rootName;
@@ -459,22 +638,22 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
     int pidFd;
     int nsFd;
     struct stat sb;
-    pid_t child;
 
     static bool apexUpdatable = android::sysprop::ApexProperties::updatable().value_or(false);
 
     if (!(dir = opendir("/proc"))) {
-        PLOG(ERROR) << "Failed to opendir";
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to opendir");
-        return -1;
+        return false;
     }
 
     // Figure out root namespace to compare against below
     if (!android::vold::Readlinkat(dirfd(dir), "1/ns/mnt", &rootName)) {
-        PLOG(ERROR) << "Failed to read root namespace";
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to read root namespace");
         closedir(dir);
-        return -1;
+        return false;
     }
 
+    async_safe_format_log(ANDROID_LOG_INFO, "vold", "Start scanning all processes");
     // Poke through all running PIDs look for apps running as UID
     while ((de = readdir(dir))) {
         pid_t pid;
@@ -489,21 +668,23 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
             goto next;
         }
         if (fstat(pidFd, &sb) != 0) {
-            PLOG(WARNING) << "Failed to stat " << de->d_name;
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to stat %s", de->d_name);
             goto next;
         }
-        if (sb.st_uid != uid) {
+        if (uid != 0 && sb.st_uid != uid) {
+            goto next;
+        }
+        if (userId != static_cast<userid_t>(-1) && multiuser_get_user_id(sb.st_uid) != userId) {
             goto next;
         }
 
         // Matches so far, but refuse to touch if in root namespace
-        LOG(DEBUG) << "Found matching PID " << de->d_name;
         if (!android::vold::Readlinkat(pidFd, "ns/mnt", &pidName)) {
-            PLOG(WARNING) << "Failed to read namespace for " << de->d_name;
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold",
+                    "Failed to read namespacefor %s", de->d_name);
             goto next;
         }
         if (rootName == pidName) {
-            LOG(WARNING) << "Skipping due to root namespace";
             goto next;
         }
 
@@ -518,11 +699,9 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
             // non-Java process whose UID is < AID_APP_START. (The UID condition
             // is required to not filter out child processes spawned by apps.)
             if (!android::vold::Readlinkat(pidFd, "exe", &exeName)) {
-                PLOG(WARNING) << "Failed to read exe name for " << de->d_name;
                 goto next;
             }
             if (!StartsWith(exeName, "/system/bin/app_process") && sb.st_uid < AID_APP_START) {
-                LOG(WARNING) << "Skipping due to native system process";
                 goto next;
             }
         }
@@ -531,58 +710,13 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
         // NOLINTNEXTLINE(android-cloexec-open): Deliberately not O_CLOEXEC
         nsFd = openat(pidFd, "ns/mnt", O_RDONLY);
         if (nsFd < 0) {
-            PLOG(WARNING) << "Failed to open namespace for " << de->d_name;
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold",
+                    "Failed to open namespace for %s", de->d_name);
             goto next;
         }
 
-        if (!(child = fork())) {
+        if (!callback(sb.st_uid, pid, nsFd, de->d_name, params)) {
-            if (setns(nsFd, CLONE_NEWNS) != 0) {
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed in callback");
-                PLOG(ERROR) << "Failed to setns for " << de->d_name;
-                _exit(1);
-            }
-
-            android::vold::UnmountTree("/storage/");
-
-            std::string storageSource;
-            if (mode == "default") {
-                storageSource = "/mnt/runtime/default";
-            } else if (mode == "read") {
-                storageSource = "/mnt/runtime/read";
-            } else if (mode == "write") {
-                storageSource = "/mnt/runtime/write";
-            } else if (mode == "full") {
-                storageSource = "/mnt/runtime/full";
-            } else {
-                // Sane default of no storage visible
-                _exit(0);
-            }
-            if (TEMP_FAILURE_RETRY(
-                    mount(storageSource.c_str(), "/storage", NULL, MS_BIND | MS_REC, NULL)) == -1) {
-                PLOG(ERROR) << "Failed to mount " << storageSource << " for " << de->d_name;
-                _exit(1);
-            }
-            if (TEMP_FAILURE_RETRY(mount(NULL, "/storage", NULL, MS_REC | MS_SLAVE, NULL)) == -1) {
-                PLOG(ERROR) << "Failed to set MS_SLAVE to /storage for " << de->d_name;
-                _exit(1);
-            }
-
-            // Mount user-specific symlink helper into place
-            userid_t user_id = multiuser_get_user_id(uid);
-            std::string userSource(StringPrintf("/mnt/user/%d", user_id));
-            if (TEMP_FAILURE_RETRY(
-                    mount(userSource.c_str(), "/storage/self", NULL, MS_BIND, NULL)) == -1) {
-                PLOG(ERROR) << "Failed to mount " << userSource << " for " << de->d_name;
-                _exit(1);
-            }
-
-            _exit(0);
-        }
-
-        if (child == -1) {
-            PLOG(ERROR) << "Failed to fork";
-            goto next;
-        } else {
-            TEMP_FAILURE_RETRY(waitpid(child, nullptr, 0));
         }
 
     next:
@@ -590,16 +724,199 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
         close(pidFd);
     }
     closedir(dir);
+    async_safe_format_log(ANDROID_LOG_INFO, "vold", "Finished scanning all processes");
+    return true;
+}
+
+int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
+    if (GetBoolProperty(android::vold::kPropFuse, false)) {
+        // TODO(135341433): Implement fuse specific logic.
+        return 0;
+    }
+    return scanProcProcesses(uid, static_cast<userid_t>(-1),
+            forkAndRemountChild, &mountMode) ? 0 : -1;
+}
+
+
+// In each app's namespace, mount tmpfs on obb and data dir, and bind mount obb and data
+// package dirs.
+static bool remountStorageDirs(int nsFd, const char* android_data_dir, const char* android_obb_dir,
+        int uid, const char* sources[], const char* targets[], int size) {
+    // This code is executed after a fork so it's very important that the set of
+    // methods we call here is strictly limited.
+    if (setns(nsFd, CLONE_NEWNS) != 0) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to setns %s", strerror(errno));
+        return false;
+    }
+
+    // Mount tmpfs on Android/data and Android/obb
+    if (TEMP_FAILURE_RETRY(mount("tmpfs", android_data_dir, "tmpfs",
+            MS_NOSUID | MS_NODEV | MS_NOEXEC, "uid=0,gid=0,mode=0751")) == -1) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount tmpfs to %s :%s",
+                        android_data_dir, strerror(errno));
+        return false;
+    }
+    if (TEMP_FAILURE_RETRY(mount("tmpfs", android_obb_dir, "tmpfs",
+            MS_NOSUID | MS_NODEV | MS_NOEXEC, "uid=0,gid=0,mode=0751")) == -1) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount tmpfs to %s :%s",
+                android_obb_dir, strerror(errno));
+        return false;
+    }
+
+    for (int i = 0; i < size; i++) {
+        // Create package dir and bind mount it to the actual one.
+        if (TEMP_FAILURE_RETRY(mkdir(targets[i], 0700)) == -1) {
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mkdir %s %s",
+                    targets[i], strerror(errno));
+            return false;
+        }
+        if (TEMP_FAILURE_RETRY(mount(sources[i], targets[i], NULL, MS_BIND | MS_REC, NULL)) == -1) {
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s to %s :%s",
+                                  sources[i], targets[i], strerror(errno));
+            return false;
+        }
+    }
+    return true;
+}
+
+static std::string getStorageDirSrc(userid_t userId, const std::string& dirName,
+        const std::string& packageName) {
+    if (IsSdcardfsUsed()) {
+        return StringPrintf("/mnt/runtime/default/emulated/%d/%s/%s",
+                userId, dirName.c_str(), packageName.c_str());
+    } else {
+        return StringPrintf("/mnt/pass_through/%d/emulated/%d/%s/%s",
+                userId, userId, dirName.c_str(), packageName.c_str());
+    }
+}
+
+static std::string getStorageDirTarget(userid_t userId, std::string dirName,
+        std::string packageName) {
+    return StringPrintf("/storage/emulated/%d/%s/%s",
+            userId, dirName.c_str(), packageName.c_str());
+}
+
+// Fork the process and remount storage
+bool VolumeManager::forkAndRemountStorage(int uid, int pid,
+                                          const std::vector<std::string>& packageNames) {
+    userid_t userId = multiuser_get_user_id(uid);
+    std::string mnt_path = StringPrintf("/proc/%d/ns/mnt", pid);
+    android::base::unique_fd nsFd(
+            TEMP_FAILURE_RETRY(open(mnt_path.c_str(), O_RDONLY | O_CLOEXEC)));
+    if (nsFd == -1) {
+        PLOG(ERROR) << "Unable to open " << mnt_path.c_str();
+        return false;
+    }
+    // Storing both Android/obb and Android/data paths.
+    int size = packageNames.size() * 2;
+
+    std::unique_ptr<std::string[]> sources(new std::string[size]);
+    std::unique_ptr<std::string[]> targets(new std::string[size]);
+    std::unique_ptr<const char*[]> sources_uptr(new const char*[size]);
+    std::unique_ptr<const char*[]> targets_uptr(new const char*[size]);
+    const char** sources_cstr = sources_uptr.get();
+    const char** targets_cstr = targets_uptr.get();
+
+    for (int i = 0; i < size; i += 2) {
+        std::string const& packageName = packageNames[i/2];
+        sources[i] = getStorageDirSrc(userId, "Android/data", packageName);
+        targets[i] = getStorageDirTarget(userId, "Android/data", packageName);
+        sources[i+1] = getStorageDirSrc(userId, "Android/obb", packageName);
+        targets[i+1] = getStorageDirTarget(userId, "Android/obb", packageName);
+
+        sources_cstr[i] = sources[i].c_str();
+        targets_cstr[i] = targets[i].c_str();
+        sources_cstr[i+1] = sources[i+1].c_str();
+        targets_cstr[i+1] = targets[i+1].c_str();
+    }
+
+    for (int i = 0; i < size; i++) {
+        auto status = EnsureDirExists(sources_cstr[i], 0771, AID_MEDIA_RW, AID_MEDIA_RW);
+        if (status != OK) {
+            PLOG(ERROR) << "Failed to create dir: " << sources_cstr[i];
+            return false;
+        }
+        // Make sure /storage/emulated/... paths are setup correctly
+        status = setupAppDir(targets_cstr[i], uid, false /* fixupExistingOnly */);
+        if (status != OK) {
+            PLOG(ERROR) << "Failed to create dir: " << targets_cstr[i];
+            return false;
+        }
+    }
+
+    char android_data_dir[PATH_MAX];
+    char android_obb_dir[PATH_MAX];
+    snprintf(android_data_dir, PATH_MAX, "/storage/emulated/%d/Android/data", userId);
+    snprintf(android_obb_dir, PATH_MAX, "/storage/emulated/%d/Android/obb", userId);
+
+    pid_t child;
+    // Fork a child to mount Android/obb android Android/data dirs, as we don't want it to affect
+    // original vold process mount namespace.
+    if (!(child = fork())) {
+        if (remountStorageDirs(nsFd, android_data_dir, android_obb_dir, uid,
+                sources_cstr, targets_cstr, size)) {
+            _exit(0);
+        } else {
+            _exit(1);
+        }
+    }
+
+    if (child == -1) {
+        PLOG(ERROR) << "Failed to fork";
+        return false;
+    } else {
+        int status;
+        if (TEMP_FAILURE_RETRY(waitpid(child, &status, 0)) == -1) {
+            PLOG(ERROR) << "Failed to waitpid: " << child;
+            return false;
+        }
+        if (!WIFEXITED(status)) {
+            PLOG(ERROR) << "Process did not exit normally, status: " << status;
+            return false;
+        }
+        if (WEXITSTATUS(status)) {
+            PLOG(ERROR) << "Process exited with code: " << WEXITSTATUS(status);
+            return false;
+        }
+    }
+    return true;
+}
+
+int VolumeManager::remountAppStorageDirs(int uid, int pid,
+        const std::vector<std::string>& packageNames) {
+    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
+        return 0;
+    }
+    // Only run the remount if fuse is mounted for that user.
+    userid_t userId = multiuser_get_user_id(uid);
+    bool fuseMounted = false;
+    for (auto& vol : mInternalEmulatedVolumes) {
+        if (vol->getMountUserId() == userId && vol->getState() == VolumeBase::State::kMounted) {
+            auto* emulatedVol = static_cast<android::vold::EmulatedVolume*>(vol.get());
+            if (emulatedVol) {
+                fuseMounted = emulatedVol->isFuseMounted();
+            }
+            break;
+        }
+    }
+    if (fuseMounted) {
+        forkAndRemountStorage(uid, pid, packageNames);
+    }
     return 0;
 }
 
+int VolumeManager::abortFuse() {
+    return android::vold::AbortFuseConnections();
+}
+
 int VolumeManager::reset() {
     // Tear down all existing disks/volumes and start from a blank slate so
     // newly connected framework hears all events.
-    if (mInternalEmulated != nullptr) {
+    for (const auto& vol : mInternalEmulatedVolumes) {
-        mInternalEmulated->destroy();
+        vol->destroy();
-        mInternalEmulated->create();
     }
+    mInternalEmulatedVolumes.clear();
+
     for (const auto& disk : mDisks) {
         disk->destroy();
         disk->create();
@@ -612,19 +929,22 @@ int VolumeManager::reset() {
 
 // Can be called twice (sequentially) during shutdown. should be safe for that.
 int VolumeManager::shutdown() {
-    if (mInternalEmulated == nullptr) {
+    if (mInternalEmulatedVolumes.empty()) {
         return 0;  // already shutdown
     }
     android::vold::sSleepOnUnmount = false;
-    mInternalEmulated->destroy();
+    for (const auto& vol : mInternalEmulatedVolumes) {
-    mInternalEmulated = nullptr;
+        vol->destroy();
+    }
     for (const auto& disk : mDisks) {
         disk->destroy();
     }
-    mStubVolumes.clear();
+
+    mInternalEmulatedVolumes.clear();
     mDisks.clear();
     mPendingDisks.clear();
     android::vold::sSleepOnUnmount = true;
+
     return 0;
 }
 
@@ -633,11 +953,8 @@ int VolumeManager::unmountAll() {
     ATRACE_NAME("VolumeManager::unmountAll()");
 
     // First, try gracefully unmounting all known devices
-    if (mInternalEmulated != nullptr) {
+    for (const auto& vol : mInternalEmulatedVolumes) {
-        mInternalEmulated->unmount();
+        vol->unmount();
-    }
-    for (const auto& stub : mStubVolumes) {
-        stub->unmount();
     }
     for (const auto& disk : mDisks) {
         disk->unmountAll();
@@ -661,7 +978,8 @@ int VolumeManager::unmountAll() {
 #ifdef __ANDROID_DEBUGGABLE__
              !StartsWith(test, "/mnt/scratch") &&
 #endif
-             !StartsWith(test, "/mnt/vendor") && !StartsWith(test, "/mnt/product")) ||
+             !StartsWith(test, "/mnt/vendor") && !StartsWith(test, "/mnt/product") &&
+             !StartsWith(test, "/mnt/installer") && !StartsWith(test, "/mnt/androidwritable")) ||
             StartsWith(test, "/storage/")) {
             toUnmount.push_front(test);
         }
@@ -676,15 +994,72 @@ int VolumeManager::unmountAll() {
     return 0;
 }
 
-int VolumeManager::mkdirs(const std::string& path) {
+int VolumeManager::setupAppDir(const std::string& path, int32_t appUid, bool fixupExistingOnly) {
     // Only offer to create directories for paths managed by vold
-    if (StartsWith(path, "/storage/")) {
+    if (!StartsWith(path, "/storage/")) {
-        // fs_mkdirs() does symlink checking and relative path enforcement
-        return fs_mkdirs(path.c_str(), 0700);
-    } else {
         LOG(ERROR) << "Failed to find mounted volume for " << path;
         return -EINVAL;
     }
+
+    // Find the volume it belongs to
+    auto filter_fn = [&](const VolumeBase& vol) {
+        if (vol.getState() != VolumeBase::State::kMounted) {
+            // The volume must be mounted
+            return false;
+        }
+        if ((vol.getMountFlags() & VolumeBase::MountFlags::kVisible) == 0) {
+            // and visible
+            return false;
+        }
+        if (vol.getInternalPath().empty()) {
+            return false;
+        }
+        if (vol.getMountUserId() != USER_UNKNOWN &&
+            vol.getMountUserId() != multiuser_get_user_id(appUid)) {
+            // The app dir must be created on a volume with the same user-id
+            return false;
+        }
+        if (!path.empty() && StartsWith(path, vol.getPath())) {
+            return true;
+        }
+
+        return false;
+    };
+    auto volume = findVolumeWithFilter(filter_fn);
+    if (volume == nullptr) {
+        LOG(ERROR) << "Failed to find mounted volume for " << path;
+        return -EINVAL;
+    }
+    // Convert paths to lower filesystem paths to avoid making FUSE requests for these reasons:
+    // 1. A FUSE request from vold puts vold at risk of hanging if the FUSE daemon is down
+    // 2. The FUSE daemon prevents requests on /mnt/user/0/emulated/<userid != 0> and a request
+    // on /storage/emulated/10 means /mnt/user/0/emulated/10
+    const std::string lowerPath =
+            volume->getInternalPath() + path.substr(volume->getPath().length());
+
+    const std::string volumeRoot = volume->getRootPath();  // eg /data/media/0
+
+    if (fixupExistingOnly && (access(lowerPath.c_str(), F_OK) != 0)) {
+        // Nothing to fixup
+        return OK;
+    }
+
+    if (volume->getType() == VolumeBase::Type::kPublic) {
+        // On public volumes, we don't need to setup permissions, as everything goes through
+        // FUSE; just create the dirs and be done with it.
+        return fs_mkdirs(lowerPath.c_str(), 0700);
+    }
+
+    // Create the app paths we need from the root
+    return PrepareAppDirFromRoot(lowerPath, volumeRoot, appUid, fixupExistingOnly);
+}
+
+int VolumeManager::fixupAppDir(const std::string& path, int32_t appUid) {
+    if (IsSdcardfsUsed()) {
+        //sdcardfs magically does this for us
+        return OK;
+    }
+    return setupAppDir(path, appUid, true /* fixupExistingOnly */);
 }
 
 int VolumeManager::createObb(const std::string& sourcePath, const std::string& sourceKey,
@@ -715,27 +1090,31 @@ int VolumeManager::destroyObb(const std::string& volId) {
 
 int VolumeManager::createStubVolume(const std::string& sourcePath, const std::string& mountPath,
                                     const std::string& fsType, const std::string& fsUuid,
-                                    const std::string& fsLabel, std::string* outVolId) {
+                                    const std::string& fsLabel, int32_t flags,
-    int id = mNextStubVolumeId++;
+                                    std::string* outVolId) {
-    auto vol = std::shared_ptr<android::vold::VolumeBase>(
+    dev_t stubId = --mNextStubId;
-        new android::vold::StubVolume(id, sourcePath, mountPath, fsType, fsUuid, fsLabel));
+    auto vol = std::shared_ptr<android::vold::StubVolume>(
-    vol->create();
+            new android::vold::StubVolume(stubId, sourcePath, mountPath, fsType, fsUuid, fsLabel));
 
-    mStubVolumes.push_back(vol);
+    int32_t passedFlags = android::vold::Disk::Flags::kStub;
+    passedFlags |= (flags & android::vold::Disk::Flags::kUsb);
+    passedFlags |= (flags & android::vold::Disk::Flags::kSd);
+    // StubDisk doesn't have device node corresponds to it. So, a fake device
+    // number is used.
+    auto disk = std::shared_ptr<android::vold::Disk>(
+            new android::vold::Disk("stub", stubId, "stub", passedFlags));
+    disk->initializePartition(vol);
+    handleDiskAdded(disk);
     *outVolId = vol->getId();
     return android::OK;
 }
 
 int VolumeManager::destroyStubVolume(const std::string& volId) {
-    auto i = mStubVolumes.begin();
+    auto tokens = android::base::Split(volId, ":");
-    while (i != mStubVolumes.end()) {
+    CHECK(tokens.size() == 2);
-        if ((*i)->getId() == volId) {
+    dev_t stubId;
-            (*i)->destroy();
+    CHECK(android::base::ParseUint(tokens[1], &stubId));
-            i = mStubVolumes.erase(i);
+    handleDiskRemoved(stubId);
-        } else {
-            ++i;
-        }
-    }
     return android::OK;
 }
 

VolumeManager.h

@@ -23,6 +23,7 @@
 
 #include <list>
 #include <mutex>
+#include <set>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -82,8 +83,28 @@ class VolumeManager {
     std::shared_ptr<android::vold::Disk> findDisk(const std::string& id);
     std::shared_ptr<android::vold::VolumeBase> findVolume(const std::string& id);
 
+    template <typename Fn>
+    std::shared_ptr<android::vold::VolumeBase> findVolumeWithFilter(Fn fn) {
+        for (const auto& vol : mInternalEmulatedVolumes) {
+            if (fn(*vol)) {
+                return vol;
+            }
+        }
+        for (const auto& disk : mDisks) {
+            for (const auto& vol : disk->getVolumes()) {
+                if (fn(*vol)) {
+                    return vol;
+                }
+            }
+        }
+
+        return nullptr;
+    }
+
     void listVolumes(android::vold::VolumeBase::Type type, std::list<std::string>& list) const;
 
+    const std::set<userid_t>& getStartedUsers() const { return mStartedUsers; }
+
     int forgetPartition(const std::string& partGuid, const std::string& fsUuid);
 
     int onUserAdded(userid_t userId, int userSerialNumber);
@@ -91,12 +112,16 @@ class VolumeManager {
     int onUserStarted(userid_t userId);
     int onUserStopped(userid_t userId);
 
+    void createPendingDisksIfNeeded();
     int onSecureKeyguardStateChanged(bool isShowing);
 
     int setPrimary(const std::shared_ptr<android::vold::VolumeBase>& vol);
 
     int remountUid(uid_t uid, int32_t remountMode);
+    int remountAppStorageDirs(int uid, int pid, const std::vector<std::string>& packageNames);
 
+    /* Aborts all FUSE filesystems, in case the FUSE daemon is no longer up. */
+    int abortFuse();
     /* Reset all internal state, typically during framework boot */
     int reset();
     /* Prepare for device shutdown, safely unmounting all devices */
@@ -107,16 +132,53 @@ class VolumeManager {
     int updateVirtualDisk();
     int setDebug(bool enable);
 
+    bool forkAndRemountStorage(int uid, int pid, const std::vector<std::string>& packageNames);
+
     static VolumeManager* Instance();
 
     /*
-     * Ensure that all directories along given path exist, creating parent
+     * Creates a directory 'path' for an application, automatically creating
-     * directories as needed.  Validates that given path is absolute and that
+     * directories along the given path if they don't exist yet.
-     * it contains no relative "." or ".." paths or symlinks.  Last path segment
+     *
-     * is treated as filename and ignored, unless the path ends with "/".  Also
+     * Example:
-     * ensures that path belongs to a volume managed by vold.
+     *   path = /storage/emulated/0/Android/data/com.foo/files/
+     *
+     * This function will first match the first part of the path with the volume
+     * root of any known volumes; in this case, "/storage/emulated/0" matches
+     * with the volume root of the emulated volume for user 0.
+     *
+     * The subseqent part of the path must start with one of the well-known
+     * Android/ data directories, /Android/data, /Android/obb or
+     * /Android/media.
+     *
+     * The final part of the path is application specific. This function will
+     * create all directories, including the application-specific ones, and
+     * set the UID of all app-specific directories below the well-known data
+     * directories to the 'appUid' argument. In the given example, the UID
+     * of /storage/emulated/0/Android/data/com.foo and
+     * /storage/emulated/0/Android/data/com.foo/files would be set to 'appUid'.
+     *
+     * The UID/GID of the parent directories will be set according to the
+     * requirements of the underlying filesystem and are of no concern to the
+     * caller.
+     *
+     * If fixupExistingOnly is set, we make sure to fixup any existing dirs and
+     * files in the passed in path, but only if that path exists; if it doesn't
+     * exist, this function doesn't create them.
+     *
+     * Validates that given paths are absolute and that they contain no relative
+     * "." or ".." paths or symlinks.  Last path segment is treated as filename
+     * and ignored, unless the path ends with "/".  Also ensures that path
+     * belongs to a volume managed by vold.
      */
-    int mkdirs(const std::string& path);
+    int setupAppDir(const std::string& path, int32_t appUid, bool fixupExistingOnly = false);
+
+    /**
+     * Fixes up an existing application directory, as if it was created with
+     * setupAppDir() above. This includes fixing up the UID/GID, permissions and
+     * project IDs of the contained files and directories.
+     */
+    int fixupAppDir(const std::string& path, int32_t appUid);
 
     int createObb(const std::string& path, const std::string& key, int32_t ownerGid,
                   std::string* outVolId);
@@ -124,7 +186,7 @@ class VolumeManager {
 
     int createStubVolume(const std::string& sourcePath, const std::string& mountPath,
                          const std::string& fsType, const std::string& fsUuid,
-                         const std::string& fsLabel, std::string* outVolId);
+                         const std::string& fsLabel, int32_t flags, std::string* outVolId);
     int destroyStubVolume(const std::string& volId);
 
     int mountAppFuse(uid_t uid, int mountId, android::base::unique_fd* device_fd);
@@ -137,10 +199,15 @@ class VolumeManager {
 
     int linkPrimary(userid_t userId);
 
+    void createEmulatedVolumesForUser(userid_t userId);
+    void destroyEmulatedVolumesForUser(userid_t userId);
+
     void handleDiskAdded(const std::shared_ptr<android::vold::Disk>& disk);
     void handleDiskChanged(dev_t device);
     void handleDiskRemoved(dev_t device);
 
+    bool updateFuseMountedProperty();
+
     std::mutex mLock;
     std::mutex mCryptLock;
 
@@ -150,18 +217,19 @@ class VolumeManager {
     std::list<std::shared_ptr<android::vold::Disk>> mDisks;
     std::list<std::shared_ptr<android::vold::Disk>> mPendingDisks;
     std::list<std::shared_ptr<android::vold::VolumeBase>> mObbVolumes;
-    std::list<std::shared_ptr<android::vold::VolumeBase>> mStubVolumes;
+    std::list<std::shared_ptr<android::vold::VolumeBase>> mInternalEmulatedVolumes;
 
     std::unordered_map<userid_t, int> mAddedUsers;
-    std::unordered_set<userid_t> mStartedUsers;
+    // This needs to be a regular set because we care about the ordering here;
+    // user 0 should always go first, because it is responsible for sdcardfs.
+    std::set<userid_t> mStartedUsers;
 
     std::string mVirtualDiskPath;
     std::shared_ptr<android::vold::Disk> mVirtualDisk;
-    std::shared_ptr<android::vold::VolumeBase> mInternalEmulated;
     std::shared_ptr<android::vold::VolumeBase> mPrimary;
 
     int mNextObbId;
-    int mNextStubVolumeId;
+    int mNextStubId;
     bool mSecureKeyguardShowing;
 };
 

binder/android/os/IVold.aidl

@@ -18,12 +18,14 @@ package android.os;
 
 import android.os.incremental.IncrementalFileSystemControlParcel;
 import android.os.IVoldListener;
+import android.os.IVoldMountCallback;
 import android.os.IVoldTaskListener;
 
 /** {@hide} */
 interface IVold {
     void setListener(IVoldListener listener);
 
+    void abortFuse();
     void monitor();
     void reset();
     void shutdown();
@@ -41,7 +43,8 @@ interface IVold {
     void partition(@utf8InCpp String diskId, int partitionType, int ratio);
     void forgetPartition(@utf8InCpp String partGuid, @utf8InCpp String fsUuid);
 
-    void mount(@utf8InCpp String volId, int mountFlags, int mountUserId);
+    void mount(@utf8InCpp String volId, int mountFlags, int mountUserId,
+         @nullable IVoldMountCallback callback);
     void unmount(@utf8InCpp String volId);
     void format(@utf8InCpp String volId, @utf8InCpp String fsType);
     void benchmark(@utf8InCpp String volId, IVoldTaskListener listener);
@@ -50,8 +53,10 @@ interface IVold {
                      IVoldTaskListener listener);
 
     void remountUid(int uid, int remountMode);
+    void remountAppStorageDirs(int uid, int pid, in @utf8InCpp String[] packageNames);
 
-    void mkdirs(@utf8InCpp String path);
+    void setupAppDir(@utf8InCpp String path, int appUid);
+    void fixupAppDir(@utf8InCpp String path, int appUid);
 
     @utf8InCpp String createObb(@utf8InCpp String sourcePath, @utf8InCpp String sourceKey,
                                 int ownerGid);
@@ -123,7 +128,7 @@ interface IVold {
 
     @utf8InCpp String createStubVolume(@utf8InCpp String sourcePath,
             @utf8InCpp String mountPath, @utf8InCpp String fsType,
-            @utf8InCpp String fsUuid, @utf8InCpp String fsLabel);
+            @utf8InCpp String fsUuid, @utf8InCpp String fsLabel, int flags);
     void destroyStubVolume(@utf8InCpp String volId);
 
     FileDescriptor openAppFuseFile(int uid, int mountId, int fileId, int flags);
@@ -131,6 +136,7 @@ interface IVold {
     boolean incFsEnabled();
     IncrementalFileSystemControlParcel mountIncFs(@utf8InCpp String backingPath, @utf8InCpp String targetDir, int flags);
     void unmountIncFs(@utf8InCpp String dir);
+    void setIncFsMountOptions(in IncrementalFileSystemControlParcel control, boolean enableReadLogs);
     void bindMount(@utf8InCpp String sourceDir, @utf8InCpp String targetDir);
 
     const int ENCRYPTION_FLAG_NO_UI = 4;
@@ -166,6 +172,8 @@ interface IVold {
     const int REMOUNT_MODE_LEGACY = 4;
     const int REMOUNT_MODE_INSTALLER = 5;
     const int REMOUNT_MODE_FULL = 6;
+    const int REMOUNT_MODE_PASS_THROUGH = 7;
+    const int REMOUNT_MODE_ANDROID_WRITABLE = 8;
 
     const int VOLUME_STATE_UNMOUNTED = 0;
     const int VOLUME_STATE_CHECKING = 1;

binder/android/os/IVoldListener.aidl

@@ -25,7 +25,7 @@ oneway interface IVoldListener {
     void onDiskDestroyed(@utf8InCpp String diskId);
 
     void onVolumeCreated(@utf8InCpp String volId,
-            int type, @utf8InCpp String diskId, @utf8InCpp String partGuid);
+            int type, @utf8InCpp String diskId, @utf8InCpp String partGuid, int userId);
     void onVolumeStateChanged(@utf8InCpp String volId, int state);
     void onVolumeMetadataChanged(@utf8InCpp String volId,
             @utf8InCpp String fsType, @utf8InCpp String fsUuid, @utf8InCpp String fsLabel);

binder/android/os/IVoldMountCallback.aidl

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.os;
+
+/** {@hide} */
+interface IVoldMountCallback {
+    boolean onVolumeChecking(FileDescriptor fuseFd, @utf8InCpp String path,
+        @utf8InCpp String internalPath);
+}

cryptfs.cpp

@@ -67,6 +67,9 @@
 #include <time.h>
 #include <unistd.h>
 
+#include <chrono>
+#include <thread>
+
 extern "C" {
 #include <crypto_scrypt.h>
 }
@@ -1131,9 +1134,22 @@ static int create_crypto_blk_dev(struct crypt_mnt_ftr* crypt_ftr, const unsigned
 }
 
 static int delete_crypto_blk_dev(const std::string& name) {
+    bool ret;
     auto& dm = DeviceMapper::Instance();
-    if (!dm.DeleteDevice(name)) {
+    // TODO(b/149396179) there appears to be a race somewhere in the system where trying
-        SLOGE("Cannot remove dm-crypt device %s: %s\n", name.c_str(), strerror(errno));
+    // to delete the device fails with EBUSY; for now, work around this by retrying.
+    int tries = 5;
+    while (tries-- > 0) {
+        ret = dm.DeleteDevice(name);
+        if (ret || errno != EBUSY) {
+            break;
+        }
+        SLOGW("DM_DEV Cannot remove dm-crypt device %s: %s, retrying...\n", name.c_str(),
+              strerror(errno));
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    }
+    if (!ret) {
+        SLOGE("DM_DEV Cannot remove dm-crypt device %s: %s\n", name.c_str(), strerror(errno));
         return -1;
     }
     return 0;

fs/F2fs.cpp

@@ -94,6 +94,20 @@ status_t Format(const std::string& source) {
     cmd.push_back("-O");
     cmd.push_back("verity");
 
+    const bool needs_casefold =
+            android::base::GetBoolProperty("external_storage.casefold.enabled", false);
+    const bool needs_projid =
+            android::base::GetBoolProperty("external_storage.projid.enabled", false);
+    if (needs_projid) {
+        cmd.push_back("-O");
+        cmd.push_back("project_quota,extra_attr");
+    }
+    if (needs_casefold) {
+        cmd.push_back("-O");
+        cmd.push_back("casefold");
+        cmd.push_back("-C");
+        cmd.push_back("utf8");
+    }
     cmd.push_back(source);
     return ForkExecvp(cmd);
 }

model/Disk.cpp

@@ -132,6 +132,17 @@ void Disk::listVolumes(VolumeBase::Type type, std::list<std::string>& list) cons
     }
 }
 
+std::vector<std::shared_ptr<VolumeBase>> Disk::getVolumes() const {
+    std::vector<std::shared_ptr<VolumeBase>> vols;
+    for (const auto& vol : mVolumes) {
+        vols.push_back(vol);
+        auto stackedVolumes = vol->getVolumes();
+        vols.insert(vols.end(), stackedVolumes.begin(), stackedVolumes.end());
+    }
+
+    return vols;
+}
+
 status_t Disk::create() {
     CHECK(!mCreated);
     mCreated = true;
@@ -139,6 +150,10 @@ status_t Disk::create() {
     auto listener = VolumeManager::Instance()->getListener();
     if (listener) listener->onDiskCreated(getId(), mFlags);
 
+    if (isStub()) {
+        createStubVolume();
+        return OK;
+    }
     readMetadata();
     readPartitions();
     return OK;
@@ -203,6 +218,15 @@ void Disk::createPrivateVolume(dev_t device, const std::string& partGuid) {
     vol->create();
 }
 
+void Disk::createStubVolume() {
+    CHECK(mVolumes.size() == 1);
+    auto listener = VolumeManager::Instance()->getListener();
+    if (listener) listener->onDiskMetadataChanged(getId(), mSize, mLabel, mSysPath);
+    if (listener) listener->onDiskScanned(getId());
+    mVolumes[0]->setDiskId(getId());
+    mVolumes[0]->create();
+}
+
 void Disk::destroyAllVolumes() {
     for (const auto& vol : mVolumes) {
         vol->destroy();
@@ -403,6 +427,12 @@ status_t Disk::readPartitions() {
     return OK;
 }
 
+void Disk::initializePartition(std::shared_ptr<StubVolume> vol) {
+    CHECK(isStub());
+    CHECK(mVolumes.empty());
+    mVolumes.push_back(vol);
+}
+
 status_t Disk::unmountAll() {
     for (const auto& vol : mVolumes) {
         vol->unmount();

model/Disk.h

@@ -17,6 +17,7 @@
 #ifndef ANDROID_VOLD_DISK_H
 #define ANDROID_VOLD_DISK_H
 
+#include "StubVolume.h"
 #include "Utils.h"
 #include "VolumeBase.h"
 
@@ -52,6 +53,9 @@ class Disk {
         kUsb = 1 << 3,
         /* Flag that disk is EMMC internal */
         kEmmc = 1 << 4,
+        /* Flag that disk is Stub disk, i.e., disk that is managed from outside
+         * Android (e.g., ARC++). */
+        kStub = 1 << 5,
     };
 
     const std::string& getId() const { return mId; }
@@ -67,11 +71,14 @@ class Disk {
 
     void listVolumes(VolumeBase::Type type, std::list<std::string>& list) const;
 
+    std::vector<std::shared_ptr<VolumeBase>> getVolumes() const;
+
     status_t create();
     status_t destroy();
 
     status_t readMetadata();
     status_t readPartitions();
+    void initializePartition(std::shared_ptr<StubVolume> vol);
 
     status_t unmountAll();
 
@@ -107,11 +114,14 @@ class Disk {
 
     void createPublicVolume(dev_t device);
     void createPrivateVolume(dev_t device, const std::string& partGuid);
+    void createStubVolume();
 
     void destroyAllVolumes();
 
     int getMaxMinors();
 
+    bool isStub() { return mFlags & kStub; }
+
     DISALLOW_COPY_AND_ASSIGN(Disk);
 };
 

model/EmulatedVolume.cpp

@@ -15,10 +15,14 @@
  */
 
 #include "EmulatedVolume.h"
+
+#include "AppFuseUtil.h"
 #include "Utils.h"
 #include "VolumeManager.h"
 
 #include <android-base/logging.h>
+#include <android-base/properties.h>
+#include <android-base/scopeguard.h>
 #include <android-base/stringprintf.h>
 #include <cutils/fs.h>
 #include <private/android_filesystem_config.h>
@@ -37,116 +41,438 @@ using android::base::StringPrintf;
 namespace android {
 namespace vold {
 
-static const char* kFusePath = "/system/bin/sdcard";
+static const char* kSdcardFsPath = "/system/bin/sdcard";
 
-EmulatedVolume::EmulatedVolume(const std::string& rawPath)
+EmulatedVolume::EmulatedVolume(const std::string& rawPath, int userId)
-    : VolumeBase(Type::kEmulated), mFusePid(0) {
+    : VolumeBase(Type::kEmulated) {
-    setId("emulated");
+    setId(StringPrintf("emulated;%u", userId));
     mRawPath = rawPath;
     mLabel = "emulated";
+    mFuseMounted = false;
+    mUseSdcardFs = IsSdcardfsUsed();
+    mAppDataIsolationEnabled = base::GetBoolProperty(kVoldAppDataIsolationEnabled, false);
 }
 
-EmulatedVolume::EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid)
+EmulatedVolume::EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid,
-    : VolumeBase(Type::kEmulated), mFusePid(0) {
+                               int userId)
-    setId(StringPrintf("emulated:%u,%u", major(device), minor(device)));
+    : VolumeBase(Type::kEmulated) {
+    setId(StringPrintf("emulated:%u,%u;%u", major(device), minor(device), userId));
     mRawPath = rawPath;
     mLabel = fsUuid;
+    mFuseMounted = false;
+    mUseSdcardFs = IsSdcardfsUsed();
+    mAppDataIsolationEnabled = base::GetBoolProperty(kVoldAppDataIsolationEnabled, false);
 }
 
 EmulatedVolume::~EmulatedVolume() {}
 
-status_t EmulatedVolume::doMount() {
+std::string EmulatedVolume::getLabel() {
     // We could have migrated storage to an adopted private volume, so always
     // call primary storage "emulated" to avoid media rescans.
-    std::string label = mLabel;
     if (getMountFlags() & MountFlags::kPrimary) {
-        label = "emulated";
+        return "emulated";
+    } else {
+        return mLabel;
+    }
+}
+
+// Creates a bind mount from source to target
+static status_t doFuseBindMount(const std::string& source, const std::string& target,
+                                std::list<std::string>& pathsToUnmount) {
+    LOG(INFO) << "Bind mounting " << source << " on " << target;
+    auto status = BindMount(source, target);
+    if (status != OK) {
+        return status;
+    }
+    LOG(INFO) << "Bind mounted " << source << " on " << target;
+    pathsToUnmount.push_front(target);
+    return OK;
+}
+
+status_t EmulatedVolume::mountFuseBindMounts() {
+    std::string androidSource;
+    std::string label = getLabel();
+    int userId = getMountUserId();
+    std::list<std::string> pathsToUnmount;
+
+    auto unmounter = [&]() {
+        LOG(INFO) << "mountFuseBindMounts() unmount scope_guard running";
+        for (const auto& path : pathsToUnmount) {
+            LOG(INFO) << "Unmounting " << path;
+            auto status = UnmountTree(path);
+            if (status != OK) {
+                LOG(INFO) << "Failed to unmount " << path;
+            } else {
+                LOG(INFO) << "Unmounted " << path;
+            }
+        }
+    };
+    auto unmount_guard = android::base::make_scope_guard(unmounter);
+
+    if (mUseSdcardFs) {
+        androidSource = StringPrintf("/mnt/runtime/default/%s/%d/Android", label.c_str(), userId);
+    } else {
+        androidSource = StringPrintf("/%s/%d/Android", mRawPath.c_str(), userId);
     }
 
-    mFuseDefault = StringPrintf("/mnt/runtime/default/%s", label.c_str());
+    status_t status = OK;
-    mFuseRead = StringPrintf("/mnt/runtime/read/%s", label.c_str());
+    // When app data isolation is enabled, obb/ will be mounted per app, otherwise we should
-    mFuseWrite = StringPrintf("/mnt/runtime/write/%s", label.c_str());
+    // bind mount the whole Android/ to speed up reading.
-    mFuseFull = StringPrintf("/mnt/runtime/full/%s", label.c_str());
+    if (!mAppDataIsolationEnabled) {
+        std::string androidDataSource = StringPrintf("%s/data", androidSource.c_str());
+        std::string androidDataTarget(
+                StringPrintf("/mnt/user/%d/%s/%d/Android/data", userId, label.c_str(), userId));
+        status = doFuseBindMount(androidDataSource, androidDataTarget, pathsToUnmount);
+        if (status != OK) {
+            return status;
+        }
+
+        std::string androidObbSource = StringPrintf("%s/obb", androidSource.c_str());
+        std::string androidObbTarget(
+                StringPrintf("/mnt/user/%d/%s/%d/Android/obb", userId, label.c_str(), userId));
+        status = doFuseBindMount(androidObbSource, androidObbTarget, pathsToUnmount);
+        if (status != OK) {
+            return status;
+        }
+    }
+
+    // Installers get the same view as all other apps, with the sole exception that the
+    // OBB dirs (Android/obb) are writable to them. On sdcardfs devices, this requires
+    // a special bind mount, since app-private and OBB dirs share the same GID, but we
+    // only want to give access to the latter.
+    if (mUseSdcardFs) {
+        std::string obbSource(StringPrintf("/mnt/runtime/write/%s/%d/Android/obb",
+                label.c_str(), userId));
+        std::string obbInstallerTarget(StringPrintf("/mnt/installer/%d/%s/%d/Android/obb",
+                userId, label.c_str(), userId));
+
+        status = doFuseBindMount(obbSource, obbInstallerTarget, pathsToUnmount);
+        if (status != OK) {
+            return status;
+        }
+    } else if (mAppDataIsolationEnabled) {
+        std::string obbSource(StringPrintf("%s/obb", androidSource.c_str()));
+        std::string obbInstallerTarget(StringPrintf("/mnt/installer/%d/%s/%d/Android/obb",
+                userId, label.c_str(), userId));
+
+        status = doFuseBindMount(obbSource, obbInstallerTarget, pathsToUnmount);
+        if (status != OK) {
+            return status;
+        }
+    }
+
+    // /mnt/androidwriteable is similar to /mnt/installer, but it's for
+    // MOUNT_EXTERNAL_ANDROID_WRITABLE apps and it can also access DATA (Android/data) dirs.
+    if (mAppDataIsolationEnabled) {
+        std::string obbSource = mUseSdcardFs ?
+            StringPrintf("/mnt/runtime/write/%s/%d/Android/obb", label.c_str(), userId)
+            : StringPrintf("%s/obb", androidSource.c_str());
+
+        std::string obbAndroidWritableTarget(
+                StringPrintf("/mnt/androidwritable/%d/%s/%d/Android/obb",
+                userId, label.c_str(), userId));
+
+        status = doFuseBindMount(obbSource, obbAndroidWritableTarget, pathsToUnmount);
+        if (status != OK) {
+            return status;
+        }
+
+        std::string dataSource = mUseSdcardFs ?
+                StringPrintf("/mnt/runtime/write/%s/%d/Android/data", label.c_str(), userId)
+                : StringPrintf("%s/data", androidSource.c_str());
+        std::string dataTarget(StringPrintf("/mnt/androidwritable/%d/%s/%d/Android/data",
+                userId, label.c_str(), userId));
+
+        status = doFuseBindMount(dataSource, dataTarget, pathsToUnmount);
+        if (status != OK) {
+            return status;
+        }
+    }
+    unmount_guard.Disable();
+    return OK;
+}
+
+status_t EmulatedVolume::unmountFuseBindMounts() {
+    std::string label = getLabel();
+    int userId = getMountUserId();
+
+    if (mUseSdcardFs || mAppDataIsolationEnabled) {
+        std::string installerTarget(
+                StringPrintf("/mnt/installer/%d/%s/%d/Android/obb", userId, label.c_str(), userId));
+        LOG(INFO) << "Unmounting " << installerTarget;
+        auto status = UnmountTree(installerTarget);
+        if (status != OK) {
+            LOG(ERROR) << "Failed to unmount " << installerTarget;
+            // Intentional continue to try to unmount the other bind mount
+        }
+    }
+    if (mAppDataIsolationEnabled) {
+        std::string obbTarget( StringPrintf("/mnt/androidwritable/%d/%s/%d/Android/obb",
+                userId, label.c_str(), userId));
+        LOG(INFO) << "Unmounting " << obbTarget;
+        auto status = UnmountTree(obbTarget);
+        if (status != OK) {
+            LOG(ERROR) << "Failed to unmount " << obbTarget;
+            // Intentional continue to try to unmount the other bind mount
+        }
+        std::string dataTarget(StringPrintf("/mnt/androidwritable/%d/%s/%d/Android/data",
+                userId, label.c_str(), userId));
+        LOG(INFO) << "Unmounting " << dataTarget;
+        status = UnmountTree(dataTarget);
+        if (status != OK) {
+            LOG(ERROR) << "Failed to unmount " << dataTarget;
+            // Intentional continue to try to unmount the other bind mount
+        }
+    }
+
+    // When app data isolation is enabled, kill all apps that obb/ is mounted, otherwise we should
+    // umount the whole Android/ dir.
+    if (mAppDataIsolationEnabled) {
+        std::string appObbDir(StringPrintf("%s/%d/Android/obb", getPath().c_str(), userId));
+        KillProcessesWithMountPrefix(appObbDir);
+    } else {
+        std::string androidDataTarget(
+                StringPrintf("/mnt/user/%d/%s/%d/Android/data", userId, label.c_str(), userId));
+
+        LOG(INFO) << "Unmounting " << androidDataTarget;
+        auto status = UnmountTree(androidDataTarget);
+        if (status != OK) {
+            return status;
+        }
+        LOG(INFO) << "Unmounted " << androidDataTarget;
+
+        std::string androidObbTarget(
+                StringPrintf("/mnt/user/%d/%s/%d/Android/obb", userId, label.c_str(), userId));
+
+        LOG(INFO) << "Unmounting " << androidObbTarget;
+        status = UnmountTree(androidObbTarget);
+        if (status != OK) {
+            return status;
+        }
+        LOG(INFO) << "Unmounted " << androidObbTarget;
+    }
+    return OK;
+}
+
+status_t EmulatedVolume::unmountSdcardFs() {
+    if (!mUseSdcardFs || getMountUserId() != 0) {
+        // For sdcardfs, only unmount for user 0, since user 0 will always be running
+        // and the paths don't change for different users.
+        return OK;
+    }
+
+    ForceUnmount(mSdcardFsDefault);
+    ForceUnmount(mSdcardFsRead);
+    ForceUnmount(mSdcardFsWrite);
+    ForceUnmount(mSdcardFsFull);
+
+    rmdir(mSdcardFsDefault.c_str());
+    rmdir(mSdcardFsRead.c_str());
+    rmdir(mSdcardFsWrite.c_str());
+    rmdir(mSdcardFsFull.c_str());
+
+    mSdcardFsDefault.clear();
+    mSdcardFsRead.clear();
+    mSdcardFsWrite.clear();
+    mSdcardFsFull.clear();
+
+    return OK;
+}
+
+status_t EmulatedVolume::doMount() {
+    std::string label = getLabel();
+    bool isVisible = getMountFlags() & MountFlags::kVisible;
+
+    mSdcardFsDefault = StringPrintf("/mnt/runtime/default/%s", label.c_str());
+    mSdcardFsRead = StringPrintf("/mnt/runtime/read/%s", label.c_str());
+    mSdcardFsWrite = StringPrintf("/mnt/runtime/write/%s", label.c_str());
+    mSdcardFsFull = StringPrintf("/mnt/runtime/full/%s", label.c_str());
 
     setInternalPath(mRawPath);
     setPath(StringPrintf("/storage/%s", label.c_str()));
 
-    if (fs_prepare_dir(mFuseDefault.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+    if (fs_prepare_dir(mSdcardFsDefault.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseRead.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+        fs_prepare_dir(mSdcardFsRead.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseWrite.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+        fs_prepare_dir(mSdcardFsWrite.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseFull.c_str(), 0700, AID_ROOT, AID_ROOT)) {
+        fs_prepare_dir(mSdcardFsFull.c_str(), 0700, AID_ROOT, AID_ROOT)) {
         PLOG(ERROR) << getId() << " failed to create mount points";
         return -errno;
     }
 
-    dev_t before = GetDevice(mFuseFull);
+    dev_t before = GetDevice(mSdcardFsFull);
 
-    if (!(mFusePid = fork())) {
+    bool isFuse = base::GetBoolProperty(kPropFuse, false);
-        // clang-format off
+
-        if (execl(kFusePath, kFusePath,
+    // Mount sdcardfs regardless of FUSE, since we need it to bind-mount on top of the
-                "-u", "1023", // AID_MEDIA_RW
+    // FUSE volume for various reasons.
-                "-g", "1023", // AID_MEDIA_RW
+    if (mUseSdcardFs && getMountUserId() == 0) {
-                "-m",
+        LOG(INFO) << "Executing sdcardfs";
-                "-w",
+        int sdcardFsPid;
-                "-G",
+        if (!(sdcardFsPid = fork())) {
-                "-i",
+            // clang-format off
-                "-o",
+            if (execl(kSdcardFsPath, kSdcardFsPath,
-                mRawPath.c_str(),
+                    "-u", "1023", // AID_MEDIA_RW
-                label.c_str(),
+                    "-g", "1023", // AID_MEDIA_RW
-                NULL)) {
+                    "-m",
-            // clang-format on
+                    "-w",
-            PLOG(ERROR) << "Failed to exec";
+                    "-G",
+                    "-i",
+                    "-o",
+                    mRawPath.c_str(),
+                    label.c_str(),
+                    NULL)) {
+                // clang-format on
+                PLOG(ERROR) << "Failed to exec";
+            }
+
+            LOG(ERROR) << "sdcardfs exiting";
+            _exit(1);
         }
 
-        LOG(ERROR) << "FUSE exiting";
+        if (sdcardFsPid == -1) {
-        _exit(1);
+            PLOG(ERROR) << getId() << " failed to fork";
-    }
+            return -errno;
-
-    if (mFusePid == -1) {
-        PLOG(ERROR) << getId() << " failed to fork";
-        return -errno;
-    }
-
-    nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
-    while (before == GetDevice(mFuseFull)) {
-        LOG(DEBUG) << "Waiting for FUSE to spin up...";
-        usleep(50000);  // 50ms
-
-        nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
-        if (nanoseconds_to_milliseconds(now - start) > 5000) {
-            LOG(WARNING) << "Timed out while waiting for FUSE to spin up";
-            return -ETIMEDOUT;
         }
+
+        nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
+        while (before == GetDevice(mSdcardFsFull)) {
+            LOG(DEBUG) << "Waiting for sdcardfs to spin up...";
+            usleep(50000);  // 50ms
+
+            nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
+            if (nanoseconds_to_milliseconds(now - start) > 5000) {
+                LOG(WARNING) << "Timed out while waiting for sdcardfs to spin up";
+                return -ETIMEDOUT;
+            }
+        }
+        /* sdcardfs will have exited already. The filesystem will still be running */
+        TEMP_FAILURE_RETRY(waitpid(sdcardFsPid, nullptr, 0));
+        sdcardFsPid = 0;
+    }
+
+    if (isFuse && isVisible) {
+        // Make sure we unmount sdcardfs if we bail out with an error below
+        auto sdcardfs_unmounter = [&]() {
+            LOG(INFO) << "sdcardfs_unmounter scope_guard running";
+            unmountSdcardFs();
+        };
+        auto sdcardfs_guard = android::base::make_scope_guard(sdcardfs_unmounter);
+
+        LOG(INFO) << "Mounting emulated fuse volume";
+        android::base::unique_fd fd;
+        int user_id = getMountUserId();
+        auto volumeRoot = getRootPath();
+
+        // Make sure Android/ dirs exist for bind mounting
+        status_t res = PrepareAndroidDirs(volumeRoot);
+        if (res != OK) {
+            LOG(ERROR) << "Failed to prepare Android/ directories";
+            return res;
+        }
+
+        res = MountUserFuse(user_id, getInternalPath(), label, &fd);
+        if (res != 0) {
+            PLOG(ERROR) << "Failed to mount emulated fuse volume";
+            return res;
+        }
+
+        mFuseMounted = true;
+        auto fuse_unmounter = [&]() {
+            LOG(INFO) << "fuse_unmounter scope_guard running";
+            fd.reset();
+            if (UnmountUserFuse(user_id, getInternalPath(), label) != OK) {
+                PLOG(INFO) << "UnmountUserFuse failed on emulated fuse volume";
+            }
+            mFuseMounted = false;
+        };
+        auto fuse_guard = android::base::make_scope_guard(fuse_unmounter);
+
+        auto callback = getMountCallback();
+        if (callback) {
+            bool is_ready = false;
+            callback->onVolumeChecking(std::move(fd), getPath(), getInternalPath(), &is_ready);
+            if (!is_ready) {
+                return -EIO;
+            }
+        }
+
+        // Only do the bind-mounts when we know for sure the FUSE daemon can resolve the path.
+        res = mountFuseBindMounts();
+        if (res != OK) {
+            return res;
+        }
+
+        ConfigureReadAheadForFuse(GetFuseMountPathForUser(user_id, label), 256u);
+
+        // By default, FUSE has a max_dirty ratio of 1%. This means that out of
+        // all dirty pages in the system, only 1% is allowed to belong to any
+        // FUSE filesystem. The reason this is in place is that FUSE
+        // filesystems shouldn't be trusted by default; a FUSE filesystem could
+        // take up say 100% of dirty pages, and subsequently refuse to write
+        // them back to storage.  The kernel will then apply rate-limiting, and
+        // block other tasks from writing.  For this particular FUSE filesystem
+        // however, we trust the implementation, because it is a part of the
+        // Android platform. So use the default ratio of 100%.
+        //
+        // The reason we're setting this is that there's a suspicion that the
+        // kernel starts rate-limiting the FUSE filesystem under extreme
+        // memory pressure scenarios. While the kernel will only rate limit if
+        // the writeback can't keep up with the write rate, under extreme
+        // memory pressure the write rate may dip as well, in which case FUSE
+        // writes to a 1% max_ratio filesystem are throttled to an extreme amount.
+        //
+        // To prevent this, just give FUSE 40% max_ratio, meaning it can take
+        // up to 40% of all dirty pages in the system.
+        ConfigureMaxDirtyRatioForFuse(GetFuseMountPathForUser(user_id, label), 40u);
+
+        // All mounts where successful, disable scope guards
+        sdcardfs_guard.Disable();
+        fuse_guard.Disable();
     }
-    /* sdcardfs will have exited already. FUSE will still be running */
-    TEMP_FAILURE_RETRY(waitpid(mFusePid, nullptr, 0));
-    mFusePid = 0;
 
     return OK;
 }
 
 status_t EmulatedVolume::doUnmount() {
-    // Unmount the storage before we kill the FUSE process. If we kill
+    int userId = getMountUserId();
-    // the FUSE process first, most file system operations will return
+
+    // Kill all processes using the filesystem before we unmount it. If we
+    // unmount the filesystem first, most file system operations will return
     // ENOTCONN until the unmount completes. This is an exotic and unusual
     // error code and might cause broken behaviour in applications.
-    KillProcessesUsingPath(getPath());
+    if (mFuseMounted) {
-    ForceUnmount(mFuseDefault);
+        // For FUSE specifically, we have an emulated volume per user, so only kill
-    ForceUnmount(mFuseRead);
+        // processes using files from this particular user.
-    ForceUnmount(mFuseWrite);
+        std::string user_path(StringPrintf("%s/%d", getPath().c_str(), getMountUserId()));
-    ForceUnmount(mFuseFull);
+        LOG(INFO) << "Killing all processes referencing " << user_path;
+        KillProcessesUsingPath(user_path);
+    } else {
+        KillProcessesUsingPath(getPath());
+    }
 
-    rmdir(mFuseDefault.c_str());
+    if (mFuseMounted) {
-    rmdir(mFuseRead.c_str());
+        std::string label = getLabel();
-    rmdir(mFuseWrite.c_str());
-    rmdir(mFuseFull.c_str());
 
-    mFuseDefault.clear();
+        // Ignoring unmount return status because we do want to try to unmount
-    mFuseRead.clear();
+        // the rest cleanly.
-    mFuseWrite.clear();
+        unmountFuseBindMounts();
-    mFuseFull.clear();
 
-    return OK;
+        if (UnmountUserFuse(userId, getInternalPath(), label) != OK) {
+            PLOG(INFO) << "UnmountUserFuse failed on emulated fuse volume";
+            return -errno;
+        }
+
+        mFuseMounted = false;
+    }
+
+    return unmountSdcardFs();
+}
+
+std::string EmulatedVolume::getRootPath() const {
+    int user_id = getMountUserId();
+    std::string volumeRoot = StringPrintf("%s/%d", getInternalPath().c_str(), user_id);
+
+    return volumeRoot;
 }
 
 }  // namespace vold

model/EmulatedVolume.h

@@ -27,7 +27,7 @@ namespace vold {
 /*
  * Shared storage emulated on top of private storage.
  *
- * Knows how to spawn a FUSE daemon to synthesize permissions.  ObbVolume
+ * Knows how to spawn a sdcardfs daemon to synthesize permissions.  ObbVolume
  * can be stacked above it.
  *
  * This volume is always multi-user aware, but is only binds itself to
@@ -37,25 +37,38 @@ namespace vold {
  */
 class EmulatedVolume : public VolumeBase {
   public:
-    explicit EmulatedVolume(const std::string& rawPath);
+    explicit EmulatedVolume(const std::string& rawPath, int userId);
-    EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid);
+    EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid, int userId);
     virtual ~EmulatedVolume();
+    std::string getRootPath() const override;
+    bool isFuseMounted() const { return mFuseMounted; }
 
   protected:
     status_t doMount() override;
     status_t doUnmount() override;
 
   private:
+    status_t unmountSdcardFs();
+    status_t mountFuseBindMounts();
+    status_t unmountFuseBindMounts();
+
+    std::string getLabel();
     std::string mRawPath;
     std::string mLabel;
 
-    std::string mFuseDefault;
+    std::string mSdcardFsDefault;
-    std::string mFuseRead;
+    std::string mSdcardFsRead;
-    std::string mFuseWrite;
+    std::string mSdcardFsWrite;
-    std::string mFuseFull;
+    std::string mSdcardFsFull;
 
-    /* PID of FUSE wrapper */
+    /* Whether we mounted FUSE for this volume */
-    pid_t mFusePid;
+    bool mFuseMounted;
+
+    /* Whether to use sdcardfs for this volume */
+    bool mUseSdcardFs;
+
+    /* Whether to use app data isolation is enabled tor this volume */
+    bool mAppDataIsolationEnabled;
 
     DISALLOW_COPY_AND_ASSIGN(EmulatedVolume);
 };

model/PrivateVolume.cpp

@@ -36,6 +36,7 @@
 #include <sys/sysmacros.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <thread>
 
 using android::base::StringPrintf;
 using android::vold::IsVirtioBlkDevice;
@@ -43,6 +44,7 @@ using android::vold::IsVirtioBlkDevice;
 namespace android {
 namespace vold {
 
+static const unsigned int kMajorBlockLoop = 7;
 static const unsigned int kMajorBlockMmc = 179;
 
 PrivateVolume::PrivateVolume(dev_t device, const KeyBuffer& keyRaw)
@@ -69,8 +71,19 @@ status_t PrivateVolume::doCreate() {
 
     // Recover from stale vold by tearing down any old mappings
     auto& dm = dm::DeviceMapper::Instance();
-    if (!dm.DeleteDeviceIfExists(getId())) {
+    // TODO(b/149396179) there appears to be a race somewhere in the system where trying
+    // to delete the device fails with EBUSY; for now, work around this by retrying.
+    bool ret;
+    int tries = 10;
+    while (tries-- > 0) {
+        ret = dm.DeleteDeviceIfExists(getId());
+        if (ret || errno != EBUSY) {
+            break;
+        }
         PLOG(ERROR) << "Cannot remove dm device " << getId();
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    }
+    if (!ret) {
         return -EIO;
     }
 
@@ -86,8 +99,19 @@ status_t PrivateVolume::doCreate() {
 
 status_t PrivateVolume::doDestroy() {
     auto& dm = dm::DeviceMapper::Instance();
-    if (!dm.DeleteDevice(getId())) {
+    // TODO(b/149396179) there appears to be a race somewhere in the system where trying
+    // to delete the device fails with EBUSY; for now, work around this by retrying.
+    bool ret;
+    int tries = 10;
+    while (tries-- > 0) {
+        ret = dm.DeleteDevice(getId());
+        if (ret || errno != EBUSY) {
+            break;
+        }
         PLOG(ERROR) << "Cannot remove dm device " << getId();
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    }
+    if (!ret) {
         return -EIO;
     }
     return DestroyDeviceNode(mRawDevPath);
@@ -154,16 +178,24 @@ status_t PrivateVolume::doMount() {
         return -EIO;
     }
 
-    // Create a new emulated volume stacked above us, it will automatically
-    // be destroyed during unmount
-    std::string mediaPath(mPath + "/media");
-    auto vol = std::shared_ptr<VolumeBase>(new EmulatedVolume(mediaPath, mRawDevice, mFsUuid));
-    addVolume(vol);
-    vol->create();
-
     return OK;
 }
 
+void PrivateVolume::doPostMount() {
+    auto vol_manager = VolumeManager::Instance();
+    std::string mediaPath(mPath + "/media");
+
+    // Create a new emulated volume stacked above us for all added users, they will automatically
+    // be destroyed during unmount
+    for (userid_t user : vol_manager->getStartedUsers()) {
+        auto vol = std::shared_ptr<VolumeBase>(
+                new EmulatedVolume(mediaPath, mRawDevice, mFsUuid, user));
+        vol->setMountUserId(user);
+        addVolume(vol);
+        vol->create();
+    }
+}
+
 status_t PrivateVolume::doUnmount() {
     ForceUnmount(mPath);
 
@@ -179,7 +211,9 @@ status_t PrivateVolume::doFormat(const std::string& fsType) {
     if (fsType == "auto") {
         // For now, assume that all MMC devices are flash-based SD cards, and
         // give everyone else ext4 because sysfs rotational isn't reliable.
+        // Additionally, prefer f2fs for loop-based devices
         if ((major(mRawDevice) == kMajorBlockMmc ||
+             major(mRawDevice) == kMajorBlockLoop ||
              IsVirtioBlkDevice(major(mRawDevice))) && f2fs::IsSupported()) {
             resolvedFsType = "f2fs";
         } else {

model/PrivateVolume.h

@@ -42,11 +42,14 @@ class PrivateVolume : public VolumeBase {
     const std::string& getFsType() const { return mFsType; };
     const std::string& getRawDevPath() const { return mRawDevPath; };
     const std::string& getRawDmDevPath() const { return mDmDevPath; };
+    const std::string& getFsUuid() const { return mFsUuid; };
+    dev_t getRawDevice() const { return mRawDevice; };
 
   protected:
     status_t doCreate() override;
     status_t doDestroy() override;
     status_t doMount() override;
+    void doPostMount() override;
     status_t doUnmount() override;
     status_t doFormat(const std::string& fsType) override;
 

model/PublicVolume.cpp

@@ -15,6 +15,8 @@
  */
 
 #include "PublicVolume.h"
+
+#include "AppFuseUtil.h"
 #include "Utils.h"
 #include "VolumeManager.h"
 #include "fs/Exfat.h"
@@ -41,13 +43,15 @@ using android::base::StringPrintf;
 namespace android {
 namespace vold {
 
-static const char* kFusePath = "/system/bin/sdcard";
+static const char* kSdcardFsPath = "/system/bin/sdcard";
 
 static const char* kAsecPath = "/mnt/secure/asec";
 
-PublicVolume::PublicVolume(dev_t device) : VolumeBase(Type::kPublic), mDevice(device), mFusePid(0) {
+PublicVolume::PublicVolume(dev_t device) : VolumeBase(Type::kPublic), mDevice(device) {
     setId(StringPrintf("public:%u,%u", major(device), minor(device)));
     mDevPath = StringPrintf("/dev/block/vold/%s", getId().c_str());
+    mFuseMounted = false;
+    mUseSdcardFs = IsSdcardfsUsed();
 }
 
 PublicVolume::~PublicVolume() {}
@@ -93,6 +97,7 @@ status_t PublicVolume::doDestroy() {
 }
 
 status_t PublicVolume::doMount() {
+    bool isVisible = getMountFlags() & MountFlags::kVisible;
     readMetadata();
 
     if (mFsType == "vfat" && vfat::IsSupported()) {
@@ -118,13 +123,13 @@ status_t PublicVolume::doMount() {
 
     mRawPath = StringPrintf("/mnt/media_rw/%s", stableName.c_str());
 
-    mFuseDefault = StringPrintf("/mnt/runtime/default/%s", stableName.c_str());
+    mSdcardFsDefault = StringPrintf("/mnt/runtime/default/%s", stableName.c_str());
-    mFuseRead = StringPrintf("/mnt/runtime/read/%s", stableName.c_str());
+    mSdcardFsRead = StringPrintf("/mnt/runtime/read/%s", stableName.c_str());
-    mFuseWrite = StringPrintf("/mnt/runtime/write/%s", stableName.c_str());
+    mSdcardFsWrite = StringPrintf("/mnt/runtime/write/%s", stableName.c_str());
-    mFuseFull = StringPrintf("/mnt/runtime/full/%s", stableName.c_str());
+    mSdcardFsFull = StringPrintf("/mnt/runtime/full/%s", stableName.c_str());
 
     setInternalPath(mRawPath);
-    if (getMountFlags() & MountFlags::kVisible) {
+    if (isVisible) {
         setPath(StringPrintf("/storage/%s", stableName.c_str()));
     } else {
         setPath(mRawPath);
@@ -136,13 +141,14 @@ status_t PublicVolume::doMount() {
     }
 
     if (mFsType == "vfat") {
-        if (vfat::Mount(mDevPath, mRawPath, false, false, false, AID_MEDIA_RW, AID_MEDIA_RW, 0007,
+        if (vfat::Mount(mDevPath, mRawPath, false, false, false, AID_ROOT,
-                        true)) {
+                        (isVisible ? AID_MEDIA_RW : AID_EXTERNAL_STORAGE), 0007, true)) {
             PLOG(ERROR) << getId() << " failed to mount " << mDevPath;
             return -EIO;
         }
     } else if (mFsType == "exfat") {
-        if (exfat::Mount(mDevPath, mRawPath, AID_MEDIA_RW, AID_MEDIA_RW, 0007)) {
+        if (exfat::Mount(mDevPath, mRawPath, AID_ROOT,
+                         (isVisible ? AID_MEDIA_RW : AID_EXTERNAL_STORAGE), 0007)) {
             PLOG(ERROR) << getId() << " failed to mount " << mDevPath;
             return -EIO;
         }
@@ -152,73 +158,108 @@ status_t PublicVolume::doMount() {
         initAsecStage();
     }
 
-    if (!(getMountFlags() & MountFlags::kVisible)) {
+    if (!isVisible) {
-        // Not visible to apps, so no need to spin up FUSE
+        // Not visible to apps, so no need to spin up sdcardfs or FUSE
         return OK;
     }
 
-    if (fs_prepare_dir(mFuseDefault.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+    if (mUseSdcardFs) {
-        fs_prepare_dir(mFuseRead.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+        if (fs_prepare_dir(mSdcardFsDefault.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseWrite.c_str(), 0700, AID_ROOT, AID_ROOT) ||
+            fs_prepare_dir(mSdcardFsRead.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        fs_prepare_dir(mFuseFull.c_str(), 0700, AID_ROOT, AID_ROOT)) {
+            fs_prepare_dir(mSdcardFsWrite.c_str(), 0700, AID_ROOT, AID_ROOT) ||
-        PLOG(ERROR) << getId() << " failed to create FUSE mount points";
+            fs_prepare_dir(mSdcardFsFull.c_str(), 0700, AID_ROOT, AID_ROOT)) {
-        return -errno;
+            PLOG(ERROR) << getId() << " failed to create sdcardfs mount points";
+            return -errno;
+        }
+
+        dev_t before = GetDevice(mSdcardFsFull);
+
+        int sdcardFsPid;
+        if (!(sdcardFsPid = fork())) {
+            if (getMountFlags() & MountFlags::kPrimary) {
+                // clang-format off
+                if (execl(kSdcardFsPath, kSdcardFsPath,
+                        "-u", "1023", // AID_MEDIA_RW
+                        "-g", "1023", // AID_MEDIA_RW
+                        "-U", std::to_string(getMountUserId()).c_str(),
+                        "-w",
+                        mRawPath.c_str(),
+                        stableName.c_str(),
+                        NULL)) {
+                    // clang-format on
+                    PLOG(ERROR) << "Failed to exec";
+                }
+            } else {
+                // clang-format off
+                if (execl(kSdcardFsPath, kSdcardFsPath,
+                        "-u", "1023", // AID_MEDIA_RW
+                        "-g", "1023", // AID_MEDIA_RW
+                        "-U", std::to_string(getMountUserId()).c_str(),
+                        mRawPath.c_str(),
+                        stableName.c_str(),
+                        NULL)) {
+                    // clang-format on
+                    PLOG(ERROR) << "Failed to exec";
+                }
+            }
+
+            LOG(ERROR) << "sdcardfs exiting";
+            _exit(1);
+        }
+
+        if (sdcardFsPid == -1) {
+            PLOG(ERROR) << getId() << " failed to fork";
+            return -errno;
+        }
+
+        nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
+        while (before == GetDevice(mSdcardFsFull)) {
+            LOG(DEBUG) << "Waiting for sdcardfs to spin up...";
+            usleep(50000);  // 50ms
+
+            nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
+            if (nanoseconds_to_milliseconds(now - start) > 5000) {
+                LOG(WARNING) << "Timed out while waiting for sdcardfs to spin up";
+                return -ETIMEDOUT;
+            }
+        }
+        /* sdcardfs will have exited already. The filesystem will still be running */
+        TEMP_FAILURE_RETRY(waitpid(sdcardFsPid, nullptr, 0));
     }
 
-    dev_t before = GetDevice(mFuseFull);
+    bool isFuse = base::GetBoolProperty(kPropFuse, false);
+    if (isFuse) {
+        // We need to mount FUSE *after* sdcardfs, since the FUSE daemon may depend
+        // on sdcardfs being up.
+        LOG(INFO) << "Mounting public fuse volume";
+        android::base::unique_fd fd;
+        int user_id = getMountUserId();
+        int result = MountUserFuse(user_id, getInternalPath(), stableName, &fd);
 
-    if (!(mFusePid = fork())) {
+        if (result != 0) {
-        if (getMountFlags() & MountFlags::kPrimary) {
+            LOG(ERROR) << "Failed to mount public fuse volume";
-            // clang-format off
+            doUnmount();
-            if (execl(kFusePath, kFusePath,
+            return -result;
-                    "-u", "1023", // AID_MEDIA_RW
+        }
-                    "-g", "1023", // AID_MEDIA_RW
+
-                    "-U", std::to_string(getMountUserId()).c_str(),
+        mFuseMounted = true;
-                    "-w",
+        auto callback = getMountCallback();
-                    mRawPath.c_str(),
+        if (callback) {
-                    stableName.c_str(),
+            bool is_ready = false;
-                    NULL)) {
+            callback->onVolumeChecking(std::move(fd), getPath(), getInternalPath(), &is_ready);
-                // clang-format on
+            if (!is_ready) {
-                PLOG(ERROR) << "Failed to exec";
+                LOG(ERROR) << "Failed to complete public volume mount";
-            }
+                doUnmount();
-        } else {
+                return -EIO;
-            // clang-format off
-            if (execl(kFusePath, kFusePath,
-                    "-u", "1023", // AID_MEDIA_RW
-                    "-g", "1023", // AID_MEDIA_RW
-                    "-U", std::to_string(getMountUserId()).c_str(),
-                    mRawPath.c_str(),
-                    stableName.c_str(),
-                    NULL)) {
-                // clang-format on
-                PLOG(ERROR) << "Failed to exec";
             }
         }
 
-        LOG(ERROR) << "FUSE exiting";
+        ConfigureReadAheadForFuse(GetFuseMountPathForUser(user_id, stableName), 256u);
-        _exit(1);
-    }
 
-    if (mFusePid == -1) {
+        // See comment in model/EmulatedVolume.cpp
-        PLOG(ERROR) << getId() << " failed to fork";
+        ConfigureMaxDirtyRatioForFuse(GetFuseMountPathForUser(user_id, stableName), 40u);
-        return -errno;
     }
 
-    nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
-    while (before == GetDevice(mFuseFull)) {
-        LOG(DEBUG) << "Waiting for FUSE to spin up...";
-        usleep(50000);  // 50ms
-
-        nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
-        if (nanoseconds_to_milliseconds(now - start) > 5000) {
-            LOG(WARNING) << "Timed out while waiting for FUSE to spin up";
-            return -ETIMEDOUT;
-        }
-    }
-    /* sdcardfs will have exited already. FUSE will still be running */
-    TEMP_FAILURE_RETRY(waitpid(mFusePid, nullptr, 0));
-    mFusePid = 0;
-
     return OK;
 }
 
@@ -229,24 +270,41 @@ status_t PublicVolume::doUnmount() {
     // error code and might cause broken behaviour in applications.
     KillProcessesUsingPath(getPath());
 
+    if (mFuseMounted) {
+        // Use UUID as stable name, if available
+        std::string stableName = getId();
+        if (!mFsUuid.empty()) {
+            stableName = mFsUuid;
+        }
+
+        if (UnmountUserFuse(getMountUserId(), getInternalPath(), stableName) != OK) {
+            PLOG(INFO) << "UnmountUserFuse failed on public fuse volume";
+            return -errno;
+        }
+
+        mFuseMounted = false;
+    }
+
     ForceUnmount(kAsecPath);
 
-    ForceUnmount(mFuseDefault);
+    if (mUseSdcardFs) {
-    ForceUnmount(mFuseRead);
+        ForceUnmount(mSdcardFsDefault);
-    ForceUnmount(mFuseWrite);
+        ForceUnmount(mSdcardFsRead);
-    ForceUnmount(mFuseFull);
+        ForceUnmount(mSdcardFsWrite);
+        ForceUnmount(mSdcardFsFull);
+
+        rmdir(mSdcardFsDefault.c_str());
+        rmdir(mSdcardFsRead.c_str());
+        rmdir(mSdcardFsWrite.c_str());
+        rmdir(mSdcardFsFull.c_str());
+
+        mSdcardFsDefault.clear();
+        mSdcardFsRead.clear();
+        mSdcardFsWrite.clear();
+        mSdcardFsFull.clear();
+    }
     ForceUnmount(mRawPath);
-
-    rmdir(mFuseDefault.c_str());
-    rmdir(mFuseRead.c_str());
-    rmdir(mFuseWrite.c_str());
-    rmdir(mFuseFull.c_str());
     rmdir(mRawPath.c_str());
-
-    mFuseDefault.clear();
-    mFuseRead.clear();
-    mFuseWrite.clear();
-    mFuseFull.clear();
     mRawPath.clear();
 
     return OK;

model/PublicVolume.h

@@ -27,7 +27,7 @@ namespace vold {
 /*
  * Shared storage provided by public (vfat) partition.
  *
- * Knows how to mount itself and then spawn a FUSE daemon to synthesize
+ * Knows how to mount itself and then spawn a sdcardfs daemon to synthesize
  * permissions.  AsecVolume and ObbVolume can be stacked above it.
  *
  * This volume is not inherently multi-user aware, so it has two possible
@@ -60,13 +60,16 @@ class PublicVolume : public VolumeBase {
     /* Mount point of raw partition */
     std::string mRawPath;
 
-    std::string mFuseDefault;
+    std::string mSdcardFsDefault;
-    std::string mFuseRead;
+    std::string mSdcardFsRead;
-    std::string mFuseWrite;
+    std::string mSdcardFsWrite;
-    std::string mFuseFull;
+    std::string mSdcardFsFull;
 
-    /* PID of FUSE wrapper */
+    /* Whether we mounted FUSE for this volume */
-    pid_t mFusePid;
+    bool mFuseMounted;
+
+    /* Whether to use sdcardfs for this volume */
+    bool mUseSdcardFs;
 
     /* Filesystem type */
     std::string mFsType;

model/StubVolume.cpp

@@ -16,6 +16,8 @@
 
 #include "StubVolume.h"
 
+#include <inttypes.h>
+
 #include <android-base/logging.h>
 #include <android-base/stringprintf.h>
 
@@ -24,7 +26,7 @@ using android::base::StringPrintf;
 namespace android {
 namespace vold {
 
-StubVolume::StubVolume(int id, const std::string& sourcePath, const std::string& mountPath,
+StubVolume::StubVolume(dev_t id, const std::string& sourcePath, const std::string& mountPath,
                        const std::string& fsType, const std::string& fsUuid,
                        const std::string& fsLabel)
     : VolumeBase(Type::kStub),
@@ -33,7 +35,7 @@ StubVolume::StubVolume(int id, const std::string& sourcePath, const std::string&
       mFsType(fsType),
       mFsUuid(fsUuid),
       mFsLabel(fsLabel) {
-    setId(StringPrintf("stub:%d", id));
+    setId(StringPrintf("stub:%llu", (unsigned long long)id));
 }
 
 StubVolume::~StubVolume() {}

model/StubVolume.h

@@ -31,7 +31,7 @@ namespace vold {
  */
 class StubVolume : public VolumeBase {
   public:
-    StubVolume(int id, const std::string& sourcePath, const std::string& mountPath,
+    StubVolume(dev_t id, const std::string& sourcePath, const std::string& mountPath,
                const std::string& fsType, const std::string& fsUuid, const std::string& fsLabel);
     virtual ~StubVolume();
 

model/VolumeBase.cpp

@@ -143,6 +143,16 @@ status_t VolumeBase::setInternalPath(const std::string& internalPath) {
     return OK;
 }
 
+status_t VolumeBase::setMountCallback(
+        const android::sp<android::os::IVoldMountCallback>& callback) {
+    mMountCallback = callback;
+    return OK;
+}
+
+sp<android::os::IVoldMountCallback> VolumeBase::getMountCallback() const {
+    return mMountCallback;
+}
+
 android::sp<android::os::IVoldListener> VolumeBase::getListener() const {
     if (mSilent) {
         return nullptr;
@@ -176,7 +186,8 @@ status_t VolumeBase::create() {
 
     auto listener = getListener();
     if (listener) {
-        listener->onVolumeCreated(getId(), static_cast<int32_t>(mType), mDiskId, mPartGuid);
+        listener->onVolumeCreated(getId(), static_cast<int32_t>(mType), mDiskId, mPartGuid,
+                                  mMountUserId);
     }
 
     setState(State::kUnmounted);
@@ -221,9 +232,14 @@ status_t VolumeBase::mount() {
     status_t res = doMount();
     setState(res == OK ? State::kMounted : State::kUnmountable);
 
+    if (res == OK) {
+        doPostMount();
+    }
     return res;
 }
 
+void VolumeBase::doPostMount() {}
+
 status_t VolumeBase::unmount() {
     if (mState != State::kMounted) {
         LOG(WARNING) << getId() << " unmount requires state mounted";
@@ -263,6 +279,11 @@ status_t VolumeBase::doFormat(const std::string& fsType) {
     return -ENOTSUP;
 }
 
+std::string VolumeBase::getRootPath() const {
+    // Usually the same as the internal path, except for emulated volumes.
+    return getInternalPath();
+}
+
 std::ostream& VolumeBase::operator<<(std::ostream& stream) const {
     return stream << " VolumeBase{id=" << mId << ",mountFlags=" << mMountFlags
                   << ",mountUserId=" << mMountUserId << "}";

model/VolumeBase.h

@@ -19,6 +19,7 @@
 
 #include "Utils.h"
 #include "android/os/IVoldListener.h"
+#include "android/os/IVoldMountCallback.h"
 
 #include <cutils/multiuser.h>
 #include <utils/Errors.h>
@@ -87,11 +88,13 @@ class VolumeBase {
     State getState() const { return mState; }
     const std::string& getPath() const { return mPath; }
     const std::string& getInternalPath() const { return mInternalPath; }
+    const std::list<std::shared_ptr<VolumeBase>>& getVolumes() const { return mVolumes; }
 
     status_t setDiskId(const std::string& diskId);
     status_t setPartGuid(const std::string& partGuid);
     status_t setMountFlags(int mountFlags);
     status_t setMountUserId(userid_t mountUserId);
+    status_t setMountCallback(const android::sp<android::os::IVoldMountCallback>& callback);
     status_t setSilent(bool silent);
 
     void addVolume(const std::shared_ptr<VolumeBase>& volume);
@@ -107,6 +110,8 @@ class VolumeBase {
     status_t unmount();
     status_t format(const std::string& fsType);
 
+    virtual std::string getRootPath() const;
+
     std::ostream& operator<<(std::ostream& stream) const;
 
   protected:
@@ -115,6 +120,7 @@ class VolumeBase {
     virtual status_t doCreate();
     virtual status_t doDestroy();
     virtual status_t doMount() = 0;
+    virtual void doPostMount();
     virtual status_t doUnmount() = 0;
     virtual status_t doFormat(const std::string& fsType);
 
@@ -123,6 +129,7 @@ class VolumeBase {
     status_t setInternalPath(const std::string& internalPath);
 
     android::sp<android::os::IVoldListener> getListener() const;
+    android::sp<android::os::IVoldMountCallback> getMountCallback() const;
 
   private:
     /* ID that uniquely references volume while alive */
@@ -147,6 +154,7 @@ class VolumeBase {
     std::string mInternalPath;
     /* Flag indicating that volume should emit no events */
     bool mSilent;
+    android::sp<android::os::IVoldMountCallback> mMountCallback;
 
     /* Volumes stacked on top of this volume */
     std::list<std::shared_ptr<VolumeBase>> mVolumes;

tests/VoldNativeServiceValidation_test.cpp

@@ -24,9 +24,9 @@ using namespace std::literals;
 
 namespace android::vold {
 
-class VoldNativeServiceValidationTest : public testing::Test {};
+class VoldServiceValidationTest : public testing::Test {};
 
-TEST_F(VoldNativeServiceValidationTest, CheckArgumentPathTest) {
+TEST_F(VoldServiceValidationTest, CheckArgumentPathTest) {
     EXPECT_TRUE(CheckArgumentPath("/").isOk());
     EXPECT_TRUE(CheckArgumentPath("/1/2").isOk());
     EXPECT_TRUE(CheckArgumentPath("/1/2/").isOk());

vdc.cpp

@@ -99,6 +99,8 @@ int main(int argc, char** argv) {
         checkStatus(args, vold->fdeEnable(passwordType, "", encryptionFlags));
     } else if (args[0] == "cryptfs" && args[1] == "mountdefaultencrypted") {
         checkStatus(args, vold->mountDefaultEncrypted());
+    } else if (args[0] == "volume" && args[1] == "abort_fuse") {
+        checkStatus(args, vold->abortFuse());
     } else if (args[0] == "volume" && args[1] == "shutdown") {
         checkStatus(args, vold->shutdown());
     } else if (args[0] == "volume" && args[1] == "reset") {