init: Wait for daemon to fully spin up all threads

During selinux transition, daemon will notify `init` process
by writing to file "/metadata/ota/daemon-alive-indicator".

Init will wait until daemon notifies it. Furthermore, daemon
will only write to that file once all threads are spin up
and attached to dm-user misc devices.

Once snapshot-merge is completed, this file will be removed.
Additionally, during boot, init will also ensure that
there are no stale files and will try to remove just
before selinux transition.

Bug: 262407519
Test: OTA on Pixel - Verify new file exits and init waits until daemon
is fully up.
Change-Id: Iabef58ad282d80a7afa493e9df9468ae41a13e44
Signed-off-by: Akilesh Kailash <akailash@google.com>

fs_mgr/libsnapshot/snapshot.cpp

@@ -1498,6 +1498,7 @@ void SnapshotManager::AcknowledgeMergeSuccess(LockedFile* lock) {
     if (UpdateUsesUserSnapshots(lock) && !device()->IsTestDevice()) {
         if (snapuserd_client_) {
             snapuserd_client_->DetachSnapuserd();
+            snapuserd_client_->RemoveTransitionedDaemonIndicator();
             snapuserd_client_ = nullptr;
         }
     }

fs_mgr/libsnapshot/snapuserd/Android.bp

@@ -37,11 +37,13 @@ cc_defaults {
 cc_library_static {
     name: "libsnapshot_snapuserd",
     defaults: [
+        "fs_mgr_defaults",
         "libsnapshot_snapuserd_defaults",
     ],
     recovery_available: true,
     static_libs: [
         "libcutils_sockets",
+        "libfs_mgr",
     ],
     shared_libs: [
         "libbase",
@@ -49,6 +51,7 @@ cc_library_static {
     ],
     export_include_dirs: ["include"],
     ramdisk_available: true,
+    vendor_ramdisk_available: true,
 }
 
 cc_defaults {
@@ -86,6 +89,7 @@ cc_defaults {
         "libgflags",
         "liblog",
         "libsnapshot_cow",
+        "libsnapshot_snapuserd",
         "libz",
         "liblz4",
         "libext4_utils",

fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_client.h

@@ -32,6 +32,7 @@ static constexpr uint32_t PACKET_SIZE = 512;
 
 static constexpr char kSnapuserdSocket[] = "snapuserd";
 static constexpr char kSnapuserdSocketProxy[] = "snapuserd_proxy";
+static constexpr char kDaemonAliveIndicator[] = "daemon-alive-indicator";
 
 // Ensure that the second-stage daemon for snapuserd is running.
 bool EnsureSnapuserdStarted();
@@ -44,9 +45,11 @@ class SnapuserdClient {
     std::string Receivemsg();
 
     bool ValidateConnection();
+    std::string GetDaemonAliveIndicatorPath();
 
   public:
     explicit SnapuserdClient(android::base::unique_fd&& sockfd);
+    SnapuserdClient(){};
 
     static std::unique_ptr<SnapuserdClient> Connect(const std::string& socket_name,
                                                     std::chrono::milliseconds timeout_ms);
@@ -91,6 +94,17 @@ class SnapuserdClient {
     // Check the update verification status - invoked by update_verifier during
     // boot
     bool QueryUpdateVerification();
+
+    // Check if Snapuser daemon is ready post selinux transition after OTA boot
+    // This is invoked only by init as there is no sockets setup yet during
+    // selinux transition
+    bool IsTransitionedDaemonReady();
+
+    // Remove the daemon-alive-indicator path post snapshot merge
+    bool RemoveTransitionedDaemonIndicator();
+
+    // Notify init that snapuserd daemon is ready post selinux transition
+    void NotifyTransitionDaemonIsReady();
 };
 
 }  // namespace snapshot

fs_mgr/libsnapshot/snapuserd/snapuserd_client.cpp

@@ -29,10 +29,12 @@
 #include <chrono>
 #include <sstream>
 
+#include <android-base/file.h>
 #include <android-base/logging.h>
 #include <android-base/parseint.h>
 #include <android-base/properties.h>
 #include <android-base/strings.h>
+#include <fs_mgr/file_wait.h>
 #include <snapuserd/snapuserd_client.h>
 
 namespace android {
@@ -279,5 +281,42 @@ bool SnapuserdClient::QueryUpdateVerification() {
     return response == "success";
 }
 
+std::string SnapuserdClient::GetDaemonAliveIndicatorPath() {
+    return "/metadata/ota/" + std::string(kDaemonAliveIndicator);
+}
+
+bool SnapuserdClient::IsTransitionedDaemonReady() {
+    if (!android::fs_mgr::WaitForFile(GetDaemonAliveIndicatorPath(), 10s)) {
+        LOG(ERROR) << "Timed out waiting for daemon indicator path: "
+                   << GetDaemonAliveIndicatorPath();
+        return false;
+    }
+
+    return true;
+}
+
+bool SnapuserdClient::RemoveTransitionedDaemonIndicator() {
+    std::string error;
+    std::string filePath = GetDaemonAliveIndicatorPath();
+    if (!android::base::RemoveFileIfExists(filePath, &error)) {
+        LOG(ERROR) << "Failed to remove DaemonAliveIndicatorPath - error: " << error;
+        return false;
+    }
+
+    if (!android::fs_mgr::WaitForFileDeleted(filePath, 5s)) {
+        LOG(ERROR) << "Timed out waiting for " << filePath << " to unlink";
+        return false;
+    }
+
+    return true;
+}
+
+void SnapuserdClient::NotifyTransitionDaemonIsReady() {
+    if (!android::base::WriteStringToFile("1", GetDaemonAliveIndicatorPath())) {
+        PLOG(ERROR) << "Unable to write daemon alive indicator path: "
+                    << GetDaemonAliveIndicatorPath();
+    }
+}
+
 }  // namespace snapshot
 }  // namespace android

fs_mgr/libsnapshot/snapuserd/snapuserd_daemon.cpp

@@ -119,6 +119,12 @@ bool Daemon::StartServerForUserspaceSnapshots(int arg_start, int argc, char** ar
         }
     }
 
+    // We reach this point only during selinux transition during device boot.
+    // At this point, all threads are spin up and are ready to serve the I/O
+    // requests for dm-user. Lets inform init.
+    auto client = std::make_unique<SnapuserdClient>();
+    client->NotifyTransitionDaemonIsReady();
+
     // Skip the accept() call to avoid spurious log spam. The server will still
     // run until all handlers have completed.
     return user_server_.WaitForSocket();

init/snapuserd_transition.cpp

@@ -112,6 +112,10 @@ void LaunchFirstStageSnapuserd(SnapshotDriver driver) {
 
     setenv(kSnapuserdFirstStagePidVar, std::to_string(pid).c_str(), 1);
 
+    if (!client->RemoveTransitionedDaemonIndicator()) {
+        LOG(ERROR) << "RemoveTransitionedDaemonIndicator failed";
+    }
+
     LOG(INFO) << "Relaunched snapuserd with pid: " << pid;
 }
 
@@ -263,6 +267,19 @@ void SnapuserdSelinuxHelper::FinishTransition() {
  * we may see audit logs.
  */
 bool SnapuserdSelinuxHelper::TestSnapuserdIsReady() {
+    // Wait for the daemon to be fully up. Daemon will write to path
+    // /metadata/ota/daemon-alive-indicator only when all the threads
+    // are ready and attached to dm-user.
+    //
+    // This check will fail for GRF devices with vendor on Android S.
+    // snapuserd binary from Android S won't be able to communicate
+    // and hence, we will fallback and issue I/O to verify
+    // the presence of daemon.
+    auto client = std::make_unique<SnapuserdClient>();
+    if (!client->IsTransitionedDaemonReady()) {
+        LOG(ERROR) << "IsTransitionedDaemonReady failed";
+    }
+
     std::string dev = "/dev/block/mapper/system"s + fs_mgr_get_slot_suffix();
     android::base::unique_fd fd(open(dev.c_str(), O_RDONLY | O_DIRECT));
     if (fd < 0) {