debuggerd: report crashes even when out of file descriptors.

Use nasty clone hacks to let us close random file descriptors to be
able to connect to debuggerd when the fd table is full.

Bug: http://b/32013594
Test: crasher exhaustfd-SIGSEGV
Change-Id: I47772e9a5994da4473bd935b105d9c36827c017a

debuggerd/client/debuggerd_client.cpp

@@ -31,6 +31,7 @@
 #include <errno.h>
 #include <inttypes.h>
 #include <pthread.h>
+#include <sched.h>
 #include <signal.h>
 #include <stddef.h>
 #include <stdio.h>
@@ -41,6 +42,7 @@
 #include <sys/socket.h>
 #include <sys/syscall.h>
 #include <sys/un.h>
+#include <sys/wait.h>
 #include <unistd.h>
 
 #include "private/libc_logging.h"
@@ -56,6 +58,13 @@
 
 static debuggerd_callbacks_t g_callbacks;
 
+// Don't use __libc_fatal because it exits via abort, which might put us back into a signal handler.
+#define fatal(...)                                             \
+  do {                                                         \
+    __libc_format_log(ANDROID_LOG_FATAL, "libc", __VA_ARGS__); \
+    _exit(1);                                                  \
+  } while (0)
+
 static int socket_abstract_client(const char* name, int type) {
   sockaddr_un addr;
 
@@ -188,7 +197,7 @@ static bool have_siginfo(int signum) {
   return result;
 }
 
-static void send_debuggerd_packet() {
+static void send_debuggerd_packet(pid_t crashing_tid, pid_t pseudothread_tid) {
   // Mutex to prevent multiple crashing threads from trying to talk
   // to debuggerd at the same time.
   static pthread_mutex_t crash_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -218,7 +227,8 @@ static void send_debuggerd_packet() {
   // that's actually in our process.
   debugger_msg_t msg;
   msg.action = DEBUGGER_ACTION_CRASH;
-  msg.tid = gettid();
+  msg.tid = crashing_tid;
+  msg.ignore_tid = pseudothread_tid;
   msg.abort_msg_address = 0;
 
   if (g_callbacks.get_abort_message) {
@@ -229,11 +239,9 @@ static void send_debuggerd_packet() {
   if (ret == sizeof(msg)) {
     char debuggerd_ack;
     ret = TEMP_FAILURE_RETRY(read(s, &debuggerd_ack, 1));
-    int saved_errno = errno;
     if (g_callbacks.post_dump) {
       g_callbacks.post_dump();
     }
-    errno = saved_errno;
   } else {
     // read or write failed -- broken connection?
     __libc_format_log(ANDROID_LOG_FATAL, "libc", "Failed while talking to debuggerd: %s",
@@ -243,6 +251,33 @@ static void send_debuggerd_packet() {
   close(s);
 }
 
+struct debugger_thread_info {
+  pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+  pid_t crashing_tid;
+  pid_t pseudothread_tid;
+  int signal_number;
+  siginfo_t* info;
+};
+
+// Logging and contacting debuggerd requires free file descriptors, which we might not have.
+// Work around this by spawning a "thread" that shares its parent's address space, but not its file
+// descriptor table, so that we can close random file descriptors without affecting the original
+// process. Note that this doesn't go through pthread_create, so TLS is shared with the spawning
+// process.
+static void* pseudothread_stack;
+static int debuggerd_dispatch_pseudothread(void* arg) {
+  debugger_thread_info* thread_info = static_cast<debugger_thread_info*>(arg);
+
+  for (int i = 3; i < 1024; ++i) {
+    close(i);
+  }
+
+  log_signal_summary(thread_info->signal_number, thread_info->info);
+  send_debuggerd_packet(thread_info->crashing_tid, thread_info->pseudothread_tid);
+  pthread_mutex_unlock(&thread_info->mutex);
+  return 0;
+}
+
 /*
  * Catches fatal signals so we can ask debuggerd to ptrace us before
  * we crash.
@@ -254,9 +289,25 @@ static void debuggerd_signal_handler(int signal_number, siginfo_t* info, void*)
     info = nullptr;
   }
 
-  log_signal_summary(signal_number, info);
+  debugger_thread_info thread_info = {
+    .crashing_tid = gettid(),
+    .signal_number = signal_number,
+    .info = info
+  };
+
+  pthread_mutex_lock(&thread_info.mutex);
+  pid_t child_pid = clone(debuggerd_dispatch_pseudothread, pseudothread_stack,
+                          CLONE_THREAD | CLONE_SIGHAND | CLONE_VM | CLONE_CHILD_SETTID,
+                          &thread_info, nullptr, nullptr, &thread_info.pseudothread_tid);
+
+  if (child_pid == -1) {
+    fatal("failed to spawn debuggerd dispatch thread: %s", strerror(errno));
+  }
+
+  // Wait for the child to finish and unlock the mutex.
+  // This relies on bionic behavior that isn't guaranteed by the standard.
+  pthread_mutex_lock(&thread_info.mutex);
 
-  send_debuggerd_packet();
 
   // We need to return from the signal handler so that debuggerd can dump the
   // thread that crashed, but returning here does not guarantee that the signal
@@ -281,9 +332,7 @@ static void debuggerd_signal_handler(int signal_number, siginfo_t* info, void*)
 
   int rc = syscall(SYS_rt_tgsigqueueinfo, getpid(), gettid(), signal_number, info);
   if (rc != 0) {
-    __libc_format_log(ANDROID_LOG_FATAL, "libc", "failed to resend signal during crash: %s",
-                      strerror(errno));
-    _exit(0);
+    fatal("failed to resend signal during crash: %s", strerror(errno));
   }
 }
 
@@ -292,6 +341,23 @@ void debuggerd_init(debuggerd_callbacks_t* callbacks) {
     g_callbacks = *callbacks;
   }
 
+  void* thread_stack_allocation =
+    mmap(nullptr, PAGE_SIZE * 3, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  if (thread_stack_allocation == MAP_FAILED) {
+    fatal("failed to allocate debuggerd thread stack");
+  }
+
+  char* stack = static_cast<char*>(thread_stack_allocation) + PAGE_SIZE;
+  if (mprotect(stack, PAGE_SIZE, PROT_READ | PROT_WRITE) != 0) {
+    fatal("failed to mprotect debuggerd thread stack");
+  }
+
+  // Stack grows negatively, set it to the last byte in the page...
+  stack = (stack + PAGE_SIZE - 1);
+  // and align it.
+  stack -= 15;
+  pseudothread_stack = stack;
+
   struct sigaction action;
   memset(&action, 0, sizeof(action));
   sigemptyset(&action.sa_mask);

debuggerd/crasher.cpp

@@ -1,5 +1,6 @@
 #include <assert.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <pthread.h>
 #include <sched.h>
 #include <signal.h>
@@ -141,7 +142,13 @@ static int do_action(const char* arg)
 {
     fprintf(stderr, "%s: init pid=%d tid=%d\n", __progname, getpid(), gettid());
 
-    if (!strncmp(arg, "thread-", strlen("thread-"))) {
+    if (!strncmp(arg, "exhaustfd-", strlen("exhaustfd-"))) {
+      errno = 0;
+      while (errno != EMFILE) {
+        open("/dev/null", O_RDONLY);
+      }
+      return do_action(arg + strlen("exhaustfd-"));
+    } else if (!strncmp(arg, "thread-", strlen("thread-"))) {
         return do_action_on_thread(arg + strlen("thread-"));
     } else if (!strcmp(arg, "SIGSEGV-non-null")) {
         sigsegv_non_null();
@@ -208,6 +215,8 @@ static int do_action(const char* arg)
     fprintf(stderr, "  SIGTRAP               cause a SIGTRAP\n");
     fprintf(stderr, "prefix any of the above with 'thread-' to not run\n");
     fprintf(stderr, "on the process' main thread.\n");
+    fprintf(stderr, "prefix any of the above with 'exhaustfd-' to exhaust\n");
+    fprintf(stderr, "all available file descriptors before crashing.\n");
     return EXIT_SUCCESS;
 }
 

debuggerd/debuggerd.cpp

@@ -69,6 +69,7 @@ struct debugger_request_t {
   debugger_action_t action;
   pid_t pid, tid;
   uid_t uid, gid;
+  pid_t ignore_tid;
   uintptr_t abort_msg_address;
 };
 
@@ -215,6 +216,7 @@ static int read_request(int fd, debugger_request_t* out_request) {
 
   out_request->action = static_cast<debugger_action_t>(msg.action);
   out_request->tid = msg.tid;
+  out_request->ignore_tid = msg.ignore_tid;
   out_request->pid = cr.pid;
   out_request->uid = cr.uid;
   out_request->gid = cr.gid;
@@ -408,7 +410,7 @@ static void redirect_to_32(int fd, debugger_request_t* request) {
 }
 #endif
 
-static void ptrace_siblings(pid_t pid, pid_t main_tid, std::set<pid_t>& tids) {
+static void ptrace_siblings(pid_t pid, pid_t main_tid, pid_t ignore_tid, std::set<pid_t>& tids) {
   char task_path[64];
 
   snprintf(task_path, sizeof(task_path), "/proc/%d/task", pid);
@@ -434,7 +436,7 @@ static void ptrace_siblings(pid_t pid, pid_t main_tid, std::set<pid_t>& tids) {
       continue;
     }
 
-    if (tid == main_tid) {
+    if (tid == main_tid || tid == ignore_tid) {
       continue;
     }
 
@@ -583,7 +585,7 @@ static void worker_process(int fd, debugger_request_t& request) {
 
   std::set<pid_t> siblings;
   if (!attach_gdb) {
-    ptrace_siblings(request.pid, request.tid, siblings);
+    ptrace_siblings(request.pid, request.tid, request.ignore_tid, siblings);
   }
 
   // Generate the backtrace map before dropping privileges.

debuggerd/include/debuggerd/client.h

@@ -43,6 +43,7 @@ typedef enum {
 typedef struct __attribute__((packed)) {
   int32_t action;
   pid_t tid;
+  pid_t ignore_tid;
   uint64_t abort_msg_address;
 } debugger_msg_t;