From b999b82eb79b6b8aaabcca2a65028158fcb246f3 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Wed, 9 Feb 2022 17:57:21 -0800 Subject: [PATCH] Dump threads in tombstone fallback path. When dumping a tombstone using the fallback path, only the main thread was showing up. Modify the code to dump the threads using a slightly different path for the tombstone generation code. In addition, while looking at this code, two MTE variables were not set in the tombstone fallback code. Added those variables so MTE devices will work properly in this fallback path. Modified the tombstone unit tests for seccomp to have multiple threads and verify those threads show up in the tombstone. Bug: 208933016 Test: Ran unit tests. Test: Ran debuggerd on a privileged process and verified Test: all threads dumped. Also verified that the tagged_addr_ctrl Test: variable is present on the raven device. Change-Id: I16eadb0cc2c37a7dbc5cac16af9b5051008b5127 --- debuggerd/debuggerd_test.cpp | 27 ++- debuggerd/handler/debuggerd_fallback.cpp | 61 ++---- debuggerd/libdebuggerd/tombstone.cpp | 37 +++- debuggerd/libdebuggerd/tombstone_proto.cpp | 192 ++++++++++-------- .../seccomp_policy/crash_dump.arm64.policy | 2 +- .../seccomp_policy/crash_dump.policy.def | 2 +- debuggerd/util.cpp | 22 ++ debuggerd/util.h | 2 + 8 files changed, 196 insertions(+), 149 deletions(-) diff --git a/debuggerd/debuggerd_test.cpp b/debuggerd/debuggerd_test.cpp index 0737612e0..85adbeaa5 100644 --- a/debuggerd/debuggerd_test.cpp +++ b/debuggerd/debuggerd_test.cpp @@ -1409,6 +1409,16 @@ __attribute__((noinline)) extern "C" bool raise_debugger_signal(DebuggerdDumpTyp return true; } +extern "C" void foo() { + LOG(INFO) << "foo"; + std::this_thread::sleep_for(1s); +} + +extern "C" void bar() { + LOG(INFO) << "bar"; + std::this_thread::sleep_for(1s); +} + TEST_F(CrasherTest, seccomp_tombstone) { int intercept_result; unique_fd output_fd; @@ -1416,6 +1426,11 @@ TEST_F(CrasherTest, seccomp_tombstone) { static const auto dump_type = kDebuggerdTombstone; StartProcess( []() { + std::thread a(foo); + std::thread b(bar); + + std::this_thread::sleep_for(100ms); + raise_debugger_signal(dump_type); _exit(0); }, @@ -1430,16 +1445,8 @@ TEST_F(CrasherTest, seccomp_tombstone) { std::string result; ConsumeFd(std::move(output_fd), &result); ASSERT_BACKTRACE_FRAME(result, "raise_debugger_signal"); -} - -extern "C" void foo() { - LOG(INFO) << "foo"; - std::this_thread::sleep_for(1s); -} - -extern "C" void bar() { - LOG(INFO) << "bar"; - std::this_thread::sleep_for(1s); + ASSERT_BACKTRACE_FRAME(result, "foo"); + ASSERT_BACKTRACE_FRAME(result, "bar"); } TEST_F(CrasherTest, seccomp_backtrace) { diff --git a/debuggerd/handler/debuggerd_fallback.cpp b/debuggerd/handler/debuggerd_fallback.cpp index baf994fa1..4c1f9eb47 100644 --- a/debuggerd/handler/debuggerd_fallback.cpp +++ b/debuggerd/handler/debuggerd_fallback.cpp @@ -98,32 +98,6 @@ static void debuggerd_fallback_tombstone(int output_fd, int proto_fd, ucontext_t __linker_disable_fallback_allocator(); } -static void iterate_siblings(bool (*callback)(pid_t, int), int output_fd) { - pid_t current_tid = gettid(); - char buf[BUFSIZ]; - snprintf(buf, sizeof(buf), "/proc/%d/task", current_tid); - DIR* dir = opendir(buf); - - if (!dir) { - async_safe_format_log(ANDROID_LOG_ERROR, "libc", "failed to open %s: %s", buf, strerror(errno)); - return; - } - - struct dirent* ent; - while ((ent = readdir(dir))) { - char* end; - long tid = strtol(ent->d_name, &end, 10); - if (end == ent->d_name || *end != '\0') { - continue; - } - - if (tid != current_tid) { - callback(tid, output_fd); - } - } - closedir(dir); -} - static bool forward_output(int src_fd, int dst_fd, pid_t expected_tid) { // Make sure the thread actually got the signal. struct pollfd pfd = { @@ -216,21 +190,21 @@ static void trace_handler(siginfo_t* info, ucontext_t* ucontext) { } // Only allow one thread to perform a trace at a time. - static pthread_mutex_t trace_mutex = PTHREAD_MUTEX_INITIALIZER; - int ret = pthread_mutex_trylock(&trace_mutex); - if (ret != 0) { - async_safe_format_log(ANDROID_LOG_INFO, "libc", "pthread_mutex_try_lock failed: %s", - strerror(ret)); + static std::mutex trace_mutex; + if (!trace_mutex.try_lock()) { + async_safe_format_log(ANDROID_LOG_INFO, "libc", "trace lock failed"); return; } + std::lock_guard scoped_lock(trace_mutex, std::adopt_lock); + // Fetch output fd from tombstoned. unique_fd tombstone_socket, output_fd; if (!tombstoned_connect(getpid(), &tombstone_socket, &output_fd, nullptr, kDebuggerdNativeBacktrace)) { async_safe_format_log(ANDROID_LOG_ERROR, "libc", "missing crash_dump_fallback() in selinux policy?"); - goto exit; + return; } dump_backtrace_header(output_fd.get()); @@ -239,15 +213,15 @@ static void trace_handler(siginfo_t* info, ucontext_t* ucontext) { debuggerd_fallback_trace(output_fd.get(), ucontext); // Send a signal to all of our siblings, asking them to dump their stack. - iterate_siblings( - [](pid_t tid, int output_fd) { + pid_t current_tid = gettid(); + if (!iterate_tids(current_tid, [&output_fd](pid_t tid) { // Use a pipe, to be able to detect situations where the thread gracefully exits before // receiving our signal. unique_fd pipe_read, pipe_write; if (!Pipe(&pipe_read, &pipe_write)) { async_safe_format_log(ANDROID_LOG_ERROR, "libc", "failed to create pipe: %s", strerror(errno)); - return false; + return; } uint64_t expected = pack_thread_fd(-1, -1); @@ -257,7 +231,7 @@ static void trace_handler(siginfo_t* info, ucontext_t* ucontext) { async_safe_format_log(ANDROID_LOG_ERROR, "libc", "thread %d is already outputting to fd %d?", tid, fd); close(sent_fd); - return false; + return; } siginfo_t siginfo = {}; @@ -269,10 +243,10 @@ static void trace_handler(siginfo_t* info, ucontext_t* ucontext) { if (syscall(__NR_rt_tgsigqueueinfo, getpid(), tid, BIONIC_SIGNAL_DEBUGGER, &siginfo) != 0) { async_safe_format_log(ANDROID_LOG_ERROR, "libc", "failed to send trace signal to %d: %s", tid, strerror(errno)); - return false; + return; } - bool success = forward_output(pipe_read.get(), output_fd, tid); + bool success = forward_output(pipe_read.get(), output_fd.get(), tid); if (!success) { async_safe_format_log(ANDROID_LOG_ERROR, "libc", "timeout expired while waiting for thread %d to dump", tid); @@ -288,15 +262,14 @@ static void trace_handler(siginfo_t* info, ucontext_t* ucontext) { } } - return true; - }, - output_fd.get()); + return; + })) { + async_safe_format_log(ANDROID_LOG_ERROR, "libc", "failed to open /proc/%d/task: %s", + current_tid, strerror(errno)); + } dump_backtrace_footer(output_fd.get()); tombstoned_notify_completion(tombstone_socket.get()); - -exit: - pthread_mutex_unlock(&trace_mutex); } static void crash_handler(siginfo_t* info, ucontext_t* ucontext, void* abort_message) { diff --git a/debuggerd/libdebuggerd/tombstone.cpp b/debuggerd/libdebuggerd/tombstone.cpp index f21a20378..14caaf6d6 100644 --- a/debuggerd/libdebuggerd/tombstone.cpp +++ b/debuggerd/libdebuggerd/tombstone.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -73,22 +74,40 @@ void engrave_tombstone_ucontext(int tombstone_fd, int proto_fd, uint64_t abort_m std::map threads; threads[tid] = ThreadInfo{ - .registers = std::move(regs), - .uid = uid, - .tid = tid, - .thread_name = std::move(thread_name), - .pid = pid, - .command_line = std::move(command_line), - .selinux_label = std::move(selinux_label), - .siginfo = siginfo, + .registers = std::move(regs), .uid = uid, .tid = tid, .thread_name = std::move(thread_name), + .pid = pid, .command_line = std::move(command_line), .selinux_label = std::move(selinux_label), + .siginfo = siginfo, +#if defined(__aarch64__) + // Only supported on aarch64 for now. + .tagged_addr_ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), + .pac_enabled_keys = prctl(PR_PAC_GET_ENABLED_KEYS, 0, 0, 0, 0), +#endif }; + if (pid == tid) { + const ThreadInfo& thread = threads[pid]; + if (!iterate_tids(pid, [&threads, &thread](pid_t tid) { + threads[tid] = ThreadInfo{ + .uid = thread.uid, + .tid = tid, + .pid = thread.pid, + .command_line = thread.command_line, + .thread_name = get_thread_name(tid), + .tagged_addr_ctrl = thread.tagged_addr_ctrl, + .pac_enabled_keys = thread.pac_enabled_keys, + }; + })) { + async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, "failed to open /proc/%d/task: %s", pid, + strerror(errno)); + } + } unwindstack::UnwinderFromPid unwinder(kMaxFrames, pid, unwindstack::Regs::CurrentArch()); auto process_memory = unwindstack::Memory::CreateProcessMemoryCached(getpid()); unwinder.SetProcessMemory(process_memory); if (!unwinder.Init()) { - async_safe_fatal("failed to init unwinder object"); + async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, "failed to init unwinder object"); + return; } ProcessInfo process_info; diff --git a/debuggerd/libdebuggerd/tombstone_proto.cpp b/debuggerd/libdebuggerd/tombstone_proto.cpp index b7d5bc452..3e31bb790 100644 --- a/debuggerd/libdebuggerd/tombstone_proto.cpp +++ b/debuggerd/libdebuggerd/tombstone_proto.cpp @@ -48,6 +48,7 @@ #include #include +#include #include #include #include @@ -346,6 +347,93 @@ void fill_in_backtrace_frame(BacktraceFrame* f, const unwindstack::FrameData& fr f->set_build_id(frame.map_info->GetPrintableBuildID()); } +static void dump_registers(unwindstack::Unwinder* unwinder, + const std::unique_ptr& regs, Thread& thread, + bool memory_dump) { + if (regs == nullptr) { + return; + } + + unwindstack::Maps* maps = unwinder->GetMaps(); + unwindstack::Memory* memory = unwinder->GetProcessMemory().get(); + + regs->IterateRegisters([&thread, memory_dump, maps, memory](const char* name, uint64_t value) { + Register r; + r.set_name(name); + r.set_u64(value); + *thread.add_registers() = r; + + if (memory_dump) { + MemoryDump dump; + + dump.set_register_name(name); + std::shared_ptr map_info = maps->Find(untag_address(value)); + if (map_info) { + dump.set_mapping_name(map_info->name()); + } + + constexpr size_t kNumBytesAroundRegister = 256; + constexpr size_t kNumTagsAroundRegister = kNumBytesAroundRegister / kTagGranuleSize; + char buf[kNumBytesAroundRegister]; + uint8_t tags[kNumTagsAroundRegister]; + ssize_t bytes = dump_memory(buf, sizeof(buf), tags, sizeof(tags), &value, memory); + if (bytes == -1) { + return; + } + dump.set_begin_address(value); + dump.set_memory(buf, bytes); + + bool has_tags = false; +#if defined(__aarch64__) + for (size_t i = 0; i < kNumTagsAroundRegister; ++i) { + if (tags[i] != 0) { + has_tags = true; + } + } +#endif // defined(__aarch64__) + + if (has_tags) { + dump.mutable_arm_mte_metadata()->set_memory_tags(tags, kNumTagsAroundRegister); + } + + *thread.add_memory_dump() = std::move(dump); + } + }); +} + +static void log_unwinder_error(unwindstack::Unwinder* unwinder) { + if (unwinder->LastErrorCode() == unwindstack::ERROR_NONE) { + return; + } + + async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, " error code: %s", + unwinder->LastErrorCodeString()); + async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, " error address: 0x%" PRIx64, + unwinder->LastErrorAddress()); +} + +static void dump_thread_backtrace(unwindstack::Unwinder* unwinder, Thread& thread) { + if (unwinder->NumFrames() == 0) { + async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, "failed to unwind"); + log_unwinder_error(unwinder); + return; + } + + if (unwinder->elf_from_memory_not_file()) { + auto backtrace_note = thread.mutable_backtrace_note(); + *backtrace_note->Add() = + "Function names and BuildId information is missing for some frames due"; + *backtrace_note->Add() = "to unreadable libraries. For unwinds of apps, only shared libraries"; + *backtrace_note->Add() = "found under the lib/ directory are readable."; + *backtrace_note->Add() = "On this device, run setenforce 0 to make the libraries readable."; + } + unwinder->SetDisplayBuildID(true); + for (const auto& frame : unwinder->frames()) { + BacktraceFrame* f = thread.add_current_backtrace(); + fill_in_backtrace_frame(f, frame); + } +} + static void dump_thread(Tombstone* tombstone, unwindstack::Unwinder* unwinder, const ThreadInfo& thread_info, bool memory_dump = false) { Thread thread; @@ -355,97 +443,32 @@ static void dump_thread(Tombstone* tombstone, unwindstack::Unwinder* unwinder, thread.set_tagged_addr_ctrl(thread_info.tagged_addr_ctrl); thread.set_pac_enabled_keys(thread_info.pac_enabled_keys); - unwindstack::Maps* maps = unwinder->GetMaps(); - unwindstack::Memory* memory = unwinder->GetProcessMemory().get(); - - thread_info.registers->IterateRegisters( - [&thread, memory_dump, maps, memory](const char* name, uint64_t value) { - Register r; - r.set_name(name); - r.set_u64(value); - *thread.add_registers() = r; - - if (memory_dump) { - MemoryDump dump; - - dump.set_register_name(name); - std::shared_ptr map_info = maps->Find(untag_address(value)); - if (map_info) { - dump.set_mapping_name(map_info->name()); - } - - constexpr size_t kNumBytesAroundRegister = 256; - constexpr size_t kNumTagsAroundRegister = kNumBytesAroundRegister / kTagGranuleSize; - char buf[kNumBytesAroundRegister]; - uint8_t tags[kNumTagsAroundRegister]; - size_t start_offset = 0; - ssize_t bytes = dump_memory(buf, sizeof(buf), tags, sizeof(tags), &value, memory); - if (bytes == -1) { - return; - } - dump.set_begin_address(value); - - if (start_offset + bytes > sizeof(buf)) { - async_safe_fatal("dump_memory overflowed? start offset = %zu, bytes read = %zd", - start_offset, bytes); - } - - dump.set_memory(buf, bytes); - - bool has_tags = false; -#if defined(__aarch64__) - for (size_t i = 0; i < kNumTagsAroundRegister; ++i) { - if (tags[i] != 0) { - has_tags = true; - } - } -#endif // defined(__aarch64__) - - if (has_tags) { - dump.mutable_arm_mte_metadata()->set_memory_tags(tags, kNumTagsAroundRegister); - } - - *thread.add_memory_dump() = std::move(dump); - } - }); - - std::unique_ptr regs_copy(thread_info.registers->Clone()); - unwinder->SetRegs(regs_copy.get()); - unwinder->Unwind(); - if (unwinder->NumFrames() == 0) { - async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, "failed to unwind"); - if (unwinder->LastErrorCode() != unwindstack::ERROR_NONE) { - async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, " error code: %s", - unwinder->LastErrorCodeString()); - async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, " error address: 0x%" PRIx64, - unwinder->LastErrorAddress()); + if (thread_info.pid == getpid() && thread_info.pid != thread_info.tid) { + // Fallback path for non-main thread, doing unwind from running process. + unwindstack::ThreadUnwinder thread_unwinder(kMaxFrames, unwinder->GetMaps()); + if (!thread_unwinder.Init()) { + async_safe_format_log(ANDROID_LOG_ERROR, LOG_TAG, + "Unable to initialize ThreadUnwinder object."); + log_unwinder_error(&thread_unwinder); + return; } + + std::unique_ptr initial_regs; + thread_unwinder.UnwindWithSignal(BIONIC_SIGNAL_BACKTRACE, thread_info.tid, &initial_regs); + dump_registers(&thread_unwinder, initial_regs, thread, memory_dump); + dump_thread_backtrace(&thread_unwinder, thread); } else { - if (unwinder->elf_from_memory_not_file()) { - auto backtrace_note = thread.mutable_backtrace_note(); - *backtrace_note->Add() = - "Function names and BuildId information is missing for some frames due"; - *backtrace_note->Add() = - "to unreadable libraries. For unwinds of apps, only shared libraries"; - *backtrace_note->Add() = "found under the lib/ directory are readable."; - *backtrace_note->Add() = "On this device, run setenforce 0 to make the libraries readable."; - } - unwinder->SetDisplayBuildID(true); - for (const auto& frame : unwinder->frames()) { - BacktraceFrame* f = thread.add_current_backtrace(); - fill_in_backtrace_frame(f, frame); - } + dump_registers(unwinder, thread_info.registers, thread, memory_dump); + std::unique_ptr regs_copy(thread_info.registers->Clone()); + unwinder->SetRegs(regs_copy.get()); + unwinder->Unwind(); + dump_thread_backtrace(unwinder, thread); } auto& threads = *tombstone->mutable_threads(); threads[thread_info.tid] = thread; } -static void dump_main_thread(Tombstone* tombstone, unwindstack::Unwinder* unwinder, - const ThreadInfo& thread_info) { - dump_thread(tombstone, unwinder, thread_info, true); -} - static void dump_mappings(Tombstone* tombstone, unwindstack::Unwinder* unwinder) { unwindstack::Maps* maps = unwinder->GetMaps(); std::shared_ptr process_memory = unwinder->GetProcessMemory(); @@ -663,7 +686,8 @@ void engrave_tombstone_proto(Tombstone* tombstone, unwindstack::Unwinder* unwind dump_abort_message(&result, unwinder, process_info); - dump_main_thread(&result, unwinder, main_thread); + // Dump the main thread, but save the memory around the registers. + dump_thread(&result, unwinder, main_thread, /* memory_dump */ true); for (const auto& [tid, thread_info] : threads) { if (tid != target_thread) { diff --git a/debuggerd/seccomp_policy/crash_dump.arm64.policy b/debuggerd/seccomp_policy/crash_dump.arm64.policy index 858a338bd..4e8fdf9fb 100644 --- a/debuggerd/seccomp_policy/crash_dump.arm64.policy +++ b/debuggerd/seccomp_policy/crash_dump.arm64.policy @@ -25,7 +25,7 @@ tgkill: 1 rt_sigprocmask: 1 rt_sigaction: 1 rt_tgsigqueueinfo: 1 -prctl: arg0 == PR_GET_NO_NEW_PRIVS || arg0 == 0x53564d41 || arg0 == PR_PAC_RESET_KEYS +prctl: arg0 == PR_GET_NO_NEW_PRIVS || arg0 == 0x53564d41 || arg0 == PR_PAC_RESET_KEYS || arg0 == PR_GET_TAGGED_ADDR_CTRL || arg0 == PR_PAC_GET_ENABLED_KEYS madvise: 1 mprotect: arg2 in 0x1|0x2 munmap: 1 diff --git a/debuggerd/seccomp_policy/crash_dump.policy.def b/debuggerd/seccomp_policy/crash_dump.policy.def index 152697cf3..4eb996eb2 100644 --- a/debuggerd/seccomp_policy/crash_dump.policy.def +++ b/debuggerd/seccomp_policy/crash_dump.policy.def @@ -37,7 +37,7 @@ rt_tgsigqueueinfo: 1 #define PR_SET_VMA 0x53564d41 #if defined(__aarch64__) // PR_PAC_RESET_KEYS happens on aarch64 in pthread_create path. -prctl: arg0 == PR_GET_NO_NEW_PRIVS || arg0 == PR_SET_VMA || arg0 == PR_PAC_RESET_KEYS +prctl: arg0 == PR_GET_NO_NEW_PRIVS || arg0 == PR_SET_VMA || arg0 == PR_PAC_RESET_KEYS || arg0 == PR_GET_TAGGED_ADDR_CTRL || arg0 == PR_PAC_GET_ENABLED_KEYS #else prctl: arg0 == PR_GET_NO_NEW_PRIVS || arg0 == PR_SET_VMA #endif diff --git a/debuggerd/util.cpp b/debuggerd/util.cpp index ce0fd30f7..5c6abc94b 100644 --- a/debuggerd/util.cpp +++ b/debuggerd/util.cpp @@ -18,6 +18,7 @@ #include +#include #include #include @@ -74,3 +75,24 @@ std::string get_timestamp() { n = strftime(s, sz, "%z", &tm), s += n, sz -= n; return buf; } + +bool iterate_tids(pid_t pid, std::function callback) { + char buf[BUFSIZ]; + snprintf(buf, sizeof(buf), "/proc/%d/task", pid); + std::unique_ptr dir(opendir(buf), closedir); + if (dir == nullptr) { + return false; + } + + struct dirent* entry; + while ((entry = readdir(dir.get())) != nullptr) { + pid_t tid = atoi(entry->d_name); + if (tid == 0) { + continue; + } + if (pid != tid) { + callback(tid); + } + } + return true; +} diff --git a/debuggerd/util.h b/debuggerd/util.h index ec2862a2a..43758702f 100644 --- a/debuggerd/util.h +++ b/debuggerd/util.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include @@ -27,3 +28,4 @@ std::string get_process_name(pid_t pid); std::string get_thread_name(pid_t tid); std::string get_timestamp(); +bool iterate_tids(pid_t, std::function);