Merge "Reland "[MTE] remap stacks with PROT_MTE when requested by dlopened library"" into main am: a732e2aef5

Original change: https://android-review.googlesource.com/c/platform/bionic/+/2955914

Change-Id: I91f22862517b6c3b41893ac913a282a600a9f398
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

libc/bionic/heap_tagging.cpp

@@ -57,6 +57,7 @@ void SetDefaultHeapTaggingLevel() {
         break;
       case M_HEAP_TAGGING_LEVEL_SYNC:
       case M_HEAP_TAGGING_LEVEL_ASYNC:
+        atomic_store(&globals->memtag, true);
         atomic_store(&globals->memtag_stack, __libc_shared_globals()->initial_memtag_stack);
         break;
       default:
@@ -113,6 +114,7 @@ bool SetHeapTaggingLevel(HeapTaggingLevel tag_level) {
           globals->heap_pointer_tag = static_cast<uintptr_t>(0xffull << UNTAG_SHIFT);
         }
         atomic_store(&globals->memtag_stack, false);
+        atomic_store(&globals->memtag, false);
       });
 
       if (heap_tagging_level != M_HEAP_TAGGING_LEVEL_TBI) {

libc/bionic/libc_init_dynamic.cpp

@@ -39,11 +39,12 @@
  *   all dynamic linking has been performed.
  */
 
+#include <elf.h>
 #include <stddef.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdint.h>
-#include <elf.h>
+#include "bionic/pthread_internal.h"
 #include "libc_init_common.h"
 
 #include "private/bionic_defs.h"
@@ -59,6 +60,10 @@ extern "C" {
   extern int __cxa_atexit(void (*)(void *), void *, void *);
 };
 
+void memtag_stack_dlopen_callback() {
+  __pthread_internal_remap_stack_with_mte();
+}
+
 // Use an initializer so __libc_sysinfo will have a fallback implementation
 // while .preinit_array constructors run.
 #if defined(__i386__)
@@ -156,6 +161,10 @@ __noreturn void __libc_init(void* raw_args,
 
   __libc_init_mte_late();
 
+  // This roundabout way is needed so we don't use the static libc linked into the linker, which
+  // will not affect the process.
+  __libc_shared_globals()->memtag_stack_dlopen_callback = memtag_stack_dlopen_callback;
+
   exit(slingshot(args.argc - __libc_shared_globals()->initial_linker_arg_count,
                  args.argv + __libc_shared_globals()->initial_linker_arg_count,
                  args.envp));

libc/bionic/pthread_attr.cpp

@@ -155,36 +155,6 @@ int pthread_attr_setstack(pthread_attr_t* attr, void* stack_base, size_t stack_s
   return 0;
 }
 
-static uintptr_t __get_main_stack_startstack() {
-  FILE* fp = fopen("/proc/self/stat", "re");
-  if (fp == nullptr) {
-    async_safe_fatal("couldn't open /proc/self/stat: %m");
-  }
-
-  char line[BUFSIZ];
-  if (fgets(line, sizeof(line), fp) == nullptr) {
-    async_safe_fatal("couldn't read /proc/self/stat: %m");
-  }
-
-  fclose(fp);
-
-  // See man 5 proc. There's no reason comm can't contain ' ' or ')',
-  // so we search backwards for the end of it. We're looking for this field:
-  //
-  //  startstack %lu (28) The address of the start (i.e., bottom) of the stack.
-  uintptr_t startstack = 0;
-  const char* end_of_comm = strrchr(line, ')');
-  if (sscanf(end_of_comm + 1, " %*c "
-             "%*d %*d %*d %*d %*d "
-             "%*u %*u %*u %*u %*u %*u %*u "
-             "%*d %*d %*d %*d %*d %*d "
-             "%*u %*u %*d %*u %*u %*u %" SCNuPTR, &startstack) != 1) {
-    async_safe_fatal("couldn't parse /proc/self/stat");
-  }
-
-  return startstack;
-}
-
 static int __pthread_attr_getstack_main_thread(void** stack_base, size_t* stack_size) {
   ErrnoRestorer errno_restorer;
 
@@ -198,28 +168,11 @@ static int __pthread_attr_getstack_main_thread(void** stack_base, size_t* stack_
   if (stack_limit.rlim_cur == RLIM_INFINITY) {
     stack_limit.rlim_cur = 8 * 1024 * 1024;
   }
-
-  // Ask the kernel where our main thread's stack started.
-  uintptr_t startstack = __get_main_stack_startstack();
-
-  // Hunt for the region that contains that address.
-  FILE* fp = fopen("/proc/self/maps", "re");
-  if (fp == nullptr) {
-    async_safe_fatal("couldn't open /proc/self/maps: %m");
-  }
-  char line[BUFSIZ];
-  while (fgets(line, sizeof(line), fp) != nullptr) {
-    uintptr_t lo, hi;
-    if (sscanf(line, "%" SCNxPTR "-%" SCNxPTR, &lo, &hi) == 2) {
-      if (lo <= startstack && startstack <= hi) {
-        *stack_size = stack_limit.rlim_cur;
-        *stack_base = reinterpret_cast<void*>(hi - *stack_size);
-        fclose(fp);
-        return 0;
-      }
-    }
-  }
-  async_safe_fatal("stack not found in /proc/self/maps");
+  uintptr_t lo, hi;
+  __find_main_stack_limits(&lo, &hi);
+  *stack_size = stack_limit.rlim_cur;
+  *stack_base = reinterpret_cast<void*>(hi - *stack_size);
+  return 0;
 }
 
 __BIONIC_WEAK_FOR_NATIVE_BRIDGE

libc/bionic/pthread_internal.cpp

@@ -40,6 +40,7 @@
 #include "private/ErrnoRestorer.h"
 #include "private/ScopedRWLock.h"
 #include "private/bionic_futex.h"
+#include "private/bionic_globals.h"
 #include "private/bionic_tls.h"
 
 static pthread_internal_t* g_thread_list = nullptr;
@@ -119,6 +120,89 @@ pthread_internal_t* __pthread_internal_find(pthread_t thread_id, const char* cal
   return nullptr;
 }
 
+static uintptr_t __get_main_stack_startstack() {
+  FILE* fp = fopen("/proc/self/stat", "re");
+  if (fp == nullptr) {
+    async_safe_fatal("couldn't open /proc/self/stat: %m");
+  }
+
+  char line[BUFSIZ];
+  if (fgets(line, sizeof(line), fp) == nullptr) {
+    async_safe_fatal("couldn't read /proc/self/stat: %m");
+  }
+
+  fclose(fp);
+
+  // See man 5 proc. There's no reason comm can't contain ' ' or ')',
+  // so we search backwards for the end of it. We're looking for this field:
+  //
+  //  startstack %lu (28) The address of the start (i.e., bottom) of the stack.
+  uintptr_t startstack = 0;
+  const char* end_of_comm = strrchr(line, ')');
+  if (sscanf(end_of_comm + 1,
+             " %*c "
+             "%*d %*d %*d %*d %*d "
+             "%*u %*u %*u %*u %*u %*u %*u "
+             "%*d %*d %*d %*d %*d %*d "
+             "%*u %*u %*d %*u %*u %*u %" SCNuPTR,
+             &startstack) != 1) {
+    async_safe_fatal("couldn't parse /proc/self/stat");
+  }
+
+  return startstack;
+}
+
+void __find_main_stack_limits(uintptr_t* low, uintptr_t* high) {
+  // Ask the kernel where our main thread's stack started.
+  uintptr_t startstack = __get_main_stack_startstack();
+
+  // Hunt for the region that contains that address.
+  FILE* fp = fopen("/proc/self/maps", "re");
+  if (fp == nullptr) {
+    async_safe_fatal("couldn't open /proc/self/maps: %m");
+  }
+  char line[BUFSIZ];
+  while (fgets(line, sizeof(line), fp) != nullptr) {
+    uintptr_t lo, hi;
+    if (sscanf(line, "%" SCNxPTR "-%" SCNxPTR, &lo, &hi) == 2) {
+      if (lo <= startstack && startstack <= hi) {
+        *low = lo;
+        *high = hi;
+        fclose(fp);
+        return;
+      }
+    }
+  }
+  async_safe_fatal("stack not found in /proc/self/maps");
+}
+
+void __pthread_internal_remap_stack_with_mte() {
+#if defined(__aarch64__)
+  // If process doesn't have MTE enabled, we don't need to do anything.
+  if (!__libc_globals->memtag) return;
+  bool prev = true;
+  __libc_globals.mutate(
+      [&prev](libc_globals* globals) { prev = atomic_exchange(&globals->memtag_stack, true); });
+  if (prev) return;
+  uintptr_t lo, hi;
+  __find_main_stack_limits(&lo, &hi);
+
+  if (mprotect(reinterpret_cast<void*>(lo), hi - lo,
+               PROT_READ | PROT_WRITE | PROT_MTE | PROT_GROWSDOWN)) {
+    async_safe_fatal("error: failed to set PROT_MTE on main thread");
+  }
+  ScopedWriteLock creation_locker(&g_thread_creation_lock);
+  ScopedReadLock list_locker(&g_thread_list_lock);
+  for (pthread_internal_t* t = g_thread_list; t != nullptr; t = t->next) {
+    if (t->terminating || t->is_main()) continue;
+    if (mprotect(t->mmap_base_unguarded, t->mmap_size_unguarded,
+                 PROT_READ | PROT_WRITE | PROT_MTE)) {
+      async_safe_fatal("error: failed to set PROT_MTE on thread: %d", t->tid);
+    }
+  }
+#endif
+}
+
 bool android_run_on_all_threads(bool (*func)(void*), void* arg) {
   // Take the locks in this order to avoid inversion (pthread_create ->
   // __pthread_internal_add).

libc/bionic/pthread_internal.h

@@ -178,6 +178,7 @@ class pthread_internal_t {
   bionic_tls* bionic_tls;
 
   int errno_value;
+  bool is_main() { return start_routine == nullptr; }
 };
 
 struct ThreadMapping {
@@ -207,6 +208,7 @@ __LIBC_HIDDEN__ pthread_internal_t* __pthread_internal_find(pthread_t pthread_id
 __LIBC_HIDDEN__ pid_t __pthread_internal_gettid(pthread_t pthread_id, const char* caller);
 __LIBC_HIDDEN__ void __pthread_internal_remove(pthread_internal_t* thread);
 __LIBC_HIDDEN__ void __pthread_internal_remove_and_free(pthread_internal_t* thread);
+__LIBC_HIDDEN__ void __find_main_stack_limits(uintptr_t* low, uintptr_t* high);
 
 static inline __always_inline bionic_tcb* __get_bionic_tcb() {
   return reinterpret_cast<bionic_tcb*>(&__get_tls()[MIN_TLS_SLOT]);
@@ -266,6 +268,9 @@ __LIBC_HIDDEN__ extern void __bionic_atfork_run_prepare();
 __LIBC_HIDDEN__ extern void __bionic_atfork_run_child();
 __LIBC_HIDDEN__ extern void __bionic_atfork_run_parent();
 
+// Re-map all threads and successively launched threads with PROT_MTE.
+__LIBC_HIDDEN__ void __pthread_internal_remap_stack_with_mte();
+
 extern "C" bool android_run_on_all_threads(bool (*func)(void*), void* arg);
 
 extern pthread_rwlock_t g_thread_creation_lock;

libc/private/bionic_globals.h

@@ -50,6 +50,7 @@ struct libc_globals {
   uintptr_t heap_pointer_tag;
   _Atomic(bool) memtag_stack;
   _Atomic(bool) decay_time_enabled;
+  _Atomic(bool) memtag;
 
   // In order to allow a complete switch between dispatch tables without
   // the need for copying each function by function in the structure,
@@ -135,6 +136,8 @@ struct libc_shared_globals {
   HeapTaggingLevel initial_heap_tagging_level = M_HEAP_TAGGING_LEVEL_NONE;
   bool initial_memtag_stack = false;
   int64_t heap_tagging_upgrade_timer_sec = 0;
+
+  void (*memtag_stack_dlopen_callback)() = nullptr;
 };
 
 __LIBC_HIDDEN__ libc_shared_globals* __libc_shared_globals();

linker/linker.cpp

@@ -2213,6 +2213,14 @@ void* do_dlopen(const char* name, int flags,
   loading_trace.End();
 
   if (si != nullptr) {
+    if (si->has_min_version(7) && si->memtag_stack()) {
+      LD_LOG(kLogDlopen, "... dlopen enabling MTE for: realpath=\"%s\", soname=\"%s\"",
+             si->get_realpath(), si->get_soname());
+      if (auto* cb = __libc_shared_globals()->memtag_stack_dlopen_callback) {
+        cb();
+      }
+    }
+
     void* handle = si->to_handle();
     LD_LOG(kLogDlopen,
            "... dlopen calling constructors: realpath=\"%s\", soname=\"%s\", handle=%p",

tests/Android.bp

@@ -1107,6 +1107,31 @@ cc_test {
     test_suites: ["device-tests"],
 }
 
+cc_test {
+    name: "memtag_stack_dlopen_test",
+    enabled: false,
+    // This does not use bionic_tests_defaults because it is not supported on
+    // host.
+    arch: {
+        arm64: {
+            enabled: true,
+        },
+    },
+    sanitize: {
+        memtag_heap: true,
+        memtag_stack: false,
+    },
+    srcs: [
+        "memtag_stack_dlopen_test.cpp",
+    ],
+    shared_libs: [
+        "libbase",
+    ],
+    data_libs: ["libtest_simple_memtag_stack"],
+    header_libs: ["bionic_libc_platform_headers"],
+    test_suites: ["device-tests"],
+}
+
 cc_test {
     name: "bionic-stress-tests",
     defaults: [

tests/libs/Android.bp

@@ -233,6 +233,17 @@ cc_test_library {
     srcs: ["dlopen_testlib_simple.cpp"],
 }
 
+// -----------------------------------------------------------------------------
+// Library used by memtag_stack_dlopen_test tests
+// -----------------------------------------------------------------------------
+cc_test_library {
+    name: "libtest_simple_memtag_stack",
+    sanitize: {
+        memtag_stack: true,
+    },
+    srcs: ["dlopen_testlib_simple.cpp"],
+}
+
 // -----------------------------------------------------------------------------
 // Libraries used by hwasan_test
 // -----------------------------------------------------------------------------

tests/memtag_stack_dlopen_test.cpp

@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <thread>
+
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#include <gtest/gtest.h>
+
+#include <android-base/silent_death_test.h>
+#include <android-base/test_utils.h>
+#include "utils.h"
+
+#if defined(__BIONIC__) && defined(__aarch64__)
+__attribute__((target("mte"))) bool is_stack_mte_on() {
+  alignas(16) int x = 0;
+  void* p = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(&x) + (1UL << 57));
+  void* p_cpy = p;
+  __builtin_arm_stg(p);
+  p = __builtin_arm_ldg(p);
+  __builtin_arm_stg(&x);
+  return p == p_cpy;
+}
+
+// We can't use pthread_getattr_np because that uses the rlimit rather than the actual mapping
+// bounds.
+static void find_main_stack_limits(uintptr_t* low, uintptr_t* high) {
+  uintptr_t startstack = reinterpret_cast<uintptr_t>(__builtin_frame_address(0));
+
+  // Hunt for the region that contains that address.
+  FILE* fp = fopen("/proc/self/maps", "re");
+  if (fp == nullptr) {
+    abort();
+  }
+  char line[BUFSIZ];
+  while (fgets(line, sizeof(line), fp) != nullptr) {
+    uintptr_t lo, hi;
+    if (sscanf(line, "%" SCNxPTR "-%" SCNxPTR, &lo, &hi) == 2) {
+      if (lo <= startstack && startstack <= hi) {
+        *low = lo;
+        *high = hi;
+        fclose(fp);
+        return;
+      }
+    }
+  }
+  abort();
+}
+
+template <typename Fn>
+unsigned int fault_new_stack_page(uintptr_t low, Fn f) {
+  uintptr_t new_low;
+  uintptr_t new_high;
+  volatile char buf[4096];
+  buf[4095] = 1;
+  find_main_stack_limits(&new_low, &new_high);
+  if (new_low < low) {
+    f();
+    return new_high;
+  }
+  // Useless, but should defeat TCO.
+  return new_low + fault_new_stack_page(low, f);
+}
+
+#endif
+
+enum State { kInit, kThreadStarted, kStackRemapped };
+
+TEST(MemtagStackDlopenTest, DlopenRemapsStack) {
+#if defined(__BIONIC__) && defined(__aarch64__)
+  if (!running_with_mte()) GTEST_SKIP() << "Test requires MTE.";
+
+  std::string path = android::base::GetExecutableDirectory() + "/libtest_simple_memtag_stack.so";
+  ASSERT_EQ(0, access(path.c_str(), F_OK));  // Verify test setup.
+  EXPECT_FALSE(is_stack_mte_on());
+  std::mutex m;
+  std::condition_variable cv;
+  State state = kInit;
+
+  bool is_early_thread_mte_on = false;
+  std::thread early_th([&] {
+    {
+      std::lock_guard lk(m);
+      state = kThreadStarted;
+    }
+    cv.notify_one();
+    {
+      std::unique_lock lk(m);
+      cv.wait(lk, [&] { return state == kStackRemapped; });
+    }
+    is_early_thread_mte_on = is_stack_mte_on();
+  });
+  {
+    std::unique_lock lk(m);
+    cv.wait(lk, [&] { return state == kThreadStarted; });
+  }
+  void* handle = dlopen(path.c_str(), RTLD_NOW);
+  {
+    std::lock_guard lk(m);
+    state = kStackRemapped;
+  }
+  cv.notify_one();
+  ASSERT_NE(handle, nullptr);
+  EXPECT_TRUE(is_stack_mte_on());
+
+  bool new_stack_page_mte_on = false;
+  uintptr_t low;
+  uintptr_t high;
+  find_main_stack_limits(&low, &high);
+  fault_new_stack_page(low, [&] { new_stack_page_mte_on = is_stack_mte_on(); });
+  EXPECT_TRUE(new_stack_page_mte_on);
+
+  bool is_late_thread_mte_on = false;
+  std::thread late_th([&] { is_late_thread_mte_on = is_stack_mte_on(); });
+  late_th.join();
+  early_th.join();
+  EXPECT_TRUE(is_early_thread_mte_on);
+  EXPECT_TRUE(is_late_thread_mte_on);
+#else
+  GTEST_SKIP() << "requires bionic arm64";
+#endif
+}