Implement dynamic TLS accesses and allocation

Initialize a thread's DTV to an empty zeroed DTV. Allocate the DTV and
any ELF module's TLS segment on-demand in __tls_get_addr. Use a generation
counter, incremented in the linker, to signal when threads should
update/reallocate their DTV objects.

A generation count of 0 always indicates the constant zero DTV.

Once a DTV is allocated, it isn't freed until the thread exits, because
a signal handler could interrupt the fast path of __tls_get_addr between
accessing the DTV slot and reading a field of the DTV. Bionic keeps a
linked list of DTV objects so it can free them at thread-exit.

Dynamic TLS memory is allocated using a BionicAllocator instance in
libc_shared_globals. For async-signal safety, access to the
linker/libc-shared state is protected by first blocking signals, then by
acquiring the reader-writer lock, TlsModules::rwlock. A write lock is
needed to allocate or free memory.

In pthread_exit, unconditionally block signals before freeing dynamic
TLS memory or freeing the shadow call stack.

ndk_cruft.cpp: Avoid including pthread_internal.h inside an extern "C".
(The header now includes a C++ template that doesn't compile inside
extern "C".)

Bug: http://b/78026329
Bug: http://b/123094171
Test: bionic unit tests
Change-Id: I3c9b12921c9e68b33dcc1d1dd276bff364eff5d7

libc/bionic/__libc_init_main_thread.cpp

@@ -74,6 +74,7 @@ extern "C" void __libc_init_main_thread_early(const KernelArgumentBlock& args,
   __libc_init_sysinfo(); // uses AT_SYSINFO auxv entry
 #endif
   __init_tcb(temp_tcb, &main_thread);
+  __init_tcb_dtv(temp_tcb);
   __set_tls(&temp_tcb->tls_slot(0));
   main_thread.tid = __getpid();
   main_thread.set_cached_pid(main_thread.tid);

libc/bionic/bionic_elf_tls.cpp

@@ -34,9 +34,22 @@
 #include <unistd.h>
 
 #include "private/ScopedRWLock.h"
+#include "private/ScopedSignalBlocker.h"
 #include "private/bionic_globals.h"
 #include "private/bionic_macros.h"
 #include "private/bionic_tls.h"
+#include "pthread_internal.h"
+
+// Every call to __tls_get_addr needs to check the generation counter, so
+// accesses to the counter need to be as fast as possible. Keep a copy of it in
+// a hidden variable, which can be accessed without using the GOT. The linker
+// will update this variable when it updates its counter.
+//
+// To allow the linker to update this variable, libc.so's constructor passes its
+// address to the linker. To accommodate a possible __tls_get_addr call before
+// libc.so's constructor, this local copy is initialized to SIZE_MAX, forcing
+// __tls_get_addr to initially use the slow path.
+__LIBC_HIDDEN__ _Atomic(size_t) __libc_tls_generation_copy = SIZE_MAX;
 
 // Search for a TLS segment in the given phdr table. Returns true if it has a
 // TLS segment and false otherwise.
@@ -168,6 +181,7 @@ void __init_static_tls(void* static_tls) {
   // moving the initial part. If this locking is too slow, we can duplicate the
   // static part of the table.
   TlsModules& modules = __libc_shared_globals()->tls_modules;
+  ScopedSignalBlocker ssb;
   ScopedReadLock locker(&modules.rwlock);
 
   for (size_t i = 0; i < modules.module_count; ++i) {
@@ -187,3 +201,166 @@ void __init_static_tls(void* static_tls) {
            module.segment.init_size);
   }
 }
+
+static inline size_t dtv_size_in_bytes(size_t module_count) {
+  return sizeof(TlsDtv) + module_count * sizeof(void*);
+}
+
+// Calculates the number of module slots to allocate in a new DTV. For small
+// objects (up to 1KiB), the TLS allocator allocates memory in power-of-2 sizes,
+// so for better space usage, ensure that the DTV size (header + slots) is a
+// power of 2.
+//
+// The lock on TlsModules must be held.
+static size_t calculate_new_dtv_count() {
+  size_t loaded_cnt = __libc_shared_globals()->tls_modules.module_count;
+  size_t bytes = dtv_size_in_bytes(MAX(1, loaded_cnt));
+  if (!powerof2(bytes)) {
+    bytes = BIONIC_ROUND_UP_POWER_OF_2(bytes);
+  }
+  return (bytes - sizeof(TlsDtv)) / sizeof(void*);
+}
+
+// This function must be called with signals blocked and a write lock on
+// TlsModules held.
+static void update_tls_dtv(bionic_tcb* tcb) {
+  const TlsModules& modules = __libc_shared_globals()->tls_modules;
+  BionicAllocator& allocator = __libc_shared_globals()->tls_allocator;
+
+  // Use the generation counter from the shared globals instead of the local
+  // copy, which won't be initialized yet if __tls_get_addr is called before
+  // libc.so's constructor.
+  if (__get_tcb_dtv(tcb)->generation == atomic_load(&modules.generation)) {
+    return;
+  }
+
+  const size_t old_cnt = __get_tcb_dtv(tcb)->count;
+
+  // If the DTV isn't large enough, allocate a larger one. Because a signal
+  // handler could interrupt the fast path of __tls_get_addr, we don't free the
+  // old DTV. Instead, we add the old DTV to a list, then free all of a thread's
+  // DTVs at thread-exit. Each time the DTV is reallocated, its size at least
+  // doubles.
+  if (modules.module_count > old_cnt) {
+    size_t new_cnt = calculate_new_dtv_count();
+    TlsDtv* const old_dtv = __get_tcb_dtv(tcb);
+    TlsDtv* const new_dtv = static_cast<TlsDtv*>(allocator.alloc(dtv_size_in_bytes(new_cnt)));
+    memcpy(new_dtv, old_dtv, dtv_size_in_bytes(old_cnt));
+    new_dtv->count = new_cnt;
+    new_dtv->next = old_dtv;
+    __set_tcb_dtv(tcb, new_dtv);
+  }
+
+  TlsDtv* const dtv = __get_tcb_dtv(tcb);
+
+  const StaticTlsLayout& layout = __libc_shared_globals()->static_tls_layout;
+  char* static_tls = reinterpret_cast<char*>(tcb) - layout.offset_bionic_tcb();
+
+  // Initialize static TLS modules and free unloaded modules.
+  for (size_t i = 0; i < dtv->count; ++i) {
+    if (i < modules.module_count) {
+      const TlsModule& mod = modules.module_table[i];
+      if (mod.static_offset != SIZE_MAX) {
+        dtv->modules[i] = static_tls + mod.static_offset;
+        continue;
+      }
+      if (mod.first_generation != kTlsGenerationNone &&
+          mod.first_generation <= dtv->generation) {
+        continue;
+      }
+    }
+    allocator.free(dtv->modules[i]);
+    dtv->modules[i] = nullptr;
+  }
+
+  dtv->generation = atomic_load(&modules.generation);
+}
+
+__attribute__((noinline)) static void* tls_get_addr_slow_path(const TlsIndex* ti) {
+  TlsModules& modules = __libc_shared_globals()->tls_modules;
+  bionic_tcb* tcb = __get_bionic_tcb();
+
+  // Block signals and lock TlsModules. We may need the allocator, so take
+  // a write lock.
+  ScopedSignalBlocker ssb;
+  ScopedWriteLock locker(&modules.rwlock);
+
+  update_tls_dtv(tcb);
+
+  TlsDtv* dtv = __get_tcb_dtv(tcb);
+  const size_t module_idx = __tls_module_id_to_idx(ti->module_id);
+  void* mod_ptr = dtv->modules[module_idx];
+  if (mod_ptr == nullptr) {
+    const TlsSegment& segment = modules.module_table[module_idx].segment;
+    mod_ptr = __libc_shared_globals()->tls_allocator.memalign(segment.alignment, segment.size);
+    if (segment.init_size > 0) {
+      memcpy(mod_ptr, segment.init_ptr, segment.init_size);
+    }
+    dtv->modules[module_idx] = mod_ptr;
+  }
+
+  return static_cast<char*>(mod_ptr) + ti->offset;
+}
+
+// Returns the address of a thread's TLS memory given a module ID and an offset
+// into that module's TLS segment. This function is called on every access to a
+// dynamic TLS variable on targets that don't use TLSDESC. arm64 uses TLSDESC,
+// so it only calls this function on a thread's first access to a module's TLS
+// segment.
+//
+// On most targets, this accessor function is __tls_get_addr and
+// TLS_GET_ADDR_CCONV is unset. 32-bit x86 uses ___tls_get_addr instead and a
+// regparm() calling convention.
+extern "C" void* TLS_GET_ADDR(const TlsIndex* ti) TLS_GET_ADDR_CCONV {
+  TlsDtv* dtv = __get_tcb_dtv(__get_bionic_tcb());
+
+  // TODO: See if we can use a relaxed memory ordering here instead.
+  size_t generation = atomic_load(&__libc_tls_generation_copy);
+  if (__predict_true(generation == dtv->generation)) {
+    void* mod_ptr = dtv->modules[__tls_module_id_to_idx(ti->module_id)];
+    if (__predict_true(mod_ptr != nullptr)) {
+      return static_cast<char*>(mod_ptr) + ti->offset;
+    }
+  }
+
+  return tls_get_addr_slow_path(ti);
+}
+
+// This function frees:
+//  - TLS modules referenced by the current DTV.
+//  - The list of DTV objects associated with the current thread.
+//
+// The caller must have already blocked signals.
+void __free_dynamic_tls(bionic_tcb* tcb) {
+  TlsModules& modules = __libc_shared_globals()->tls_modules;
+  BionicAllocator& allocator = __libc_shared_globals()->tls_allocator;
+
+  // If we didn't allocate any dynamic memory, skip out early without taking
+  // the lock.
+  TlsDtv* dtv = __get_tcb_dtv(tcb);
+  if (dtv->generation == kTlsGenerationNone) {
+    return;
+  }
+
+  // We need the write lock to use the allocator.
+  ScopedWriteLock locker(&modules.rwlock);
+
+  // First free everything in the current DTV.
+  for (size_t i = 0; i < dtv->count; ++i) {
+    if (i < modules.module_count && modules.module_table[i].static_offset != SIZE_MAX) {
+      // This module's TLS memory is allocated statically, so don't free it here.
+      continue;
+    }
+    allocator.free(dtv->modules[i]);
+  }
+
+  // Now free the thread's list of DTVs.
+  while (dtv->generation != kTlsGenerationNone) {
+    TlsDtv* next = dtv->next;
+    allocator.free(dtv);
+    dtv = next;
+  }
+
+  // Clear the DTV slot. The DTV must not be used again with this thread.
+  tcb->tls_slot(TLS_SLOT_DTV) = nullptr;
+}

libc/bionic/libc_init_dynamic.cpp

@@ -51,6 +51,7 @@
 #include <elf.h>
 #include "libc_init_common.h"
 
+#include "private/bionic_elf_tls.h"
 #include "private/bionic_globals.h"
 #include "private/bionic_macros.h"
 #include "private/bionic_ssp.h"
@@ -82,6 +83,12 @@ static void __libc_preinit_impl() {
   __libc_init_sysinfo();
 #endif
 
+  // Register libc.so's copy of the TLS generation variable so the linker can
+  // update it when it loads or unloads a shared object.
+  TlsModules& tls_modules = __libc_shared_globals()->tls_modules;
+  tls_modules.generation_libc_so = &__libc_tls_generation_copy;
+  __libc_tls_generation_copy = tls_modules.generation;
+
   __libc_init_globals();
   __libc_init_common();
 

libc/bionic/libc_init_static.cpp

@@ -92,19 +92,22 @@ static void layout_static_tls(KernelArgumentBlock& args) {
   size_t phdr_ct = getauxval(AT_PHNUM);
 
   static TlsModule mod;
+  TlsModules& modules = __libc_shared_globals()->tls_modules;
   if (__bionic_get_tls_segment(phdr_start, phdr_ct, 0, &mod.segment)) {
     if (!__bionic_check_tls_alignment(&mod.segment.alignment)) {
       async_safe_fatal("error: TLS segment alignment in \"%s\" is not a power of 2: %zu\n",
                        progname, mod.segment.alignment);
     }
     mod.static_offset = layout.reserve_exe_segment_and_tcb(&mod.segment, progname);
-    mod.first_generation = 1;
-    __libc_shared_globals()->tls_modules.generation = 1;
-    __libc_shared_globals()->tls_modules.module_count = 1;
-    __libc_shared_globals()->tls_modules.module_table = &mod;
+    mod.first_generation = kTlsGenerationFirst;
+
+    modules.module_count = 1;
+    modules.module_table = &mod;
   } else {
     layout.reserve_exe_segment_and_tcb(nullptr, progname);
   }
+  // Enable the fast path in __tls_get_addr.
+  __libc_tls_generation_copy = modules.generation;
 
   layout.finish_layout();
 }

libc/bionic/ndk_cruft.cpp

@@ -355,9 +355,14 @@ void* dlmalloc(size_t size) {
   return malloc(size);
 }
 
+} // extern "C"
+
 #define __get_thread __real_get_thread
 #include "pthread_internal.h"
 #undef __get_thread
+
+extern "C" {
+
 // Various third-party apps contain a backport of our pthread_rwlock implementation that uses this.
 pthread_internal_t* __get_thread() {
   return __real_get_thread();

libc/bionic/pthread_create.cpp

@@ -70,6 +70,14 @@ void __init_tcb_stack_guard(bionic_tcb* tcb) {
   tcb->tls_slot(TLS_SLOT_STACK_GUARD) = reinterpret_cast<void*>(__stack_chk_guard);
 }
 
+__attribute__((no_stack_protector))
+void __init_tcb_dtv(bionic_tcb* tcb) {
+  // Initialize the DTV slot to a statically-allocated empty DTV. The first
+  // access to a dynamic TLS variable allocates a new DTV.
+  static const TlsDtv zero_dtv = {};
+  __set_tcb_dtv(tcb, const_cast<TlsDtv*>(&zero_dtv));
+}
+
 void __init_bionic_tls_ptrs(bionic_tcb* tcb, bionic_tls* tls) {
   tcb->thread()->bionic_tls = tls;
   tcb->tls_slot(TLS_SLOT_BIONIC_TLS) = tls;
@@ -291,6 +299,7 @@ static int __allocate_thread(pthread_attr_t* attr, bionic_tcb** tcbp, void** chi
   // Initialize TLS memory.
   __init_static_tls(mapping.static_tls);
   __init_tcb(tcb, thread);
+  __init_tcb_dtv(tcb);
   __init_tcb_stack_guard(tcb);
   __init_bionic_tls_ptrs(tcb, tls);
 

libc/bionic/pthread_exit.cpp

@@ -98,15 +98,22 @@ void pthread_exit(void* return_value) {
     thread->alternate_signal_stack = nullptr;
   }
 
+  ThreadJoinState old_state = THREAD_NOT_JOINED;
+  while (old_state == THREAD_NOT_JOINED &&
+         !atomic_compare_exchange_weak(&thread->join_state, &old_state, THREAD_EXITED_NOT_JOINED)) {
+  }
+
+  // We don't want to take a signal after unmapping the stack, the shadow call
+  // stack, or dynamic TLS memory.
+  ScopedSignalBlocker ssb;
+
 #ifdef __aarch64__
   // Free the shadow call stack and guard pages.
   munmap(thread->shadow_call_stack_guard_region, SCS_GUARD_REGION_SIZE);
 #endif
 
-  ThreadJoinState old_state = THREAD_NOT_JOINED;
-  while (old_state == THREAD_NOT_JOINED &&
-         !atomic_compare_exchange_weak(&thread->join_state, &old_state, THREAD_EXITED_NOT_JOINED)) {
-  }
+  // Free the ELF TLS DTV and all dynamically-allocated ELF TLS memory.
+  __free_dynamic_tls(__get_bionic_tcb());
 
   if (old_state == THREAD_DETACHED) {
     // The thread is detached, no one will use pthread_internal_t after pthread_exit.
@@ -121,10 +128,6 @@ void pthread_exit(void* return_value) {
     if (thread->mmap_size != 0) {
       // We need to free mapped space for detached threads when they exit.
       // That's not something we can do in C.
-
-      // We don't want to take a signal after we've unmapped the stack.
-      // That's one last thing we can do before dropping to assembler.
-      ScopedSignalBlocker ssb;
       __hwasan_thread_exit();
       _exit_with_stack_teardown(thread->mmap_base, thread->mmap_size);
     }

libc/bionic/pthread_internal.h

@@ -38,6 +38,7 @@
 #define __hwasan_thread_exit()
 #endif
 
+#include "private/bionic_elf_tls.h"
 #include "private/bionic_lock.h"
 #include "private/bionic_tls.h"
 
@@ -154,6 +155,7 @@ struct ThreadMapping {
 
 __LIBC_HIDDEN__ void __init_tcb(bionic_tcb* tcb, pthread_internal_t* thread);
 __LIBC_HIDDEN__ void __init_tcb_stack_guard(bionic_tcb* tcb);
+__LIBC_HIDDEN__ void __init_tcb_dtv(bionic_tcb* tcb);
 __LIBC_HIDDEN__ void __init_bionic_tls_ptrs(bionic_tcb* tcb, bionic_tls* tls);
 __LIBC_HIDDEN__ bionic_tls* __allocate_temp_bionic_tls();
 __LIBC_HIDDEN__ void __free_temp_bionic_tls(bionic_tls* tls);
@@ -179,6 +181,15 @@ static inline __always_inline bionic_tls& __get_bionic_tls() {
   return *static_cast<bionic_tls*>(__get_tls()[TLS_SLOT_BIONIC_TLS]);
 }
 
+static inline __always_inline TlsDtv* __get_tcb_dtv(bionic_tcb* tcb) {
+  uintptr_t dtv_slot = reinterpret_cast<uintptr_t>(tcb->tls_slot(TLS_SLOT_DTV));
+  return reinterpret_cast<TlsDtv*>(dtv_slot - offsetof(TlsDtv, generation));
+}
+
+static inline void __set_tcb_dtv(bionic_tcb* tcb, TlsDtv* val) {
+  tcb->tls_slot(TLS_SLOT_DTV) = &val->generation;
+}
+
 extern "C" __LIBC_HIDDEN__ int __set_tls(void* ptr);
 
 __LIBC_HIDDEN__ void pthread_key_clean_all(void);

libc/libc.map.txt

@@ -1446,8 +1446,10 @@ LIBC_P { # introduced=P
 
 LIBC_Q { # introduced=Q
   global:
+    ___tls_get_addr; # x86
     __aeabi_read_tp; # arm
     __res_randomid;
+    __tls_get_addr; # arm x86_64
     android_fdsan_close_with_tag;
     android_fdsan_create_owner_tag;
     android_fdsan_exchange_owner_tag;

libc/private/bionic_elf_tls.h

@@ -34,6 +34,8 @@
 #include <stdint.h>
 #include <sys/cdefs.h>
 
+__LIBC_HIDDEN__ extern _Atomic(size_t) __libc_tls_generation_copy;
+
 struct TlsSegment {
   size_t size = 0;
   size_t alignment = 1;
@@ -84,6 +86,16 @@ private:
   size_t round_up_with_overflow_check(size_t value, size_t alignment);
 };
 
+static constexpr size_t kTlsGenerationNone = 0;
+static constexpr size_t kTlsGenerationFirst = 1;
+
+// The first ELF TLS module has ID 1. Zero is reserved for the first word of
+// the DTV, a generation count. Unresolved weak symbols also use module ID 0.
+static constexpr size_t kTlsUninitializedModuleId = 0;
+
+static inline size_t __tls_module_id_to_idx(size_t id) { return id - 1; }
+static inline size_t __tls_module_idx_to_id(size_t idx) { return idx + 1; }
+
 // A descriptor for a single ELF TLS module.
 struct TlsModule {
   TlsSegment segment;
@@ -93,7 +105,7 @@ struct TlsModule {
 
   // The generation in which this module was loaded. Dynamic TLS lookups use
   // this field to detect when a module has been unloaded.
-  size_t first_generation = 0;
+  size_t first_generation = kTlsGenerationNone;
 
   // Used by the dynamic linker to track the associated soinfo* object.
   void* soinfo_ptr = nullptr;
@@ -105,9 +117,10 @@ struct TlsModule {
 struct TlsModules {
   constexpr TlsModules() {}
 
-  // A generation counter. The value is incremented each time an solib is loaded
-  // or unloaded.
-  _Atomic(size_t) generation = 0;
+  // A pointer to the TLS generation counter in libc.so. The counter is
+  // incremented each time an solib is loaded or unloaded.
+  _Atomic(size_t) generation = kTlsGenerationFirst;
+  _Atomic(size_t) *generation_libc_so = nullptr;
 
   // Access to the TlsModule[] table requires taking this lock.
   pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER;
@@ -119,3 +132,46 @@ struct TlsModules {
 };
 
 void __init_static_tls(void* static_tls);
+
+// Dynamic Thread Vector. Each thread has a different DTV. For each module
+// (executable or solib), the DTV has a pointer to that module's TLS memory. The
+// DTV is initially empty and is allocated on-demand. It grows as more modules
+// are dlopen'ed. See https://www.akkadia.org/drepper/tls.pdf.
+//
+// The layout of the DTV is specified in various documents, but it is not part
+// of Bionic's public ABI. A compiler can't generate code to access it directly,
+// because it can't access libc's global generation counter.
+struct TlsDtv {
+  // Number of elements in this object's modules field.
+  size_t count;
+
+  // A pointer to an older TlsDtv object that should be freed when the thread
+  // exits. The objects aren't immediately freed because a DTV could be
+  // reallocated by a signal handler that interrupted __tls_get_addr's fast
+  // path.
+  TlsDtv* next;
+
+  // The DTV slot points at this field, which allows omitting an add instruction
+  // on the fast path for a TLS lookup. The arm64 tlsdesc_resolver.S depends on
+  // the layout of fields past this point.
+  size_t generation;
+  void* modules[];
+};
+
+struct TlsIndex {
+  size_t module_id;
+  size_t offset;
+};
+
+#if defined(__i386__)
+#define TLS_GET_ADDR_CCONV __attribute__((regparm(1)))
+#define TLS_GET_ADDR ___tls_get_addr
+#else
+#define TLS_GET_ADDR_CCONV
+#define TLS_GET_ADDR __tls_get_addr
+#endif
+
+extern "C" void* TLS_GET_ADDR(const TlsIndex* ti) TLS_GET_ADDR_CCONV;
+
+struct bionic_tcb;
+void __free_dynamic_tls(bionic_tcb* tcb);

libc/private/bionic_globals.h

@@ -33,6 +33,7 @@
 #include <link.h>
 #include <pthread.h>
 
+#include "private/bionic_allocator.h"
 #include "private/bionic_elf_tls.h"
 #include "private/bionic_fdsan.h"
 #include "private/bionic_malloc_dispatch.h"
@@ -70,6 +71,7 @@ struct libc_shared_globals {
 
   StaticTlsLayout static_tls_layout;
   TlsModules tls_modules;
+  BionicAllocator tls_allocator;
 
   // Values passed from the linker to libc.so.
   const char* init_progname = nullptr;

linker/linker_soinfo.h

@@ -35,6 +35,7 @@
 
 #include "private/bionic_elf_tls.h"
 #include "linker_namespaces.h"
+#include "linker_tls.h"
 
 #define FLAG_LINKED           0x00000001
 #define FLAG_EXE              0x00000004 // The main executable
@@ -102,14 +103,9 @@ struct version_info {
 // TODO(dimitry): remove reference from soinfo member functions to this class.
 class VersionTracker;
 
-// The first ELF TLS module has ID 1. Zero is reserved for the first word of
-// the DTV, a generation count, and unresolved weak symbols also use module
-// ID 0.
-static constexpr size_t kUninitializedModuleId = 0;
-
 struct soinfo_tls {
   TlsSegment segment;
-  size_t module_id = kUninitializedModuleId;
+  size_t module_id = kTlsUninitializedModuleId;
 };
 
 #if defined(__work_around_b_24465209__)

linker/linker_tls.cpp

@@ -31,6 +31,7 @@
 #include <vector>
 
 #include "private/ScopedRWLock.h"
+#include "private/ScopedSignalBlocker.h"
 #include "private/bionic_defs.h"
 #include "private/bionic_elf_tls.h"
 #include "private/bionic_globals.h"
@@ -41,9 +42,6 @@
 static bool g_static_tls_finished;
 static std::vector<TlsModule> g_tls_modules;
 
-static inline size_t module_id_to_idx(size_t id) { return id - 1; }
-static inline size_t module_idx_to_id(size_t idx) { return idx + 1; }
-
 static size_t get_unused_module_index() {
   for (size_t i = 0; i < g_tls_modules.size(); ++i) {
     if (g_tls_modules[i].soinfo_ptr == nullptr) {
@@ -57,37 +55,47 @@ static size_t get_unused_module_index() {
 }
 
 static void register_tls_module(soinfo* si, size_t static_offset) {
+  TlsModules& libc_modules = __libc_shared_globals()->tls_modules;
+
   // The global TLS module table points at the std::vector of modules declared
   // in this file, so acquire a write lock before modifying the std::vector.
-  ScopedWriteLock locker(&__libc_shared_globals()->tls_modules.rwlock);
+  ScopedSignalBlocker ssb;
+  ScopedWriteLock locker(&libc_modules.rwlock);
 
   size_t module_idx = get_unused_module_index();
 
   soinfo_tls* si_tls = si->get_tls();
-  si_tls->module_id = module_idx_to_id(module_idx);
+  si_tls->module_id = __tls_module_idx_to_id(module_idx);
+
+  const size_t new_generation = ++libc_modules.generation;
+  __libc_tls_generation_copy = new_generation;
+  if (libc_modules.generation_libc_so != nullptr) {
+    *libc_modules.generation_libc_so = new_generation;
+  }
 
   g_tls_modules[module_idx] = {
     .segment = si_tls->segment,
     .static_offset = static_offset,
-    .first_generation = ++__libc_shared_globals()->tls_modules.generation,
+    .first_generation = new_generation,
     .soinfo_ptr = si,
   };
 }
 
 static void unregister_tls_module(soinfo* si) {
+  ScopedSignalBlocker ssb;
   ScopedWriteLock locker(&__libc_shared_globals()->tls_modules.rwlock);
 
   soinfo_tls* si_tls = si->get_tls();
-  TlsModule& mod = g_tls_modules[module_id_to_idx(si_tls->module_id)];
+  TlsModule& mod = g_tls_modules[__tls_module_id_to_idx(si_tls->module_id)];
   CHECK(mod.static_offset == SIZE_MAX);
   CHECK(mod.soinfo_ptr == si);
   mod = {};
-  si_tls->module_id = kUninitializedModuleId;
+  si_tls->module_id = kTlsUninitializedModuleId;
 }
 
 // The reference is valid until a TLS module is registered or unregistered.
 const TlsModule& get_tls_module(size_t module_id) {
-  size_t module_idx = module_id_to_idx(module_id);
+  size_t module_idx = __tls_module_id_to_idx(module_id);
   CHECK(module_idx < g_tls_modules.size());
   return g_tls_modules[module_idx];
 }
@@ -123,7 +131,7 @@ void linker_finalize_static_tls() {
 
 void register_soinfo_tls(soinfo* si) {
   soinfo_tls* si_tls = si->get_tls();
-  if (si_tls == nullptr || si_tls->module_id != kUninitializedModuleId) {
+  if (si_tls == nullptr || si_tls->module_id != kTlsUninitializedModuleId) {
     return;
   }
   size_t static_offset = SIZE_MAX;
@@ -136,7 +144,7 @@ void register_soinfo_tls(soinfo* si) {
 
 void unregister_soinfo_tls(soinfo* si) {
   soinfo_tls* si_tls = si->get_tls();
-  if (si_tls == nullptr || si_tls->module_id == kUninitializedModuleId) {
+  if (si_tls == nullptr || si_tls->module_id == kTlsUninitializedModuleId) {
     return;
   }
   return unregister_tls_module(si);