Merge "Move VDSO pointers to a shared globals struct."

libc/bionic/libc_init_common.cpp

@@ -42,10 +42,12 @@
 #include <unistd.h>
 
 #include "private/bionic_auxv.h"
+#include "private/bionic_globals.h"
 #include "private/bionic_ssp.h"
 #include "private/bionic_tls.h"
 #include "private/KernelArgumentBlock.h"
 #include "private/libc_logging.h"
+#include "private/WriteProtected.h"
 #include "pthread_internal.h"
 
 extern "C" abort_msg_t** __abort_message_ptr;
@@ -54,7 +56,7 @@ extern "C" int __system_properties_init(void);
 extern "C" int __set_tls(void* ptr);
 extern "C" int __set_tid_address(int* tid_address);
 
-__LIBC_HIDDEN__ void __libc_init_vdso();
+__LIBC_HIDDEN__ WriteProtected<libc_globals> __libc_globals;
 
 // Not public, but well-known in the BSDs.
 const char* __progname;
@@ -105,6 +107,16 @@ void __libc_init_main_thread(KernelArgumentBlock& args) {
   __init_alternate_signal_stack(&main_thread);
 }
 
+void __libc_init_globals(KernelArgumentBlock& args) {
+  // Initialize libc globals that are needed in both the linker and in libc.
+  // In dynamic binaries, this is run at least twice for different copies of the
+  // globals, once for the linker's copy and once for the one in libc.so.
+  __libc_globals.initialize();
+  __libc_globals.mutate([&args](libc_globals* globals) {
+    __libc_init_vdso(globals, args);
+  });
+}
+
 void __libc_init_common(KernelArgumentBlock& args) {
   // Initialize various globals.
   environ = args.envp;
@@ -121,9 +133,7 @@ void __libc_init_common(KernelArgumentBlock& args) {
   __pthread_internal_add(main_thread);
 
   __system_properties_init(); // Requires 'environ'.
-
   __bionic_setjmp_cookie_init();
-  __libc_init_vdso();
 }
 
 __noreturn static void __early_abort(int line) {

libc/bionic/libc_init_common.h

@@ -51,6 +51,9 @@ __END_DECLS
 #if defined(__cplusplus)
 
 class KernelArgumentBlock;
+
+__LIBC_HIDDEN__ void __libc_init_globals(KernelArgumentBlock& args);
+
 __LIBC_HIDDEN__ void __libc_init_common(KernelArgumentBlock& args);
 
 __LIBC_HIDDEN__ void __libc_init_AT_SECURE(KernelArgumentBlock& args);

libc/bionic/libc_init_dynamic.cpp

@@ -74,6 +74,7 @@ __attribute__((constructor)) static void __libc_preinit() {
   // __libc_init_common() will change the TLS area so the old one won't be accessible anyway.
   *args_slot = NULL;
 
+  __libc_init_globals(*args);
   __libc_init_common(*args);
 
   // Hooks for various libraries to let them know that we're starting up.

libc/bionic/libc_init_static.cpp

@@ -84,7 +84,12 @@ __noreturn void __libc_init(void* raw_args,
                             int (*slingshot)(int, char**, char**),
                             structors_array_t const * const structors) {
   KernelArgumentBlock args(raw_args);
+
   __libc_init_main_thread(args);
+
+  // Initializing the globals requires TLS to be available for errno.
+  __libc_init_globals(args);
+
   __libc_init_AT_SECURE(args);
   __libc_init_common(args);
 

libc/bionic/vdso.cpp

@@ -14,66 +14,47 @@
  * limitations under the License.
  */
 
-#include <link.h>
-#include <string.h>
-#include <sys/auxv.h>
-#include <unistd.h>
+#include "private/bionic_globals.h"
+#include "private/bionic_vdso.h"
 
 #if defined(__aarch64__) || defined(__x86_64__) || defined (__i386__)
 
-#if defined(__aarch64__)
-#define VDSO_CLOCK_GETTIME_SYMBOL "__kernel_clock_gettime"
-#define VDSO_GETTIMEOFDAY_SYMBOL  "__kernel_gettimeofday"
-#elif defined(__x86_64__) || defined(__i386__)
-#define VDSO_CLOCK_GETTIME_SYMBOL "__vdso_clock_gettime"
-#define VDSO_GETTIMEOFDAY_SYMBOL  "__vdso_gettimeofday"
-#endif
-
-#include <errno.h>
 #include <limits.h>
-#include <sys/mman.h>
+#include <link.h>
+#include <string.h>
+#include <sys/cdefs.h>
+#include <sys/time.h>
 #include <time.h>
-
-#include "private/bionic_prctl.h"
-#include "private/libc_logging.h"
-
-extern "C" int __clock_gettime(int, timespec*);
-extern "C" int __gettimeofday(timeval*, struct timezone*);
-
-struct vdso_entry {
-  const char* name;
-  void* fn;
-};
-
-enum {
-  VDSO_CLOCK_GETTIME = 0,
-  VDSO_GETTIMEOFDAY,
-  VDSO_END
-};
-
-static union {
-  vdso_entry entries[VDSO_END];
-  char padding[PAGE_SIZE];
-} vdso __attribute__((aligned(PAGE_SIZE))) = {{
-  [VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL, reinterpret_cast<void*>(__clock_gettime) },
-  [VDSO_GETTIMEOFDAY] = { VDSO_GETTIMEOFDAY_SYMBOL, reinterpret_cast<void*>(__gettimeofday) },
-}};
+#include <unistd.h>
+#include "private/KernelArgumentBlock.h"
 
 int clock_gettime(int clock_id, timespec* tp) {
-  int (*vdso_clock_gettime)(int, timespec*) =
-      reinterpret_cast<int (*)(int, timespec*)>(vdso.entries[VDSO_CLOCK_GETTIME].fn);
-  return vdso_clock_gettime(clock_id, tp);
+  auto vdso_clock_gettime = reinterpret_cast<decltype(&clock_gettime)>(
+    __libc_globals->vdso[VDSO_CLOCK_GETTIME].fn);
+  if (__predict_true(vdso_clock_gettime)) {
+    return vdso_clock_gettime(clock_id, tp);
+  }
+  return __clock_gettime(clock_id, tp);
 }
 
 int gettimeofday(timeval* tv, struct timezone* tz) {
-  int (*vdso_gettimeofday)(timeval*, struct timezone*) =
-      reinterpret_cast<int (*)(timeval*, struct timezone*)>(vdso.entries[VDSO_GETTIMEOFDAY].fn);
-  return vdso_gettimeofday(tv, tz);
+  auto vdso_gettimeofday = reinterpret_cast<decltype(&gettimeofday)>(
+    __libc_globals->vdso[VDSO_GETTIMEOFDAY].fn);
+  if (__predict_true(vdso_gettimeofday)) {
+    return vdso_gettimeofday(tv, tz);
+  }
+  return __gettimeofday(tv, tz);
 }
 
-static void __libc_init_vdso_entries() {
+void __libc_init_vdso(libc_globals* globals, KernelArgumentBlock& args) {
+  auto&& vdso = globals->vdso;
+  vdso[VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL,
+                               reinterpret_cast<void*>(__clock_gettime) };
+  vdso[VDSO_GETTIMEOFDAY] = { VDSO_GETTIMEOFDAY_SYMBOL,
+                              reinterpret_cast<void*>(__gettimeofday) };
+
   // Do we have a vdso?
-  uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR);
+  uintptr_t vdso_ehdr_addr = args.getauxval(AT_SYSINFO_EHDR);
   ElfW(Ehdr)* vdso_ehdr = reinterpret_cast<ElfW(Ehdr)*>(vdso_ehdr_addr);
   if (vdso_ehdr == nullptr) {
     return;
@@ -123,27 +104,16 @@ static void __libc_init_vdso_entries() {
   // Are there any symbols we want?
   for (size_t i = 0; i < symbol_count; ++i) {
     for (size_t j = 0; j < VDSO_END; ++j) {
-      if (strcmp(vdso.entries[j].name, strtab + symtab[i].st_name) == 0) {
-        vdso.entries[j].fn = reinterpret_cast<void*>(vdso_addr + symtab[i].st_value);
+      if (strcmp(vdso[j].name, strtab + symtab[i].st_name) == 0) {
+        vdso[j].fn = reinterpret_cast<void*>(vdso_addr + symtab[i].st_value);
       }
     }
   }
 }
 
-void __libc_init_vdso() {
-  __libc_init_vdso_entries();
-
-  // We can't use PR_SET_VMA because this isn't an anonymous region.
-  // Long-term we should be able to replace all of this with ifuncs.
-  static_assert(PAGE_SIZE == sizeof(vdso), "sizeof(vdso) too large");
-  if (mprotect(vdso.entries, sizeof(vdso), PROT_READ) == -1) {
-    __libc_fatal("failed to mprotect PROT_READ vdso function pointer table: %s", strerror(errno));
-  }
-}
-
 #else
 
-void __libc_init_vdso() {
+void __libc_init_vdso(libc_globals*, KernelArgumentBlock&) {
 }
 
 #endif

libc/private/WriteProtected.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _PRIVATE_WRITEPROTECTED_H
+#define _PRIVATE_WRITEPROTECTED_H
+
+#include <errno.h>
+#include <string.h>
+#include <sys/cdefs.h>
+#include <sys/mman.h>
+#include <sys/user.h>
+
+#include "private/bionic_macros.h"
+#include "private/bionic_prctl.h"
+#include "private/libc_logging.h"
+
+template <typename T>
+union WriteProtectedContents {
+  T value;
+  char padding[PAGE_SIZE];
+
+  WriteProtectedContents() = default;
+  DISALLOW_COPY_AND_ASSIGN(WriteProtectedContents);
+} __attribute__((aligned(PAGE_SIZE)));
+
+// Write protected wrapper class that aligns its contents to a page boundary,
+// and sets the memory protection to be non-writable, except when being modified
+// explicitly.
+template <typename T>
+class WriteProtected {
+  static_assert(sizeof(T) < PAGE_SIZE,
+                "WriteProtected only supports contents up to PAGE_SIZE");
+  static_assert(__is_pod(T), "WriteProtected only supports POD contents");
+
+  WriteProtectedContents<T> contents;
+
+ public:
+  WriteProtected() = default;
+  DISALLOW_COPY_AND_ASSIGN(WriteProtected);
+
+  void initialize() {
+    // Not strictly necessary, but this will hopefully segfault if we initialize
+    // multiple times by accident.
+    memset(&contents, 0, sizeof(contents));
+
+    if (mprotect(&contents, PAGE_SIZE, PROT_READ)) {
+      __libc_fatal("failed to make WriteProtected nonwritable in initialize");
+    }
+  }
+
+  const T* operator->() {
+    return &contents.value;
+  }
+
+  const T& operator*() {
+    return contents.value;
+  }
+
+  template <typename Mutator>
+  void mutate(Mutator mutator) {
+    if (mprotect(&contents, PAGE_SIZE, PROT_READ | PROT_WRITE) != 0) {
+      __libc_fatal("failed to make WriteProtected writable in mutate: %s",
+                   strerror(errno));
+    }
+    mutator(&contents.value);
+    if (mprotect(&contents, PAGE_SIZE, PROT_READ) != 0) {
+      __libc_fatal("failed to make WriteProtected nonwritable in mutate: %s",
+                   strerror(errno));
+    }
+  }
+};
+
+#endif

libc/private/bionic_globals.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#ifndef _PRIVATE_BIONIC_GLOBALS_H
+#define _PRIVATE_BIONIC_GLOBALS_H
+
+#include <sys/cdefs.h>
+#include "private/bionic_vdso.h"
+#include "private/WriteProtected.h"
+
+struct libc_globals {
+  vdso_entry vdso[VDSO_END];
+};
+
+__LIBC_HIDDEN__ extern WriteProtected<libc_globals> __libc_globals;
+
+class KernelArgumentBlock;
+__LIBC_HIDDEN__ void __libc_init_vdso(libc_globals* globals,
+                                      KernelArgumentBlock& args);
+
+#endif

libc/private/bionic_vdso.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#ifndef _PRIVATE_BIONIC_VDSO_H
+#define _PRIVATE_BIONIC_VDSO_H
+
+#include <time.h>
+
+#if defined(__aarch64__)
+#define VDSO_CLOCK_GETTIME_SYMBOL "__kernel_clock_gettime"
+#define VDSO_GETTIMEOFDAY_SYMBOL  "__kernel_gettimeofday"
+#elif defined(__x86_64__) || defined(__i386__)
+#define VDSO_CLOCK_GETTIME_SYMBOL "__vdso_clock_gettime"
+#define VDSO_GETTIMEOFDAY_SYMBOL  "__vdso_gettimeofday"
+#endif
+
+extern "C" int __clock_gettime(int, timespec*);
+extern "C" int __gettimeofday(timeval*, struct timezone*);
+
+struct vdso_entry {
+  const char* name;
+  void* fn;
+};
+
+enum {
+  VDSO_CLOCK_GETTIME = 0,
+  VDSO_GETTIMEOFDAY,
+  VDSO_END
+};
+
+#endif  // _PRIVATE_BIONIC_VDSO_H

linker/linker.cpp

@@ -63,6 +63,7 @@
 #include "base/strings.h"
 #include "ziparchive/zip_archive.h"
 
+extern void __libc_init_globals(KernelArgumentBlock&);
 extern void __libc_init_AT_SECURE(KernelArgumentBlock&);
 
 // Override macros to use C++ style casts.
@@ -3579,6 +3580,9 @@ extern "C" ElfW(Addr) __linker_init(void* raw_args) {
 
   __libc_init_main_thread(args);
 
+  // Initialize the linker's static libc's globals
+  __libc_init_globals(args);
+
   // Initialize the linker's own global variables
   linker_so.call_constructors();