Move VDSO pointers to a shared globals struct.
Change-Id: I01cbc9cf0917dc1fac52d9205bda2c68529d12ef
diff --git a/libc/bionic/libc_init_common.cpp b/libc/bionic/libc_init_common.cpp
index 3bbb210..5b9edaf 100644
--- a/libc/bionic/libc_init_common.cpp
+++ b/libc/bionic/libc_init_common.cpp
@@ -42,10 +42,12 @@
#include <unistd.h>
#include "private/bionic_auxv.h"
+#include "private/bionic_globals.h"
#include "private/bionic_ssp.h"
#include "private/bionic_tls.h"
#include "private/KernelArgumentBlock.h"
#include "private/libc_logging.h"
+#include "private/WriteProtected.h"
#include "pthread_internal.h"
extern "C" abort_msg_t** __abort_message_ptr;
@@ -54,7 +56,7 @@
extern "C" int __set_tls(void* ptr);
extern "C" int __set_tid_address(int* tid_address);
-__LIBC_HIDDEN__ void __libc_init_vdso();
+__LIBC_HIDDEN__ WriteProtected<libc_globals> __libc_globals;
// Not public, but well-known in the BSDs.
const char* __progname;
@@ -105,6 +107,16 @@
__init_alternate_signal_stack(&main_thread);
}
+void __libc_init_globals(KernelArgumentBlock& args) {
+ // Initialize libc globals that are needed in both the linker and in libc.
+ // In dynamic binaries, this is run at least twice for different copies of the
+ // globals, once for the linker's copy and once for the one in libc.so.
+ __libc_globals.initialize();
+ __libc_globals.mutate([&args](libc_globals* globals) {
+ __libc_init_vdso(globals, args);
+ });
+}
+
void __libc_init_common(KernelArgumentBlock& args) {
// Initialize various globals.
environ = args.envp;
@@ -121,9 +133,7 @@
__pthread_internal_add(main_thread);
__system_properties_init(); // Requires 'environ'.
-
__bionic_setjmp_cookie_init();
- __libc_init_vdso();
}
__noreturn static void __early_abort(int line) {
diff --git a/libc/bionic/libc_init_common.h b/libc/bionic/libc_init_common.h
index 673ad5b..5066652 100644
--- a/libc/bionic/libc_init_common.h
+++ b/libc/bionic/libc_init_common.h
@@ -51,6 +51,9 @@
#if defined(__cplusplus)
class KernelArgumentBlock;
+
+__LIBC_HIDDEN__ void __libc_init_globals(KernelArgumentBlock& args);
+
__LIBC_HIDDEN__ void __libc_init_common(KernelArgumentBlock& args);
__LIBC_HIDDEN__ void __libc_init_AT_SECURE(KernelArgumentBlock& args);
diff --git a/libc/bionic/libc_init_dynamic.cpp b/libc/bionic/libc_init_dynamic.cpp
index 78125f9..3bb6e89 100644
--- a/libc/bionic/libc_init_dynamic.cpp
+++ b/libc/bionic/libc_init_dynamic.cpp
@@ -74,6 +74,7 @@
// __libc_init_common() will change the TLS area so the old one won't be accessible anyway.
*args_slot = NULL;
+ __libc_init_globals(*args);
__libc_init_common(*args);
// Hooks for various libraries to let them know that we're starting up.
diff --git a/libc/bionic/libc_init_static.cpp b/libc/bionic/libc_init_static.cpp
index 2b1a8cb..2fe86d0 100644
--- a/libc/bionic/libc_init_static.cpp
+++ b/libc/bionic/libc_init_static.cpp
@@ -84,7 +84,12 @@
int (*slingshot)(int, char**, char**),
structors_array_t const * const structors) {
KernelArgumentBlock args(raw_args);
+
__libc_init_main_thread(args);
+
+ // Initializing the globals requires TLS to be available for errno.
+ __libc_init_globals(args);
+
__libc_init_AT_SECURE(args);
__libc_init_common(args);
diff --git a/libc/bionic/vdso.cpp b/libc/bionic/vdso.cpp
index 9eae3d5..029c148 100644
--- a/libc/bionic/vdso.cpp
+++ b/libc/bionic/vdso.cpp
@@ -14,66 +14,47 @@
* limitations under the License.
*/
-#include <link.h>
-#include <string.h>
-#include <sys/auxv.h>
-#include <unistd.h>
+#include "private/bionic_globals.h"
+#include "private/bionic_vdso.h"
#if defined(__aarch64__) || defined(__x86_64__) || defined (__i386__)
-#if defined(__aarch64__)
-#define VDSO_CLOCK_GETTIME_SYMBOL "__kernel_clock_gettime"
-#define VDSO_GETTIMEOFDAY_SYMBOL "__kernel_gettimeofday"
-#elif defined(__x86_64__) || defined(__i386__)
-#define VDSO_CLOCK_GETTIME_SYMBOL "__vdso_clock_gettime"
-#define VDSO_GETTIMEOFDAY_SYMBOL "__vdso_gettimeofday"
-#endif
-
-#include <errno.h>
#include <limits.h>
-#include <sys/mman.h>
+#include <link.h>
+#include <string.h>
+#include <sys/cdefs.h>
+#include <sys/time.h>
#include <time.h>
-
-#include "private/bionic_prctl.h"
-#include "private/libc_logging.h"
-
-extern "C" int __clock_gettime(int, timespec*);
-extern "C" int __gettimeofday(timeval*, struct timezone*);
-
-struct vdso_entry {
- const char* name;
- void* fn;
-};
-
-enum {
- VDSO_CLOCK_GETTIME = 0,
- VDSO_GETTIMEOFDAY,
- VDSO_END
-};
-
-static union {
- vdso_entry entries[VDSO_END];
- char padding[PAGE_SIZE];
-} vdso __attribute__((aligned(PAGE_SIZE))) = {{
- [VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL, reinterpret_cast<void*>(__clock_gettime) },
- [VDSO_GETTIMEOFDAY] = { VDSO_GETTIMEOFDAY_SYMBOL, reinterpret_cast<void*>(__gettimeofday) },
-}};
+#include <unistd.h>
+#include "private/KernelArgumentBlock.h"
int clock_gettime(int clock_id, timespec* tp) {
- int (*vdso_clock_gettime)(int, timespec*) =
- reinterpret_cast<int (*)(int, timespec*)>(vdso.entries[VDSO_CLOCK_GETTIME].fn);
- return vdso_clock_gettime(clock_id, tp);
+ auto vdso_clock_gettime = reinterpret_cast<decltype(&clock_gettime)>(
+ __libc_globals->vdso[VDSO_CLOCK_GETTIME].fn);
+ if (__predict_true(vdso_clock_gettime)) {
+ return vdso_clock_gettime(clock_id, tp);
+ }
+ return __clock_gettime(clock_id, tp);
}
int gettimeofday(timeval* tv, struct timezone* tz) {
- int (*vdso_gettimeofday)(timeval*, struct timezone*) =
- reinterpret_cast<int (*)(timeval*, struct timezone*)>(vdso.entries[VDSO_GETTIMEOFDAY].fn);
- return vdso_gettimeofday(tv, tz);
+ auto vdso_gettimeofday = reinterpret_cast<decltype(&gettimeofday)>(
+ __libc_globals->vdso[VDSO_GETTIMEOFDAY].fn);
+ if (__predict_true(vdso_gettimeofday)) {
+ return vdso_gettimeofday(tv, tz);
+ }
+ return __gettimeofday(tv, tz);
}
-static void __libc_init_vdso_entries() {
+void __libc_init_vdso(libc_globals* globals, KernelArgumentBlock& args) {
+ auto&& vdso = globals->vdso;
+ vdso[VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL,
+ reinterpret_cast<void*>(__clock_gettime) };
+ vdso[VDSO_GETTIMEOFDAY] = { VDSO_GETTIMEOFDAY_SYMBOL,
+ reinterpret_cast<void*>(__gettimeofday) };
+
// Do we have a vdso?
- uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR);
+ uintptr_t vdso_ehdr_addr = args.getauxval(AT_SYSINFO_EHDR);
ElfW(Ehdr)* vdso_ehdr = reinterpret_cast<ElfW(Ehdr)*>(vdso_ehdr_addr);
if (vdso_ehdr == nullptr) {
return;
@@ -123,27 +104,16 @@
// Are there any symbols we want?
for (size_t i = 0; i < symbol_count; ++i) {
for (size_t j = 0; j < VDSO_END; ++j) {
- if (strcmp(vdso.entries[j].name, strtab + symtab[i].st_name) == 0) {
- vdso.entries[j].fn = reinterpret_cast<void*>(vdso_addr + symtab[i].st_value);
+ if (strcmp(vdso[j].name, strtab + symtab[i].st_name) == 0) {
+ vdso[j].fn = reinterpret_cast<void*>(vdso_addr + symtab[i].st_value);
}
}
}
}
-void __libc_init_vdso() {
- __libc_init_vdso_entries();
-
- // We can't use PR_SET_VMA because this isn't an anonymous region.
- // Long-term we should be able to replace all of this with ifuncs.
- static_assert(PAGE_SIZE == sizeof(vdso), "sizeof(vdso) too large");
- if (mprotect(vdso.entries, sizeof(vdso), PROT_READ) == -1) {
- __libc_fatal("failed to mprotect PROT_READ vdso function pointer table: %s", strerror(errno));
- }
-}
-
#else
-void __libc_init_vdso() {
+void __libc_init_vdso(libc_globals*, KernelArgumentBlock&) {
}
#endif