Make safepoint thread local.

Preparing for using the safepoint to deliver SIGINT, which requires distinguish
between master and worker threads.

src/cgutils.cpp

@@ -455,7 +455,7 @@ static bool deserves_sret(jl_value_t *dt, Type *T)
 static Value *emit_nthptr_addr(Value *v, ssize_t n)
 {
     return builder.CreateGEP(builder.CreateBitCast(v, T_ppjlvalue),
-                             ConstantInt::get(T_size, (ssize_t)n));
+                             ConstantInt::get(T_size, n));
 }
 
 static Value *emit_nthptr_addr(Value *v, Value *idx)
@@ -477,6 +477,13 @@ static Value *emit_nthptr_recast(Value *v, Value *idx, MDNode *tbaa, Type *ptype
     return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false));
 }
 
+static Value *emit_nthptr_recast(Value *v, ssize_t n, MDNode *tbaa, Type *ptype)
+{
+    // p = (jl_value_t**)v; *(ptype)&p[n]
+    Value *vptr = emit_nthptr_addr(v, n);
+    return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false));
+}
+
 static Value *emit_typeptr_addr(Value *p)
 {
     ssize_t offset = (sizeof(jl_taggedvalue_t) -

src/codegen.cpp

@@ -343,9 +343,6 @@ static GlobalVariable *jltls_states_var;
 static GlobalVariable *jltls_states_func_ptr = NULL;
 static size_t jltls_states_func_idx = 0;
 #endif
-// Imaging mode only (non-imaging mode use pointer directly)
-static GlobalVariable *jl_safepoint_page_ptr = NULL;
-static size_t jl_safepoint_page_idx = 0;
 
 // important functions
 static Function *jlnew_func;
@@ -3430,14 +3427,9 @@ static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx)
 {
     // allocate a placeholder gc instruction
     ctx->ptlsStates = builder.CreateCall(prepare_call(jltls_states_func));
-    if (imaging_mode) {
-        ctx->signalPage =
-            tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jl_safepoint_page_ptr)));
-    }
-    else {
-        ctx->signalPage = builder.CreateIntToPtr(
-            ConstantInt::get(T_size, (uintptr_t)jl_safepoint_page), T_pint8);
-    }
+    int nthfield = offsetof(jl_tls_states_t, safepoint) / sizeof(void*);
+    ctx->signalPage = emit_nthptr_recast(ctx->ptlsStates, nthfield, tbaa_const,
+                                         PointerType::get(T_psize, 0));
 }
 
 void jl_codegen_finalize_temp_arg(CallInst *ptlsStates, Type *T_pjlvalue,
@@ -5170,12 +5162,6 @@ static void init_julia_llvm_env(Module *m)
                                 jltls_states_func_idx);
     }
 #endif
-    if (imaging_mode) {
-        jl_safepoint_page_ptr =
-            jl_emit_sysimg_slot(m, T_pint8, "jl_safepoint_page.ptr",
-                                (uintptr_t)jl_safepoint_page,
-                                jl_safepoint_page_idx);
-    }
 
     std::vector<Type*> args1(0);
     args1.push_back(T_pint8);

src/dump.c

@@ -232,9 +232,6 @@ static int jl_load_sysimg_so(void)
         *sysimg_gvars[tls_getter_idx - 1] =
             (jl_value_t*)jl_get_ptls_states_getter();
 #endif
-        size_t safepoint_idx = *(size_t*)jl_dlsym(jl_sysimg_handle,
-                                                  "jl_safepoint_page_idx");
-        *sysimg_gvars[safepoint_idx - 1] = (jl_value_t*)jl_safepoint_page;
         const char *cpu_target = (const char*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_target");
         if (strcmp(cpu_target,jl_options.cpu_target) != 0)
             jl_error("Julia and the system image were compiled for different architectures.\n"

src/jitlayers.cpp

@@ -812,10 +812,11 @@ static void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL
 #endif
 }
 
+#ifdef JULIA_ENABLE_THREADING
 // Emit a slot in the system image to be filled at sysimg init time.
 // Returns the global var. Fill `idx` with 1-base index in the sysimg gv.
 // Use as an optimization for runtime constant addresses to have one less
-// load.
+// load. (Used only by threading).
 static GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name,
                                            uintptr_t init, size_t &idx)
 {
@@ -839,6 +840,7 @@ static GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *nam
     idx = jl_sysimg_gvars.size();
     return gv;
 }
+#endif
 
 static void* jl_get_global(GlobalVariable *gv)
 {
@@ -914,12 +916,6 @@ static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap,
                                  ConstantInt::get(T_size, jltls_states_func_idx),
                                  "jl_ptls_states_getter_idx"));
 #endif
-    addComdat(new GlobalVariable(*mod,
-                                 T_size,
-                                 true,
-                                 GlobalVariable::ExternalLinkage,
-                                 ConstantInt::get(T_size, jl_safepoint_page_idx),
-                                 "jl_safepoint_page_idx"));
 
     Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target);
     addComdat(new GlobalVariable(*mod,

src/julia_internal.h

@@ -268,6 +268,12 @@ void jl_start_threads(void);
 void jl_shutdown_threading(void);
 
 // Whether the GC is running
+extern char *jl_safepoint_pages;
+STATIC_INLINE int jl_addr_is_safepoint(uintptr_t addr)
+{
+    uintptr_t safepoint_addr = (uintptr_t)jl_safepoint_pages;
+    return addr >= safepoint_addr && addr < safepoint_addr + jl_page_size * 3;
+}
 extern volatile uint32_t jl_gc_running;
 // All the functions are safe to be called from within a signal handler
 // provided that the thread will not be interrupted by another asynchronous

src/julia_threads.h

@@ -32,6 +32,7 @@
 typedef struct _jl_tls_states_t {
     struct _jl_gcframe_t *pgcstack;
     struct _jl_value_t *exception_in_transit;
+    volatile size_t *safepoint;
     // Whether it is safe to execute GC at the same time.
 #define JL_GC_STATE_WAITING 1
     // gc_state = 1 means the thread is doing GC or is waiting for the GC to
@@ -285,15 +286,14 @@ JL_DLLEXPORT void (jl_cpu_wake)(void);
 
 // Accessing the tls variables, gc safepoint and gc states
 JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void);
-JL_DLLEXPORT extern volatile size_t *jl_safepoint_page;
 // This triggers a SegFault when we are in GC
 // Assign it to a variable to make sure the compiler emit the load
 // and to avoid Clang warning for -Wunused-volatile-lvalue
-#define jl_gc_safepoint() do {                          \
-        jl_signal_fence();                              \
-        size_t safepoint_load = *jl_safepoint_page;     \
-        jl_signal_fence();                              \
-        (void)safepoint_load;                           \
+#define jl_gc_safepoint() do {                                          \
+        jl_signal_fence();                                              \
+        size_t safepoint_load = *jl_get_ptls_states()->safepoint;       \
+        jl_signal_fence();                                              \
+        (void)safepoint_load;                                           \
     } while (0)
 #ifndef JULIA_ENABLE_THREADING
 extern JL_DLLEXPORT jl_tls_states_t jl_tls_states;

src/safepoint.c

@@ -16,10 +16,16 @@ extern "C" {
 
 /* static uint32_t jl_signal_pending = 0; */
 volatile uint32_t jl_gc_running = 0;
-JL_DLLEXPORT volatile size_t *jl_safepoint_page = NULL;
-// The number of safepoints enabled on the two pages.
-// The first page, the one before `jl_safepoint_page` is the SIGINT page.
-// The second page, the one pointed to by `jl_safepoint_page` is the GC page.
+char *jl_safepoint_pages = NULL;
+// The number of safepoints enabled on the three pages.
+// The first page, is the SIGINT page, only used by the master thread.
+// The second page, is the GC page for the master thread, this is where
+// the `safepoint` tls pointer points to for the master thread.
+// The third page is the GC page for the other threads. The thread's
+// `safepoint` tls pointer points the beginning of this page + `sizeof(size_t)`
+// so that both safepoint load and pending signal load falls in this page.
+// The initialization of the `safepoint` pointer is done `ti_initthread`
+// in `threading.c`.
 uint8_t jl_safepoint_enable_cnt[3] = {0, 0, 0};
 
 // This lock should be acquired before enabling/disabling the safepoint
@@ -48,7 +54,7 @@ static void jl_safepoint_enable(int idx)
         return;
     }
     // Now that we are requested to mprotect the page and it wasn't already.
-    char *pageaddr = (char*)jl_safepoint_page + jl_page_size * (idx - 1);
+    char *pageaddr = jl_safepoint_pages + jl_page_size * idx;
 #ifdef _OS_WINDOWS_
     DWORD old_prot;
     VirtualProtect(pageaddr, jl_page_size, PAGE_NOACCESS, &old_prot);
@@ -67,7 +73,7 @@ static void jl_safepoint_disable(int idx)
     }
     // Now that we are requested to un-mprotect the page and no one else
     // want it to be kept protected.
-    char *pageaddr = (char*)jl_safepoint_page + jl_page_size * (idx - 1);
+    char *pageaddr = jl_safepoint_pages + jl_page_size * idx;
 #ifdef _OS_WINDOWS_
     DWORD old_prot;
     VirtualProtect(pageaddr, jl_page_size, PAGE_READONLY, &old_prot);
@@ -95,7 +101,7 @@ void jl_safepoint_init(void)
     }
     // The signal page is for the gc safepoint.
     // The page before it is the sigint pending flag.
-    jl_safepoint_page = (size_t*)(addr + pgsz);
+    jl_safepoint_pages = addr;
 }
 
 int jl_safepoint_start_gc(void)

src/signals-mach.c

@@ -166,7 +166,7 @@ kern_return_t catch_exception_raise(mach_port_t            exception_port,
     HANDLE_MACH_ERROR("thread_get_state", ret);
     uint64_t fault_addr = exc_state.__faultvaddr;
 #ifdef JULIA_ENABLE_THREADING
-    if (fault_addr == (uintptr_t)jl_safepoint_page) {
+    if (jl_addr_is_safepoint(fault_addr)) {
         jl_mutex_lock_nogc(&safepoint_lock);
         if (!jl_gc_running) {
             // GC is done before we get the message, do nothing and return

src/signals-unix.c

@@ -90,7 +90,7 @@ static void segv_handler(int sig, siginfo_t *info, void *context)
     assert(sig == SIGSEGV || sig == SIGBUS);
 
 #ifdef JULIA_ENABLE_THREADING
-    if (info->si_addr == jl_safepoint_page) {
+    if (jl_addr_is_safepoint((uintptr_t)info->si_addr)) {
         jl_unblock_signal(sig);
         jl_set_gc_and_wait();
         return;

src/signals-win.c

@@ -169,8 +169,7 @@ static LONG WINAPI _exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo,
                 return EXCEPTION_CONTINUE_EXECUTION;
             case EXCEPTION_ACCESS_VIOLATION:
 #ifdef JULIA_ENABLE_THREADING
-                if (ExceptionInfo->ExceptionRecord->ExceptionInformation[1] ==
-                    (intptr_t)jl_safepoint_page) {
+                if (jl_addr_is_safepoint(ExceptionInfo->ExceptionRecord->ExceptionInformation[1])) {
                     jl_set_gc_and_wait();
                     return EXCEPTION_CONTINUE_EXECUTION;
                 }

src/threading.c

@@ -104,6 +104,15 @@ static void ti_initthread(int16_t tid)
     ptls->tid = tid;
     ptls->pgcstack = NULL;
     ptls->gc_state = 0; // GC unsafe
+    // Conditionally initialize the safepoint address. See comment in
+    // `safepoint.c`
+    if (tid == 0) {
+        ptls->safepoint = (size_t*)(jl_safepoint_pages + jl_page_size);
+    }
+    else {
+        ptls->safepoint = (size_t*)(jl_safepoint_pages + jl_page_size * 2 +
+                                    sizeof(size_t));
+    }
     ptls->current_module = NULL;
     void *bt_data = malloc(sizeof(uintptr_t) * (JL_MAX_BT_SIZE + 1));
     if (bt_data == NULL) {