Split out safepoint.c

Add note and function prototypes about using safepoint to deliver SIGINT.

src/Makefile

@@ -29,7 +29,7 @@ SRCS := \
 	jltypes gf typemap ast builtins module interpreter \
 	alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \
 	simplevector APInt-C runtime_intrinsics runtime_ccall \
-	threadgroup threading stackwalk gc
+	threadgroup threading stackwalk gc safepoint
 
 ifeq ($(JULIACODEGEN),LLVM)
 SRCS += codegen disasm debuginfo llvm-simdloop llvm-gcroot

src/codegen.cpp

@@ -344,8 +344,8 @@ static GlobalVariable *jltls_states_func_ptr = NULL;
 static size_t jltls_states_func_idx = 0;
 #endif
 // Imaging mode only (non-imaging mode use pointer directly)
-static GlobalVariable *jl_gc_signal_page_ptr = NULL;
-static size_t jl_gc_signal_page_idx = 0;
+static GlobalVariable *jl_safepoint_page_ptr = NULL;
+static size_t jl_safepoint_page_idx = 0;
 
 // important functions
 static Function *jlnew_func;
@@ -3432,11 +3432,11 @@ static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx)
     ctx->ptlsStates = builder.CreateCall(prepare_call(jltls_states_func));
     if (imaging_mode) {
         ctx->signalPage =
-            tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jl_gc_signal_page_ptr)));
+            tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jl_safepoint_page_ptr)));
     }
     else {
         ctx->signalPage = builder.CreateIntToPtr(
-            ConstantInt::get(T_size, (uintptr_t)jl_gc_signal_page), T_pint8);
+            ConstantInt::get(T_size, (uintptr_t)jl_safepoint_page), T_pint8);
     }
 }
 
@@ -5171,10 +5171,10 @@ static void init_julia_llvm_env(Module *m)
     }
 #endif
     if (imaging_mode) {
-        jl_gc_signal_page_ptr =
-            jl_emit_sysimg_slot(m, T_pint8, "jl_gc_signal_page.ptr",
-                                (uintptr_t)jl_gc_signal_page,
-                                jl_gc_signal_page_idx);
+        jl_safepoint_page_ptr =
+            jl_emit_sysimg_slot(m, T_pint8, "jl_safepoint_page.ptr",
+                                (uintptr_t)jl_safepoint_page,
+                                jl_safepoint_page_idx);
     }
 
     std::vector<Type*> args1(0);

src/dump.c

@@ -232,9 +232,9 @@ static int jl_load_sysimg_so(void)
         *sysimg_gvars[tls_getter_idx - 1] =
             (jl_value_t*)jl_get_ptls_states_getter();
 #endif
-        size_t signal_page_idx = *(size_t*)jl_dlsym(jl_sysimg_handle,
-                                                    "jl_gc_signal_page_idx");
-        *sysimg_gvars[signal_page_idx - 1] = (jl_value_t*)jl_gc_signal_page;
+        size_t safepoint_idx = *(size_t*)jl_dlsym(jl_sysimg_handle,
+                                                  "jl_safepoint_page_idx");
+        *sysimg_gvars[safepoint_idx - 1] = (jl_value_t*)jl_safepoint_page;
         const char *cpu_target = (const char*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_target");
         if (strcmp(cpu_target,jl_options.cpu_target) != 0)
             jl_error("Julia and the system image were compiled for different architectures.\n"

src/gc.c

@@ -45,7 +45,8 @@ static jl_mutex_t finalizers_lock;
  * threads that enters `jl_gc_collect()` at the same time (or later calling
  * from unmanaged code) will wait in `jl_gc_collect()` until the GC is finished.
  *
- * Before starting the mark phase the GC thread calls `jl_gc_signal_begin()`
+ * Before starting the mark phase the GC thread calls `jl_safepoint_gc_start()`
+ * and `jl_gc_wait_for_the_world()`
  * to make sure all the thread are in a safe state for the GC. The function
  * activates the safepoint and wait for all the threads to get ready for the
  * GC (`gc_state != 0`). It also acquires the `finalizers` lock so that no
@@ -354,95 +355,26 @@ NOINLINE static uintptr_t gc_get_stack_ptr(void)
 
 #include "gc-debug.c"
 
-// Only one thread can be doing the collection right now. That thread set
-// `jl_running_gc` to one on entering the GC and set it back afterward.
-static volatile uint64_t jl_gc_running = 0;
-
-JL_DLLEXPORT volatile size_t *jl_gc_signal_page = NULL;
-
-void jl_gc_signal_init(void)
-{
-    // jl_page_size isn't available yet.
-#ifdef _OS_WINDOWS_
-    jl_gc_signal_page = (size_t*)VirtualAlloc(NULL, jl_getpagesize(),
-                                              MEM_COMMIT, PAGE_READONLY);
-#else
-    jl_gc_signal_page = (size_t*)mmap(0, jl_getpagesize(), PROT_READ,
-                                      MAP_NORESERVE | MAP_PRIVATE |
-                                      MAP_ANONYMOUS, -1, 0);
-    if (jl_gc_signal_page == MAP_FAILED)
-        jl_gc_signal_page = NULL;
-#endif
-    if (jl_gc_signal_page == NULL) {
-        jl_printf(JL_STDERR, "could not allocate GC synchronization page\n");
-        gc_debug_critical_error();
-        abort();
-    }
-}
-
 #ifdef JULIA_ENABLE_THREADING
-static void jl_wait_for_gc(void)
-{
-    while (jl_gc_running) {
-        jl_cpu_pause(); // yield?
-    }
-}
-
-void jl_gc_signal_wait(void)
-{
-    int8_t state = jl_gc_state();
-    jl_get_ptls_states()->gc_state = JL_GC_STATE_WAITING;
-    jl_wait_for_gc();
-    jl_get_ptls_states()->gc_state = state;
-}
-
 static void jl_gc_wait_for_the_world(void)
 {
     for (int i = 0;i < jl_n_threads;i++) {
         jl_tls_states_t *ptls = jl_all_task_states[i].ptls;
-        while (!ptls->gc_state) {
+        // FIXME: The acquire load pairs with the release stores
+        // in the signal handler of safepoint so we are sure that
+        // all the stores on those threads are visible. However,
+        // we're currently not using atomic stores in mutator threads.
+        // We should either use atomic store release there too or use signals
+        // to flush the memory operations on those threads.
+        while (!ptls->gc_state || !jl_atomic_load_acquire(&ptls->gc_state)) {
             jl_cpu_pause(); // yield?
         }
     }
 }
-
-static void jl_gc_signal_begin(void)
-{
-#ifdef __APPLE__
-    // This needs to be after setting `jl_gc_running` so that only one thread
-    // can talk to the signal handler
-    jl_mach_gc_begin();
-#endif
-#ifdef _OS_WINDOWS_
-    DWORD old_prot;
-    VirtualProtect((void*)jl_gc_signal_page, jl_page_size,
-                   PAGE_NOACCESS, &old_prot);
 #else
-    mprotect((void*)jl_gc_signal_page, jl_page_size, PROT_NONE);
-#endif
-    jl_gc_wait_for_the_world();
-    JL_LOCK_NOGC(&finalizers_lock);
-}
-
-static void jl_gc_signal_end(void)
+static inline void jl_gc_wait_for_the_world(void)
 {
-    JL_UNLOCK_NOGC(&finalizers_lock);
-#ifdef _OS_WINDOWS_
-    DWORD old_prot;
-    VirtualProtect((void*)jl_gc_signal_page, jl_page_size,
-                   PAGE_READONLY, &old_prot);
-#else
-    mprotect((void*)jl_gc_signal_page, jl_page_size, PROT_READ);
-#endif
-#ifdef __APPLE__
-    jl_mach_gc_end();
-#endif
 }
-#else
-
-#define jl_gc_signal_begin()
-#define jl_gc_signal_end()
-
 #endif
 
 static int jl_gc_finalizers_inhibited; // don't run finalizers during codegen #11956
@@ -2326,37 +2258,27 @@ JL_DLLEXPORT void jl_gc_collect(int full)
         return;
     char *stack_hi = (char*)gc_get_stack_ptr();
     gc_debug_print();
-    JL_SIGATOMIC_BEGIN();
 
     int8_t old_state = jl_gc_state();
     jl_get_ptls_states()->gc_state = JL_GC_STATE_WAITING;
-    // In case multiple threads enter the GC at the same time, only allow
-    // one of them to actually run the collection. We can't just let the
-    // master thread do the GC since it might be running unmanaged code
-    // and can take arbitrarily long time before hitting a safe point.
-    if (jl_atomic_compare_exchange(&jl_gc_running, 0, 1) != 0) {
-#ifdef JULIA_ENABLE_THREADING
-        JL_SIGATOMIC_END();
-        jl_wait_for_gc();
+    // `jl_safepoint_start_gc()` makes sure only one thread can
+    // run the GC.
+    if (!jl_safepoint_start_gc()) {
+        // Multithread only. See assertion in `safepoint.c`
         jl_gc_state_set(old_state, JL_GC_STATE_WAITING);
-#else
-        // For single thread, GC should not call itself (in finalizers) before
-        // setting jl_gc_running to false so this should never happen.
-        assert(0 && "GC synchronization failure");
-#endif
         return;
     }
-    jl_gc_signal_begin();
+    // no-op for non-threading
+    jl_gc_wait_for_the_world();
 
-    if (!jl_gc_disable_counter)
+    if (!jl_gc_disable_counter) {
+        JL_LOCK_NOGC(&finalizers_lock);
         _jl_gc_collect(full, stack_hi);
+        JL_UNLOCK_NOGC(&finalizers_lock);
+    }
 
-    // Need to reset the page protection before resetting the flag since
-    // the thread will trigger a segfault immediately after returning from
-    // the signal handler.
-    jl_gc_signal_end();
-    jl_gc_running = 0;
-    JL_SIGATOMIC_END();
+    // no-op for non-threading
+    jl_safepoint_end_gc();
     jl_gc_state_set(old_state, JL_GC_STATE_WAITING);
 
     if (!jl_gc_finalizers_inhibited) {

src/init.c

@@ -534,7 +534,7 @@ void _julia_init(JL_IMAGE_SEARCH rel)
     // Make sure we finalize the tls callback before starting any threads.
     jl_get_ptls_states_getter();
 #endif
-    jl_gc_signal_init();
+    jl_safepoint_init();
     libsupport_init();
     jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.),
                                     // best to call this first, since it also initializes libuv

src/jitlayers.cpp

@@ -918,8 +918,8 @@ static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap,
                                  T_size,
                                  true,
                                  GlobalVariable::ExternalLinkage,
-                                 ConstantInt::get(T_size, jl_gc_signal_page_idx),
-                                 "jl_gc_signal_page_idx"));
+                                 ConstantInt::get(T_size, jl_safepoint_page_idx),
+                                 "jl_safepoint_page_idx"));
 
     Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target);
     addComdat(new GlobalVariable(*mod,

src/julia.h

@@ -1483,10 +1483,11 @@ static inline void jl_lock_frame_pop(void)
 
 STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh)
 {
+    // This function should **NOT** have any safepoint before restoring
+    // the GC state at the end.
     JL_SIGATOMIC_BEGIN();
     jl_current_task->eh = eh->prev;
     jl_pgcstack = eh->gcstack;
-    jl_gc_state_save_and_set(eh->gc_state);
 #ifdef JULIA_ENABLE_THREADING
     arraylist_t *locks = &jl_current_task->locks;
     if (locks->len > eh->locks_len) {
@@ -1495,6 +1496,9 @@ STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh)
         locks->len = eh->locks_len;
     }
 #endif
+    // This should be the last since this can trigger a safepoint
+    // that throws a SIGINT.
+    jl_gc_state_save_and_set(eh->gc_state);
     JL_SIGATOMIC_END();
 }
 

src/julia_internal.h

@@ -266,10 +266,56 @@ void jl_set_base_ctx(char *__stk);
 void jl_init_threading(void);
 void jl_start_threads(void);
 void jl_shutdown_threading(void);
-void jl_gc_signal_init(void);
+
+// Whether the GC is running
+extern volatile uint32_t jl_gc_running;
+// All the functions are safe to be called from within a signal handler
+// provided that the thread will not be interrupted by another asynchronous
+// signal.
+// Initialize the safepoint
+void jl_safepoint_init(void);
+// Start the GC, return `1` if the thread should be running the GC.
+// Otherwise, the thread will wait in this function until the GC finishes on
+// another thread and return `0`.
+// The caller should have saved the `gc_state` and set it to `WAITING`
+// before calling this function. If the calling thread is to run the GC,
+// it should also wait for the mutator threads to hit a safepoint **AFTER**
+// this function returns
+int jl_safepoint_start_gc(void);
+// Can only be called by the thread that have got a `1` return value from
+// `jl_safepoint_start_gc()`. This disables the safepoint (for GC,
+// the `mprotect` may not be removed if there's pending SIGINT) and wake
+// up waiting threads if there's any.
+// The caller should restore `gc_state` **AFTER** calling this function.
+void jl_safepoint_end_gc(void);
+// Wait for the GC to finish
+// This function does **NOT** modify the `gc_state` to inform the GC thread
+// The caller should set it **BEFORE** calling this function.
+void jl_safepoint_wait_gc(void);
+
+// Set pending sigint and enable the mechanisms to deliver the sigint.
+void jl_safepoint_enable_sigint(void);
+// If the safepoint is enabled to deliver sigint, disable it
+// so that the thread won't repeatedly trigger it in a sigatomic region
+// while not being able to actually throw the exception.
+void jl_safepoint_defer_sigint(void);
+// Clear the sigint pending flag and disable the mechanism to deliver sigint.
+// Return `1` if the sigint should be delivered and `0` if there's no sigint
+// to be delivered.
+int jl_safepoint_consume_sigint(void);
+
 #ifdef JULIA_ENABLE_THREADING
 jl_get_ptls_states_func jl_get_ptls_states_getter(void);
-void jl_gc_signal_wait(void);
+static inline void jl_set_gc_and_wait(void)
+{
+    // reading own gc state doesn't need atomic ops since no one else
+    // should store to it.
+    int8_t state = jl_gc_state();
+    jl_atomic_store_release(&jl_get_ptls_states()->gc_state,
+                            JL_GC_STATE_WAITING);
+    jl_safepoint_wait_gc();
+    jl_atomic_store_release(&jl_get_ptls_states()->gc_state, state);
+}
 #endif
 
 void jl_dump_bitcode(char *fname, const char *sysimg_data, size_t sysimg_len);
@@ -481,11 +527,11 @@ JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a);
 
 extern jl_mutex_t typecache_lock;
 extern jl_mutex_t codegen_lock;
+extern jl_mutex_t safepoint_lock;
 
 // -- gc.c -- //
 
 #if defined(__APPLE__) && defined(JULIA_ENABLE_THREADING)
-void jl_mach_gc_begin(void);
 void jl_mach_gc_end(void);
 #endif
 

src/julia_threads.h

@@ -285,13 +285,13 @@ JL_DLLEXPORT void (jl_cpu_wake)(void);
 
 // Accessing the tls variables, gc safepoint and gc states
 JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void);
-JL_DLLEXPORT extern volatile size_t *jl_gc_signal_page;
+JL_DLLEXPORT extern volatile size_t *jl_safepoint_page;
 // This triggers a SegFault when we are in GC
 // Assign it to a variable to make sure the compiler emit the load
 // and to avoid Clang warning for -Wunused-volatile-lvalue
 #define jl_gc_safepoint() do {                          \
         jl_signal_fence();                              \
-        size_t safepoint_load = *jl_gc_signal_page;     \
+        size_t safepoint_load = *jl_safepoint_page;     \
         jl_signal_fence();                              \
         (void)safepoint_load;                           \
     } while (0)