[jit] Rework the managed code suspend polling and add COOP_GC support.

This change enables managed polling to work with both NaCL and coop as they are exactly the same except for the variables used.

We cleanup how safepoints are injects to be done on a separate pass instead of having in spread around the JIT.

We now inject safepoints at EH handler entry as those did not previously have them.

The generated AOT code was greatly optimized by having the polling variable checked before doing the icall. This more
than doubles the performance of the generated code.

mono/mini/aot-compiler.c

@@ -5291,6 +5291,8 @@ encode_patch (MonoAotCompile *acfg, MonoJumpInfo *patch_info, guint8 *buf, guint
 		encode_klass_ref (acfg, patch_info->data.virt_method->klass, p, &p);
 		encode_method_ref (acfg, patch_info->data.virt_method->method, p, &p);
 		break;
+	case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG:
+		break;
 	default:
 		g_warning ("unable to handle jump info %d", patch_info->type);
 		g_assert_not_reached ();

mono/mini/aot-runtime.c

@@ -3460,6 +3460,8 @@ decode_patch (MonoAotModule *aot_module, MonoMemPool *mp, MonoJumpInfo *ji, guin
 		ji->data.target = info;
 		break;
 	}
+	case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG:
+		break;
 	default:
 		g_warning ("unhandled type %d", ji->type);
 		g_assert_not_reached ();

mono/mini/cpu-amd64.md

@@ -295,7 +295,7 @@ move_f_to_i8: dest:i src1:f len:5
 move_i8_to_f: dest:f src1:i len:5
 call_handler: len:14 clob:c nacl:52
 aot_const: dest:i len:10
-nacl_gc_safe_point: clob:c
+gc_safe_point: clob:c src1:i len:40
 x86_test_null: src1:i len:5
 x86_compare_membase_reg: src1:b src2:i len:9
 x86_compare_membase_imm: src1:b len:13

mono/mini/cpu-x86.md

@@ -251,7 +251,7 @@ call_handler: len:11 clob:c
 aot_const: dest:i len:5
 load_gotaddr: dest:i len:64
 got_entry: dest:i src1:b len:7
-nacl_gc_safe_point: clob:c
+gc_safe_point: clob:c src1:i len:20
 x86_test_null: src1:i len:2
 x86_compare_membase_reg: src1:b src2:i len:7
 x86_compare_membase_imm: src1:b len:11

mono/mini/method-to-ir.c

@@ -7919,11 +7919,6 @@ mono_method_to_ir (MonoCompile *cfg, MonoMethod *method, MonoBasicBlock *start_b
 		cfg->bb_entry = start_bblock;
 		start_bblock->cil_code = NULL;
 		start_bblock->cil_length = 0;
-#if defined(__native_client_codegen__)
-		MONO_INST_NEW (cfg, ins, OP_NACL_GC_SAFE_POINT);
-		ins->dreg = alloc_dreg (cfg, STACK_I4);
-		MONO_ADD_INS (start_bblock, ins);
-#endif
 
 		/* EXIT BLOCK */
 		NEW_BBLOCK (cfg, end_bblock);

mono/mini/mini-amd64.c

@@ -32,6 +32,7 @@
 #include <mono/utils/mono-memory-model.h>
 #include <mono/utils/mono-tls.h>
 #include <mono/utils/mono-hwcap-x86.h>
+#include <mono/utils/mono-threads.h>
 
 #include "trace.h"
 #include "ir-emit.h"
@@ -6555,22 +6556,26 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
 			MONO_VARINFO (cfg, ins->inst_c0)->live_range_end = code - cfg->native_code;
 			break;
 		}
-		case OP_NACL_GC_SAFE_POINT: {
-#if defined(__native_client_codegen__) && defined(__native_client_gc__)
-			if (cfg->compile_aot)
-				code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc, TRUE);
-			else {
-				guint8 *br [1];
-
-				amd64_mov_reg_imm_size (code, AMD64_R11, (gpointer)&__nacl_thread_suspension_needed, 4);
-				amd64_test_membase_imm_size (code, AMD64_R11, 0, 0xFFFFFFFF, 4);
-				br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE);
-				code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc, TRUE);
-				amd64_patch (br[0], code);
-			}
+		case OP_GC_SAFE_POINT: {
+			gpointer polling_func = NULL;
+			int compare_val;
+#if defined (USE_COOP_GC)
+			polling_func = (gpointer)mono_threads_state_poll;
+			compare_val = 1;
+#elif defined(__native_client_codegen__) && defined(__native_client_gc__)
+			polling_func = (gpointer)mono_nacl_gc;
+			compare_val = 0xFFFFFFFF;
 #endif
+			if (!polling_func)
+				break;
+
+			amd64_test_membase_imm_size (code, ins->sreg1, 0, compare_val, 4);
+			br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE);
+			code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, polling_func, TRUE);
+			amd64_patch (br[0], code);
 			break;
 		}
+
 		case OP_GC_LIVENESS_DEF:
 		case OP_GC_LIVENESS_USE:
 		case OP_GC_PARAM_SLOT_LIVENESS_DEF:

mono/mini/mini-ops.h

@@ -1071,8 +1071,8 @@ MINI_OP(OP_GC_PARAM_SLOT_LIVENESS_DEF, "gc_param_slot_liveness_def", NONE, NONE,
 /* #if defined(__native_client_codegen__) || defined(__native_client__) */
 /* We have to define these in terms of the TARGET defines, not NaCl defines */
 /* because genmdesc.pl doesn't have multiple defines per platform.          */
-#if defined(TARGET_AMD64) || defined(TARGET_X86) || defined(TARGET_ARM)
-MINI_OP(OP_NACL_GC_SAFE_POINT,     "nacl_gc_safe_point", IREG, NONE, NONE)
+#if defined(TARGET_AMD64) || defined(TARGET_X86)
+MINI_OP(OP_GC_SAFE_POINT,     "gc_safe_point", NONE, IREG, NONE)
 #endif
 
 #if defined(TARGET_X86) || defined(TARGET_AMD64)

mono/mini/mini-runtime.c

@@ -1175,6 +1175,7 @@ mono_patch_info_hash (gconstpointer data)
 	case MONO_PATCH_INFO_MONITOR_ENTER_V4:
 	case MONO_PATCH_INFO_MONITOR_EXIT:
 	case MONO_PATCH_INFO_GOT_OFFSET:
+	case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG:
 		return (ji->type << 8);
 	case MONO_PATCH_INFO_CASTCLASS_CACHE:
 		return (ji->type << 8) | (ji->data.index);
@@ -1355,6 +1356,13 @@ mono_resolve_patch_target (MonoMethod *method, MonoDomain *domain, guint8 *code,
 		target = code_slot;
 		break;
 	}
+	case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG:
+#if defined(__native_client_codegen__)
+		target = (gpointer)&__nacl_thread_suspension_needed;
+#else
+		target = (gpointer)&mono_polling_required;
+#endif
+		break;
 	case MONO_PATCH_INFO_SWITCH: {
 		gpointer *jump_table;
 		int i;
@@ -3205,6 +3213,10 @@ register_icalls (void)
 #if defined(__native_client__) || defined(__native_client_codegen__)
 	register_icall (mono_nacl_gc, "mono_nacl_gc", "void", TRUE);
 #endif
+#if defined(USE_COOP_GC)
+	register_icall (mono_threads_state_poll, "mono_threads_state_poll", "void", TRUE);
+#endif
+
 #ifndef MONO_ARCH_NO_EMULATE_LONG_MUL_OPTS
 	register_opcode_emulation (OP_LMUL, "__emul_lmul", "long long long", mono_llmult, "mono_llmult", TRUE);
 	register_opcode_emulation (OP_LDIV, "__emul_ldiv", "long long long", mono_lldiv, "mono_lldiv", FALSE);

mono/mini/mini-x86.c

@@ -29,6 +29,7 @@
 #include <mono/utils/mono-mmap.h>
 #include <mono/utils/mono-memory-model.h>
 #include <mono/utils/mono-hwcap-x86.h>
+#include <mono/utils/mono-threads.h>
 
 #include "trace.h"
 #include "mini-x86.h"
@@ -5023,19 +5024,26 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
 			MONO_VARINFO (cfg, ins->inst_c0)->live_range_end = code - cfg->native_code;
 			break;
 		}
-		case OP_NACL_GC_SAFE_POINT: {
-#if defined(__native_client_codegen__) && defined(__native_client_gc__)
-			if (cfg->compile_aot)
-				code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc);
-			else {
-				guint8 *br [1];
-
-				x86_test_mem_imm8 (code, (gpointer)&__nacl_thread_suspension_needed, 0xFFFFFFFF);
-				br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE);
-				code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc);
-				x86_patch (br[0], code);
-			}
+		case OP_GC_SAFE_POINT: {
+			gpointer polling_func = NULL;
+			int compare_val;
+			guint8 *br [1];
+
+#if defined (USE_COOP_GC)
+			polling_func = (gpointer)mono_threads_state_poll;
+			compare_val = 1;
+#elif defined(__native_client_codegen__) && defined(__native_client_gc__)
+			polling_func = (gpointer)mono_nacl_gc;
+			compare_val = 0xFFFFFFFF;
 #endif
+			if (!polling_func)
+				break;
+
+			x86_test_membase_imm (code, ins->sreg1, 0, compare_val);
+			br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE);
+			code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, polling_func);
+			x86_patch (br [0], code);
+
 			break;
 		}
 		case OP_GC_LIVENESS_DEF:

mono/mini/mini.c

@@ -3005,6 +3005,71 @@ is_open_method (MonoMethod *method)
 }
 
 #ifndef DISABLE_JIT
+
+static void
+mono_create_gc_safepoint (MonoCompile *cfg, MonoBasicBlock *bblock)
+{
+#if defined(__native_client_codegen__) || USE_COOP_GC
+
+	MonoInst *poll_addr, *ins;
+	if (cfg->verbose_level)
+		printf ("ADDING SAFE POINT TO BB %d\n", bblock->block_num);
+
+#if defined(__native_client_codegen__)
+	NEW_AOTCONST (cfg, poll_addr, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG, (gpointer)&__nacl_thread_suspension_needed);
+#else
+	NEW_AOTCONST (cfg, poll_addr, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG, (gpointer)&mono_polling_required);
+#endif
+
+	MONO_INST_NEW (cfg, ins, OP_GC_SAFE_POINT);
+	ins->sreg1 = poll_addr->dreg;
+
+	 if (bblock->flags & BB_EXCEPTION_HANDLER) {
+		MonoInst *eh_op = bblock->code;
+
+		// we only skip the ops that start EH blocks.
+		if (eh_op && eh_op->opcode != OP_START_HANDLER && eh_op->opcode != OP_GET_EX_OBJ)
+			eh_op = NULL;
+
+		mono_bblock_insert_after_ins (bblock, eh_op, poll_addr);
+		mono_bblock_insert_after_ins (bblock, poll_addr, ins);
+	} else if (bblock == cfg->bb_entry) {
+		mono_bblock_insert_after_ins (bblock, bblock->last_ins, poll_addr);
+		mono_bblock_insert_after_ins (bblock, poll_addr, ins);
+
+	} else {
+		mono_bblock_insert_before_ins (bblock, NULL, poll_addr);
+		mono_bblock_insert_after_ins (bblock, poll_addr, ins);
+	}
+
+#endif
+}
+
+/*
+This code inserts safepoints into managed code at important code paths.
+Those are:
+
+-the first basic block
+-landing BB for exception handlers
+-loop body starts.
+
+*/
+static void
+mono_insert_safepoints (MonoCompile *cfg)
+{
+#if defined(__native_client_codegen__) || defined(USE_COOP_GC)
+	MonoBasicBlock *bb;
+
+	if (cfg->verbose_level)
+		printf ("INSERTING SAFEPOINTS\n");
+
+	for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
+		if (bb->loop_body_start || bb == cfg->bb_entry || bb->flags & BB_EXCEPTION_HANDLER)
+			mono_create_gc_safepoint (cfg, bb);
+	}
+#endif
+}
+
 /*
  * mini_method_compile:
  * @method: the method to compile
@@ -3126,6 +3191,10 @@ mini_method_compile (MonoMethod *method, guint32 opts, MonoDomain *domain, JitFl
 		cfg->gen_sdb_seq_points = FALSE;
 	}
 #endif
+	/* coop / nacl requires loop detection to happen */
+#if defined(__native_client_codegen__) || defined(USE_COOP_GC)
+	cfg->opt |= MONO_OPT_LOOP;
+#endif
 
 	cfg->explicit_null_checks = debug_options.explicit_null_checks;
 	cfg->soft_breakpoints = debug_options.soft_breakpoints;
@@ -3550,6 +3619,8 @@ mini_method_compile (MonoMethod *method, guint32 opts, MonoDomain *domain, JitFl
 		mono_compute_natural_loops (cfg);
 	}
 
+	mono_insert_safepoints (cfg);
+
 	/* after method_to_ir */
 	if (parts == 1) {
 		if (MONO_METHOD_COMPILE_END_ENABLED ())

mono/mini/mini.h

@@ -105,7 +105,7 @@
 #endif
 
 /* Version number of the AOT file format */
-#define MONO_AOT_FILE_VERSION 114
+#define MONO_AOT_FILE_VERSION 115
 
 //TODO: This is x86/amd64 specific.
 #define mono_simd_shuffle_mask(a,b,c,d) ((a) | ((b) << 2) | ((c) << 4) | ((d) << 6))

mono/mini/patch-info.h

@@ -55,4 +55,5 @@ PATCH_INFO(METHOD_CODE_SLOT, "method_code_slot")
 PATCH_INFO(LDSTR_LIT, "ldstr_lit")
 PATCH_INFO(GC_NURSERY_START, "gc_nursery_start")
 PATCH_INFO(VIRT_METHOD, "virt_method")
+PATCH_INFO(GC_SAFE_POINT_FLAG, "gc_safe_point_flag")
 PATCH_INFO(NONE, "none")