[aot] Add aot support for write barrier.

We emit two write barriers, for the case of the concurrent and non-concurrent collector in the mscorlib image. All calls to the write barrier are made through the plt, calls which are resolved according to the GC configuration at runtime.

mono/cil/cil-opcodes.xml

@@ -313,4 +313,6 @@
 <opcode name="mono_jit_detach" input="Pop0" output="Push0" args="InlineNone" o1="0xF0" o2="0x12" flow="next" />
 <opcode name="mono_jit_icall_addr" input="Pop0" output="PushI" args="InlineI" o1="0xF0" o2="0x13" flow="next" />
 <opcode name="mono_ldptr_int_req_flag" input="Pop0" output="PushI" args="InlineNone" o1="0xF0" o2="0x14" flow="next" />
+<opcode name="mono_ldptr_card_table" input="Pop0" output="PushI" args="InlineNone" o1="0xF0" o2="0x15" flow="next" />
+<opcode name="mono_ldptr_nursery_start" input="Pop0" output="PushI" args="InlineNone" o1="0xF0" o2="0x16" flow="next" />
 </opdesc>

mono/cil/opcode.def

@@ -314,6 +314,8 @@ OPDEF(CEE_MONO_JIT_ATTACH, "mono_jit_attach", Pop0, Push0, InlineNone, X, 2, 0xF
 OPDEF(CEE_MONO_JIT_DETACH, "mono_jit_detach", Pop0, Push0, InlineNone, X, 2, 0xF0, 0x12, NEXT)
 OPDEF(CEE_MONO_JIT_ICALL_ADDR, "mono_jit_icall_addr", Pop0, Push0, InlineNone, X, 2, 0xF0, 0x13, NEXT)
 OPDEF(CEE_MONO_LDPTR_INT_REQ_FLAG, "mono_ldptr_int_req_flag", Pop0, PushI, InlineNone, X, 2, 0xF0, 0x14, NEXT)
+OPDEF(CEE_MONO_LDPTR_CARD_TABLE, "mono_ldptr_card_table", Pop0, PushI, InlineNone, X, 2, 0xF0, 0x15, NEXT)
+OPDEF(CEE_MONO_LDPTR_NURSERY_START, "mono_ldptr_nursery_start", Pop0, PushI, InlineNone, X, 2, 0xF0, 0x16, NEXT)
 #ifndef OPALIAS
 #define _MONO_CIL_OPALIAS_DEFINED_
 #define OPALIAS(a,s,r)

mono/metadata/method-builder.c

@@ -329,6 +329,19 @@ mono_mb_emit_i4 (MonoMethodBuilder *mb, gint32 data)
 	mb->pos += 4;
 }
 
+void
+mono_mb_emit_i8 (MonoMethodBuilder *mb, gint64 data)
+{
+	if ((mb->pos + 8) >= mb->code_size) {
+		mb->code_size += mb->code_size >> 1;
+		mb->code = g_realloc (mb->code, mb->code_size);
+	}
+
+	mono_mb_patch_addr (mb, mb->pos, data);
+	mono_mb_patch_addr (mb, mb->pos + 4, data >> 32);
+	mb->pos += 8;
+}
+
 void
 mono_mb_emit_i2 (MonoMethodBuilder *mb, gint16 data)
 {
@@ -440,6 +453,13 @@ mono_mb_emit_icon (MonoMethodBuilder *mb, gint32 value)
 	}
 }
 
+void
+mono_mb_emit_icon8 (MonoMethodBuilder *mb, gint64 value)
+{
+	mono_mb_emit_byte (mb, CEE_LDC_I8);
+	mono_mb_emit_i8 (mb, value);
+}
+
 int
 mono_mb_get_label (MonoMethodBuilder *mb)
 {

mono/metadata/method-builder.h

@@ -113,6 +113,9 @@ mono_mb_emit_exception_full (MonoMethodBuilder *mb, const char *exc_nspace, cons
 void
 mono_mb_emit_icon (MonoMethodBuilder *mb, gint32 value);
 
+void
+mono_mb_emit_icon8 (MonoMethodBuilder *mb, gint64 value);
+
 guint32
 mono_mb_emit_branch (MonoMethodBuilder *mb, guint8 op);
 
@@ -137,6 +140,9 @@ mono_mb_emit_i2 (MonoMethodBuilder *mb, gint16 data);
 void
 mono_mb_emit_i4 (MonoMethodBuilder *mb, gint32 data);
 
+void
+mono_mb_emit_i8 (MonoMethodBuilder *mb, gint64 data);
+
 void
 mono_mb_emit_op (MonoMethodBuilder *mb, guint8 op, gpointer data);
 

mono/metadata/sgen-gc.c

@@ -5132,16 +5132,24 @@ sgen_has_critical_method (void)
 static void
 emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels, gboolean is_concurrent)
 {
+	int shifted_nursery_start = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+
 	memset (nursery_check_return_labels, 0, sizeof (int) * 2);
 	// if (ptr_in_nursery (ptr)) return;
 	/*
 	 * Masking out the bits might be faster, but we would have to use 64 bit
 	 * immediates, which might be slower.
 	 */
+	mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
+	mono_mb_emit_byte (mb, CEE_MONO_LDPTR_NURSERY_START);
+	mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+	mono_mb_emit_byte (mb, CEE_SHR_UN);
+	mono_mb_emit_stloc (mb, shifted_nursery_start);
+
 	mono_mb_emit_ldarg (mb, 0);
 	mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
 	mono_mb_emit_byte (mb, CEE_SHR_UN);
-	mono_mb_emit_ptr (mb, (gpointer)((mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS));
+	mono_mb_emit_ldloc (mb, shifted_nursery_start);
 	nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
 
 	if (!is_concurrent) {
@@ -5150,7 +5158,7 @@ emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels, gbo
 		mono_mb_emit_byte (mb, CEE_LDIND_I);
 		mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
 		mono_mb_emit_byte (mb, CEE_SHR_UN);
-		mono_mb_emit_ptr (mb, (gpointer)((mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS));
+		mono_mb_emit_ldloc (mb, shifted_nursery_start);
 		nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
 	}
 }
@@ -5216,12 +5224,19 @@ mono_gc_get_specific_write_barrier (gboolean is_concurrent)
 	ldc_i4_1
 	stind_i1
 	*/
-	mono_mb_emit_ptr (mb, sgen_cardtable);
+	mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
+	mono_mb_emit_byte (mb, CEE_MONO_LDPTR_CARD_TABLE);
 	mono_mb_emit_ldarg (mb, 0);
 	mono_mb_emit_icon (mb, CARD_BITS);
 	mono_mb_emit_byte (mb, CEE_SHR_UN);
+	mono_mb_emit_byte (mb, CEE_CONV_I);
 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
-	mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
+#if SIZEOF_VOID_P == 8
+	mono_mb_emit_icon8 (mb, CARD_MASK);
+#else
+	mono_mb_emit_icon (mb, CARD_MASK);
+#endif
+	mono_mb_emit_byte (mb, CEE_CONV_I);
 	mono_mb_emit_byte (mb, CEE_AND);
 #endif
 	mono_mb_emit_byte (mb, CEE_ADD);

mono/mini/aot-compiler.c

@@ -3545,6 +3545,12 @@ add_wrappers (MonoAotCompile *acfg)
 			}
 		}
 
+		/* write barriers */
+		if (mono_gc_is_moving ()) {
+			add_method (acfg, mono_gc_get_specific_write_barrier (FALSE));
+			add_method (acfg, mono_gc_get_specific_write_barrier (TRUE));
+		}
+
 		/* Stelemref wrappers */
 		{
 			MonoMethod **wrappers;
@@ -4501,6 +4507,9 @@ is_direct_callable (MonoAotCompile *acfg, MonoMethod *method, MonoJumpInfo *patc
 			if (callee_cfg->method->wrapper_type == MONO_WRAPPER_ALLOC)
 				/* sgen does some initialization when the allocator method is created */
 				direct_callable = FALSE;
+			if (callee_cfg->method->wrapper_type == MONO_WRAPPER_WRITE_BARRIER)
+				/* we don't know at compile time whether sgen is concurrent or not */
+				direct_callable = FALSE;
 
 			if (direct_callable)
 				return TRUE;
@@ -5075,6 +5084,7 @@ encode_patch (MonoAotCompile *acfg, MonoJumpInfo *patch_info, guint8 *buf, guint
 	case MONO_PATCH_INFO_MSCORLIB_GOT_ADDR:
 	case MONO_PATCH_INFO_JIT_TLS_ID:
 	case MONO_PATCH_INFO_GC_CARD_TABLE_ADDR:
+	case MONO_PATCH_INFO_GC_NURSERY_START:
 		break;
 	case MONO_PATCH_INFO_CASTCLASS_CACHE:
 		encode_value (patch_info->data.index, p, &p);
@@ -5349,7 +5359,8 @@ emit_method_info (MonoAotCompile *acfg, MonoCompile *cfg)
 			continue;
 		}
 
-		if (patch_info->type == MONO_PATCH_INFO_GC_CARD_TABLE_ADDR) {
+		if (patch_info->type == MONO_PATCH_INFO_GC_CARD_TABLE_ADDR ||
+				patch_info->type == MONO_PATCH_INFO_GC_NURSERY_START) {
 			/* Stored in a GOT slot initialized at module load time */
 			patch_info->type = MONO_PATCH_INFO_NONE;
 			continue;
@@ -6871,6 +6882,7 @@ compile_method (MonoAotCompile *acfg, MonoMethod *method)
 		case MONO_PATCH_INFO_GOT_OFFSET:
 		case MONO_PATCH_INFO_NONE:
 		case MONO_PATCH_INFO_GC_CARD_TABLE_ADDR:
+		case MONO_PATCH_INFO_GC_NURSERY_START:
 			break;
 		case MONO_PATCH_INFO_IMAGE:
 			/* The assembly is stored in GOT slot 0 */
@@ -9345,6 +9357,11 @@ mono_compile_assembly (MonoAssembly *ass, guint32 opts, const char *aot_options)
 		get_got_offset (acfg, FALSE, ji);
 		get_got_offset (acfg, TRUE, ji);
 
+		ji = mono_mempool_alloc0 (acfg->mempool, sizeof (MonoJumpInfo));
+		ji->type = MONO_PATCH_INFO_GC_NURSERY_START;
+		get_got_offset (acfg, FALSE, ji);
+		get_got_offset (acfg, TRUE, ji);
+
 		ji = mono_mempool_alloc0 (acfg->mempool, sizeof (MonoJumpInfo));
 		ji->type = MONO_PATCH_INFO_JIT_TLS_ID;
 		get_got_offset (acfg, FALSE, ji);

mono/mini/aot-runtime.c

@@ -2020,14 +2020,18 @@ load_aot_module (MonoAssembly *assembly, gpointer user_data)
 
 		memset (&ji, 0, sizeof (ji));
 		ji.type = MONO_PATCH_INFO_GC_CARD_TABLE_ADDR;
-
 		amodule->got [2] = mono_resolve_patch_target (NULL, mono_get_root_domain (), NULL, &ji, FALSE);
+
+		memset (&ji, 0, sizeof (ji));
+		ji.type = MONO_PATCH_INFO_GC_NURSERY_START;
+		amodule->got [3] = mono_resolve_patch_target (NULL, mono_get_root_domain (), NULL, &ji, FALSE);
 	}
 
 	if (amodule->llvm_got) {
 		amodule->llvm_got [0] = amodule->got [0];
 		amodule->llvm_got [1] = amodule->got [1];
 		amodule->llvm_got [2] = amodule->got [2];
+		amodule->llvm_got [3] = amodule->got [3];
 	}
 
 	/*
@@ -3317,6 +3321,7 @@ decode_patch (MonoAotModule *aot_module, MonoMemPool *mp, MonoJumpInfo *ji, guin
 	case MONO_PATCH_INFO_MONITOR_ENTER_V4:
 	case MONO_PATCH_INFO_MONITOR_EXIT:
 	case MONO_PATCH_INFO_GC_CARD_TABLE_ADDR:
+	case MONO_PATCH_INFO_GC_NURSERY_START:
 	case MONO_PATCH_INFO_JIT_TLS_ID:
 		break;
 	case MONO_PATCH_INFO_CASTCLASS_CACHE:

mono/mini/method-to-ir.c

@@ -11985,6 +11985,36 @@ mono_method_to_ir (MonoCompile *cfg, MonoMethod *method, MonoBasicBlock *start_b
 
 				break;
 			}
+			case CEE_MONO_LDPTR_CARD_TABLE: {
+				int shift_bits;
+				gpointer card_mask;
+				CHECK_STACK_OVF (1);
+
+				if (cfg->compile_aot)
+					EMIT_NEW_AOTCONST (cfg, ins, MONO_PATCH_INFO_GC_CARD_TABLE_ADDR, NULL);
+				else
+					EMIT_NEW_PCONST (cfg, ins, mono_gc_get_card_table (&shift_bits, &card_mask));
+
+				*sp++ = ins;
+				ip += 2;
+				inline_costs += 10 * num_calls++;
+				break;
+			}
+			case CEE_MONO_LDPTR_NURSERY_START: {
+				int shift_bits;
+				size_t size;
+				CHECK_STACK_OVF (1);
+
+				if (cfg->compile_aot)
+					EMIT_NEW_AOTCONST (cfg, ins, MONO_PATCH_INFO_GC_NURSERY_START, NULL);
+				else
+					EMIT_NEW_PCONST (cfg, ins, mono_gc_get_nursery (&shift_bits, &size));
+
+				*sp++ = ins;
+				ip += 2;
+				inline_costs += 10 * num_calls++;
+				break;
+			}
 			case CEE_MONO_LDPTR_INT_REQ_FLAG: {
 				CHECK_STACK_OVF (1);
 

mono/mini/mini-runtime.c

@@ -1149,6 +1149,7 @@ mono_patch_info_hash (gconstpointer data)
 	case MONO_PATCH_INFO_INTERRUPTION_REQUEST_FLAG:
 	case MONO_PATCH_INFO_MSCORLIB_GOT_ADDR:
 	case MONO_PATCH_INFO_GC_CARD_TABLE_ADDR:
+	case MONO_PATCH_INFO_GC_NURSERY_START:
 	case MONO_PATCH_INFO_JIT_TLS_ID:
 	case MONO_PATCH_INFO_MONITOR_ENTER:
 	case MONO_PATCH_INFO_MONITOR_ENTER_V4:
@@ -1606,6 +1607,13 @@ mono_resolve_patch_target (MonoMethod *method, MonoDomain *domain, guint8 *code,
 		target = mono_gc_get_card_table (&card_table_shift_bits, &card_table_mask);
 		break;
 	}
+	case MONO_PATCH_INFO_GC_NURSERY_START: {
+		int shift_bits;
+		size_t size;
+
+		target = mono_gc_get_nursery (&shift_bits, &size);
+		break;
+	}
 	case MONO_PATCH_INFO_CASTCLASS_CACHE: {
 		target = mono_domain_alloc0 (domain, sizeof (gpointer));
 		break;

mono/mini/patch-info.h

@@ -53,4 +53,5 @@ PATCH_INFO(TLS_OFFSET, "tls_offset")
 PATCH_INFO(OBJC_SELECTOR_REF, "objc_selector_ref")
 PATCH_INFO(METHOD_CODE_SLOT, "method_code_slot")
 PATCH_INFO(LDSTR_LIT, "ldstr_lit")
+PATCH_INFO(GC_NURSERY_START, "gc_nursery_start")
 PATCH_INFO(NONE, "none")