Arm64: Implement region write barriers

src/coreclr/vm/arm64/asmhelpers.S

@@ -180,7 +180,7 @@ PATCH_LABEL ThePreStubPatchLabel
     ret lr
 LEAF_END ThePreStubPatch, _TEXT
 
-// void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset)
+// void JIT_UpdateWriteBarrierState(size_t writeableOffset)
 //
 // Update shadow copies of the various state info required for barrier
 //
@@ -194,12 +194,11 @@ LEAF_END ThePreStubPatch, _TEXT
 LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT
     PROLOG_SAVE_REG_PAIR_INDEXED   fp, lr, -16
 
-    // x0-x7, x10 will contain intended new state
-    // x8 will preserve skipEphemeralCheck
+    // x0-x7, x10-x11, x13-x14 will contain intended new state
+    // x9 will preserve writeableOffset
     // x12 will be used for pointers
 
-    mov x8, x0
-    mov x9, x1
+    mov x9, x0
 
     PREPARE_EXTERNAL_VAR g_card_table, x12
     ldr  x0, [x12]
@@ -220,23 +219,27 @@ LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT
     PREPARE_EXTERNAL_VAR g_ephemeral_high, x12
     ldr  x4, [x12]
 
-    cbz  x8, LOCAL_LABEL(EphemeralCheckEnabled)
-    movz x3, #0
-    movn x4, #0
-LOCAL_LABEL(EphemeralCheckEnabled):
-
     PREPARE_EXTERNAL_VAR g_lowest_address, x12
     ldr  x5, [x12]
 
     PREPARE_EXTERNAL_VAR g_highest_address, x12
     ldr  x6, [x12]
 
+    PREPARE_EXTERNAL_VAR g_region_to_generation_table, x12
+    ldr  x7, [x12]
+
+    PREPARE_EXTERNAL_VAR g_region_shr, x12
+    ldr  w10, [x12]
+
+    PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, x12
+    ldr  w11, [x12]
+
 #ifdef WRITE_BARRIER_CHECK
     PREPARE_EXTERNAL_VAR g_GCShadow, x12
-    ldr  x7, [x12]
+    ldr  x13, [x12]
 
     PREPARE_EXTERNAL_VAR g_GCShadowEnd, x12
-    ldr  x10, [x12]
+    ldr  x14, [x12]
 #endif
 
     // Update wbs state
@@ -247,12 +250,12 @@ LOCAL_LABEL(EphemeralCheckEnabled):
     stp  x0, x1, [x12], 16
     stp  x2, x3, [x12], 16
     stp  x4, x5, [x12], 16
-    str  x6, [x12], 8
+    stp  x6, x7, [x12], 16
+    stp  w10, w11, [x12], 8
 #ifdef WRITE_BARRIER_CHECK
-    stp  x7, x10, [x12], 16
+    stp  x13, x14, [x12], 16
 #endif
 
-
     EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 16
     EPILOG_RETURN
 LEAF_END JIT_UpdateWriteBarrierState

src/coreclr/vm/arm64/patchedcode.S

@@ -142,33 +142,79 @@ LOCAL_LABEL(ShadowUpdateEnd):
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
     // Update the write watch table if necessary
     ldr  x12, LOCAL_LABEL(wbs_sw_ww_table)
-    cbz  x12, LOCAL_LABEL(CheckCardTable)
+    cbz  x12, LOCAL_LABEL(CheckCardTableBounds)
     add  x12, x12, x14, lsr #0xc  // SoftwareWriteWatch::AddressToTableByteIndexShift
     ldrb w17, [x12]
-    cbnz x17, LOCAL_LABEL(CheckCardTable)
+    cbnz x17, LOCAL_LABEL(CheckCardTableBounds)
     mov  w17, #0xFF
     strb w17, [x12]
 #endif
 
-LOCAL_LABEL(CheckCardTable):
-    // Branch to Exit if the reference is not in the Gen0 heap
+LOCAL_LABEL(CheckCardTableBounds):
+    // Branch to Exit if the reference is not in the heap
     ldr  x12, LOCAL_LABEL(wbs_ephemeral_low)
     ldr  x17, LOCAL_LABEL(wbs_ephemeral_high)
     cmp  x15, x12
     ccmp x15, x17, #0x2, hs
     bhs  LOCAL_LABEL(Exit)
 
+    // Region Checks
+
+    // Check if using regions
+    ldr  x17, LOCAL_LABEL(wbs_region_to_generation_table)
+    cbz  x17, LOCAL_LABEL(CheckCardTable)
+
+    // Calculate region locations
+    ldr  w12, LOCAL_LABEL(wbs_region_shr)
+    lsr  x15, x15, x12
+    add  x15, x15, x17 // x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table
+    lsr  x12, x14, x12
+    add  x12, x12, x17 // x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table
+
+    // Check whether the region we're storing into is gen 0 - nothing to do in this case
+    ldrb w12, [x12]
+    cbz  w12, LOCAL_LABEL(Exit)
+
+    // Check this is going from old to young
+    ldrb w15, [x15]
+    cmp  w15, w12
+    bhs  LOCAL_LABEL(Exit)
+
+    // Bitwise write barriers only
+    ldr  w17, LOCAL_LABEL(wbs_region_use_bitwise_write_barrier)
+    cbz  w17, LOCAL_LABEL(CheckCardTable)
+
     // Check if we need to update the card table
+    lsr w17, w14, 8
+    and w17, w17, 7
+    movz w15, 1
+    lsl w17, w15, w17  // w17 = 1 << (RHS >> 8 && 7)
     ldr  x12, LOCAL_LABEL(wbs_card_table)
     add  x15, x12, x14, lsr #11
-    ldrb w12, [x15]
+    ldrb w12, [x15]  // w12 = [(RHS >> 11) + g_card_table]
+    tst  w12, w17
+    bne  LOCAL_LABEL(Exit)
+
+    // Atomically update the card table
+    // Requires LSE, but the code is only compiled for 8.0
+    .word 0x383131FF // stsetb w17, [x15]
+    b  LOCAL_LABEL(CheckCardBundleTable)
+
+    // End of Region Checks
+
+LOCAL_LABEL(CheckCardTable):
+    // Check if we need to update the card table
+    ldr  x12, LOCAL_LABEL(wbs_card_table)
+    add  x15, x12, x14, lsr #11
+    ldrb w12, [x15]  // w12 = [(RHS >> 11) + g_card_table]
     cmp  x12, 0xFF
     beq  LOCAL_LABEL(Exit)
 
     // Update the card table
     mov  x12, 0xFF
     strb w12, [x15]
 
+LOCAL_LABEL(CheckCardBundleTable):
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
     // Check if we need to update the card bundle table
     ldr  x12, LOCAL_LABEL(wbs_card_bundle_table)
@@ -208,6 +254,12 @@ LOCAL_LABEL(wbs_lowest_address):
     .quad 0
 LOCAL_LABEL(wbs_highest_address):
     .quad 0
+LOCAL_LABEL(wbs_region_to_generation_table):
+    .quad 0
+LOCAL_LABEL(wbs_region_shr):
+    .word 0
+LOCAL_LABEL(wbs_region_use_bitwise_write_barrier):
+    .word 0
 #ifdef WRITE_BARRIER_CHECK
 LOCAL_LABEL(wbs_GCShadow):
     .quad 0

src/coreclr/vm/arm64/stubs.cpp

@@ -865,12 +865,12 @@ void JIT_TailCall()
 }
 
 #if !defined(DACCESS_COMPILE)
-EXTERN_C void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset);
+EXTERN_C void JIT_UpdateWriteBarrierState(size_t writeableOffset);
 
 extern "C" void STDCALL JIT_PatchedCodeStart();
 extern "C" void STDCALL JIT_PatchedCodeLast();
 
-static void UpdateWriteBarrierState(bool skipEphemeralCheck)
+static void UpdateWriteBarrierState()
 {
     BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart);
     BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart;
@@ -880,7 +880,8 @@ static void UpdateWriteBarrierState(bool skipEphemeralCheck)
         writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart);
         writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW();
     }
-    JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart);
+
+    JIT_UpdateWriteBarrierState(writeBarrierCodeStartRW - writeBarrierCodeStart);
 }
 
 void InitJITHelpers1()
@@ -909,12 +910,12 @@ void InitJITHelpers1()
         }
     }
 
-    UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
+    UpdateWriteBarrierState();
 }
 
 
 #else
-void UpdateWriteBarrierState(bool) {}
+void UpdateWriteBarrierState() {}
 #endif // !defined(DACCESS_COMPILE)
 
 PTR_CONTEXT GetCONTEXTFromRedirectedStubStackFrame(T_DISPATCHER_CONTEXT * pDispatcherContext)
@@ -1076,26 +1077,26 @@ void FlushWriteBarrierInstructionCache()
 
 int StompWriteBarrierEphemeral(bool isRuntimeSuspended)
 {
-    UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
+    UpdateWriteBarrierState();
     return SWB_PASS;
 }
 
 int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
 {
-    UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
+    UpdateWriteBarrierState();
     return SWB_PASS;
 }
 
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 int SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
 {
-    UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
+    UpdateWriteBarrierState();
     return SWB_PASS;
 }
 
 int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
 {
-    UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
+    UpdateWriteBarrierState();
     return SWB_PASS;
 }
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP

src/coreclr/vm/gcenv.ee.cpp

@@ -1053,6 +1053,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
             ThreadSuspend::RestartEE(FALSE, TRUE);
         }
         return; // unlike other branches we have already done cleanup so bailing out here
+
     case WriteBarrierOp::StompEphemeral:
         assert(args->is_runtime_suspended && "the runtime must be suspended here!");
         // StompEphemeral requires a new ephemeral low and a new ephemeral high
@@ -1063,8 +1064,22 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         g_region_to_generation_table = args->region_to_generation_table;
         g_region_shr = args->region_shr;
         g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier;
+#if defined(HOST_ARM64)
+        // Only allow bitwise write barriers if LSE atomics are present
+        if (!g_arm64_atomics_present)
+        {
+            g_region_use_bitwise_write_barrier = false;
+        }
+        // Skip ephemeral checks for regionless server GC
+        if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr)
+        {
+            g_ephemeral_low = 0;
+            g_ephemeral_high = 0;
+        }
+#endif
         stompWBCompleteActions |= ::StompWriteBarrierEphemeral(args->is_runtime_suspended);
         break;
+
     case WriteBarrierOp::Initialize:
         assert(args->is_runtime_suspended && "the runtime must be suspended here!");
         // This operation should only be invoked once, upon initialization.
@@ -1090,16 +1105,32 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         g_region_to_generation_table = args->region_to_generation_table;
         g_region_shr = args->region_shr;
         g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier;
+        g_ephemeral_low = args->ephemeral_low;
+        g_ephemeral_high = args->ephemeral_high;
+#if defined(HOST_ARM64)
+        // Only allow bitwise write barriers if LSE atomics are present
+        if (!g_arm64_atomics_present)
+        {
+            g_region_use_bitwise_write_barrier = false;
+        }
+        // Skip ephemeral checks for regionless server GC
+        if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr)
+        {
+            g_ephemeral_low = 0;
+            g_ephemeral_high = 0;
+        }
+#endif
         stompWBCompleteActions |= ::StompWriteBarrierResize(true, false);
 
+#if !defined(HOST_ARM64)
         // StompWriteBarrierResize does not necessarily bash g_ephemeral_low
         // usages, so we must do so here. This is particularly true on x86,
         // where StompWriteBarrierResize will not bash g_ephemeral_low when
         // called with the parameters (true, false), as it is above.
-        g_ephemeral_low = args->ephemeral_low;
-        g_ephemeral_high = args->ephemeral_high;
         stompWBCompleteActions |= ::StompWriteBarrierEphemeral(true);
+#endif
         break;
+
     case WriteBarrierOp::SwitchToWriteWatch:
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         assert(args->is_runtime_suspended && "the runtime must be suspended here!");
@@ -1111,6 +1142,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP");
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         break;
+
     case WriteBarrierOp::SwitchToNonWriteWatch:
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         assert(args->is_runtime_suspended && "the runtime must be suspended here!");
@@ -1121,6 +1153,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP");
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         break;
+
     default:
         assert(!"unknown WriteBarrierOp enum");
     }