Arm64: Implement region write barriers (#111636)

Extend the Arm64 writebarrier function to support regions and use the WriteBarrierManager, similar to Amd64. This results in 10 different versions of the JIT_WriteBarrier, with the WriteBarrierManager deciding on which version to use.

Pseudo code for the writebarrier is included in GC-write-barriers.md

This is expected to make the writebarrier slower, but improve the performance of the GC. DOTNET_GCWriteBarrier=3 can be used give the same functionality as before this change.

The behavior of the writebarrier is:
Before the PR: check ephemeral bounds, update a byte in the card table, mark the card bundle
After the PR:
DOTNET_GCWriteBarrier=1 (default, bit region write barriers): check ephemeral bounds, check regions, update a bit in the card table, mark the card bundle
DOTNET_GCWriteBarrier=2 (byte region write barriers): check ephemeral bounds, check regions, update a byte in the card table, mark the card bundle
DOTNET_GCWriteBarrier=3 (server write barriers): check ephemeral bounds, update a byte in the card table, mark the card bundle. This is the same as before the PR.
DOTNET_gcServer=1: update a byte in the card table, mark the card bundle.

Test results on an 8 core Cobalt 100.

Ephemeral test (dotnet/performance)
WB_nonephemeral : -20%
WB_ephemeral: -16%

WKS GC is calculating the generation of regions in addition to comparing with g_ephemeral_low/high". So while it might set fewer cards, it is more expensive and it shows.

With DOTNET_GCWriteBarrier=3:
WB_nonephemeral : +15%
WB_ephemeral: +1%

SVR GC WB also became more expensive but it sets way fewer cards (for nonephemeral it should set almost no cards).

GCPerfsim
Flags: -tc 2 -tagb 200 -tlgb 2 -lohpi 0 -sohsi 50 -ramb 20 -rlmb 0.2 -sohpi 0

No environment variables set:
Gen0 pause: -21.06%. Gen1 pause -14.25%

DOTNET_GCWriteBarrier=2:
Gen0 pause: -6.7%. Gen1 pause -2.78%

DOTNET_GCWriteBarrier=3 :
Gen0 pause: -1.37%. Gen1 pause -1.26%

DOTNET_gcServer=1 DOTNET_GCHeapCount=8:
Gen0 pause: -7.24%. Gen1 pause -3.49%

Above are linux numbers. On windows for no env var set we are seeing not as much but still quite noticeable pause improvements around 8% to 10% for this config of GCPerfSim.

Author: a74nh

docs/design/coreclr/jit/GC-write-barriers.md

@@ -0,0 +1,101 @@
+# GC write barriers
+
+The GC write barrier function (JIT_WriteBarrier) is generally the hottest function in CoreCLR and is written in assembly. The full pseudo code for the function is as follows:
+
+
+````
+JIT_WriteBarrier(Object **dst, Object *ref)
+    Set *dst = ref
+
+    // Shadow Heap update
+    ifdef WRITE_BARRIER_CHECK: // Only set in DEBUG mode
+        if g_GCShadow != 0:
+            long *shadow_dst = g_GCShadow + (dst - g_lowest_address)
+            // Check shadow heap location is within shadow heap
+            if shadow_dst < g_GCShadowEnd:
+                *shadow_dst = ref
+                atomic: wait for stores to complete
+                if *dst != ref:
+                    *shadow_dst = INVALIDGCVALUE
+
+    // Update the write watch table, if it's in use
+    ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP:
+        if g_sw_ww_table != 0:
+            char *ww_table_dst = g_sw_ww_table + (dst>>11)
+            if *ww_table_dst != 0:
+                *ww_table_dst =  0xff
+
+    // Return if the reference is not in ephemeral generations
+    if ref < g_ephemeral_low || ref >= g_ephemeral_high:
+        return
+
+    // Region Checks
+    if g_region_to_generation_table != 0:
+
+        // Calculate region generations
+        char reg_loc_dst = *((dst >> g_region_shr) + g_region_to_generation_table)
+        char reg_loc_ref = *((ref >> g_region_shr) + g_region_to_generation_table)
+
+        // Return if the region we're storing into is Gen 0
+        if reg_loc_dst == 0:
+            return
+
+        // Return if the new reference is not from old to young
+        if reg_loc_ref >= reg_loc_dst:
+            return
+
+        // Bitwise write barriers only
+        if g_region_use_bitwise_write_barrier:
+
+            char *card_table_dst = (dst >> 11) + g_card_table
+            char dst_bit = 1 << (dst >> 8 && 7)
+
+            // Check if we need to update the card table
+            if *card_table_dst & dst_bit == 0:
+                return
+            
+            // Atomically update the card table
+            lock: *card_table_dst |= dst_bit
+
+            goto CardBundle
+
+    // Check if we need to update the card table
+    char *card_table_dst = (dst >> 11) + g_card_table
+    if *card_table_dst == 0xff:
+        return
+
+    // Update the card table
+    *card_table_dst = 0xff
+
+CardBundle:
+
+    // Mark the card bundle table as dirty
+    Ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES:
+        char card_bundle_dst = (dst >> 21) + g_card_bundle_table
+        if *card_bundle_dst != 0xff:
+            *card_bundle_dst = 0xff
+
+````
+
+The Checked Write Barrier has additional checks:
+
+````
+JIT_CheckedWriteBarrier(Object **dst, Object *ref)
+
+    // Return if the destination is not on the heap
+    if ref < g_lowest_address || ref >= g_highest_address:
+        return
+
+    return JIT_WriteBarrier(dst, ref)
+````
+
+## WriteBarrierManager
+
+On AMD64 and Arm64, there several different implementations of the write barrier function. Each version is a subset of the `JIT_WriteBarrier` above, assuming different state, meaning most `if` checks can be skipped. The actual write barrier that is called is a copy of one of these implementations.
+
+The WriteBarrierManager keeps track of which implementation is currently being used. As internal state changes, the WriteBarrierManager updates the copy to the correct implementation. In practice, most of the internal state is fixed on startup, with only changes to/from use of write watch barriers changing during runtime.
+
+`WRITE_BARRIER_CHECK` is only set in `DEBUG` mode. On Arm64 `WRITE_BARRIER_CHECK` checks exist at the top of each version of the function when `DEBUG` mode is enabled. On `Amd64` these checks do not exist. Instead, a special `JIT_WriteBarrier_Debug` version of the function exists, which contains most of the functionality of `JIT_WriteBarrier` pseudo code and is used exclusively when `DEBUG` mode is enabled.
+
+On Arm64, `g_region_use_bitwise_write_barrier` is only set if LSE atomics are present on the hardware, as only LSE provides a single instruction to atomically update a byte via a bitwise OR.
+

src/coreclr/pal/inc/unixasmmacrosarm64.inc

@@ -37,6 +37,13 @@ C_FUNC(\Name):
 C_FUNC(\Name):
 .endm
 
+// On MacOS, local labels cannot be used in arithmetic expressions.
+#if defined(__APPLE__)
+#define FIXUP_LABEL(name) name
+#else
+#define FIXUP_LABEL(name) .L##name
+#endif
+
 .macro LEAF_ENTRY Name, Section
         .global C_FUNC(\Name)
 #if defined(__APPLE__)

src/coreclr/vm/CMakeLists.txt

@@ -785,11 +785,11 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64)
     )
 
     set(VM_SOURCES_WKS_ARCH
-        ${ARCH_SOURCES_DIR}/jitinterfaceamd64.cpp
         ${ARCH_SOURCES_DIR}/profiler.cpp
         exceptionhandling.cpp
         gcinfodecoder.cpp
         jitinterfacegen.cpp
+        writebarriermanager.cpp
     )
 
     set(VM_HEADERS_WKS_ARCH
@@ -856,6 +856,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64)
     set(VM_SOURCES_WKS_ARCH
         ${ARCH_SOURCES_DIR}/profiler.cpp
         gcinfodecoder.cpp
+        writebarriermanager.cpp
     )
 
     if(CLR_CMAKE_HOST_UNIX)

src/coreclr/vm/arm64/asmhelpers.S

@@ -180,83 +180,6 @@ PATCH_LABEL ThePreStubPatchLabel
     ret lr
 LEAF_END ThePreStubPatch, _TEXT
 
-// void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset)
-//
-// Update shadow copies of the various state info required for barrier
-//
-// State info is contained in a literal pool at the end of the function
-// Placed in text section so that it is close enough to use ldr literal and still
-// be relocatable. Eliminates need for PREPARE_EXTERNAL_VAR in hot code.
-//
-// Align and group state info together so it fits in a single cache line
-// and each entry can be written atomically
-//
-LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT
-    PROLOG_SAVE_REG_PAIR_INDEXED   fp, lr, -16
-
-    // x0-x7, x10 will contain intended new state
-    // x8 will preserve skipEphemeralCheck
-    // x12 will be used for pointers
-
-    mov x8, x0
-    mov x9, x1
-
-    PREPARE_EXTERNAL_VAR g_card_table, x12
-    ldr  x0, [x12]
-
-#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
-    PREPARE_EXTERNAL_VAR g_card_bundle_table, x12
-    ldr  x1, [x12]
-#endif
-
-#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-    PREPARE_EXTERNAL_VAR g_write_watch_table, x12
-    ldr  x2, [x12]
-#endif
-
-    PREPARE_EXTERNAL_VAR g_ephemeral_low, x12
-    ldr  x3, [x12]
-
-    PREPARE_EXTERNAL_VAR g_ephemeral_high, x12
-    ldr  x4, [x12]
-
-    cbz  x8, LOCAL_LABEL(EphemeralCheckEnabled)
-    movz x3, #0
-    movn x4, #0
-LOCAL_LABEL(EphemeralCheckEnabled):
-
-    PREPARE_EXTERNAL_VAR g_lowest_address, x12
-    ldr  x5, [x12]
-
-    PREPARE_EXTERNAL_VAR g_highest_address, x12
-    ldr  x6, [x12]
-
-#ifdef WRITE_BARRIER_CHECK
-    PREPARE_EXTERNAL_VAR g_GCShadow, x12
-    ldr  x7, [x12]
-
-    PREPARE_EXTERNAL_VAR g_GCShadowEnd, x12
-    ldr  x10, [x12]
-#endif
-
-    // Update wbs state
-    PREPARE_EXTERNAL_VAR JIT_WriteBarrier_Table_Loc, x12
-    ldr  x12, [x12]
-    add  x12, x12, x9
-
-    stp  x0, x1, [x12], 16
-    stp  x2, x3, [x12], 16
-    stp  x4, x5, [x12], 16
-    str  x6, [x12], 8
-#ifdef WRITE_BARRIER_CHECK
-    stp  x7, x10, [x12], 16
-#endif
-
-
-    EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 16
-    EPILOG_RETURN
-LEAF_END JIT_UpdateWriteBarrierState
-
 // ------------------------// ------------------------------------------------------------------
 // __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val)
 LEAF_ENTRY  JIT_WriteBarrier_Callable, _TEXT

src/coreclr/vm/arm64/asmhelpers.asm

@@ -248,84 +248,6 @@ ThePreStubPatchLabel
         ret             lr
         LEAF_END
 
-;-----------------------------------------------------------------------------
-; void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset)
-;
-; Update shadow copies of the various state info required for barrier
-;
-; State info is contained in a literal pool at the end of the function
-; Placed in text section so that it is close enough to use ldr literal and still
-; be relocatable. Eliminates need for PREPARE_EXTERNAL_VAR in hot code.
-;
-; Align and group state info together so it fits in a single cache line
-; and each entry can be written atomically
-;
-    LEAF_ENTRY JIT_UpdateWriteBarrierState
-        PROLOG_SAVE_REG_PAIR   fp, lr, #-16!
-
-        ; x0-x7, x10 will contain intended new state
-        ; x8 will preserve skipEphemeralCheck
-        ; x12 will be used for pointers
-
-        mov      x8, x0
-        mov      x9, x1
-
-        adrp     x12, g_card_table
-        ldr      x0, [x12, g_card_table]
-
-#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
-        adrp     x12, g_card_bundle_table
-        ldr      x1, [x12, g_card_bundle_table]
-#endif
-
-#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-        adrp     x12, g_write_watch_table
-        ldr      x2, [x12, g_write_watch_table]
-#endif
-
-        adrp     x12, g_ephemeral_low
-        ldr      x3, [x12, g_ephemeral_low]
-
-        adrp     x12, g_ephemeral_high
-        ldr      x4, [x12, g_ephemeral_high]
-
-        ; Check skipEphemeralCheck
-        cbz      x8, EphemeralCheckEnabled
-        movz     x3, #0
-        movn     x4, #0
-
-EphemeralCheckEnabled
-        adrp     x12, g_lowest_address
-        ldr      x5, [x12, g_lowest_address]
-
-        adrp     x12, g_highest_address
-        ldr      x6, [x12, g_highest_address]
-
-#ifdef WRITE_BARRIER_CHECK
-        adrp     x12, $g_GCShadow
-        ldr      x7, [x12, $g_GCShadow]
-
-        adrp     x12, $g_GCShadowEnd
-        ldr      x10, [x12, $g_GCShadowEnd]
-#endif
-
-        ; Update wbs state
-        adrp     x12, JIT_WriteBarrier_Table_Loc
-        ldr      x12, [x12, JIT_WriteBarrier_Table_Loc]
-        add      x12, x12, x9
-        stp      x0, x1, [x12], 16
-        stp      x2, x3, [x12], 16
-        stp      x4, x5, [x12], 16
-        str      x6, [x12], 8
-#ifdef WRITE_BARRIER_CHECK
-        stp     x7, x10, [x12], 16
-#endif
-
-        EPILOG_RESTORE_REG_PAIR fp, lr, #16!
-        EPILOG_RETURN
-
-    LEAF_END JIT_UpdateWriteBarrierState
-
 #ifdef FEATURE_COMINTEROP
 
 ; ------------------------------------------------------------------