diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 659452eec91a2a..3658e5c5c61650 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -254,6 +254,7 @@ CONFIG_DWORD_INFO(INTERNAL_FastGCCheckStack, W("FastGCCheckStack"), 0, "") CONFIG_DWORD_INFO(INTERNAL_FastGCStress, W("FastGCStress"), 0, "Reduce the number of GCs done by enabling GCStress") RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_GCBreakOnOOM, W("GCBreakOnOOM"), 0, "Does a DebugBreak at the soonest time we detect an OOM") RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConcurrent, W("gcConcurrent"), (DWORD)-1, "Enables/Disables concurrent GC") +RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_UseGCWriteBarrierCopy, W("UseGCWriteBarrierCopy"), 1, "Use a copy of the write barrier for the GC. This is somewhat faster and for optimizations where the barrier is mutated as the program runs. Setting this to 0 removes scenarios where the write barrier is ever mutable.") #ifdef FEATURE_CONSERVATIVE_GC RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConservative, W("gcConservative"), 0, "Enables/Disables conservative GC") diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index 35994841224f3f..62c9c63a00c924 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -184,11 +184,11 @@ JITHELPER(CORINFO_HELP_CHECK_OBJ, JIT_CheckObj, METHOD__NIL) // GC Write barrier support - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF, JIT_WriteBarrier, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF, JIT_CheckedWriteBarrier,METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF, RhpAssignRef, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF, RhpCheckedAssignRef,METHOD__NIL) JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, RhpByRefAssignRef,METHOD__NIL) DYNAMICJITHELPER(CORINFO_HELP_BULK_WRITEBARRIER, NULL, METHOD__BUFFER__MEMCOPYGC) // Accessing fields @@ -286,19 +286,19 @@ #endif // !FEATURE_EH_FUNCLETS #ifdef TARGET_X86 - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, JIT_WriteBarrierEAX, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, JIT_WriteBarrierEBX, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ECX, JIT_WriteBarrierECX, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ESI, JIT_WriteBarrierESI, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EDI, JIT_WriteBarrierEDI, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBP, JIT_WriteBarrierEBP, METHOD__NIL) - - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EAX, JIT_CheckedWriteBarrierEAX, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBX, JIT_CheckedWriteBarrierEBX, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ECX, JIT_CheckedWriteBarrierECX, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ESI, JIT_CheckedWriteBarrierESI, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EDI, JIT_CheckedWriteBarrierEDI, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBP, JIT_CheckedWriteBarrierEBP, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, RhpAssignRefEAX, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, RhpAssignRefEBX, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ECX, RhpAssignRefECX, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ESI, RhpAssignRefESI, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EDI, RhpAssignRefEDI, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBP, RhpAssignRefEBP, METHOD__NIL) + + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EAX, RhpCheckedAssignRefEAX, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBX, RhpCheckedAssignRefEBX, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ECX, RhpCheckedAssignRefECX, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ESI, RhpCheckedAssignRefESI, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EDI, RhpCheckedAssignRefEDI, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBP, RhpCheckedAssignRefEBP, METHOD__NIL) #else JITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, NULL, METHOD__NIL) JITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, NULL, METHOD__NIL) diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index cd3d29fafef33d..0cb5ff9aa27fca 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -138,7 +138,8 @@ // On exit: // r0: trashed // r3: trashed - // CORINFO_HELP_ASSIGN_BYREF (JIT_ByRefWriteBarrier): + // r12: trashed +// CORINFO_HELP_ASSIGN_BYREF (JIT_ByRefWriteBarrier): // On entry: // r0: the destination address (object reference written here) // r1: the source address (points to object reference to write) @@ -147,6 +148,7 @@ // r1: incremented by 4 // r2: trashed // r3: trashed + // r12: trashed #define REG_WRITE_BARRIER_DST REG_ARG_0 #define RBM_WRITE_BARRIER_DST RBM_ARG_0 diff --git a/src/coreclr/nativeaot/Runtime/CMakeLists.txt b/src/coreclr/nativeaot/Runtime/CMakeLists.txt index 1e78e08f50df1b..cb2af32355ce7a 100644 --- a/src/coreclr/nativeaot/Runtime/CMakeLists.txt +++ b/src/coreclr/nativeaot/Runtime/CMakeLists.txt @@ -213,7 +213,7 @@ list(APPEND RUNTIME_SOURCES_ARCH_ASM ${ARCH_SOURCES_DIR}/InteropThunksHelpers.${ASM_SUFFIX} ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/StubDispatch.${ASM_SUFFIX} ${ARCH_SOURCES_DIR}/UniversalTransition.${ASM_SUFFIX} - ${ARCH_SOURCES_DIR}/WriteBarriers.${ASM_SUFFIX} + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.${ASM_SUFFIX} ) if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64) diff --git a/src/coreclr/nativeaot/Runtime/arm/AsmMacros_Shared.h b/src/coreclr/nativeaot/Runtime/arm/AsmMacros_Shared.h new file mode 100644 index 00000000000000..1a10c1d2c72f44 --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/arm/AsmMacros_Shared.h @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +#include "AsmOffsets.inc" +#include diff --git a/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.h b/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.h new file mode 100644 index 00000000000000..51951864f299ad --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.h @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +// TODO: Implement \ No newline at end of file diff --git a/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.inc b/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.inc new file mode 100644 index 00000000000000..956d4d22e38313 --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.inc @@ -0,0 +1,6 @@ +; Licensed to the .NET Foundation under one or more agreements. +; The .NET Foundation licenses this file to you under the MIT license. + +; This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +include AsmMacros.inc \ No newline at end of file diff --git a/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.S b/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.S deleted file mode 100644 index 876f2dfbcb80d6..00000000000000 --- a/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.S +++ /dev/null @@ -1,4 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -// TODO: Implement diff --git a/src/coreclr/nativeaot/Runtime/loongarch64/AsmMacros_Shared.h b/src/coreclr/nativeaot/Runtime/loongarch64/AsmMacros_Shared.h new file mode 100644 index 00000000000000..f7df014642731d --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/loongarch64/AsmMacros_Shared.h @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +#include +#include "AsmOffsets.inc" diff --git a/src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h b/src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h new file mode 100644 index 00000000000000..f7df014642731d --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +#include +#include "AsmOffsets.inc" diff --git a/src/coreclr/pal/inc/unixasmmacrosarm.inc b/src/coreclr/pal/inc/unixasmmacrosarm.inc index f5eb32656cd3ae..226d23aebd2f75 100644 --- a/src/coreclr/pal/inc/unixasmmacrosarm.inc +++ b/src/coreclr/pal/inc/unixasmmacrosarm.inc @@ -44,10 +44,28 @@ C_FUNC(\Name\()_End): nop .endm +.macro GLOBAL_LABEL Name + .global C_FUNC(\Name) +C_FUNC(\Name): +.endm + +.macro ALTERNATE_ENTRY Name + .global C_FUNC(\Name) + .type \Name, %function +C_FUNC(\Name): +.endm + .macro PREPARE_EXTERNAL_VAR Name, HelperReg ldr \HelperReg, [pc, #C_FUNC(\Name)@GOTPCREL] .endm +.macro PREPARE_EXTERNAL_VAR_INDIRECT Name, HelperReg + movw \HelperReg, #:lower16:C_FUNC(\Name) - (. + 12) + movt \HelperReg, #:upper16:C_FUNC(\Name) - (. + 8) + add \HelperReg, pc + ldr \HelperReg, [\HelperReg] +.endm + .macro push_nonvol_reg Register push \Register .save \Register diff --git a/src/coreclr/pal/inc/unixasmmacrosloongarch64.inc b/src/coreclr/pal/inc/unixasmmacrosloongarch64.inc index 85cf42121326e2..a949491f5fabdb 100644 --- a/src/coreclr/pal/inc/unixasmmacrosloongarch64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosloongarch64.inc @@ -17,6 +17,12 @@ C_FUNC(\Name): .endm +.macro ALTERNATE_ENTRY Name + .global C_FUNC(\Name) + .hidden C_FUNC(\Name) +C_FUNC(\Name): +.endm + .macro LEAF_ENTRY Name, Section .global C_FUNC(\Name) .type \Name, %function @@ -41,6 +47,16 @@ C_FUNC(\Name\()_End): la.local \HelperReg, \Name .endm +.macro PREPARE_EXTERNAL_VAR_INDIRECT Name, HelperReg + la.local \HelperReg, \Name + ld.d \HelperReg, \HelperReg, 0 +.endm + +.macro PREPARE_EXTERNAL_VAR_INDIRECT_W Name, HelperReg + la.local \HelperReg, \Name + ld.w \HelperReg, \HelperReg, 0 +.endm + .macro PROLOG_STACK_ALLOC Size addi.d $sp, $sp, -\Size //.cfi_adjust_cfa_offset \Size diff --git a/src/coreclr/pal/inc/unixasmmacrosriscv64.inc b/src/coreclr/pal/inc/unixasmmacrosriscv64.inc index a86acf5b897d2d..1f7344c4d15741 100644 --- a/src/coreclr/pal/inc/unixasmmacrosriscv64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosriscv64.inc @@ -37,6 +37,12 @@ C_FUNC(\Name\()_End): nop .endm +.macro ALTERNATE_ENTRY Name + .global C_FUNC(\Name) + .hidden C_FUNC(\Name) +C_FUNC(\Name): +.endm + .macro PREPARE_EXTERNAL_VAR Name, HelperReg lla \HelperReg, \Name .endm diff --git a/src/coreclr/pal/inc/unixasmmacrosx86.inc b/src/coreclr/pal/inc/unixasmmacrosx86.inc index 7bc994a779c356..0de1d04a2437c5 100644 --- a/src/coreclr/pal/inc/unixasmmacrosx86.inc +++ b/src/coreclr/pal/inc/unixasmmacrosx86.inc @@ -24,6 +24,12 @@ C_FUNC(\Name): C_FUNC(\Name): .endm +.macro ALTERNATE_ENTRY Name + .global C_FUNC(\Name) + .type \Name, %function +C_FUNC(\Name): +.endm + .macro LEAF_END Name, Section .size \Name, .-\Name .cfi_endproc diff --git a/src/coreclr/runtime/amd64/StubDispatch.S b/src/coreclr/runtime/amd64/StubDispatch.S index 3af2bc6ac019b6..0ae32980c5a9b2 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.S +++ b/src/coreclr/runtime/amd64/StubDispatch.S @@ -6,6 +6,15 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH +#if defined(__APPLE__) + // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. + // Fix, but adding an empty, unused method + LEAF_ENTRY RhpStubDispatchDoNotFailToBuild, _TEXT + ret + LEAF_END RhpStubDispatchDoNotFailToBuild, _TEXT +#endif + + // trick to avoid PLT relocation at runtime which corrupts registers #define REL_C_FUNC(name) C_FUNC(name)@gotpcrel @@ -20,7 +29,15 @@ LEAF_ENTRY RhpInterfaceDispatch\entries, _TEXT mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] // Load the MethodTable from the object instance in rdi. +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries +#ifdef TARGET_APPLE +.cfi_startproc +#endif mov rax, [rdi] CurrentOffset = OFFSETOF__InterfaceDispatchCache__m_rgEntries diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S similarity index 95% rename from src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S rename to src/coreclr/runtime/amd64/WriteBarriers.S index e55e682653b512..c15ee30e5dffea 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -2,7 +2,15 @@ // The .NET Foundation licenses this file to you under the MIT license. .intel_syntax noprefix -#include +#include "AsmMacros_Shared.h" + +#if defined(__APPLE__) + // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. + // Fix, but adding an empty, unused method + LEAF_ENTRY RhpWriteBarriersDoNotFailToBuild, _TEXT + ret + LEAF_END RhpWriteBarriersDoNotFailToBuild, _TEXT +#endif #ifdef WRITE_BARRIER_CHECK @@ -261,7 +269,15 @@ LEAF_END RhpCheckedXchg, _TEXT LEAF_ENTRY RhpByRefAssignRef, _TEXT ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 mov rcx, [rsi] +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 +#ifdef TARGET_APPLE +.cfi_startproc +#endif mov [rdi], rcx // Check whether the writes were even into the heap. If not there's no card update required. diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm b/src/coreclr/runtime/amd64/WriteBarriers.asm similarity index 99% rename from src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm rename to src/coreclr/runtime/amd64/WriteBarriers.asm index 302b9e0a8b1fed..c08109d65ea62b 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm +++ b/src/coreclr/runtime/amd64/WriteBarriers.asm @@ -1,7 +1,7 @@ ;; Licensed to the .NET Foundation under one or more agreements. ;; The .NET Foundation licenses this file to you under the MIT license. -include AsmMacros.inc +include AsmMacros_Shared.inc ;; Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used ;; during garbage collections to verify that object references where never written to the heap without using a diff --git a/src/coreclr/runtime/arm/StubDispatch.S b/src/coreclr/runtime/arm/StubDispatch.S index c761df7b3f123e..3001b67453dde9 100644 --- a/src/coreclr/runtime/arm/StubDispatch.S +++ b/src/coreclr/runtime/arm/StubDispatch.S @@ -4,8 +4,7 @@ .syntax unified .thumb -#include // generated by the build from AsmOffsets.cpp -#include +#include "AsmMacros_Shared.h" #ifdef FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/nativeaot/Runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S similarity index 77% rename from src/coreclr/nativeaot/Runtime/arm/WriteBarriers.S rename to src/coreclr/runtime/arm/WriteBarriers.S index 3bb862231a347e..30afebc760d0a1 100644 --- a/src/coreclr/nativeaot/Runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -4,79 +4,12 @@ .syntax unified .thumb -#include // generated by the build from AsmOffsets.cpp -#include +#include "AsmMacros_Shared.h" #ifdef WRITE_BARRIER_CHECK .macro UPDATE_GC_SHADOW BASENAME, REFREG, DESTREG - - // If g_GCShadow is 0, don't perform the check. - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, r12 - cbz r12, LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG) - - // Save DESTREG since we're about to modify it (and we need the original value both within the macro and - // once we exit the macro). Note that this is naughty since we're altering the stack pointer outside of - // the prolog inside a method without a frame. But given that this is only debug code and generally we - // shouldn't be walking the stack at this point it seems preferable to recoding the all the barrier - // variants to set up frames. The compiler knows exactly which registers are trashed in the simple write - // barrier case, so we don't have any more scratch registers to play with (and doing so would only make - // things harder if at a later stage we want to allow multiple barrier versions based on the input - // registers). - push \DESTREG - - // Transform DESTREG into the equivalent address in the shadow heap. - PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, r12 - sub \DESTREG, r12 - cmp \DESTREG, #0 - blo LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG) - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, r12 - add \DESTREG, r12 - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, r12 - cmp \DESTREG, r12 - bhs LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG) - - // Update the shadow heap. - str \REFREG, [\DESTREG] - - // The following read must be strongly ordered wrt to the write we've just performed in order to - // prevent race conditions. - dmb - - // Now check that the real heap location still contains the value we just wrote into the shadow heap. - mov r12, \DESTREG - ldr \DESTREG, [sp] - str r12, [sp] - ldr r12, [\DESTREG] - cmp r12, \REFREG - bne LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Invalidate_\REFREG) - - // The original DESTREG value is now restored but the stack has a value (the shadow version of the - // location) pushed. Need to discard this push before we are done. - add sp, #4 - b LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG) - -LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Invalidate_\REFREG): - // Someone went and updated the real heap. We need to invalidate the shadow location since we can't - // guarantee whose shadow update won. - - // Retrieve shadow location from the stack and restore original DESTREG to the stack. This is an - // additional memory barrier we don't require but it's on the rare path and x86 doesn't have an xchg - // variant that doesn't implicitly specify the lock prefix. Note that INVALIDGCVALUE is a 32-bit - // immediate and therefore must be moved into a register before it can be written to the shadow - // location. - mov r12, \DESTREG - ldr \DESTREG, [sp] - str r12, [sp] - push \REFREG - movw \REFREG, #(INVALIDGCVALUE & 0xFFFF) - movt \REFREG, #(INVALIDGCVALUE >> 16) - str \REFREG, [\DESTREG] - pop \REFREG - -LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG): - // Restore original DESTREG value from the stack. - pop \DESTREG + // Todo: implement, debugging helper LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): @@ -89,16 +22,38 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): #endif // WRITE_BARRIER_CHECK +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg, __wbScratch + + PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, \__wbScratch + cbz \__wbScratch, 2f + add \__wbScratch, \__wbScratch, \ptrReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift + + ldrb \tmpReg, [\__wbScratch] + cmp \tmpReg, #0xff + itt ne + movne \tmpReg, 0xff + strbne \tmpReg, [\__wbScratch] + +2: +.endm +#else +.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg, __wbScratch +.endm +#endif + // There are several different helpers used depending on which register holds the object reference. Since all // the helpers have identical structure we use a macro to define this structure. Two arguments are taken, the // name of the register that points to the location to be updated and the name of the register that holds the // object reference (this should be in upper case as it's used in the definition of the name of the helper). -.macro DEFINE_UNCHECKED_WRITE_BARRIER_CORE BASENAME, REFREG +.macro DEFINE_UNCHECKED_WRITE_BARRIER_CORE BASENAME, REFREG, TMPREG // Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless // we're in a debug build and write barrier checking has been enabled). UPDATE_GC_SHADOW \BASENAME, \REFREG, r0 + UPDATE_WRITE_WATCH_TABLE r0, r12, \TMPREG + // If the reference is to an object that's not in an ephemeral generation we have no need to track it // (since the object won't be collected or moved by an ephemeral collection). PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, r12 @@ -109,12 +64,13 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): cmp \REFREG, r12 bhs LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG) + // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write // the byte if it hasn't already been done since writes are expensive and impact scaling. PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, r12 - add r0, r12, r0, lsr #LOG2_CLUMP_SIZE + add r0, r12, r0, lsr #10 ldrb r12, [r0] cmp r12, #0x0FF bne LOCAL_LABEL(\BASENAME\()_UpdateCardTable_\REFREG) @@ -170,7 +126,7 @@ GLOBAL_LABEL RhpAssignRefAVLocation .endif str \REFREG, [r0] - DEFINE_UNCHECKED_WRITE_BARRIER_CORE RhpAssignRef, \REFREG + DEFINE_UNCHECKED_WRITE_BARRIER_CORE RhpAssignRef, \REFREG, r3 bx lr LEAF_END RhpAssignRef\EXPORT_REG_NAME, _TEXT @@ -187,7 +143,7 @@ DEFINE_UNCHECKED_WRITE_BARRIER r1, r1 // collection. // -.macro DEFINE_CHECKED_WRITE_BARRIER_CORE BASENAME, REFREG +.macro DEFINE_CHECKED_WRITE_BARRIER_CORE BASENAME, REFREG, TMPREG // The location being updated might not even lie in the GC heap (a handle or stack location for instance), // in which case no write barrier is required. @@ -198,7 +154,7 @@ DEFINE_UNCHECKED_WRITE_BARRIER r1, r1 cmp r0, r12 bhs LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG) - DEFINE_UNCHECKED_WRITE_BARRIER_CORE \BASENAME, \REFREG + DEFINE_UNCHECKED_WRITE_BARRIER_CORE \BASENAME, \REFREG, \TMPREG .endm @@ -240,7 +196,7 @@ GLOBAL_LABEL RhpCheckedAssignRefAVLocation .endif str \REFREG, [r0] - DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedAssignRef, \REFREG + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedAssignRef, \REFREG, r3 bx lr LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT @@ -250,6 +206,8 @@ LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT // just one write barrier that assumes the input register is RSI. DEFINE_CHECKED_WRITE_BARRIER r1, r1 +#ifdef FEATURE_NATIVEAOT + // r0 = destination address // r1 = value // r2 = comparand @@ -267,7 +225,7 @@ LOCAL_LABEL(RhpCheckedLockCmpXchgRetry): bne LOCAL_LABEL(RhpCheckedLockCmpXchgRetry) mov r3, r2 - DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, r1 + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, r1, r2 mov r0, r3 bx lr @@ -285,13 +243,14 @@ LOCAL_LABEL(RhpCheckedXchgRetry): cmp r3, #0 bne LOCAL_LABEL(RhpCheckedXchgRetry) - DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedXchg, r1 + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedXchg, r1, r3 // The original value is currently in r2. We need to return it in r0. mov r0, r2 bx lr LEAF_END RhpCheckedXchg, _TEXT +#endif // FEATURE_NATIVEAOT // // RhpByRefAssignRef simulates movs instruction for object references. @@ -329,6 +288,8 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2 // we're in a debug build and write barrier checking has been enabled). UPDATE_GC_SHADOW BASENAME, r2, r0 + UPDATE_WRITE_WATCH_TABLE r0, r12, r3 + // If the reference is to an object that's not in an ephemeral generation we have no need to track it // (since the object won't be collected or moved by an ephemeral collection). PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, r3 @@ -348,7 +309,7 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2 // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write // the byte if it hasn't already been done since writes are expensive and impact scaling. PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, r3 - add r2, r3, r2, lsr #LOG2_CLUMP_SIZE + add r2, r3, r2, lsr #10 ldrb r3, [r2] cmp r3, #0x0FF bne LOCAL_LABEL(RhpByRefAssignRef_UpdateCardTable) diff --git a/src/coreclr/runtime/arm64/StubDispatch.S b/src/coreclr/runtime/arm64/StubDispatch.S index 1155e6ac257a1a..750a99db42638a 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.S +++ b/src/coreclr/runtime/arm64/StubDispatch.S @@ -5,6 +5,14 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH +#if defined(__APPLE__) + // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. + // Fix, but adding an empty, unused method + LEAF_ENTRY RhpStubDispatchDoNotFailToBuild, _TEXT + ret + LEAF_END RhpStubDispatchDoNotFailToBuild, _TEXT +#endif + // Macro that generates code to check a single cache entry. .macro CHECK_CACHE_ENTRY entry // Check a single entry in the cache. @@ -31,7 +39,15 @@ ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] // Load the MethodTable from the object instance in x0. +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries +#ifdef TARGET_APPLE +.cfi_startproc +#endif ldr x10, [x0] .global CurrentEntry diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S similarity index 95% rename from src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S rename to src/coreclr/runtime/arm64/WriteBarriers.S index 6948e0fa94a07b..8087b290f00272 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -1,7 +1,15 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include +#include "AsmMacros_Shared.h" + +#if defined(__APPLE__) + // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. + // Fix, but adding an empty, unused method + LEAF_ENTRY RhpWriteBarriersDoNotFailToBuild, _TEXT + ret + LEAF_END RhpWriteBarriersDoNotFailToBuild, _TEXT +#endif // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used // during garbage collections to verify that object references where never written to the heap without using a @@ -229,10 +237,17 @@ LEAF_END RhpByRefAssignRefArm64, _TEXT b C_FUNC(RhpAssignRefArm64) LOCAL_LABEL(NotInHeap): +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation +#ifdef TARGET_APPLE +.cfi_startproc +#endif str x15, [x14], 8 ret - LEAF_END RhpCheckedAssignRefArm64, _TEXT // JIT_WriteBarrier(Object** dst, Object* src) @@ -266,6 +281,7 @@ LEAF_ENTRY RhpAssignRef, _TEXT b C_FUNC(RhpAssignRefArm64) LEAF_END RhpAssignRef, _TEXT +#ifdef FEATURE_NATIVEAOT // Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon // successful updates. @@ -395,3 +411,4 @@ LOCAL_LABEL(NoBarrierXchg): #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT .arch_extension nolse #endif +#endif // FEATURE_NATIVEAOT diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm similarity index 98% rename from src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm rename to src/coreclr/runtime/arm64/WriteBarriers.asm index 05a26044dddd21..10fb789fa37fc1 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -8,7 +8,7 @@ ;; collection. ;; -#include "AsmMacros.h" +#include "AsmMacros_Shared.h" TEXTAREA @@ -57,7 +57,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD PREPARE_EXTERNAL_VAR_INDIRECT $g_GCShadow, x12 add $destReg, $destReg, x12 - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, x12 + PREPARE_EXTERNAL_VAR_INDIRECT $g_GCShadowEnd, x12 cmp $destReg, x12 bhs %ft0 @@ -206,7 +206,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1 ;; - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address - LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT + LEAF_ENTRY RhpByRefAssignRefArm64 ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ldr x15, [x13], 8 @@ -275,6 +275,7 @@ NotInHeap b RhpAssignRefArm64 LEAF_END RhpAssignRef +#ifdef FEATURE_NATIVEAOT ;; Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon ;; successful updates. @@ -388,5 +389,6 @@ NoBarrierXchg ret LEAF_END RhpCheckedXchg +#endif // FEATURE_NATIVEAOT end diff --git a/src/coreclr/runtime/i386/StubDispatch.asm b/src/coreclr/runtime/i386/StubDispatch.asm index 9b70dd727f5874..b3672d4e1eb045 100644 --- a/src/coreclr/runtime/i386/StubDispatch.asm +++ b/src/coreclr/runtime/i386/StubDispatch.asm @@ -7,7 +7,7 @@ .code -include AsmMacros.inc +include AsmMacros_Shared.inc ifdef FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/runtime/i386/WriteBarriers.S b/src/coreclr/runtime/i386/WriteBarriers.S new file mode 100644 index 00000000000000..f26590a88ab462 --- /dev/null +++ b/src/coreclr/runtime/i386/WriteBarriers.S @@ -0,0 +1,303 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +.intel_syntax noprefix +#include "AsmMacros_Shared.h" + +// TODO! This is implemented, but not tested. + +#ifdef WRITE_BARRIER_CHECK + +.macro UPDATE_GC_SHADOW BASENAME, DESTREG, REFREG, TEMPREG + + // If g_GCShadow is 0, don't perform the check. + PREPARE_EXTERNAL_VAR g_GCShadow, \TEMPREG + cmp dword ptr [\TEMPREG], 0 + je LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Done_\DESTREG\()_\REFREG) + + // Save DESTREG since we're about to modify it (and we need the original value both within the macro and + // once we exit the macro). + push \DESTREG + + // Transform DESTREG into the equivalent address in the shadow heap. + PREPARE_EXTERNAL_VAR g_lowest_address, \TEMPREG + sub \DESTREG, [\TEMPREG] + jb LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_PopThenDone_\DESTREG\()_\REFREG) + PREPARE_EXTERNAL_VAR g_GCShadow, \TEMPREG + add \DESTREG, [\TEMPREG] + PREPARE_EXTERNAL_VAR g_GCShadowEnd, \TEMPREG + cmp \DESTREG, [\TEMPREG] + jae LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_PopThenDone_\DESTREG\()_\REFREG) + + // Update the shadow heap. + mov [\DESTREG], \REFREG + + // Now check that the real heap location still contains the value we just wrote into the shadow heap. This + // read must be strongly ordered wrt to the previous write to prevent race conditions. We also need to + // recover the old value of DESTREG for the comparison so use an xchg instruction (which has an implicit lock + // prefix). + xchg [esp], \DESTREG + cmp [\DESTREG], \REFREG + jne LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Invalidate_\DESTREG\()_\REFREG) + + // The original DESTREG value is now restored but the stack has a value (the shadow version of the + // location) pushed. Need to discard this push before we are done. + add esp, 4 + jmp LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Done_\DESTREG\()_\REFREG) + +LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Invalidate_\DESTREG\()_\REFREG): + // Someone went and updated the real heap. We need to invalidate the shadow location since we can't + // guarantee whose shadow update won. + + // Retrieve shadow location from the stack and restore original DESTREG to the stack. This is an + // additional memory barrier we don't require but it's on the rare path and x86 doesn't have an xchg + // variant that doesn't implicitly specify the lock prefix. + xchg [esp], \DESTREG + mov dword ptr [\DESTREG], INVALIDGCVALUE + +LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_PopThenDone_\DESTREG\()_\REFREG): + // Restore original DESTREG value from the stack. + pop \DESTREG + +LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Done_\DESTREG\()_\REFREG): +.endm + +#else // WRITE_BARRIER_CHECK + +.macro UPDATE_GC_SHADOW BASENAME, DESTREG, REFREG, TEMPREG +.endm +#endif + +// There are several different helpers used depending on which register holds the object reference. Since all +// the helpers have identical structure we use a macro to define this structure. Two arguments are taken, the +// name of the register that points to the location to be updated and the name of the register that holds the +// object reference (this should be in upper case as it's used in the definition of the name of the helper). +.macro DEFINE_WRITE_BARRIER DESTREG, REFREG, TEMPREG + +// Define a helper with a name of the form RhpAssignRefEAX etc. (along with suitable calling standard +// decoration). The location to be updated is in DESTREG. The object reference that will be assigned into that +// location is in one of the other general registers determined by the value of REFREG. +LEAF_ENTRY RhpAssignRef\REFREG, _TEXT + + // Export the canonical write barrier under unqualified name as well + .ifc \REFREG, EDX + ALTERNATE_ENTRY RhpAssignRef + ALTERNATE_ENTRY RhpAssignRefAVLocation + .endif + + ALTERNATE_ENTRY RhpAssignRef\REFREG\()AVLocation + + // Write the reference into the location. Note that we rely on the fact that no GC can occur between here + // and the card table update we may perform below. + mov dword ptr [\DESTREG], \REFREG + + // Save a register so that we have an available register as a temporary for PREPARE_EXTERNAL_VAR + push \TEMPREG + + // Update the shadow copy of the heap with the same value (if enabled). + UPDATE_GC_SHADOW RhpAssignRef, \DESTREG, \REFREG, \TEMPREG + + // If the reference is to an object that's not in an ephemeral generation we have no need to track it + // (since the object won't be collected or moved by an ephemeral collection). + PREPARE_EXTERNAL_VAR g_ephemeral_low, \TEMPREG + cmp \REFREG, [\TEMPREG] + jb LOCAL_LABEL(WriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + PREPARE_EXTERNAL_VAR g_ephemeral_high, \TEMPREG + cmp \REFREG, [\TEMPREG] + jae LOCAL_LABEL(WriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + + // We have a location on the GC heap being updated with a reference to an ephemeral object so we must + // track this write. The location address is translated into an offset in the card table bitmap. We set + // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write + // the byte if it hasn't already been done since writes are expensive and impact scaling. + shr \DESTREG, 10 + PREPARE_EXTERNAL_VAR g_card_table, \TEMPREG + add \DESTREG, [\TEMPREG] + cmp byte ptr [\DESTREG], 0xFF + jne LOCAL_LABEL(WriteBarrier_UpdateCardTable_\DESTREG\()_\REFREG) + +LOCAL_LABEL(WriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG): + pop \TEMPREG + ret + +// We get here if it's necessary to update the card table. +LOCAL_LABEL(WriteBarrier_UpdateCardTable_\DESTREG\()_\REFREG): + pop \TEMPREG + mov byte ptr [\DESTREG], 0xFF + ret +LEAF_END RhpAssignRef\REFREG, _TEXT +.endm + +.macro DEFINE_CHECKED_WRITE_BARRIER_CORE BASENAME, DESTREG, REFREG, TEMPREG + + // The location being updated might not even lie in the GC heap (a handle or stack location for instance), + // in which case no write barrier is required. + PREPARE_EXTERNAL_VAR g_lowest_address, \TEMPREG + cmp \DESTREG, [\TEMPREG] + jb LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + PREPARE_EXTERNAL_VAR g_highest_address, \TEMPREG + cmp \DESTREG, [\TEMPREG] + jae LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + + // Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless + // we're in a debug build and write barrier checking has been enabled). + UPDATE_GC_SHADOW \BASENAME, \DESTREG, \REFREG, \TEMPREG + + // If the reference is to an object that's not in an ephemeral generation we have no need to track it + // (since the object won't be collected or moved by an ephemeral collection). + PREPARE_EXTERNAL_VAR g_ephemeral_low, \TEMPREG + cmp \REFREG, [\TEMPREG] + jb LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + PREPARE_EXTERNAL_VAR g_ephemeral_high, \TEMPREG + cmp \REFREG, [\TEMPREG] + jae LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + + // We have a location on the GC heap being updated with a reference to an ephemeral object so we must + // track this write. The location address is translated into an offset in the card table bitmap. We set + // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write + // the byte if it hasn't already been done since writes are expensive and impact scaling. + shr \DESTREG, 10 + PREPARE_EXTERNAL_VAR g_card_table, \TEMPREG + add \DESTREG, [\TEMPREG] + cmp byte ptr [\DESTREG], 0xFF + je LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + // We get here if it's necessary to update the card table. + mov byte ptr [\DESTREG], 0xFF +LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG): +.endm + +// This macro is very much like the one above except that it generates a variant of the function which also +// checks whether the destination is actually somewhere within the GC heap. +.macro DEFINE_CHECKED_WRITE_BARRIER DESTREG, REFREG, TEMPREG + +// Define a helper with a name of the form RhpCheckedAssignRefEAX etc. (along with suitable calling standard +// decoration). The location to be updated is in DESTREG. The object reference that will be assigned into +// that location is in one of the other general registers determined by the value of REFREG. + +// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: +// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen on the first instruction +// - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address +LEAF_ENTRY RhpCheckedAssignRef\REFREG, _TEXT + + // Export the canonical write barrier under unqualified name as well + .ifc \REFREG, EDX + ALTERNATE_ENTRY RhpCheckedAssignRef + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation + .endif + + ALTERNATE_ENTRY RhpCheckedAssignRef\REFREG\()AVLocation + + // Write the reference into the location. Note that we rely on the fact that no GC can occur between here + // and the card table update we may perform below. + mov dword ptr [\DESTREG], \REFREG + + // Save a register so that we have an available register as a temporary for PREPARE_EXTERNAL_VAR + push \TEMPREG + + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedAssignRef, \DESTREG, \REFREG, \TEMPREG + pop \TEMPREG + ret + +LEAF_END RhpCheckedAssignRef\REFREG, _TEXT + +.endm + +// One day we might have write barriers for all the possible argument registers but for now we have +// just one write barrier that assumes the input register is EDX. +DEFINE_CHECKED_WRITE_BARRIER ECX, EDX, EAX +DEFINE_WRITE_BARRIER ECX, EDX, EAX + +// Need some more write barriers to run CLR compiled MDIL on Redhawk - commented out for now +DEFINE_WRITE_BARRIER EDX, EAX, ECX +DEFINE_WRITE_BARRIER EDX, ECX, EAX +DEFINE_WRITE_BARRIER EDX, EBX, EAX +DEFINE_WRITE_BARRIER EDX, ESI, EAX +DEFINE_WRITE_BARRIER EDX, EDI, EAX +DEFINE_WRITE_BARRIER EDX, EBP, EAX + +DEFINE_CHECKED_WRITE_BARRIER EDX, EAX, ECX +DEFINE_CHECKED_WRITE_BARRIER EDX, ECX, EAX +DEFINE_CHECKED_WRITE_BARRIER EDX, EBX, EAX +DEFINE_CHECKED_WRITE_BARRIER EDX, ESI, EAX +DEFINE_CHECKED_WRITE_BARRIER EDX, EDI, EAX +DEFINE_CHECKED_WRITE_BARRIER EDX, EBP, EAX + +LEAF_ENTRY RhpCheckedLockCmpXchg, _TEXT + mov eax, [esp+4] + lock cmpxchg [ecx], edx + jne LOCAL_LABEL(RhpCheckedLockCmpXchg_NoWrite) + push eax + + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, ECX, EDX, EAX + pop eax +LOCAL_LABEL(RhpCheckedLockCmpXchg_NoWrite): + ret 4 +LEAF_END RhpCheckedLockCmpXchg, _TEXT + +LEAF_ENTRY RhpCheckedXchg, _TEXT + + // Setup eax with the new object for the exchange, that way it will automatically hold the correct result + // afterwards and we can leave edx unaltered ready for the GC write barrier below. + mov eax, edx + xchg [ecx], eax + push eax + + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedXchg, ECX, EDX, EAX + pop eax + ret +LEAF_END RhpCheckedXchg, _TEXT + +// +// RhpByRefAssignRef simulates movs instruction for object references. +// +// On entry: +// edi: address of ref-field (assigned to) +// esi: address of the data (source) +// +// On exit: +// edi, esi are incremented by 4, +// ecx: trashed +// +LEAF_ENTRY RhpByRefAssignRef, _TEXT +ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 + mov ecx, [esi] +ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 + mov [edi], ecx + + push eax + + // Check whether the writes were even into the heap. If not there's no card update required. + PREPARE_EXTERNAL_VAR g_lowest_address, eax + cmp edi, [eax] + jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + PREPARE_EXTERNAL_VAR g_highest_address, eax + cmp edi, [eax] + jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + + UPDATE_GC_SHADOW RhpByRefAssignRef, ecx, edi, eax + + // If the reference is to an object that's not in an ephemeral generation we have no need to track it + // (since the object won't be collected or moved by an ephemeral collection). + PREPARE_EXTERNAL_VAR g_ephemeral_low, eax + cmp ecx, [eax] + jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + PREPARE_EXTERNAL_VAR g_ephemeral_high, eax + cmp ecx, [eax] + jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + + mov ecx, edi + shr ecx, 10 + PREPARE_EXTERNAL_VAR g_card_table, eax + add ecx, [eax] + cmp byte ptr [ecx], 0xFF + je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + + mov byte ptr [ecx], 0xFF + +LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired): + // Increment the pointers before leaving + add esi,4 + add edi,4 + pop eax + ret +LEAF_END RhpByRefAssignRef, _TEXT diff --git a/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm b/src/coreclr/runtime/i386/WriteBarriers.asm similarity index 99% rename from src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm rename to src/coreclr/runtime/i386/WriteBarriers.asm index 133081bee83198..a0ea69e425212b 100644 --- a/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm +++ b/src/coreclr/runtime/i386/WriteBarriers.asm @@ -13,7 +13,7 @@ option casemap:none .code -include AsmMacros.inc +include AsmMacros_Shared.inc ;; Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used ;; during garbage collections to verify that object references where never written to the heap without using a @@ -104,6 +104,7 @@ FASTCALL_FUNC RhpAssignRef&REFREG&, 8 ;; Export the canonical write barrier under unqualified name as well ifidni , ALTERNATE_ENTRY RhpAssignRef + ALTERNATE_ENTRY @RhpAssignRef@8 ALTERNATE_ENTRY _RhpAssignRefAVLocation endif @@ -202,6 +203,7 @@ FASTCALL_FUNC RhpCheckedAssignRef&REFREG&, 8 ;; Export the canonical write barrier under unqualified name as well ifidni , ALTERNATE_ENTRY RhpCheckedAssignRef + ALTERNATE_ENTRY @RhpCheckedAssignRef@8 ALTERNATE_ENTRY _RhpCheckedAssignRefAVLocation endif diff --git a/src/coreclr/runtime/loongarch64/StubDispatch.S b/src/coreclr/runtime/loongarch64/StubDispatch.S index 45c5a293fe18de..af0ef71273abdc 100644 --- a/src/coreclr/runtime/loongarch64/StubDispatch.S +++ b/src/coreclr/runtime/loongarch64/StubDispatch.S @@ -1,8 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include -#include "AsmOffsets.inc" +#include "AsmMacros_Shared.h" #ifdef FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/nativeaot/Runtime/loongarch64/WriteBarriers.S b/src/coreclr/runtime/loongarch64/WriteBarriers.S similarity index 99% rename from src/coreclr/nativeaot/Runtime/loongarch64/WriteBarriers.S rename to src/coreclr/runtime/loongarch64/WriteBarriers.S index daa95ab5bf384a..20888056b04ea8 100644 --- a/src/coreclr/nativeaot/Runtime/loongarch64/WriteBarriers.S +++ b/src/coreclr/runtime/loongarch64/WriteBarriers.S @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include +#include "AsmMacros_Shared.h" // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used // during garbage collections to verify that object references where never written to the heap without using a diff --git a/src/coreclr/runtime/riscv64/StubDispatch.S b/src/coreclr/runtime/riscv64/StubDispatch.S index 9e49c53fbe0f54..cc4101a4a17470 100644 --- a/src/coreclr/runtime/riscv64/StubDispatch.S +++ b/src/coreclr/runtime/riscv64/StubDispatch.S @@ -1,8 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include -#include "AsmOffsets.inc" +#include "AsmMacros_Shared.h" #ifdef FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/nativeaot/Runtime/riscv64/WriteBarriers.S b/src/coreclr/runtime/riscv64/WriteBarriers.S similarity index 99% rename from src/coreclr/nativeaot/Runtime/riscv64/WriteBarriers.S rename to src/coreclr/runtime/riscv64/WriteBarriers.S index e063f549a398a1..469908665b3533 100644 --- a/src/coreclr/nativeaot/Runtime/riscv64/WriteBarriers.S +++ b/src/coreclr/runtime/riscv64/WriteBarriers.S @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include +#include "AsmMacros_Shared.h" // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used // during garbage collections to verify that object references were never written to the heap without using a diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index c3eb6e1c8bb778..1d8ff99211c2ab 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -649,6 +649,7 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64) ${ARCH_SOURCES_DIR}/UMThunkStub.asm ${ARCH_SOURCES_DIR}/VirtualCallStubAMD64.asm ${ARCH_SOURCES_DIR}/StubPrecodeDynamicHelpers.asm + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.asm ) set(VM_HEADERS_WKS_ARCH_ASM @@ -663,7 +664,8 @@ elseif(CLR_CMAKE_TARGET_ARCH_I386) ${ARCH_SOURCES_DIR}/jithelp.asm ${ARCH_SOURCES_DIR}/PInvokeStubs.asm ${ARCH_SOURCES_DIR}/thunktemplates.asm - ) + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.asm + ) set(VM_HEADERS_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmconstants.h @@ -678,6 +680,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64) ${ARCH_SOURCES_DIR}/thunktemplates.asm ${ARCH_SOURCES_DIR}/CachedInterfaceDispatchCoreCLR.asm ${ARCH_SOURCES_DIR}/StubPrecodeDynamicHelpers.asm + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.asm ) set(VM_HEADERS_WKS_ARCH_ASM @@ -709,6 +712,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/unixasmhelpers.S ${ARCH_SOURCES_DIR}/umthunkstub.S ${ARCH_SOURCES_DIR}/virtualcallstubamd64.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ${ARCH_SOURCES_DIR}/StubPrecodeDynamicHelpers.S ) elseif(CLR_CMAKE_TARGET_ARCH_I386) @@ -720,6 +724,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/umthunkstub.S ${ARCH_SOURCES_DIR}/thunktemplates.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) elseif(CLR_CMAKE_TARGET_ARCH_ARM) set(VM_SOURCES_WKS_ARCH_ASM @@ -728,6 +733,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/patchedcode.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) elseif(CLR_CMAKE_TARGET_ARCH_ARM64) set(VM_SOURCES_WKS_ARCH_ASM @@ -739,6 +745,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/StubDispatch.S ${ARCH_SOURCES_DIR}/thunktemplates.S ${ARCH_SOURCES_DIR}/StubPrecodeDynamicHelpers.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) elseif(CLR_CMAKE_TARGET_ARCH_LOONGARCH64) set(VM_SOURCES_WKS_ARCH_ASM @@ -746,6 +753,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/calldescrworkerloongarch64.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) elseif(CLR_CMAKE_TARGET_ARCH_RISCV64) set(VM_SOURCES_WKS_ARCH_ASM @@ -753,6 +761,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/calldescrworkerriscv64.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) endif() diff --git a/src/coreclr/vm/amd64/AsmMacros_Shared.inc b/src/coreclr/vm/amd64/AsmMacros_Shared.inc index c7e7ce2f562fbb..649ee1485fe5bb 100644 --- a/src/coreclr/vm/amd64/AsmMacros_Shared.inc +++ b/src/coreclr/vm/amd64/AsmMacros_Shared.inc @@ -4,4 +4,18 @@ ; This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible include AsmConstants.inc -include AsmMacros.inc \ No newline at end of file +include AsmMacros.inc + +EXTERN g_lowest_address : QWORD +EXTERN g_highest_address : QWORD +EXTERN g_ephemeral_low : QWORD +EXTERN g_ephemeral_high : QWORD +EXTERN g_card_table : QWORD + +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN g_card_bundle_table : QWORD +endif + +ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +EXTERN g_write_watch_table : QWORD +endif diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S index a3a45be29ddb75..bc0edaccea0567 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S @@ -9,6 +9,7 @@ // Stub dispatch routine for dispatch to a vtable slot LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT + // r11 currently contains the indirection cell address. // load r11 to point to the vtable offset (which is stored in the m_pCache field). mov r11, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] @@ -20,7 +21,15 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // Load the MethodTable from the object instance in rdi, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference -ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif + ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation +#ifdef TARGET_APPLE +.cfi_startproc +#endif add rax, [rdi] // Load the target address of the vtable chunk into rax diff --git a/src/coreclr/vm/amd64/JitHelpers_Fast.asm b/src/coreclr/vm/amd64/JitHelpers_Fast.asm index 7e38053605ddb1..f5b26b662441c4 100644 --- a/src/coreclr/vm/amd64/JitHelpers_Fast.asm +++ b/src/coreclr/vm/amd64/JitHelpers_Fast.asm @@ -29,7 +29,7 @@ EXTERN g_card_bundle_table:QWORD endif ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -EXTERN g_sw_ww_table:QWORD +EXTERN g_write_watch_table:QWORD EXTERN g_sw_ww_enabled_for_gc_heap:BYTE endif @@ -141,7 +141,7 @@ ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP je CheckCardTable mov rax, rdi shr rax, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift - add rax, qword ptr [g_sw_ww_table] + add rax, qword ptr [g_write_watch_table] cmp byte ptr [rax], 0h jne CheckCardTable mov byte ptr [rax], 0FFh diff --git a/src/coreclr/vm/amd64/JitHelpers_Slow.asm b/src/coreclr/vm/amd64/JitHelpers_Slow.asm index 8b1fbbb7f8d38f..d14143cf125de3 100644 --- a/src/coreclr/vm/amd64/JitHelpers_Slow.asm +++ b/src/coreclr/vm/amd64/JitHelpers_Slow.asm @@ -27,7 +27,7 @@ EXTERN g_card_bundle_table:QWORD endif ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -EXTERN g_sw_ww_table:QWORD +EXTERN g_write_watch_table:QWORD EXTERN g_sw_ww_enabled_for_gc_heap:BYTE endif @@ -117,7 +117,7 @@ ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP je CheckCardTable mov r10, rcx shr r10, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift - add r10, qword ptr [g_sw_ww_table] + add r10, qword ptr [g_write_watch_table] cmp byte ptr [r10], 0h jne CheckCardTable mov byte ptr [r10], 0FFh diff --git a/src/coreclr/vm/amd64/jithelpers_fast.S b/src/coreclr/vm/amd64/jithelpers_fast.S index b4b1df87c73bd8..37c2f5f98fd19d 100644 --- a/src/coreclr/vm/amd64/jithelpers_fast.S +++ b/src/coreclr/vm/amd64/jithelpers_fast.S @@ -98,7 +98,7 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT je LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier) mov rax, rdi shr rax, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift - add rax, qword ptr [C_VAR(g_sw_ww_table)] + add rax, qword ptr [C_VAR(g_write_watch_table)] cmp byte ptr [rax], 0x0 jne LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier) mov byte ptr [rax], 0xFF diff --git a/src/coreclr/vm/amd64/jithelpers_slow.S b/src/coreclr/vm/amd64/jithelpers_slow.S index c0c80324710c64..d955e0d039fe51 100644 --- a/src/coreclr/vm/amd64/jithelpers_slow.S +++ b/src/coreclr/vm/amd64/jithelpers_slow.S @@ -74,7 +74,7 @@ LEAF_ENTRY JIT_WriteBarrier_Debug, _TEXT je CheckCardTable_Debug mov r10, rdi shr r10, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift - PREPARE_EXTERNAL_VAR g_sw_ww_table, r11 + PREPARE_EXTERNAL_VAR g_write_watch_table, r11 add r10, qword ptr [r11] cmp byte ptr [r10], 0x0 jne CheckCardTable_Debug diff --git a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp index 7b2c853d585542..3df45e888b5dec 100644 --- a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp +++ b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp @@ -863,10 +863,10 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus #endif // FEATURE_SVR_GC case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table) + if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_write_watch_table) { ExecutableWriterHolder writeWatchTableImmediateWriterHolder((UINT64*)m_pWriteWatchTableImmediate, sizeof(UINT64)); - *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_sw_ww_table; + *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_write_watch_table; stompWBCompleteActions |= SWB_ICACHE_FLUSH; } break; @@ -1010,7 +1010,10 @@ int StompWriteBarrierEphemeral(bool isRuntimeSuspended) { WRAPPER_NO_CONTRACT; - return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); + if (IsWriteBarrierCopyEnabled()) + return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); + else + return SWB_PASS; } // This function bashes the super fast amd64 versions of the JIT_WriteBarrier @@ -1020,26 +1023,37 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) { WRAPPER_NO_CONTRACT; - return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); + if (IsWriteBarrierCopyEnabled()) + return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); + else + return SWB_PASS; } void FlushWriteBarrierInstructionCache() { - FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); + if (IsWriteBarrierCopyEnabled()) + FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); } #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) { WRAPPER_NO_CONTRACT; - return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); + if (IsWriteBarrierCopyEnabled()) + return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); + else + return SWB_PASS; } int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) { WRAPPER_NO_CONTRACT; - return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); + if (IsWriteBarrierCopyEnabled()) + return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); + else + return SWB_PASS; } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/arm/AsmMacros_Shared.h b/src/coreclr/vm/arm/AsmMacros_Shared.h new file mode 100644 index 00000000000000..afe0afb7b8ed9e --- /dev/null +++ b/src/coreclr/vm/arm/AsmMacros_Shared.h @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + + +#include "asmconstants.h" +#include "unixasmmacros.inc" diff --git a/src/coreclr/vm/arm/asmhelpers.S b/src/coreclr/vm/arm/asmhelpers.S index 973de5db340ea7..62ba0ac90e8fd6 100644 --- a/src/coreclr/vm/arm/asmhelpers.S +++ b/src/coreclr/vm/arm/asmhelpers.S @@ -609,7 +609,7 @@ LOCAL_LABEL(stackProbe_loop): #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP .macro UPDATE_WRITE_WATCH_TABLE name, ptrReg, mp, tmpReg - LOAD_GC_GLOBAL \name, __wbScratch, g_sw_ww_table + LOAD_GC_GLOBAL \name, __wbScratch, g_write_watch_table cbz __wbScratch, 2f add __wbScratch, __wbScratch, \ptrReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift @@ -719,7 +719,7 @@ LOCAL_LABEL(stackProbe_loop): .word __\name\()__g_ephemeral_high_offset .word __\name\()__g_card_table_offset #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - .word __\name\()__g_sw_ww_table_offset + .word __\name\()__g_write_watch_table_offset #endif .endm diff --git a/src/coreclr/vm/arm/stubs.cpp b/src/coreclr/vm/arm/stubs.cpp index 4dd096adc50456..5edbc415bb37d1 100644 --- a/src/coreclr/vm/arm/stubs.cpp +++ b/src/coreclr/vm/arm/stubs.cpp @@ -292,7 +292,7 @@ struct WriteBarrierDescriptor DWORD m_dw_g_ephemeral_high_offset; // Offset of the instruction reading g_ephemeral_high DWORD m_dw_g_card_table_offset; // Offset of the instruction reading g_card_table #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - DWORD m_dw_g_sw_ww_table_offset; // Offset of the instruction reading g_sw_ww_table + DWORD m_dw_g_write_watch_table_offset;// Offset of the instruction reading g_write_watch_table #endif }; @@ -463,7 +463,7 @@ void UpdateGCWriteBarriers(bool postGrow = false) GWB_PATCH_OFFSET(g_ephemeral_high); GWB_PATCH_OFFSET(g_card_table); #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - GWB_PATCH_OFFSET(g_sw_ww_table); + GWB_PATCH_OFFSET(g_write_watch_table); #endif } @@ -473,6 +473,12 @@ void UpdateGCWriteBarriers(bool postGrow = false) int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) { + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + // The runtime is not always suspended when this is called (unlike StompWriteBarrierEphemeral) but we have // no way to update the barrier code atomically on ARM since each 32-bit value we change is loaded over // two instructions. So we have to suspend the EE (which forces code out of the barrier functions) before @@ -498,6 +504,12 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) int StompWriteBarrierEphemeral(bool isRuntimeSuspended) { + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + UNREFERENCED_PARAMETER(isRuntimeSuspended); _ASSERTE(isRuntimeSuspended); UpdateGCWriteBarriers(); @@ -507,6 +519,12 @@ int StompWriteBarrierEphemeral(bool isRuntimeSuspended) #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) { + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + UNREFERENCED_PARAMETER(isRuntimeSuspended); _ASSERTE(isRuntimeSuspended); UpdateGCWriteBarriers(); @@ -515,6 +533,12 @@ int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) { + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + UNREFERENCED_PARAMETER(isRuntimeSuspended); _ASSERTE(isRuntimeSuspended); UpdateGCWriteBarriers(); diff --git a/src/coreclr/vm/arm64/AsmMacros_Shared.h b/src/coreclr/vm/arm64/AsmMacros_Shared.h index 06a05595cb977e..053145540810e4 100644 --- a/src/coreclr/vm/arm64/AsmMacros_Shared.h +++ b/src/coreclr/vm/arm64/AsmMacros_Shared.h @@ -7,7 +7,103 @@ #include "ksarm64.h" #include "asmconstants.h" #include "asmmacros.h" + + IMPORT g_lowest_address + IMPORT g_highest_address + IMPORT g_ephemeral_low + IMPORT g_ephemeral_high + IMPORT g_card_table + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + IMPORT g_card_bundle_table +#endif + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + IMPORT g_write_watch_table +#endif + +;;----------------------------------------------------------------------------- +;; Macro for loading a 64-bit constant by a minimal number of instructions +;; Since the asssembles doesn't support 64 bit arithmetics in expressions, +;; the value is passed in as lo, hi pair. + MACRO + MOVL64 $Reg, $ConstantLo, $ConstantHi + + LCLS MovInstr +MovInstr SETS "movz" + + IF ((($ConstantHi):SHR:16):AND:0xffff) != 0 + $MovInstr $Reg, #((($Constant):SHR:16):AND:0xffff), lsl #48 +MovInstr SETS "movk" + ENDIF + + IF (($ConstantHi):AND:0xffff) != 0 + $MovInstr $Reg, #(($ConstantHi):AND:0xffff), lsl #32 +MovInstr SETS "movk" + ENDIF + + IF ((($ConstantLo):SHR:16):AND:0xffff) != 0 + $MovInstr $Reg, #((($ConstantLo):SHR:16):AND:0xffff), lsl #16 +MovInstr SETS "movk" + ENDIF + + $MovInstr $Reg, #(($ConstantLo):AND:0xffff) + MEND + +;;----------------------------------------------------------------------------- +;; Macro for loading a 64bit value of a global variable into a register + MACRO + PREPARE_EXTERNAL_VAR_INDIRECT $Name, $Reg + + adrp $Reg, $Name + ldr $Reg, [$Reg, $Name] + MEND + +;; ---------------------------------------------------------------------------- - +;; Macro for loading a 32bit value of a global variable into a register + MACRO + PREPARE_EXTERNAL_VAR_INDIRECT_W $Name, $RegNum + + adrp x$RegNum, $Name + ldr w$RegNum, [x$RegNum, $Name] + MEND + +;; ---------------------------------------------------------------------------- - +;; +;; Macro to add a memory barrier. Equal to __sync_synchronize(). +;; + + MACRO + InterlockedOperationBarrier + + dmb ish + MEND + #else #include "asmconstants.h" #include "unixasmmacros.inc" + +.macro PREPARE_EXTERNAL_VAR_INDIRECT Name, HelperReg +#if defined(__APPLE__) + adrp \HelperReg, C_FUNC(\Name)@GOTPAGE + ldr \HelperReg, [\HelperReg, C_FUNC(\Name)@GOTPAGEOFF] + ldr \HelperReg, [\HelperReg] +#else + adrp \HelperReg, C_FUNC(\Name) + ldr \HelperReg, [\HelperReg, :lo12:C_FUNC(\Name)] #endif +.endm + +.macro PREPARE_EXTERNAL_VAR_INDIRECT_W Name, HelperReg +#if defined(__APPLE__) + adrp x\HelperReg, C_FUNC(\Name)@GOTPAGE + ldr x\HelperReg, [x\HelperReg, C_FUNC(\Name)@GOTPAGEOFF] + ldr w\HelperReg, [x\HelperReg] +#else + adrp x\HelperReg, C_FUNC(\Name) + ldr w\HelperReg, [x\HelperReg, :lo12:C_FUNC(\Name)] +#endif +.endm + +#endif + diff --git a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S index f0d7f3bf433017..068936ef5235e2 100644 --- a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S @@ -11,6 +11,10 @@ // LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT + // Load the MethodTable from the object instance in x0 + ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation + ldr x9, [x0] + // x11 currently contains the indirection cell address. // load x11 to point to the vtable offset (which is stored in the m_pCache field). ldr x11, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] @@ -19,10 +23,8 @@ // to get to the VTable chunk lsr x10, x11, #32 - // Load the MethodTable from the object instance in x0, and add it to the vtable offset + // Add the MethodTable to the vtable offset // to get the address in the vtable chunk list of what we want to dereference - ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation - ldr x9, [x0] add x9, x10, x9 // Load the target address of the vtable chunk into x9 diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index f974a29352bbf7..03e495606fac9f 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -210,7 +210,7 @@ LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - PREPARE_EXTERNAL_VAR g_sw_ww_table, x12 + PREPARE_EXTERNAL_VAR g_write_watch_table, x12 ldr x2, [x12] #endif diff --git a/src/coreclr/vm/arm64/asmhelpers.asm b/src/coreclr/vm/arm64/asmhelpers.asm index 9d57250de9953c..ce63e41a084db5 100644 --- a/src/coreclr/vm/arm64/asmhelpers.asm +++ b/src/coreclr/vm/arm64/asmhelpers.asm @@ -28,7 +28,7 @@ #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - IMPORT g_sw_ww_table + IMPORT g_write_watch_table #endif #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES @@ -279,8 +279,8 @@ ThePreStubPatchLabel #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - adrp x12, g_sw_ww_table - ldr x2, [x12, g_sw_ww_table] + adrp x12, g_write_watch_table + ldr x2, [x12, g_write_watch_table] #endif adrp x12, g_ephemeral_low diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index 694e7e60a75b18..bc2ae8a0607ea9 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -847,15 +847,17 @@ extern "C" void STDCALL JIT_PatchedCodeLast(); static void UpdateWriteBarrierState(bool skipEphemeralCheck) { - BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); - BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; - ExecutableWriterHolderNoLog writeBarrierWriterHolder; if (IsWriteBarrierCopyEnabled()) { - writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); + BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; + ExecutableWriterHolderNoLog writeBarrierWriterHolder; + { + writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); + writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + } + JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } - JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } void InitJITHelpers1() diff --git a/src/coreclr/vm/ceemain.cpp b/src/coreclr/vm/ceemain.cpp index bbf904a0f15f23..4605e8199f3785 100644 --- a/src/coreclr/vm/ceemain.cpp +++ b/src/coreclr/vm/ceemain.cpp @@ -659,6 +659,11 @@ void EEStartupHelper() IfFailGo(ExecutableAllocator::StaticInitialize(FatalErrorHandler)); + if (g_pConfig != NULL) + { + IfFailGoLog(g_pConfig->sync()); + } + Thread::StaticInitialize(); JITInlineTrackingMap::StaticInitialize(); @@ -749,11 +754,6 @@ void EEStartupHelper() #endif // !TARGET_UNIX InitEventStore(); - if (g_pConfig != NULL) - { - IfFailGoLog(g_pConfig->sync()); - } - // Fire the runtime information ETW event ETW::InfoLog::RuntimeInformation(ETW::InfoLog::InfoStructs::Normal); diff --git a/src/coreclr/vm/eeconfig.cpp b/src/coreclr/vm/eeconfig.cpp index fbca43f89a371f..54b62f13e4ad5c 100644 --- a/src/coreclr/vm/eeconfig.cpp +++ b/src/coreclr/vm/eeconfig.cpp @@ -112,6 +112,7 @@ HRESULT EEConfig::Init() fJitMinOpts = false; fJitEnableOptionalRelocs = false; fDisableOptimizedThreadStaticAccess = false; + fIsWriteBarrierCopyEnabled = false; fPInvokeRestoreEsp = (DWORD)-1; fStressLog = false; @@ -502,6 +503,8 @@ HRESULT EEConfig::sync() fDisableOptimizedThreadStaticAccess = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_DisableOptimizedThreadStaticAccess) != 0; + fIsWriteBarrierCopyEnabled = CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_UseGCWriteBarrierCopy) != 0; + #ifdef TARGET_X86 fPInvokeRestoreEsp = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_Jit_NetFx40PInvokeStackResilience); #endif @@ -998,7 +1001,6 @@ HRESULT TypeNamesList::Init(_In_z_ LPCWSTR str) CONTRACTL { NOTHROW; GC_NOTRIGGER; - MODE_ANY; PRECONDITION(CheckPointer(str)); INJECT_FAULT(return E_OUTOFMEMORY); } CONTRACTL_END; diff --git a/src/coreclr/vm/eeconfig.h b/src/coreclr/vm/eeconfig.h index 499a40682637bd..f50a06455c5732 100644 --- a/src/coreclr/vm/eeconfig.h +++ b/src/coreclr/vm/eeconfig.h @@ -389,6 +389,12 @@ class EEConfig && pSkipGCCoverageList->IsInList(assemblyName));} #endif + bool IsWriteBarrierCopyEnabled() const + { + LIMITED_METHOD_CONTRACT; + return fIsWriteBarrierCopyEnabled; + } + #ifdef _DEBUG inline DWORD FastGCStressLevel() const {LIMITED_METHOD_CONTRACT; return iFastGCStress;} @@ -453,6 +459,7 @@ class EEConfig bool fJitMinOpts; // Enable MinOpts for all jitted methods bool fJitEnableOptionalRelocs; // Allow optional relocs bool fDisableOptimizedThreadStaticAccess; // Disable OptimizedThreadStatic access + bool fIsWriteBarrierCopyEnabled; // Is the GC write barrier copy enabled? unsigned iJitOptimizeType; // 0=Blended,1=SmallCode,2=FastCode, default is 0=Blended diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 4d0470610d84e7..2f37091324a28e 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5894,13 +5894,78 @@ BOOL IsIPinVirtualStub(PCODE f_IP) return FALSE; #endif // FEATURE_VIRTUAL_STUB_DISPATCH } +#endif // FEATURE_EH_FUNCLETS + +typedef uint8_t CODE_LOCATION; +EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; +#if defined(HOST_X86) +EXTERN_C CODE_LOCATION RhpAssignRefEAXAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefECXAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefEBXAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefESIAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefEDIAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefEBPAVLocation; +#endif +EXTERN_C CODE_LOCATION RhpCheckedAssignRefAVLocation; +#if defined(HOST_X86) +EXTERN_C CODE_LOCATION RhpCheckedAssignRefEAXAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefECXAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBXAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefESIAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefEDIAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBPAVLocation; +#endif +EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation1; + +#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) +EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation2; +#endif + +static uintptr_t writeBarrierAVLocations[] = +{ + (uintptr_t)&RhpAssignRefAVLocation, +#if defined(HOST_X86) + (uintptr_t)&RhpAssignRefEAXAVLocation, + (uintptr_t)&RhpAssignRefECXAVLocation, + (uintptr_t)&RhpAssignRefEBXAVLocation, + (uintptr_t)&RhpAssignRefESIAVLocation, + (uintptr_t)&RhpAssignRefEDIAVLocation, + (uintptr_t)&RhpAssignRefEBPAVLocation, +#endif + (uintptr_t)&RhpCheckedAssignRefAVLocation, +#if defined(HOST_X86) + (uintptr_t)&RhpCheckedAssignRefEAXAVLocation, + (uintptr_t)&RhpCheckedAssignRefECXAVLocation, + (uintptr_t)&RhpCheckedAssignRefEBXAVLocation, + (uintptr_t)&RhpCheckedAssignRefESIAVLocation, + (uintptr_t)&RhpCheckedAssignRefEDIAVLocation, + (uintptr_t)&RhpCheckedAssignRefEBPAVLocation, +#endif + (uintptr_t)&RhpByRefAssignRefAVLocation1, +#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) + (uintptr_t)&RhpByRefAssignRefAVLocation2, +#endif +}; // Check if the passed in instruction pointer is in one of the // JIT helper functions. -bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) +bool IsIPInMarkedJitHelper(PCODE uControlPc) { LIMITED_METHOD_CONTRACT; + // compare the IP against the list of known possible AV locations in the write barrier helpers + for (size_t i = 0; i < sizeof(writeBarrierAVLocations)/sizeof(writeBarrierAVLocations[0]); i++) + { +#if defined(HOST_AMD64) || defined(HOST_X86) + // Verify that the runtime is not linked with incremental linking enabled. Incremental linking + // wraps every method symbol with a jump stub that breaks the following check. + ASSERT(*(uint8_t*)writeBarrierAVLocations[i] != 0xE9); // jmp XXXXXXXX +#endif + + if (writeBarrierAVLocations[i] == PCODEToPINSTR(uControlPc)) + return true; + } + #define CHECK_RANGE(name) \ if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; @@ -5912,10 +5977,8 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) CHECK_RANGE(JIT_StackProbe) #endif // !TARGET_ARM64 && !TARGET_LOONGARCH64 && !TARGET_RISCV64 #else -#ifdef TARGET_UNIX CHECK_RANGE(JIT_WriteBarrierGroup) CHECK_RANGE(JIT_PatchedWriteBarrierGroup) -#endif // TARGET_UNIX #endif // TARGET_X86 #if defined(TARGET_AMD64) && defined(_DEBUG) @@ -5924,7 +5987,6 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) return false; } -#endif // FEATURE_EH_FUNCLETS // Returns TRUE if caller should resume execution. BOOL @@ -5954,24 +6016,27 @@ AdjustContextForJITHelpers( if (pExceptionRecord == nullptr) { #if defined(TARGET_X86) - bool withinWriteBarrierGroup = ((ip >= (PCODE) JIT_WriteBarrierGroup) && (ip <= (PCODE) JIT_WriteBarrierGroup_End)); - bool withinPatchedWriteBarrierGroup = ((ip >= (PCODE) JIT_PatchedWriteBarrierGroup) && (ip <= (PCODE) JIT_PatchedWriteBarrierGroup_End)); - if (withinWriteBarrierGroup || withinPatchedWriteBarrierGroup) + if (IsIPInMarkedJitHelper(ip)) { DWORD* esp = (DWORD*)pContext->Esp; +#if defined(WRITE_BARRIER_CHECK) + // The assembly for patched write barriers carries the shadow gc heap writing code in such a way that the GC + // heap change is done after pushing some state and always happens in the non-patched region. + // (We don't check for this in the normal AV pathway, as in that case, we have an instruction which triggers + // the AV without writing to memory before adjusting the stack. + bool withinWriteBarrierGroup = ((ip >= (PCODE) JIT_WriteBarrierGroup) && (ip <= (PCODE) JIT_WriteBarrierGroup_End)); if (withinWriteBarrierGroup) { -#if defined(WRITE_BARRIER_CHECK) pContext->Ebp = *esp++; pContext->Ecx = *esp++; -#endif } +#endif pContext->Eip = *esp++; pContext->Esp = (DWORD)esp; return TRUE; } #elif defined(TARGET_AMD64) - if (IsIPInMarkedJitHelper((UINT_PTR)ip)) + if (IsIPInMarkedJitHelper(ip)) { Thread::VirtualUnwindToFirstManagedCallFrame(pContext); return TRUE; @@ -5985,10 +6050,9 @@ AdjustContextForJITHelpers( #endif // FEATURE_DATABREAKPOINT #if defined(TARGET_X86) && !defined(TARGET_UNIX) - void* f_IP = (void *)GetIP(pContext); + PCODE f_IP = GetIP(pContext); - if (((f_IP >= (void *) JIT_WriteBarrierGroup) && (f_IP <= (void *) JIT_WriteBarrierGroup_End)) || - ((f_IP >= (void *) JIT_PatchedWriteBarrierGroup) && (f_IP <= (void *) JIT_PatchedWriteBarrierGroup_End))) + if (IsIPInMarkedJitHelper(ip)) { // set the exception IP to be the instruction that called the write barrier void* callsite = (void *)GetAdjustedCallAddress(*dac_cast(GetSP(pContext))); @@ -5999,7 +6063,7 @@ AdjustContextForJITHelpers( SetSP(pContext, PCODE((BYTE*)GetSP(pContext) + sizeof(void*))); } - if ((f_IP >= (void *) JIT_StackProbe) && (f_IP <= (void *) JIT_StackProbe_End)) + if ((f_IP >= (PCODE) JIT_StackProbe) && (f_IP <= (PCODE) JIT_StackProbe_End)) { TADDR ebp = GetFP(pContext); void* callsite = (void *)*dac_cast(ebp + 4); @@ -6013,12 +6077,12 @@ AdjustContextForJITHelpers( return FALSE; #elif defined(FEATURE_EH_FUNCLETS) // TARGET_X86 && !TARGET_UNIX - void* f_IP = dac_cast(GetIP(pContext)); + PCODE f_IP = GetIP(pContext); CONTEXT tempContext; CONTEXT* pExceptionContext = pContext; - BOOL fExcluded = IsIPInMarkedJitHelper((UINT_PTR)f_IP); + BOOL fExcluded = IsIPInMarkedJitHelper(f_IP); if (fExcluded) { diff --git a/src/coreclr/vm/excep.h b/src/coreclr/vm/excep.h index d3235b8c2e5c98..e1d57765128b77 100644 --- a/src/coreclr/vm/excep.h +++ b/src/coreclr/vm/excep.h @@ -23,7 +23,7 @@ class Thread; BOOL IsExceptionFromManagedCode(const EXCEPTION_RECORD * pExceptionRecord); BOOL IsIPinVirtualStub(PCODE f_IP); -bool IsIPInMarkedJitHelper(UINT_PTR uControlPc); +bool IsIPInMarkedJitHelper(PCODE uControlPc); BOOL IsProcessCorruptedStateException(DWORD dwExceptionCode, OBJECTREF throwable); diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index ed476d073eea73..b62915d6c766fa 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -966,7 +966,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) if (g_sw_ww_enabled_for_gc_heap && (args->write_watch_table != nullptr)) { assert(args->is_runtime_suspended); - g_sw_ww_table = args->write_watch_table; + g_write_watch_table = args->write_watch_table; } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP @@ -1106,7 +1106,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP assert(args->is_runtime_suspended && "the runtime must be suspended here!"); assert(args->write_watch_table != nullptr); - g_sw_ww_table = args->write_watch_table; + g_write_watch_table = args->write_watch_table; g_sw_ww_enabled_for_gc_heap = true; stompWBCompleteActions |= ::SwitchToWriteWatchBarrier(true); #else @@ -1116,7 +1116,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) case WriteBarrierOp::SwitchToNonWriteWatch: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP assert(args->is_runtime_suspended && "the runtime must be suspended here!"); - g_sw_ww_table = 0; + g_write_watch_table = 0; g_sw_ww_enabled_for_gc_heap = false; stompWBCompleteActions |= ::SwitchToNonWriteWatchBarrier(true); #else diff --git a/src/coreclr/vm/gcheaputilities.cpp b/src/coreclr/vm/gcheaputilities.cpp index cffe9185d5e61b..d3c99039b78a8f 100644 --- a/src/coreclr/vm/gcheaputilities.cpp +++ b/src/coreclr/vm/gcheaputilities.cpp @@ -36,7 +36,7 @@ GPTR_IMPL(GcDacVars, g_gcDacGlobals); #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -uint8_t* g_sw_ww_table = nullptr; +uint8_t* g_write_watch_table = nullptr; bool g_sw_ww_enabled_for_gc_heap = false; #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/gcheaputilities.h b/src/coreclr/vm/gcheaputilities.h index 3a6aec2faab171..68b6c435e45bac 100644 --- a/src/coreclr/vm/gcheaputilities.h +++ b/src/coreclr/vm/gcheaputilities.h @@ -163,7 +163,7 @@ extern "C" bool g_region_use_bitwise_write_barrier; // Table containing the dirty state. This table is translated to exclude the lowest address it represents, see // TranslateTableToExcludeHeapStartAddress. -extern "C" uint8_t *g_sw_ww_table; +extern "C" uint8_t *g_write_watch_table; // Write watch may be disabled when it is not needed (between GCs for instance). This indicates whether it is enabled. extern "C" bool g_sw_ww_enabled_for_gc_heap; @@ -292,7 +292,7 @@ class GCHeapUtilities { uint8_t* end_of_write_ptr = reinterpret_cast(address) + (write_size - 1); assert(table_byte_index == reinterpret_cast(end_of_write_ptr) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift); #endif - uint8_t* table_address = &g_sw_ww_table[table_byte_index]; + uint8_t* table_address = &g_write_watch_table[table_byte_index]; if (*table_address == 0) { *table_address = 0xFF; @@ -319,7 +319,7 @@ class GCHeapUtilities { // We'll mark the entire region of memory as dirty by memsetting all entries in // the SWW table between the start and end indexes. - memset(&g_sw_ww_table[base_index], ~0, end_index - base_index + 1); + memset(&g_write_watch_table[base_index], ~0, end_index - base_index + 1); } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/i386/AsmMacros.inc b/src/coreclr/vm/i386/AsmMacros.inc index 79345b6d18b56d..5acea99ddf6b7c 100644 --- a/src/coreclr/vm/i386/AsmMacros.inc +++ b/src/coreclr/vm/i386/AsmMacros.inc @@ -5,6 +5,24 @@ ; Define macros to build unwind data for prologues. ; +FASTCALL_FUNC macro FuncName,cbArgs + FuncNameReal EQU @&FuncName&@&cbArgs + FuncNameReal proc public + FuncName label proc + PUBLIC FuncName + +endm + +FASTCALL_ENDFUNC macro + FuncNameReal endp +endm + +ALTERNATE_ENTRY macro Name + +Name label proc +PUBLIC Name + endm + __tls_array equ 2Ch ;; offsetof(TEB, ThreadLocalStoragePointer) t_CurrentThreadInfo TEXTEQU diff --git a/src/coreclr/vm/i386/AsmMacros_Shared.h b/src/coreclr/vm/i386/AsmMacros_Shared.h new file mode 100644 index 00000000000000..87920d58b2ac65 --- /dev/null +++ b/src/coreclr/vm/i386/AsmMacros_Shared.h @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +#include "unixasmmacros.inc" +#include "asmconstants.h" + diff --git a/src/coreclr/vm/i386/AsmMacros_Shared.inc b/src/coreclr/vm/i386/AsmMacros_Shared.inc new file mode 100644 index 00000000000000..968bcf2674ca5e --- /dev/null +++ b/src/coreclr/vm/i386/AsmMacros_Shared.inc @@ -0,0 +1,19 @@ +; Licensed to the .NET Foundation under one or more agreements. +; The .NET Foundation licenses this file to you under the MIT license. + +; This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +include asmconstants.inc +include asmmacros.inc + +G_LOWEST_ADDRESS equ _g_lowest_address +G_HIGHEST_ADDRESS equ _g_highest_address +G_EPHEMERAL_LOW equ _g_ephemeral_low +G_EPHEMERAL_HIGH equ _g_ephemeral_high +G_CARD_TABLE equ _g_card_table + +EXTERN G_LOWEST_ADDRESS : DWORD +EXTERN G_HIGHEST_ADDRESS : DWORD +EXTERN G_EPHEMERAL_LOW : DWORD +EXTERN G_EPHEMERAL_HIGH : DWORD +EXTERN G_CARD_TABLE : DWORD diff --git a/src/coreclr/vm/i386/jitinterfacex86.cpp b/src/coreclr/vm/i386/jitinterfacex86.cpp index ee59b847cf689b..7960eac0a948f6 100644 --- a/src/coreclr/vm/i386/jitinterfacex86.cpp +++ b/src/coreclr/vm/i386/jitinterfacex86.cpp @@ -711,61 +711,61 @@ void InitJITHelpers1() _ASSERTE_ALL_BUILDS((BYTE*)JIT_PatchedWriteBarrierGroup_End - (BYTE*)JIT_PatchedWriteBarrierGroup < (ptrdiff_t)GetOsPageSize()); // Copy the write barriers to their final resting place. - for (int iBarrier = 0; iBarrier < NUM_WRITE_BARRIERS; iBarrier++) + if (IsWriteBarrierCopyEnabled()) { - BYTE * pfunc = (BYTE *) JIT_WriteBarrierReg_PreGrow; + for (int iBarrier = 0; iBarrier < NUM_WRITE_BARRIERS; iBarrier++) + { + BYTE * pfunc = (BYTE *) JIT_WriteBarrierReg_PreGrow; - BYTE * pBuf = GetWriteBarrierCodeLocation((BYTE *)c_rgWriteBarriers[iBarrier]); - int reg = c_rgWriteBarrierRegs[iBarrier]; + BYTE * pBuf = GetWriteBarrierCodeLocation((BYTE *)c_rgWriteBarriers[iBarrier]); + int reg = c_rgWriteBarrierRegs[iBarrier]; - BYTE * pBufRW = pBuf; - ExecutableWriterHolderNoLog barrierWriterHolder; - if (IsWriteBarrierCopyEnabled()) - { + BYTE * pBufRW = pBuf; + ExecutableWriterHolderNoLog barrierWriterHolder; barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 34); pBufRW = barrierWriterHolder.GetRW(); - } - memcpy(pBufRW, pfunc, 34); + memcpy(pBufRW, pfunc, 34); - // assert the copied code ends in a ret to make sure we got the right length - _ASSERTE(pBuf[33] == 0xC3); + // assert the copied code ends in a ret to make sure we got the right length + _ASSERTE(pBuf[33] == 0xC3); - // We need to adjust registers in a couple of instructions - // It would be nice to have the template contain all zeroes for - // the register fields (corresponding to EAX), but that doesn't - // work because then we get a smaller encoding for the compares - // that only works for EAX but not the other registers. - // So we always have to clear the register fields before updating them. + // We need to adjust registers in a couple of instructions + // It would be nice to have the template contain all zeroes for + // the register fields (corresponding to EAX), but that doesn't + // work because then we get a smaller encoding for the compares + // that only works for EAX but not the other registers. + // So we always have to clear the register fields before updating them. - // First instruction to patch is a mov [edx], reg + // First instruction to patch is a mov [edx], reg - _ASSERTE(pBuf[0] == 0x89); - // Update the reg field (bits 3..5) of the ModR/M byte of this instruction - pBufRW[1] &= 0xc7; - pBufRW[1] |= reg << 3; + _ASSERTE(pBuf[0] == 0x89); + // Update the reg field (bits 3..5) of the ModR/M byte of this instruction + pBufRW[1] &= 0xc7; + pBufRW[1] |= reg << 3; - // Second instruction to patch is cmp reg, imm32 (low bound) + // Second instruction to patch is cmp reg, imm32 (low bound) - _ASSERTE(pBuf[2] == 0x81); - // Here the lowest three bits in ModR/M field are the register - pBufRW[3] &= 0xf8; - pBufRW[3] |= reg; + _ASSERTE(pBuf[2] == 0x81); + // Here the lowest three bits in ModR/M field are the register + pBufRW[3] &= 0xf8; + pBufRW[3] |= reg; #ifdef WRITE_BARRIER_CHECK - // Don't do the fancy optimization just jump to the old one - // Use the slow one for write barrier checks build because it has some good asserts - if (g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK) { - pfunc = &pBufRW[0]; - *pfunc++ = 0xE9; // JMP c_rgDebugWriteBarriers[iBarrier] - *((DWORD*) pfunc) = (BYTE*) c_rgDebugWriteBarriers[iBarrier] - (&pBuf[1] + sizeof(DWORD)); - } + // Don't do the fancy optimization just jump to the old one + // Use the slow one for write barrier checks build because it has some good asserts + if (g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK) { + pfunc = &pBufRW[0]; + *pfunc++ = 0xE9; // JMP c_rgDebugWriteBarriers[iBarrier] + *((DWORD*) pfunc) = (BYTE*) c_rgDebugWriteBarriers[iBarrier] - (&pBuf[1] + sizeof(DWORD)); + } #endif // WRITE_BARRIER_CHECK - } + } #ifndef CODECOVERAGE - ValidateWriteBarrierHelpers(); + ValidateWriteBarrierHelpers(); #endif + } // Leave the patched region writable for StompWriteBarrierEphemeral(), StompWriteBarrierResize() } @@ -853,6 +853,12 @@ int StompWriteBarrierEphemeral(bool /* isRuntimeSuspended */) int stompWBCompleteActions = SWB_PASS; + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + #ifdef WRITE_BARRIER_CHECK // Don't do the fancy optimization if we are checking write barrier if ((GetWriteBarrierCodeLocation((BYTE *)JIT_WriteBarrierEAX))[0] == 0xE9) // we are using slow write barrier @@ -866,11 +872,8 @@ int StompWriteBarrierEphemeral(bool /* isRuntimeSuspended */) BYTE * pBufRW = pBuf; ExecutableWriterHolderNoLog barrierWriterHolder; - if (IsWriteBarrierCopyEnabled()) - { - barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 42); - pBufRW = barrierWriterHolder.GetRW(); - } + barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 42); + pBufRW = barrierWriterHolder.GetRW(); // assert there is in fact a cmp r/m32, imm32 there _ASSERTE(pBuf[2] == 0x81); @@ -917,6 +920,12 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) int stompWBCompleteActions = SWB_PASS; + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + #ifdef WRITE_BARRIER_CHECK // Don't do the fancy optimization if we are checking write barrier if ((GetWriteBarrierCodeLocation((BYTE *)JIT_WriteBarrierEAX))[0] == 0xE9) // we are using slow write barrier @@ -935,11 +944,8 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) BYTE * pBufRW = pBuf; ExecutableWriterHolderNoLog barrierWriterHolder; - if (IsWriteBarrierCopyEnabled()) - { - barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 42); - pBufRW = barrierWriterHolder.GetRW(); - } + barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 42); + pBufRW = barrierWriterHolder.GetRW(); // Check if we are still using the pre-grow version of the write barrier. if (bWriteBarrierIsPreGrow) diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 1b421ab3a20281..6639751c92b032 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -171,7 +171,22 @@ extern "C" FCDECL3(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *ref, Che #else // Regular checked write barrier. extern "C" FCDECL2(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *ref); -#endif + +#ifdef TARGET_ARM64 +#define RhpCheckedAssignRef RhpCheckedAssignRefArm64 +#define RhpByRefAssignRef RhpByRefAssignRefArm64 +#define RhpAssignRef RhpAssignRefArm64 +#elif defined (TARGET_LOONGARCH64) +#define RhpAssignRef RhpAssignRefLoongArch64 +#elif defined (TARGET_RISCV64) +#define RhpAssignRef RhpAssignRefRiscV64 +#endif // TARGET_* + +#endif // FEATURE_USE_ASM_GC_WRITE_BARRIERS && defined(FEATURE_COUNT_GC_WRITE_BARRIERS) + +extern "C" FCDECL2(VOID, RhpCheckedAssignRef, Object **dst, Object *ref); +extern "C" FCDECL2(VOID, RhpByRefAssignRef, Object **dst, Object *ref); +extern "C" FCDECL2(VOID, RhpAssignRef, Object **dst, Object *ref); extern "C" FCDECL2(VOID, JIT_WriteBarrier, Object **dst, Object *ref); extern "C" FCDECL2(VOID, JIT_WriteBarrierEnsureNonHeapTarget, Object **dst, Object *ref); @@ -276,9 +291,10 @@ extern "C" // JIThelp.asm/JIThelp.s #define X86_WRITE_BARRIER_REGISTER(reg) \ - void STDCALL JIT_CheckedWriteBarrier##reg(); \ void STDCALL JIT_DebugWriteBarrier##reg(); \ - void STDCALL JIT_WriteBarrier##reg(); + void STDCALL JIT_WriteBarrier##reg(); \ + void FASTCALL RhpAssignRef##reg(Object**, Object*); \ + void FASTCALL RhpCheckedAssignRef##reg(Object**, Object*); ENUM_X86_WRITE_BARRIER_REGISTERS() #undef X86_WRITE_BARRIER_REGISTER diff --git a/src/coreclr/vm/loongarch64/AsmMacros_Shared.h b/src/coreclr/vm/loongarch64/AsmMacros_Shared.h new file mode 100644 index 00000000000000..afe0afb7b8ed9e --- /dev/null +++ b/src/coreclr/vm/loongarch64/AsmMacros_Shared.h @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + + +#include "asmconstants.h" +#include "unixasmmacros.inc" diff --git a/src/coreclr/vm/loongarch64/asmhelpers.S b/src/coreclr/vm/loongarch64/asmhelpers.S index ee33ad8599eebe..b16c58d331b79f 100644 --- a/src/coreclr/vm/loongarch64/asmhelpers.S +++ b/src/coreclr/vm/loongarch64/asmhelpers.S @@ -63,7 +63,7 @@ WRITE_BARRIER_ENTRY JIT_UpdateWriteBarrierState #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - la.local $a2, g_sw_ww_table + la.local $a2, g_write_watch_table ld.d $a2, $a2, 0 #endif diff --git a/src/coreclr/vm/loongarch64/stubs.cpp b/src/coreclr/vm/loongarch64/stubs.cpp index e763157e07b1ac..6cd8cc4430d1ae 100644 --- a/src/coreclr/vm/loongarch64/stubs.cpp +++ b/src/coreclr/vm/loongarch64/stubs.cpp @@ -860,15 +860,18 @@ extern "C" void STDCALL JIT_PatchedCodeLast(); static void UpdateWriteBarrierState(bool skipEphemeralCheck) { - BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); - BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; - ExecutableWriterHolderNoLog writeBarrierWriterHolder; if (IsWriteBarrierCopyEnabled()) { - writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); + BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; + ExecutableWriterHolderNoLog writeBarrierWriterHolder; + if (IsWriteBarrierCopyEnabled()) + { + writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); + writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + } + JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } - JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } void InitJITHelpers1() diff --git a/src/coreclr/vm/riscv64/AsmMacros_Shared.h b/src/coreclr/vm/riscv64/AsmMacros_Shared.h new file mode 100644 index 00000000000000..afe0afb7b8ed9e --- /dev/null +++ b/src/coreclr/vm/riscv64/AsmMacros_Shared.h @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + + +#include "asmconstants.h" +#include "unixasmmacros.inc" diff --git a/src/coreclr/vm/riscv64/asmhelpers.S b/src/coreclr/vm/riscv64/asmhelpers.S index f160a704b9f05e..d524ee9c2667cd 100644 --- a/src/coreclr/vm/riscv64/asmhelpers.S +++ b/src/coreclr/vm/riscv64/asmhelpers.S @@ -65,7 +65,7 @@ WRITE_BARRIER_ENTRY JIT_UpdateWriteBarrierState #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - ld a4, g_sw_ww_table + ld a4, g_write_watch_table #endif ld a5, g_ephemeral_low diff --git a/src/coreclr/vm/riscv64/stubs.cpp b/src/coreclr/vm/riscv64/stubs.cpp index c2e2d97e3ebdc4..5b2b1c75952be4 100644 --- a/src/coreclr/vm/riscv64/stubs.cpp +++ b/src/coreclr/vm/riscv64/stubs.cpp @@ -777,15 +777,18 @@ extern "C" void STDCALL JIT_PatchedCodeLast(); static void UpdateWriteBarrierState(bool skipEphemeralCheck) { - BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); - BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; - ExecutableWriterHolderNoLog writeBarrierWriterHolder; if (IsWriteBarrierCopyEnabled()) { - writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); + BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; + ExecutableWriterHolderNoLog writeBarrierWriterHolder; + if (IsWriteBarrierCopyEnabled()) + { + writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); + writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + } + JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } - JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } void InitJITHelpers1() diff --git a/src/coreclr/vm/threads.cpp b/src/coreclr/vm/threads.cpp index be643cf1652f65..1685042586057f 100644 --- a/src/coreclr/vm/threads.cpp +++ b/src/coreclr/vm/threads.cpp @@ -1132,31 +1132,24 @@ void InitThreadManager() ETW::MethodLog::StubInitialized((ULONGLONG)GetWriteBarrierCodeLocation((void*)JIT_ByRefWriteBarrier), W("@ByRefWriteBarrier")); #endif // TARGET_ARM64 || TARGET_ARM || TARGET_LOONGARCH64 || TARGET_RISCV64 +#if defined(TARGET_AMD64) + // On AMD64 the Checked/ByRef variants of the helpers jump through an indirection + // to the patched barrier, but are not part of the patched set of helpers. + SetJitHelperFunction(CORINFO_HELP_CHECKED_ASSIGN_REF, (void*)JIT_CheckedWriteBarrier); + SetJitHelperFunction(CORINFO_HELP_ASSIGN_BYREF, (void*)JIT_ByRefWriteBarrier); +#endif // TARGET_AMD64 + } else { - // I am using virtual protect to cover the entire range that this code falls in. - // - - // We could reset it to non-writeable inbetween GCs and such, but then we'd have to keep on re-writing back and forth, - // so instead we'll leave it writable from here forward. - - DWORD oldProt; - if (!ClrVirtualProtect((void *)JIT_PatchedCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart, - PAGE_EXECUTE_READWRITE, &oldProt)) - { - _ASSERTE(!"ClrVirtualProtect of code page failed"); - COMPlusThrowWin32(); - } - #ifdef TARGET_X86 - JIT_WriteBarrierEAX_Loc = (void*)JIT_WriteBarrierEAX; + JIT_WriteBarrierEAX_Loc = (void*)RhpAssignRefEAX; #else - JIT_WriteBarrier_Loc = (void*)JIT_WriteBarrier; + JIT_WriteBarrier_Loc = (void*)RhpAssignRef; #endif #if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // Store the JIT_WriteBarrier_Table copy location to a global variable so that it can be updated. - JIT_WriteBarrier_Table_Loc = (void*)&JIT_WriteBarrier_Table; + JIT_WriteBarrier_Table_Loc = NULL; #endif // TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64 } diff --git a/src/coreclr/vm/threads.h b/src/coreclr/vm/threads.h index 91db958c71a921..aebee5e6a0739c 100644 --- a/src/coreclr/vm/threads.h +++ b/src/coreclr/vm/threads.h @@ -5667,11 +5667,7 @@ inline BOOL IsWriteBarrierCopyEnabled() #ifdef DACCESS_COMPILE return FALSE; #else // DACCESS_COMPILE -#ifdef HOST_APPLE - return TRUE; -#else - return ExecutableAllocator::IsWXORXEnabled(); -#endif + return g_pConfig->IsWriteBarrierCopyEnabled(); #endif // DACCESS_COMPILE }