From bfd57cd9a56e311bb49ecd2b8615e650749da5f9 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 22 Apr 2025 15:46:36 -0700 Subject: [PATCH 01/34] Builds with write barrier moved --- src/coreclr/nativeaot/Runtime/CMakeLists.txt | 2 +- .../{nativeaot/Runtime => runtime}/amd64/WriteBarriers.S | 0 .../{nativeaot/Runtime => runtime}/amd64/WriteBarriers.asm | 0 src/coreclr/{nativeaot/Runtime => runtime}/arm/WriteBarriers.S | 0 .../{nativeaot/Runtime => runtime}/arm64/WriteBarriers.S | 0 .../{nativeaot/Runtime => runtime}/arm64/WriteBarriers.asm | 0 src/coreclr/{nativeaot/Runtime => runtime}/i386/WriteBarriers.S | 0 .../{nativeaot/Runtime => runtime}/i386/WriteBarriers.asm | 0 .../{nativeaot/Runtime => runtime}/loongarch64/WriteBarriers.S | 0 .../{nativeaot/Runtime => runtime}/riscv64/WriteBarriers.S | 0 10 files changed, 1 insertion(+), 1 deletion(-) rename src/coreclr/{nativeaot/Runtime => runtime}/amd64/WriteBarriers.S (100%) rename src/coreclr/{nativeaot/Runtime => runtime}/amd64/WriteBarriers.asm (100%) rename src/coreclr/{nativeaot/Runtime => runtime}/arm/WriteBarriers.S (100%) rename src/coreclr/{nativeaot/Runtime => runtime}/arm64/WriteBarriers.S (100%) rename src/coreclr/{nativeaot/Runtime => runtime}/arm64/WriteBarriers.asm (100%) rename src/coreclr/{nativeaot/Runtime => runtime}/i386/WriteBarriers.S (100%) rename src/coreclr/{nativeaot/Runtime => runtime}/i386/WriteBarriers.asm (100%) rename src/coreclr/{nativeaot/Runtime => runtime}/loongarch64/WriteBarriers.S (100%) rename src/coreclr/{nativeaot/Runtime => runtime}/riscv64/WriteBarriers.S (100%) diff --git a/src/coreclr/nativeaot/Runtime/CMakeLists.txt b/src/coreclr/nativeaot/Runtime/CMakeLists.txt index 43e0ead275931c..317085ff296a91 100644 --- a/src/coreclr/nativeaot/Runtime/CMakeLists.txt +++ b/src/coreclr/nativeaot/Runtime/CMakeLists.txt @@ -213,7 +213,7 @@ list(APPEND RUNTIME_SOURCES_ARCH_ASM ${ARCH_SOURCES_DIR}/InteropThunksHelpers.${ASM_SUFFIX} ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/StubDispatch.${ASM_SUFFIX} ${ARCH_SOURCES_DIR}/UniversalTransition.${ASM_SUFFIX} - ${ARCH_SOURCES_DIR}/WriteBarriers.${ASM_SUFFIX} + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.${ASM_SUFFIX} ) if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64) diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S similarity index 100% rename from src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S rename to src/coreclr/runtime/amd64/WriteBarriers.S diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm b/src/coreclr/runtime/amd64/WriteBarriers.asm similarity index 100% rename from src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm rename to src/coreclr/runtime/amd64/WriteBarriers.asm diff --git a/src/coreclr/nativeaot/Runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S similarity index 100% rename from src/coreclr/nativeaot/Runtime/arm/WriteBarriers.S rename to src/coreclr/runtime/arm/WriteBarriers.S diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S similarity index 100% rename from src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S rename to src/coreclr/runtime/arm64/WriteBarriers.S diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm similarity index 100% rename from src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm rename to src/coreclr/runtime/arm64/WriteBarriers.asm diff --git a/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.S b/src/coreclr/runtime/i386/WriteBarriers.S similarity index 100% rename from src/coreclr/nativeaot/Runtime/i386/WriteBarriers.S rename to src/coreclr/runtime/i386/WriteBarriers.S diff --git a/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm b/src/coreclr/runtime/i386/WriteBarriers.asm similarity index 100% rename from src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm rename to src/coreclr/runtime/i386/WriteBarriers.asm diff --git a/src/coreclr/nativeaot/Runtime/loongarch64/WriteBarriers.S b/src/coreclr/runtime/loongarch64/WriteBarriers.S similarity index 100% rename from src/coreclr/nativeaot/Runtime/loongarch64/WriteBarriers.S rename to src/coreclr/runtime/loongarch64/WriteBarriers.S diff --git a/src/coreclr/nativeaot/Runtime/riscv64/WriteBarriers.S b/src/coreclr/runtime/riscv64/WriteBarriers.S similarity index 100% rename from src/coreclr/nativeaot/Runtime/riscv64/WriteBarriers.S rename to src/coreclr/runtime/riscv64/WriteBarriers.S From 02b9ea6dcf6b5833347c8215af5ce20c9fc60c70 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 22 Apr 2025 15:46:41 -0700 Subject: [PATCH 02/34] More adjustment --- src/coreclr/runtime/amd64/WriteBarriers.S | 2 +- src/coreclr/runtime/amd64/WriteBarriers.asm | 2 +- src/coreclr/runtime/arm64/WriteBarriers.S | 2 +- src/coreclr/runtime/arm64/WriteBarriers.asm | 2 +- src/coreclr/vm/CMakeLists.txt | 1 + src/coreclr/vm/amd64/AsmMacros_Shared.inc | 16 +++++- src/coreclr/vm/amd64/JitHelpers_Fast.asm | 4 +- src/coreclr/vm/amd64/JitHelpers_Slow.asm | 4 +- src/coreclr/vm/amd64/jithelpers_fast.S | 2 +- src/coreclr/vm/amd64/jithelpers_slow.S | 2 +- src/coreclr/vm/amd64/jitinterfaceamd64.cpp | 4 +- src/coreclr/vm/arm/asmhelpers.S | 2 +- src/coreclr/vm/arm/stubs.cpp | 4 +- src/coreclr/vm/arm64/AsmMacros_Shared.h | 16 ++++++ src/coreclr/vm/arm64/asmhelpers.S | 2 +- src/coreclr/vm/arm64/asmhelpers.asm | 6 +- src/coreclr/vm/excep.cpp | 64 +++++++++++++++++++++ src/coreclr/vm/gcenv.ee.cpp | 6 +- src/coreclr/vm/gcheaputilities.cpp | 2 +- src/coreclr/vm/gcheaputilities.h | 6 +- src/coreclr/vm/loongarch64/asmhelpers.S | 2 +- src/coreclr/vm/riscv64/asmhelpers.S | 2 +- 22 files changed, 124 insertions(+), 29 deletions(-) diff --git a/src/coreclr/runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S index e55e682653b512..f23d29a3f14af5 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. .intel_syntax noprefix -#include +#include "AsmMacros_Shared.h" #ifdef WRITE_BARRIER_CHECK diff --git a/src/coreclr/runtime/amd64/WriteBarriers.asm b/src/coreclr/runtime/amd64/WriteBarriers.asm index 302b9e0a8b1fed..c08109d65ea62b 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.asm +++ b/src/coreclr/runtime/amd64/WriteBarriers.asm @@ -1,7 +1,7 @@ ;; Licensed to the .NET Foundation under one or more agreements. ;; The .NET Foundation licenses this file to you under the MIT license. -include AsmMacros.inc +include AsmMacros_Shared.inc ;; Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used ;; during garbage collections to verify that object references where never written to the heap without using a diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index 6948e0fa94a07b..b396ee18160367 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include +#include "AsmMacros_Shared.h" // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used // during garbage collections to verify that object references where never written to the heap without using a diff --git a/src/coreclr/runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm index 05a26044dddd21..65d84f77440219 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -8,7 +8,7 @@ ;; collection. ;; -#include "AsmMacros.h" +#include "AsmMacros_Shared.h" TEXTAREA diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index 436fc7ae5db107..c74d01416cc4e2 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -643,6 +643,7 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64) ${ARCH_SOURCES_DIR}/UMThunkStub.asm ${ARCH_SOURCES_DIR}/VirtualCallStubAMD64.asm ${ARCH_SOURCES_DIR}/StubPrecodeDynamicHelpers.asm + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.asm ) set(VM_HEADERS_WKS_ARCH_ASM diff --git a/src/coreclr/vm/amd64/AsmMacros_Shared.inc b/src/coreclr/vm/amd64/AsmMacros_Shared.inc index c7e7ce2f562fbb..649ee1485fe5bb 100644 --- a/src/coreclr/vm/amd64/AsmMacros_Shared.inc +++ b/src/coreclr/vm/amd64/AsmMacros_Shared.inc @@ -4,4 +4,18 @@ ; This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible include AsmConstants.inc -include AsmMacros.inc \ No newline at end of file +include AsmMacros.inc + +EXTERN g_lowest_address : QWORD +EXTERN g_highest_address : QWORD +EXTERN g_ephemeral_low : QWORD +EXTERN g_ephemeral_high : QWORD +EXTERN g_card_table : QWORD + +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN g_card_bundle_table : QWORD +endif + +ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +EXTERN g_write_watch_table : QWORD +endif diff --git a/src/coreclr/vm/amd64/JitHelpers_Fast.asm b/src/coreclr/vm/amd64/JitHelpers_Fast.asm index 7e38053605ddb1..f5b26b662441c4 100644 --- a/src/coreclr/vm/amd64/JitHelpers_Fast.asm +++ b/src/coreclr/vm/amd64/JitHelpers_Fast.asm @@ -29,7 +29,7 @@ EXTERN g_card_bundle_table:QWORD endif ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -EXTERN g_sw_ww_table:QWORD +EXTERN g_write_watch_table:QWORD EXTERN g_sw_ww_enabled_for_gc_heap:BYTE endif @@ -141,7 +141,7 @@ ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP je CheckCardTable mov rax, rdi shr rax, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift - add rax, qword ptr [g_sw_ww_table] + add rax, qword ptr [g_write_watch_table] cmp byte ptr [rax], 0h jne CheckCardTable mov byte ptr [rax], 0FFh diff --git a/src/coreclr/vm/amd64/JitHelpers_Slow.asm b/src/coreclr/vm/amd64/JitHelpers_Slow.asm index 63bc8cc43f1aec..fb109e016cad7e 100644 --- a/src/coreclr/vm/amd64/JitHelpers_Slow.asm +++ b/src/coreclr/vm/amd64/JitHelpers_Slow.asm @@ -27,7 +27,7 @@ EXTERN g_card_bundle_table:QWORD endif ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -EXTERN g_sw_ww_table:QWORD +EXTERN g_write_watch_table:QWORD EXTERN g_sw_ww_enabled_for_gc_heap:BYTE endif @@ -121,7 +121,7 @@ ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP je CheckCardTable mov r10, rcx shr r10, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift - add r10, qword ptr [g_sw_ww_table] + add r10, qword ptr [g_write_watch_table] cmp byte ptr [r10], 0h jne CheckCardTable mov byte ptr [r10], 0FFh diff --git a/src/coreclr/vm/amd64/jithelpers_fast.S b/src/coreclr/vm/amd64/jithelpers_fast.S index b4b1df87c73bd8..37c2f5f98fd19d 100644 --- a/src/coreclr/vm/amd64/jithelpers_fast.S +++ b/src/coreclr/vm/amd64/jithelpers_fast.S @@ -98,7 +98,7 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT je LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier) mov rax, rdi shr rax, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift - add rax, qword ptr [C_VAR(g_sw_ww_table)] + add rax, qword ptr [C_VAR(g_write_watch_table)] cmp byte ptr [rax], 0x0 jne LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier) mov byte ptr [rax], 0xFF diff --git a/src/coreclr/vm/amd64/jithelpers_slow.S b/src/coreclr/vm/amd64/jithelpers_slow.S index c0c80324710c64..d955e0d039fe51 100644 --- a/src/coreclr/vm/amd64/jithelpers_slow.S +++ b/src/coreclr/vm/amd64/jithelpers_slow.S @@ -74,7 +74,7 @@ LEAF_ENTRY JIT_WriteBarrier_Debug, _TEXT je CheckCardTable_Debug mov r10, rdi shr r10, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift - PREPARE_EXTERNAL_VAR g_sw_ww_table, r11 + PREPARE_EXTERNAL_VAR g_write_watch_table, r11 add r10, qword ptr [r11] cmp byte ptr [r10], 0x0 jne CheckCardTable_Debug diff --git a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp index f9cd2a968a02a7..e0abba6175b994 100644 --- a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp +++ b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp @@ -859,10 +859,10 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus #endif // FEATURE_SVR_GC case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table) + if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_write_watch_table) { ExecutableWriterHolder writeWatchTableImmediateWriterHolder((UINT64*)m_pWriteWatchTableImmediate, sizeof(UINT64)); - *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_sw_ww_table; + *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_write_watch_table; stompWBCompleteActions |= SWB_ICACHE_FLUSH; } break; diff --git a/src/coreclr/vm/arm/asmhelpers.S b/src/coreclr/vm/arm/asmhelpers.S index e577b1828ae8a3..3581ef6675fb0b 100644 --- a/src/coreclr/vm/arm/asmhelpers.S +++ b/src/coreclr/vm/arm/asmhelpers.S @@ -609,7 +609,7 @@ LOCAL_LABEL(stackProbe_loop): #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP .macro UPDATE_WRITE_WATCH_TABLE name, ptrReg, mp, tmpReg - LOAD_GC_GLOBAL \name, __wbScratch, g_sw_ww_table + LOAD_GC_GLOBAL \name, __wbScratch, g_write_watch_table cbz __wbScratch, 2f add __wbScratch, __wbScratch, \ptrReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift diff --git a/src/coreclr/vm/arm/stubs.cpp b/src/coreclr/vm/arm/stubs.cpp index cb665654d0cac4..126055f66c9d10 100644 --- a/src/coreclr/vm/arm/stubs.cpp +++ b/src/coreclr/vm/arm/stubs.cpp @@ -292,7 +292,7 @@ struct WriteBarrierDescriptor DWORD m_dw_g_ephemeral_high_offset; // Offset of the instruction reading g_ephemeral_high DWORD m_dw_g_card_table_offset; // Offset of the instruction reading g_card_table #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - DWORD m_dw_g_sw_ww_table_offset; // Offset of the instruction reading g_sw_ww_table + DWORD m_dw_g_sw_ww_table_offset; // Offset of the instruction reading g_write_watch_table #endif }; @@ -463,7 +463,7 @@ void UpdateGCWriteBarriers(bool postGrow = false) GWB_PATCH_OFFSET(g_ephemeral_high); GWB_PATCH_OFFSET(g_card_table); #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - GWB_PATCH_OFFSET(g_sw_ww_table); + GWB_PATCH_OFFSET(g_write_watch_table); #endif } diff --git a/src/coreclr/vm/arm64/AsmMacros_Shared.h b/src/coreclr/vm/arm64/AsmMacros_Shared.h index 06a05595cb977e..6bd6ac764d2ec5 100644 --- a/src/coreclr/vm/arm64/AsmMacros_Shared.h +++ b/src/coreclr/vm/arm64/AsmMacros_Shared.h @@ -7,7 +7,23 @@ #include "ksarm64.h" #include "asmconstants.h" #include "asmmacros.h" + +EXTERN g_lowest_address : QWORD +EXTERN g_highest_address : QWORD +EXTERN g_ephemeral_low : QWORD +EXTERN g_ephemeral_high : QWORD +EXTERN g_card_table : QWORD + +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN g_card_bundle_table : QWORD +endif + +ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +EXTERN g_write_watch_table : QWORD +endif + #else #include "asmconstants.h" #include "unixasmmacros.inc" #endif + diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index 5fac7fae7aa838..23b9b1a89c2897 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -210,7 +210,7 @@ LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - PREPARE_EXTERNAL_VAR g_sw_ww_table, x12 + PREPARE_EXTERNAL_VAR g_write_watch_table, x12 ldr x2, [x12] #endif diff --git a/src/coreclr/vm/arm64/asmhelpers.asm b/src/coreclr/vm/arm64/asmhelpers.asm index 3dd6edcb34ab2b..0fe10cab40dcbe 100644 --- a/src/coreclr/vm/arm64/asmhelpers.asm +++ b/src/coreclr/vm/arm64/asmhelpers.asm @@ -28,7 +28,7 @@ #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - IMPORT g_sw_ww_table + IMPORT g_write_watch_table #endif #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES @@ -279,8 +279,8 @@ ThePreStubPatchLabel #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - adrp x12, g_sw_ww_table - ldr x2, [x12, g_sw_ww_table] + adrp x12, g_write_watch_table + ldr x2, [x12, g_write_watch_table] #endif adrp x12, g_ephemeral_low diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 25512830088f88..156af9996d9021 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5899,12 +5899,76 @@ BOOL IsIPinVirtualStub(PCODE f_IP) #endif // FEATURE_VIRTUAL_STUB_DISPATCH } +typedef uint8_t CODE_LOCATION; +EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; +#if defined(HOST_X86) +EXTERN_C CODE_LOCATION RhpAssignRefEAXAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefECXAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefEBXAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefESIAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefEDIAVLocation; +EXTERN_C CODE_LOCATION RhpAssignRefEBPAVLocation; +#endif +EXTERN_C CODE_LOCATION RhpCheckedAssignRefAVLocation; +#if defined(HOST_X86) +EXTERN_C CODE_LOCATION RhpCheckedAssignRefEAXAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefECXAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBXAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefESIAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefEDIAVLocation; +EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBPAVLocation; +#endif +EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation1; + +#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) +EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation2; +#endif + +static uintptr_t writeBarrierAVLocations[] = +{ + (uintptr_t)&RhpAssignRefAVLocation, +#if defined(HOST_X86) + (uintptr_t)&RhpAssignRefEAXAVLocation, + (uintptr_t)&RhpAssignRefECXAVLocation, + (uintptr_t)&RhpAssignRefEBXAVLocation, + (uintptr_t)&RhpAssignRefESIAVLocation, + (uintptr_t)&RhpAssignRefEDIAVLocation, + (uintptr_t)&RhpAssignRefEBPAVLocation, +#endif + (uintptr_t)&RhpCheckedAssignRefAVLocation, +#if defined(HOST_X86) + (uintptr_t)&RhpCheckedAssignRefEAXAVLocation, + (uintptr_t)&RhpCheckedAssignRefECXAVLocation, + (uintptr_t)&RhpCheckedAssignRefEBXAVLocation, + (uintptr_t)&RhpCheckedAssignRefESIAVLocation, + (uintptr_t)&RhpCheckedAssignRefEDIAVLocation, + (uintptr_t)&RhpCheckedAssignRefEBPAVLocation, +#endif + (uintptr_t)&RhpByRefAssignRefAVLocation1, +#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) + (uintptr_t)&RhpByRefAssignRefAVLocation2, +#endif +}; + // Check if the passed in instruction pointer is in one of the // JIT helper functions. bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) { LIMITED_METHOD_CONTRACT; + // compare the IP against the list of known possible AV locations in the write barrier helpers + for (size_t i = 0; i < sizeof(writeBarrierAVLocations)/sizeof(writeBarrierAVLocations[0]); i++) + { +#if defined(HOST_AMD64) || defined(HOST_X86) + // Verify that the runtime is not linked with incremental linking enabled. Incremental linking + // wraps every method symbol with a jump stub that breaks the following check. + ASSERT(*(uint8_t*)writeBarrierAVLocations[i] != 0xE9); // jmp XXXXXXXX +#endif + + if (writeBarrierAVLocations[i] == uControlPc) + return true; + } + #define CHECK_RANGE(name) \ if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index 4738b0d7aeb8fa..0ea7f98df52770 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -964,7 +964,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) if (g_sw_ww_enabled_for_gc_heap && (args->write_watch_table != nullptr)) { assert(args->is_runtime_suspended); - g_sw_ww_table = args->write_watch_table; + g_write_watch_table = args->write_watch_table; } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP @@ -1104,7 +1104,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP assert(args->is_runtime_suspended && "the runtime must be suspended here!"); assert(args->write_watch_table != nullptr); - g_sw_ww_table = args->write_watch_table; + g_write_watch_table = args->write_watch_table; g_sw_ww_enabled_for_gc_heap = true; stompWBCompleteActions |= ::SwitchToWriteWatchBarrier(true); #else @@ -1114,7 +1114,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) case WriteBarrierOp::SwitchToNonWriteWatch: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP assert(args->is_runtime_suspended && "the runtime must be suspended here!"); - g_sw_ww_table = 0; + g_write_watch_table = 0; g_sw_ww_enabled_for_gc_heap = false; stompWBCompleteActions |= ::SwitchToNonWriteWatchBarrier(true); #else diff --git a/src/coreclr/vm/gcheaputilities.cpp b/src/coreclr/vm/gcheaputilities.cpp index cffe9185d5e61b..d3c99039b78a8f 100644 --- a/src/coreclr/vm/gcheaputilities.cpp +++ b/src/coreclr/vm/gcheaputilities.cpp @@ -36,7 +36,7 @@ GPTR_IMPL(GcDacVars, g_gcDacGlobals); #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -uint8_t* g_sw_ww_table = nullptr; +uint8_t* g_write_watch_table = nullptr; bool g_sw_ww_enabled_for_gc_heap = false; #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/gcheaputilities.h b/src/coreclr/vm/gcheaputilities.h index 8b40521c2377fa..6ddbd6874258c8 100644 --- a/src/coreclr/vm/gcheaputilities.h +++ b/src/coreclr/vm/gcheaputilities.h @@ -163,7 +163,7 @@ extern "C" bool g_region_use_bitwise_write_barrier; // Table containing the dirty state. This table is translated to exclude the lowest address it represents, see // TranslateTableToExcludeHeapStartAddress. -extern "C" uint8_t *g_sw_ww_table; +extern "C" uint8_t *g_write_watch_table; // Write watch may be disabled when it is not needed (between GCs for instance). This indicates whether it is enabled. extern "C" bool g_sw_ww_enabled_for_gc_heap; @@ -288,7 +288,7 @@ class GCHeapUtilities { uint8_t* end_of_write_ptr = reinterpret_cast(address) + (write_size - 1); assert(table_byte_index == reinterpret_cast(end_of_write_ptr) >> SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift); #endif - uint8_t* table_address = &g_sw_ww_table[table_byte_index]; + uint8_t* table_address = &g_write_watch_table[table_byte_index]; if (*table_address == 0) { *table_address = 0xFF; @@ -315,7 +315,7 @@ class GCHeapUtilities { // We'll mark the entire region of memory as dirty by memsetting all entries in // the SWW table between the start and end indexes. - memset(&g_sw_ww_table[base_index], ~0, end_index - base_index + 1); + memset(&g_write_watch_table[base_index], ~0, end_index - base_index + 1); } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/loongarch64/asmhelpers.S b/src/coreclr/vm/loongarch64/asmhelpers.S index 6f2a89383852c4..bc0bf1933a0ce4 100644 --- a/src/coreclr/vm/loongarch64/asmhelpers.S +++ b/src/coreclr/vm/loongarch64/asmhelpers.S @@ -63,7 +63,7 @@ WRITE_BARRIER_ENTRY JIT_UpdateWriteBarrierState #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - la.local $a2, g_sw_ww_table + la.local $a2, g_write_watch_table ld.d $a2, $a2, 0 #endif diff --git a/src/coreclr/vm/riscv64/asmhelpers.S b/src/coreclr/vm/riscv64/asmhelpers.S index e548ddd32d31c6..54020b66f45007 100644 --- a/src/coreclr/vm/riscv64/asmhelpers.S +++ b/src/coreclr/vm/riscv64/asmhelpers.S @@ -65,7 +65,7 @@ WRITE_BARRIER_ENTRY JIT_UpdateWriteBarrierState #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - ld a4, g_sw_ww_table + ld a4, g_write_watch_table #endif ld a5, g_ephemeral_low From 17ad508009c3d668014a820c778bdf3e95908039 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 22 Apr 2025 17:10:42 -0700 Subject: [PATCH 03/34] In theory all mucked up to make this work on Windows Amd64... Or at least, somewhat close. --- src/coreclr/inc/clrconfigvalues.h | 1 + src/coreclr/inc/jithelpers.h | 6 +++--- src/coreclr/vm/arm64/stubs.cpp | 14 ++++++++------ src/coreclr/vm/eeconfig.cpp | 12 +++++++++++- src/coreclr/vm/eeconfig.h | 7 +++++++ src/coreclr/vm/jitinterface.h | 17 ++++++++++++++++- src/coreclr/vm/loongarch64/stubs.cpp | 15 +++++++++------ src/coreclr/vm/riscv64/stubs.cpp | 15 +++++++++------ src/coreclr/vm/threads.cpp | 18 ++---------------- src/coreclr/vm/threads.h | 6 +----- 10 files changed, 67 insertions(+), 44 deletions(-) diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 8aee6fea67127e..7e28921ab80319 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -254,6 +254,7 @@ CONFIG_DWORD_INFO(INTERNAL_FastGCCheckStack, W("FastGCCheckStack"), 0, "") CONFIG_DWORD_INFO(INTERNAL_FastGCStress, W("FastGCStress"), 0, "Reduce the number of GCs done by enabling GCStress") RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_GCBreakOnOOM, W("GCBreakOnOOM"), 0, "Does a DebugBreak at the soonest time we detect an OOM") RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConcurrent, W("gcConcurrent"), (DWORD)-1, "Enables/Disables concurrent GC") +RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_ForceDisableGCWriteBarrierCopy, W("ForceDisableGCWriteBarrierCopy"), 0, "Forces the GC write barrier copy to be disabled") #ifdef FEATURE_CONSERVATIVE_GC RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConservative, W("gcConservative"), 0, "Enables/Disables conservative GC") diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index 8543f647c93b52..bbfc98dfa62db2 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -183,11 +183,11 @@ JITHELPER(CORINFO_HELP_CHECK_OBJ, JIT_CheckObj, METHOD__NIL) // GC Write barrier support - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF, JIT_WriteBarrier, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF, JIT_CheckedWriteBarrier,METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF, RhpAssignRef, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF, RhpCheckedAssignRef,METHOD__NIL) JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, RhpByRefAssignRef,METHOD__NIL) DYNAMICJITHELPER(CORINFO_HELP_BULK_WRITEBARRIER, NULL, METHOD__BUFFER__MEMCOPYGC) // Accessing fields diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index a6695fb0895b1b..24e9cfa75038a2 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -845,15 +845,17 @@ extern "C" void STDCALL JIT_PatchedCodeLast(); static void UpdateWriteBarrierState(bool skipEphemeralCheck) { - BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); - BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; - ExecutableWriterHolderNoLog writeBarrierWriterHolder; if (IsWriteBarrierCopyEnabled()) { - writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); + BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; + ExecutableWriterHolderNoLog writeBarrierWriterHolder; + { + writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); + writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + } + JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } - JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } void InitJITHelpers1() diff --git a/src/coreclr/vm/eeconfig.cpp b/src/coreclr/vm/eeconfig.cpp index fbca43f89a371f..2e197d52788221 100644 --- a/src/coreclr/vm/eeconfig.cpp +++ b/src/coreclr/vm/eeconfig.cpp @@ -112,6 +112,7 @@ HRESULT EEConfig::Init() fJitMinOpts = false; fJitEnableOptionalRelocs = false; fDisableOptimizedThreadStaticAccess = false; + fIsWriteBarrierCopyEnabled = false; fPInvokeRestoreEsp = (DWORD)-1; fStressLog = false; @@ -502,6 +503,16 @@ HRESULT EEConfig::sync() fDisableOptimizedThreadStaticAccess = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_DisableOptimizedThreadStaticAccess) != 0; +#ifdef HOST_APPLE + fIsWriteBarrierCopyEnabled = true; +#else + fIsWriteBarrierCopyEnabled = ExecutableAllocator::IsWXORXEnabled(); +#endif + if (CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_ForceDisableGCWriteBarrierCopy) != 0) + { + fIsWriteBarrierCopyEnabled = false; + } + #ifdef TARGET_X86 fPInvokeRestoreEsp = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_Jit_NetFx40PInvokeStackResilience); #endif @@ -998,7 +1009,6 @@ HRESULT TypeNamesList::Init(_In_z_ LPCWSTR str) CONTRACTL { NOTHROW; GC_NOTRIGGER; - MODE_ANY; PRECONDITION(CheckPointer(str)); INJECT_FAULT(return E_OUTOFMEMORY); } CONTRACTL_END; diff --git a/src/coreclr/vm/eeconfig.h b/src/coreclr/vm/eeconfig.h index 499a40682637bd..f50a06455c5732 100644 --- a/src/coreclr/vm/eeconfig.h +++ b/src/coreclr/vm/eeconfig.h @@ -389,6 +389,12 @@ class EEConfig && pSkipGCCoverageList->IsInList(assemblyName));} #endif + bool IsWriteBarrierCopyEnabled() const + { + LIMITED_METHOD_CONTRACT; + return fIsWriteBarrierCopyEnabled; + } + #ifdef _DEBUG inline DWORD FastGCStressLevel() const {LIMITED_METHOD_CONTRACT; return iFastGCStress;} @@ -453,6 +459,7 @@ class EEConfig bool fJitMinOpts; // Enable MinOpts for all jitted methods bool fJitEnableOptionalRelocs; // Allow optional relocs bool fDisableOptimizedThreadStaticAccess; // Disable OptimizedThreadStatic access + bool fIsWriteBarrierCopyEnabled; // Is the GC write barrier copy enabled? unsigned iJitOptimizeType; // 0=Blended,1=SmallCode,2=FastCode, default is 0=Blended diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 0974e0cd0cdeeb..2ad312d43eb21e 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -171,7 +171,22 @@ extern "C" FCDECL3(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *ref, Che #else // Regular checked write barrier. extern "C" FCDECL2(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *ref); -#endif + +#ifdef TARGET_ARM64 +#define RhpCheckedAssignRef RhpCheckedAssignRefArm64 +#define RhpByRefAssignRef RhpByRefAssignRefArm64 +#define RhpAssignRef RhpAssignRefArm64 +#elif defined (TARGET_LOONGARCH64) +#define RhpAssignRef RhpAssignRefLoongArch64 +#elif defined (TARGET_RISCV64) +#define RhpAssignRef RhpAssignRefRiscV64 +#endif // TARGET_* + +#endif // FEATURE_USE_ASM_GC_WRITE_BARRIERS && defined(FEATURE_COUNT_GC_WRITE_BARRIERS) + +extern "C" FCDECL2(VOID, RhpCheckedAssignRef, Object **dst, Object *ref); +extern "C" FCDECL2(VOID, RhpByRefAssignRef, Object **dst, Object *ref); +extern "C" FCDECL2(VOID, RhpAssignRef, Object **dst, Object *ref); extern "C" FCDECL2(VOID, JIT_WriteBarrier, Object **dst, Object *ref); extern "C" FCDECL2(VOID, JIT_WriteBarrierEnsureNonHeapTarget, Object **dst, Object *ref); diff --git a/src/coreclr/vm/loongarch64/stubs.cpp b/src/coreclr/vm/loongarch64/stubs.cpp index 920a2d892aa0b2..1ad4a670a1f42e 100644 --- a/src/coreclr/vm/loongarch64/stubs.cpp +++ b/src/coreclr/vm/loongarch64/stubs.cpp @@ -860,15 +860,18 @@ extern "C" void STDCALL JIT_PatchedCodeLast(); static void UpdateWriteBarrierState(bool skipEphemeralCheck) { - BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); - BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; - ExecutableWriterHolderNoLog writeBarrierWriterHolder; if (IsWriteBarrierCopyEnabled()) { - writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); + BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; + ExecutableWriterHolderNoLog writeBarrierWriterHolder; + if (IsWriteBarrierCopyEnabled()) + { + writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); + writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + } + JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } - JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } void InitJITHelpers1() diff --git a/src/coreclr/vm/riscv64/stubs.cpp b/src/coreclr/vm/riscv64/stubs.cpp index 5a2928efac970b..8bf8881a4257a9 100644 --- a/src/coreclr/vm/riscv64/stubs.cpp +++ b/src/coreclr/vm/riscv64/stubs.cpp @@ -777,15 +777,18 @@ extern "C" void STDCALL JIT_PatchedCodeLast(); static void UpdateWriteBarrierState(bool skipEphemeralCheck) { - BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); - BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; - ExecutableWriterHolderNoLog writeBarrierWriterHolder; if (IsWriteBarrierCopyEnabled()) { - writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); + BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; + ExecutableWriterHolderNoLog writeBarrierWriterHolder; + if (IsWriteBarrierCopyEnabled()) + { + writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); + writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); + } + JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } - JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); } void InitJITHelpers1() diff --git a/src/coreclr/vm/threads.cpp b/src/coreclr/vm/threads.cpp index 8706d4ccf76dd2..b21180f620ebb4 100644 --- a/src/coreclr/vm/threads.cpp +++ b/src/coreclr/vm/threads.cpp @@ -1131,28 +1131,14 @@ void InitThreadManager() } else { - // I am using virtual protect to cover the entire range that this code falls in. - // - - // We could reset it to non-writeable inbetween GCs and such, but then we'd have to keep on re-writing back and forth, - // so instead we'll leave it writable from here forward. - - DWORD oldProt; - if (!ClrVirtualProtect((void *)JIT_PatchedCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart, - PAGE_EXECUTE_READWRITE, &oldProt)) - { - _ASSERTE(!"ClrVirtualProtect of code page failed"); - COMPlusThrowWin32(); - } - #ifdef TARGET_X86 JIT_WriteBarrierEAX_Loc = (void*)JIT_WriteBarrierEAX; #else - JIT_WriteBarrier_Loc = (void*)JIT_WriteBarrier; + JIT_WriteBarrier_Loc = (void*)RhpByRefAssignRef; #endif #if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // Store the JIT_WriteBarrier_Table copy location to a global variable so that it can be updated. - JIT_WriteBarrier_Table_Loc = (void*)&JIT_WriteBarrier_Table; + JIT_WriteBarrier_Table_Loc = NULL; #endif // TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64 } diff --git a/src/coreclr/vm/threads.h b/src/coreclr/vm/threads.h index a1ca4943a6be8c..a2c99845851523 100644 --- a/src/coreclr/vm/threads.h +++ b/src/coreclr/vm/threads.h @@ -5662,11 +5662,7 @@ inline BOOL IsWriteBarrierCopyEnabled() #ifdef DACCESS_COMPILE return FALSE; #else // DACCESS_COMPILE -#ifdef HOST_APPLE - return TRUE; -#else - return ExecutableAllocator::IsWXORXEnabled(); -#endif + return g_pConfig->IsWriteBarrierCopyEnabled(); #endif // DACCESS_COMPILE } From 46692653ad3a121d504eb1602caa0eaf0e049911 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 23 Apr 2025 13:24:09 -0700 Subject: [PATCH 04/34] Fix config settings for write barrier --- src/coreclr/inc/clrconfigvalues.h | 2 +- src/coreclr/vm/amd64/jitinterfaceamd64.cpp | 24 +++++++++++++++++----- src/coreclr/vm/ceemain.cpp | 10 ++++----- src/coreclr/vm/eeconfig.cpp | 10 +-------- src/coreclr/vm/threads.cpp | 2 +- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 7e28921ab80319..0ca1675efaf771 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -254,7 +254,7 @@ CONFIG_DWORD_INFO(INTERNAL_FastGCCheckStack, W("FastGCCheckStack"), 0, "") CONFIG_DWORD_INFO(INTERNAL_FastGCStress, W("FastGCStress"), 0, "Reduce the number of GCs done by enabling GCStress") RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_GCBreakOnOOM, W("GCBreakOnOOM"), 0, "Does a DebugBreak at the soonest time we detect an OOM") RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConcurrent, W("gcConcurrent"), (DWORD)-1, "Enables/Disables concurrent GC") -RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_ForceDisableGCWriteBarrierCopy, W("ForceDisableGCWriteBarrierCopy"), 0, "Forces the GC write barrier copy to be disabled") +RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_UseGCWriteBarrierCopy, W("UseGCWriteBarrierCopy"), 0, "Use a copy of the write barrier for the GC. This is somewhat faster and for optimizations where the barrier is mutated as the program runs.") #ifdef FEATURE_CONSERVATIVE_GC RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConservative, W("gcConservative"), 0, "Enables/Disables conservative GC") diff --git a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp index eff53294b8a3bb..3df45e888b5dec 100644 --- a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp +++ b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp @@ -1010,7 +1010,10 @@ int StompWriteBarrierEphemeral(bool isRuntimeSuspended) { WRAPPER_NO_CONTRACT; - return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); + if (IsWriteBarrierCopyEnabled()) + return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); + else + return SWB_PASS; } // This function bashes the super fast amd64 versions of the JIT_WriteBarrier @@ -1020,26 +1023,37 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) { WRAPPER_NO_CONTRACT; - return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); + if (IsWriteBarrierCopyEnabled()) + return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); + else + return SWB_PASS; } void FlushWriteBarrierInstructionCache() { - FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); + if (IsWriteBarrierCopyEnabled()) + FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); } #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) { WRAPPER_NO_CONTRACT; - return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); + if (IsWriteBarrierCopyEnabled()) + return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); + else + return SWB_PASS; } int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) { WRAPPER_NO_CONTRACT; - return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); + if (IsWriteBarrierCopyEnabled()) + return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); + else + return SWB_PASS; } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/ceemain.cpp b/src/coreclr/vm/ceemain.cpp index db012c7c106e0e..af9d372efc87c4 100644 --- a/src/coreclr/vm/ceemain.cpp +++ b/src/coreclr/vm/ceemain.cpp @@ -659,6 +659,11 @@ void EEStartupHelper() IfFailGo(ExecutableAllocator::StaticInitialize(FatalErrorHandler)); + if (g_pConfig != NULL) + { + IfFailGoLog(g_pConfig->sync()); + } + Thread::StaticInitialize(); JITInlineTrackingMap::StaticInitialize(); @@ -749,11 +754,6 @@ void EEStartupHelper() #endif // !TARGET_UNIX InitEventStore(); - if (g_pConfig != NULL) - { - IfFailGoLog(g_pConfig->sync()); - } - // Fire the runtime information ETW event ETW::InfoLog::RuntimeInformation(ETW::InfoLog::InfoStructs::Normal); diff --git a/src/coreclr/vm/eeconfig.cpp b/src/coreclr/vm/eeconfig.cpp index 2e197d52788221..54b62f13e4ad5c 100644 --- a/src/coreclr/vm/eeconfig.cpp +++ b/src/coreclr/vm/eeconfig.cpp @@ -503,15 +503,7 @@ HRESULT EEConfig::sync() fDisableOptimizedThreadStaticAccess = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_DisableOptimizedThreadStaticAccess) != 0; -#ifdef HOST_APPLE - fIsWriteBarrierCopyEnabled = true; -#else - fIsWriteBarrierCopyEnabled = ExecutableAllocator::IsWXORXEnabled(); -#endif - if (CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_ForceDisableGCWriteBarrierCopy) != 0) - { - fIsWriteBarrierCopyEnabled = false; - } + fIsWriteBarrierCopyEnabled = CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_UseGCWriteBarrierCopy) != 0; #ifdef TARGET_X86 fPInvokeRestoreEsp = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_Jit_NetFx40PInvokeStackResilience); diff --git a/src/coreclr/vm/threads.cpp b/src/coreclr/vm/threads.cpp index b21180f620ebb4..ab2e86215cc369 100644 --- a/src/coreclr/vm/threads.cpp +++ b/src/coreclr/vm/threads.cpp @@ -1134,7 +1134,7 @@ void InitThreadManager() #ifdef TARGET_X86 JIT_WriteBarrierEAX_Loc = (void*)JIT_WriteBarrierEAX; #else - JIT_WriteBarrier_Loc = (void*)RhpByRefAssignRef; + JIT_WriteBarrier_Loc = (void*)RhpAssignRef; #endif #if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // Store the JIT_WriteBarrier_Table copy location to a global variable so that it can be updated. From a24a67e2a213fa2b4b971b8f88ef81cc2c64071c Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 23 Apr 2025 16:43:58 -0700 Subject: [PATCH 05/34] Get the Arm64 build to work on Windows --- src/coreclr/runtime/arm64/WriteBarriers.asm | 4 +- src/coreclr/vm/CMakeLists.txt | 10 +- src/coreclr/vm/arm64/AsmMacros_Shared.h | 102 +++++++++++++++++--- 3 files changed, 103 insertions(+), 13 deletions(-) diff --git a/src/coreclr/runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm index 65d84f77440219..7feb05c879c2bd 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -57,7 +57,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD PREPARE_EXTERNAL_VAR_INDIRECT $g_GCShadow, x12 add $destReg, $destReg, x12 - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, x12 + PREPARE_EXTERNAL_VAR_INDIRECT $g_GCShadowEnd, x12 cmp $destReg, x12 bhs %ft0 @@ -275,6 +275,7 @@ NotInHeap b RhpAssignRefArm64 LEAF_END RhpAssignRef +#ifdef FEATURE_NATIVEAOT ;; Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon ;; successful updates. @@ -388,5 +389,6 @@ NoBarrierXchg ret LEAF_END RhpCheckedXchg +#endif // FEATURE_NATIVEAOT end diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index 28dfb54f696402..c2f5d2b599d746 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -659,7 +659,8 @@ elseif(CLR_CMAKE_TARGET_ARCH_I386) ${ARCH_SOURCES_DIR}/jithelp.asm ${ARCH_SOURCES_DIR}/PInvokeStubs.asm ${ARCH_SOURCES_DIR}/thunktemplates.asm - ) + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.asm + ) set(VM_HEADERS_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmconstants.h @@ -674,6 +675,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64) ${ARCH_SOURCES_DIR}/thunktemplates.asm ${ARCH_SOURCES_DIR}/CachedInterfaceDispatchCoreCLR.asm ${ARCH_SOURCES_DIR}/StubPrecodeDynamicHelpers.asm + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.asm ) set(VM_HEADERS_WKS_ARCH_ASM @@ -705,6 +707,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/unixasmhelpers.S ${ARCH_SOURCES_DIR}/umthunkstub.S ${ARCH_SOURCES_DIR}/virtualcallstubamd64.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ${ARCH_SOURCES_DIR}/StubPrecodeDynamicHelpers.S ) elseif(CLR_CMAKE_TARGET_ARCH_I386) @@ -716,6 +719,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/umthunkstub.S ${ARCH_SOURCES_DIR}/thunktemplates.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) elseif(CLR_CMAKE_TARGET_ARCH_ARM) set(VM_SOURCES_WKS_ARCH_ASM @@ -724,6 +728,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/patchedcode.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) elseif(CLR_CMAKE_TARGET_ARCH_ARM64) set(VM_SOURCES_WKS_ARCH_ASM @@ -735,6 +740,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/StubDispatch.S ${ARCH_SOURCES_DIR}/thunktemplates.S ${ARCH_SOURCES_DIR}/StubPrecodeDynamicHelpers.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) elseif(CLR_CMAKE_TARGET_ARCH_LOONGARCH64) set(VM_SOURCES_WKS_ARCH_ASM @@ -742,6 +748,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/calldescrworkerloongarch64.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) elseif(CLR_CMAKE_TARGET_ARCH_RISCV64) set(VM_SOURCES_WKS_ARCH_ASM @@ -749,6 +756,7 @@ else(CLR_CMAKE_TARGET_WIN32) ${ARCH_SOURCES_DIR}/calldescrworkerriscv64.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S + ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.S ) endif() diff --git a/src/coreclr/vm/arm64/AsmMacros_Shared.h b/src/coreclr/vm/arm64/AsmMacros_Shared.h index 6bd6ac764d2ec5..053145540810e4 100644 --- a/src/coreclr/vm/arm64/AsmMacros_Shared.h +++ b/src/coreclr/vm/arm64/AsmMacros_Shared.h @@ -8,22 +8,102 @@ #include "asmconstants.h" #include "asmmacros.h" -EXTERN g_lowest_address : QWORD -EXTERN g_highest_address : QWORD -EXTERN g_ephemeral_low : QWORD -EXTERN g_ephemeral_high : QWORD -EXTERN g_card_table : QWORD + IMPORT g_lowest_address + IMPORT g_highest_address + IMPORT g_ephemeral_low + IMPORT g_ephemeral_high + IMPORT g_card_table -ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN g_card_bundle_table : QWORD -endif +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + IMPORT g_card_bundle_table +#endif + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + IMPORT g_write_watch_table +#endif + +;;----------------------------------------------------------------------------- +;; Macro for loading a 64-bit constant by a minimal number of instructions +;; Since the asssembles doesn't support 64 bit arithmetics in expressions, +;; the value is passed in as lo, hi pair. + MACRO + MOVL64 $Reg, $ConstantLo, $ConstantHi + + LCLS MovInstr +MovInstr SETS "movz" + + IF ((($ConstantHi):SHR:16):AND:0xffff) != 0 + $MovInstr $Reg, #((($Constant):SHR:16):AND:0xffff), lsl #48 +MovInstr SETS "movk" + ENDIF + + IF (($ConstantHi):AND:0xffff) != 0 + $MovInstr $Reg, #(($ConstantHi):AND:0xffff), lsl #32 +MovInstr SETS "movk" + ENDIF + + IF ((($ConstantLo):SHR:16):AND:0xffff) != 0 + $MovInstr $Reg, #((($ConstantLo):SHR:16):AND:0xffff), lsl #16 +MovInstr SETS "movk" + ENDIF + + $MovInstr $Reg, #(($ConstantLo):AND:0xffff) + MEND + +;;----------------------------------------------------------------------------- +;; Macro for loading a 64bit value of a global variable into a register + MACRO + PREPARE_EXTERNAL_VAR_INDIRECT $Name, $Reg + + adrp $Reg, $Name + ldr $Reg, [$Reg, $Name] + MEND -ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -EXTERN g_write_watch_table : QWORD -endif +;; ---------------------------------------------------------------------------- - +;; Macro for loading a 32bit value of a global variable into a register + MACRO + PREPARE_EXTERNAL_VAR_INDIRECT_W $Name, $RegNum + + adrp x$RegNum, $Name + ldr w$RegNum, [x$RegNum, $Name] + MEND + +;; ---------------------------------------------------------------------------- - +;; +;; Macro to add a memory barrier. Equal to __sync_synchronize(). +;; + + MACRO + InterlockedOperationBarrier + + dmb ish + MEND #else #include "asmconstants.h" #include "unixasmmacros.inc" + +.macro PREPARE_EXTERNAL_VAR_INDIRECT Name, HelperReg +#if defined(__APPLE__) + adrp \HelperReg, C_FUNC(\Name)@GOTPAGE + ldr \HelperReg, [\HelperReg, C_FUNC(\Name)@GOTPAGEOFF] + ldr \HelperReg, [\HelperReg] +#else + adrp \HelperReg, C_FUNC(\Name) + ldr \HelperReg, [\HelperReg, :lo12:C_FUNC(\Name)] +#endif +.endm + +.macro PREPARE_EXTERNAL_VAR_INDIRECT_W Name, HelperReg +#if defined(__APPLE__) + adrp x\HelperReg, C_FUNC(\Name)@GOTPAGE + ldr x\HelperReg, [x\HelperReg, C_FUNC(\Name)@GOTPAGEOFF] + ldr w\HelperReg, [x\HelperReg] +#else + adrp x\HelperReg, C_FUNC(\Name) + ldr w\HelperReg, [x\HelperReg, :lo12:C_FUNC(\Name)] +#endif +.endm + #endif From a858993b89b8fb65f6cf734015ebfb07c6b1ff74 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 25 Apr 2025 21:21:02 +0000 Subject: [PATCH 06/34] Fix arm64 build --- src/coreclr/runtime/arm64/WriteBarriers.S | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index b396ee18160367..2970f453dbc3f5 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -266,6 +266,7 @@ LEAF_ENTRY RhpAssignRef, _TEXT b C_FUNC(RhpAssignRefArm64) LEAF_END RhpAssignRef, _TEXT +#ifdef FEATURE_NATIVEAOT // Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon // successful updates. @@ -395,3 +396,4 @@ LOCAL_LABEL(NoBarrierXchg): #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT .arch_extension nolse #endif +#endif // FEATURE_NATIVEAOT From 1c4c79a4470c7107102986dbd668da7deb299743 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 25 Apr 2025 15:09:21 -0700 Subject: [PATCH 07/34] Fix arm build --- .../nativeaot/Runtime/arm/AsmMacros_Shared.h | 7 ++ src/coreclr/pal/inc/unixasmmacrosarm.inc | 18 +++++ src/coreclr/runtime/arm/StubDispatch.S | 3 +- src/coreclr/runtime/arm/WriteBarriers.S | 78 ++----------------- src/coreclr/vm/arm/AsmMacros_Shared.h | 8 ++ src/coreclr/vm/arm/asmhelpers.S | 2 +- src/coreclr/vm/arm/stubs.cpp | 2 +- 7 files changed, 43 insertions(+), 75 deletions(-) create mode 100644 src/coreclr/nativeaot/Runtime/arm/AsmMacros_Shared.h create mode 100644 src/coreclr/vm/arm/AsmMacros_Shared.h diff --git a/src/coreclr/nativeaot/Runtime/arm/AsmMacros_Shared.h b/src/coreclr/nativeaot/Runtime/arm/AsmMacros_Shared.h new file mode 100644 index 00000000000000..1a10c1d2c72f44 --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/arm/AsmMacros_Shared.h @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +#include "AsmOffsets.inc" +#include diff --git a/src/coreclr/pal/inc/unixasmmacrosarm.inc b/src/coreclr/pal/inc/unixasmmacrosarm.inc index f5eb32656cd3ae..226d23aebd2f75 100644 --- a/src/coreclr/pal/inc/unixasmmacrosarm.inc +++ b/src/coreclr/pal/inc/unixasmmacrosarm.inc @@ -44,10 +44,28 @@ C_FUNC(\Name\()_End): nop .endm +.macro GLOBAL_LABEL Name + .global C_FUNC(\Name) +C_FUNC(\Name): +.endm + +.macro ALTERNATE_ENTRY Name + .global C_FUNC(\Name) + .type \Name, %function +C_FUNC(\Name): +.endm + .macro PREPARE_EXTERNAL_VAR Name, HelperReg ldr \HelperReg, [pc, #C_FUNC(\Name)@GOTPCREL] .endm +.macro PREPARE_EXTERNAL_VAR_INDIRECT Name, HelperReg + movw \HelperReg, #:lower16:C_FUNC(\Name) - (. + 12) + movt \HelperReg, #:upper16:C_FUNC(\Name) - (. + 8) + add \HelperReg, pc + ldr \HelperReg, [\HelperReg] +.endm + .macro push_nonvol_reg Register push \Register .save \Register diff --git a/src/coreclr/runtime/arm/StubDispatch.S b/src/coreclr/runtime/arm/StubDispatch.S index c761df7b3f123e..3001b67453dde9 100644 --- a/src/coreclr/runtime/arm/StubDispatch.S +++ b/src/coreclr/runtime/arm/StubDispatch.S @@ -4,8 +4,7 @@ .syntax unified .thumb -#include // generated by the build from AsmOffsets.cpp -#include +#include "AsmMacros_Shared.h" #ifdef FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index 3bb862231a347e..99cef3ad302794 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -4,79 +4,12 @@ .syntax unified .thumb -#include // generated by the build from AsmOffsets.cpp -#include +#include "AsmMacros_Shared.h" #ifdef WRITE_BARRIER_CHECK .macro UPDATE_GC_SHADOW BASENAME, REFREG, DESTREG - - // If g_GCShadow is 0, don't perform the check. - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, r12 - cbz r12, LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG) - - // Save DESTREG since we're about to modify it (and we need the original value both within the macro and - // once we exit the macro). Note that this is naughty since we're altering the stack pointer outside of - // the prolog inside a method without a frame. But given that this is only debug code and generally we - // shouldn't be walking the stack at this point it seems preferable to recoding the all the barrier - // variants to set up frames. The compiler knows exactly which registers are trashed in the simple write - // barrier case, so we don't have any more scratch registers to play with (and doing so would only make - // things harder if at a later stage we want to allow multiple barrier versions based on the input - // registers). - push \DESTREG - - // Transform DESTREG into the equivalent address in the shadow heap. - PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, r12 - sub \DESTREG, r12 - cmp \DESTREG, #0 - blo LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG) - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, r12 - add \DESTREG, r12 - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, r12 - cmp \DESTREG, r12 - bhs LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG) - - // Update the shadow heap. - str \REFREG, [\DESTREG] - - // The following read must be strongly ordered wrt to the write we've just performed in order to - // prevent race conditions. - dmb - - // Now check that the real heap location still contains the value we just wrote into the shadow heap. - mov r12, \DESTREG - ldr \DESTREG, [sp] - str r12, [sp] - ldr r12, [\DESTREG] - cmp r12, \REFREG - bne LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Invalidate_\REFREG) - - // The original DESTREG value is now restored but the stack has a value (the shadow version of the - // location) pushed. Need to discard this push before we are done. - add sp, #4 - b LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG) - -LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Invalidate_\REFREG): - // Someone went and updated the real heap. We need to invalidate the shadow location since we can't - // guarantee whose shadow update won. - - // Retrieve shadow location from the stack and restore original DESTREG to the stack. This is an - // additional memory barrier we don't require but it's on the rare path and x86 doesn't have an xchg - // variant that doesn't implicitly specify the lock prefix. Note that INVALIDGCVALUE is a 32-bit - // immediate and therefore must be moved into a register before it can be written to the shadow - // location. - mov r12, \DESTREG - ldr \DESTREG, [sp] - str r12, [sp] - push \REFREG - movw \REFREG, #(INVALIDGCVALUE & 0xFFFF) - movt \REFREG, #(INVALIDGCVALUE >> 16) - str \REFREG, [\DESTREG] - pop \REFREG - -LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG): - // Restore original DESTREG value from the stack. - pop \DESTREG + // Todo: implement, debugging helper LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): @@ -114,7 +47,7 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write // the byte if it hasn't already been done since writes are expensive and impact scaling. PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, r12 - add r0, r12, r0, lsr #LOG2_CLUMP_SIZE + add r0, r12, r0, lsr #10 ldrb r12, [r0] cmp r12, #0x0FF bne LOCAL_LABEL(\BASENAME\()_UpdateCardTable_\REFREG) @@ -250,6 +183,8 @@ LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT // just one write barrier that assumes the input register is RSI. DEFINE_CHECKED_WRITE_BARRIER r1, r1 +#ifdef FEATURE_NATIVEAOT + // r0 = destination address // r1 = value // r2 = comparand @@ -292,6 +227,7 @@ LOCAL_LABEL(RhpCheckedXchgRetry): bx lr LEAF_END RhpCheckedXchg, _TEXT +#endif // FEATURE_NATIVEAOT // // RhpByRefAssignRef simulates movs instruction for object references. @@ -348,7 +284,7 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2 // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write // the byte if it hasn't already been done since writes are expensive and impact scaling. PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, r3 - add r2, r3, r2, lsr #LOG2_CLUMP_SIZE + add r2, r3, r2, lsr #10 ldrb r3, [r2] cmp r3, #0x0FF bne LOCAL_LABEL(RhpByRefAssignRef_UpdateCardTable) diff --git a/src/coreclr/vm/arm/AsmMacros_Shared.h b/src/coreclr/vm/arm/AsmMacros_Shared.h new file mode 100644 index 00000000000000..afe0afb7b8ed9e --- /dev/null +++ b/src/coreclr/vm/arm/AsmMacros_Shared.h @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + + +#include "asmconstants.h" +#include "unixasmmacros.inc" diff --git a/src/coreclr/vm/arm/asmhelpers.S b/src/coreclr/vm/arm/asmhelpers.S index 3581ef6675fb0b..b15a2ba3e248ef 100644 --- a/src/coreclr/vm/arm/asmhelpers.S +++ b/src/coreclr/vm/arm/asmhelpers.S @@ -719,7 +719,7 @@ LOCAL_LABEL(stackProbe_loop): .word __\name\()__g_ephemeral_high_offset .word __\name\()__g_card_table_offset #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - .word __\name\()__g_sw_ww_table_offset + .word __\name\()__g_write_watch_table_offset #endif .endm diff --git a/src/coreclr/vm/arm/stubs.cpp b/src/coreclr/vm/arm/stubs.cpp index 126055f66c9d10..1c47a46a5ebf32 100644 --- a/src/coreclr/vm/arm/stubs.cpp +++ b/src/coreclr/vm/arm/stubs.cpp @@ -292,7 +292,7 @@ struct WriteBarrierDescriptor DWORD m_dw_g_ephemeral_high_offset; // Offset of the instruction reading g_ephemeral_high DWORD m_dw_g_card_table_offset; // Offset of the instruction reading g_card_table #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - DWORD m_dw_g_sw_ww_table_offset; // Offset of the instruction reading g_write_watch_table + DWORD m_dw_g_write_watch_table_offset;// Offset of the instruction reading g_write_watch_table #endif }; From 999b4664688a3bea4fa95cb332c206b130b4136a Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 25 Apr 2025 15:20:20 -0700 Subject: [PATCH 08/34] Fix RiscV build --- src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h | 7 +++++++ src/coreclr/pal/inc/unixasmmacrosriscv64.inc | 6 ++++++ src/coreclr/runtime/riscv64/StubDispatch.S | 3 +-- src/coreclr/runtime/riscv64/WriteBarriers.S | 2 +- src/coreclr/vm/riscv64/AsmMacros_Shared.h | 8 ++++++++ 5 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h create mode 100644 src/coreclr/vm/riscv64/AsmMacros_Shared.h diff --git a/src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h b/src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h new file mode 100644 index 00000000000000..f7df014642731d --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +#include +#include "AsmOffsets.inc" diff --git a/src/coreclr/pal/inc/unixasmmacrosriscv64.inc b/src/coreclr/pal/inc/unixasmmacrosriscv64.inc index a86acf5b897d2d..1f7344c4d15741 100644 --- a/src/coreclr/pal/inc/unixasmmacrosriscv64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosriscv64.inc @@ -37,6 +37,12 @@ C_FUNC(\Name\()_End): nop .endm +.macro ALTERNATE_ENTRY Name + .global C_FUNC(\Name) + .hidden C_FUNC(\Name) +C_FUNC(\Name): +.endm + .macro PREPARE_EXTERNAL_VAR Name, HelperReg lla \HelperReg, \Name .endm diff --git a/src/coreclr/runtime/riscv64/StubDispatch.S b/src/coreclr/runtime/riscv64/StubDispatch.S index 9e49c53fbe0f54..cc4101a4a17470 100644 --- a/src/coreclr/runtime/riscv64/StubDispatch.S +++ b/src/coreclr/runtime/riscv64/StubDispatch.S @@ -1,8 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include -#include "AsmOffsets.inc" +#include "AsmMacros_Shared.h" #ifdef FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/runtime/riscv64/WriteBarriers.S b/src/coreclr/runtime/riscv64/WriteBarriers.S index e063f549a398a1..469908665b3533 100644 --- a/src/coreclr/runtime/riscv64/WriteBarriers.S +++ b/src/coreclr/runtime/riscv64/WriteBarriers.S @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include +#include "AsmMacros_Shared.h" // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used // during garbage collections to verify that object references were never written to the heap without using a diff --git a/src/coreclr/vm/riscv64/AsmMacros_Shared.h b/src/coreclr/vm/riscv64/AsmMacros_Shared.h new file mode 100644 index 00000000000000..afe0afb7b8ed9e --- /dev/null +++ b/src/coreclr/vm/riscv64/AsmMacros_Shared.h @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + + +#include "asmconstants.h" +#include "unixasmmacros.inc" From 547fb962d7ed4594a92e489ac43f937fb6ba3594 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 25 Apr 2025 15:31:06 -0700 Subject: [PATCH 09/34] Fix Loongarch64 build --- .../Runtime/loongarch64/AsmMacros_Shared.h | 7 +++++++ src/coreclr/pal/inc/unixasmmacrosloongarch64.inc | 16 ++++++++++++++++ src/coreclr/runtime/loongarch64/StubDispatch.S | 3 +-- src/coreclr/runtime/loongarch64/WriteBarriers.S | 2 +- src/coreclr/vm/loongarch64/AsmMacros_Shared.h | 8 ++++++++ 5 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 src/coreclr/nativeaot/Runtime/loongarch64/AsmMacros_Shared.h create mode 100644 src/coreclr/vm/loongarch64/AsmMacros_Shared.h diff --git a/src/coreclr/nativeaot/Runtime/loongarch64/AsmMacros_Shared.h b/src/coreclr/nativeaot/Runtime/loongarch64/AsmMacros_Shared.h new file mode 100644 index 00000000000000..f7df014642731d --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/loongarch64/AsmMacros_Shared.h @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +#include +#include "AsmOffsets.inc" diff --git a/src/coreclr/pal/inc/unixasmmacrosloongarch64.inc b/src/coreclr/pal/inc/unixasmmacrosloongarch64.inc index 85cf42121326e2..a949491f5fabdb 100644 --- a/src/coreclr/pal/inc/unixasmmacrosloongarch64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosloongarch64.inc @@ -17,6 +17,12 @@ C_FUNC(\Name): .endm +.macro ALTERNATE_ENTRY Name + .global C_FUNC(\Name) + .hidden C_FUNC(\Name) +C_FUNC(\Name): +.endm + .macro LEAF_ENTRY Name, Section .global C_FUNC(\Name) .type \Name, %function @@ -41,6 +47,16 @@ C_FUNC(\Name\()_End): la.local \HelperReg, \Name .endm +.macro PREPARE_EXTERNAL_VAR_INDIRECT Name, HelperReg + la.local \HelperReg, \Name + ld.d \HelperReg, \HelperReg, 0 +.endm + +.macro PREPARE_EXTERNAL_VAR_INDIRECT_W Name, HelperReg + la.local \HelperReg, \Name + ld.w \HelperReg, \HelperReg, 0 +.endm + .macro PROLOG_STACK_ALLOC Size addi.d $sp, $sp, -\Size //.cfi_adjust_cfa_offset \Size diff --git a/src/coreclr/runtime/loongarch64/StubDispatch.S b/src/coreclr/runtime/loongarch64/StubDispatch.S index 45c5a293fe18de..af0ef71273abdc 100644 --- a/src/coreclr/runtime/loongarch64/StubDispatch.S +++ b/src/coreclr/runtime/loongarch64/StubDispatch.S @@ -1,8 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include -#include "AsmOffsets.inc" +#include "AsmMacros_Shared.h" #ifdef FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/runtime/loongarch64/WriteBarriers.S b/src/coreclr/runtime/loongarch64/WriteBarriers.S index daa95ab5bf384a..20888056b04ea8 100644 --- a/src/coreclr/runtime/loongarch64/WriteBarriers.S +++ b/src/coreclr/runtime/loongarch64/WriteBarriers.S @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include +#include "AsmMacros_Shared.h" // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used // during garbage collections to verify that object references where never written to the heap without using a diff --git a/src/coreclr/vm/loongarch64/AsmMacros_Shared.h b/src/coreclr/vm/loongarch64/AsmMacros_Shared.h new file mode 100644 index 00000000000000..afe0afb7b8ed9e --- /dev/null +++ b/src/coreclr/vm/loongarch64/AsmMacros_Shared.h @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + + +#include "asmconstants.h" +#include "unixasmmacros.inc" From 9fd292185a8eb90cea8b31e3ff1cce081c65d191 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 25 Apr 2025 21:26:15 -0700 Subject: [PATCH 10/34] Fix Windows X86 build --- src/coreclr/inc/jithelpers.h | 26 ++--- .../Runtime/i386/AsmMacros_Shared.inc | 6 ++ src/coreclr/runtime/i386/StubDispatch.asm | 2 +- src/coreclr/runtime/i386/WriteBarriers.asm | 4 +- src/coreclr/vm/i386/AsmMacros.inc | 18 ++++ src/coreclr/vm/i386/AsmMacros_Shared.inc | 19 ++++ src/coreclr/vm/i386/jitinterfacex86.cpp | 100 ++++++++++-------- src/coreclr/vm/jitinterface.h | 5 +- src/coreclr/vm/threads.cpp | 9 +- 9 files changed, 124 insertions(+), 65 deletions(-) create mode 100644 src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.inc create mode 100644 src/coreclr/vm/i386/AsmMacros_Shared.inc diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index bbfc98dfa62db2..9729091e4ac613 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -285,19 +285,19 @@ #endif // !FEATURE_EH_FUNCLETS #ifdef TARGET_X86 - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, JIT_WriteBarrierEAX, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, JIT_WriteBarrierEBX, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ECX, JIT_WriteBarrierECX, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ESI, JIT_WriteBarrierESI, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EDI, JIT_WriteBarrierEDI, METHOD__NIL) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBP, JIT_WriteBarrierEBP, METHOD__NIL) - - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EAX, JIT_CheckedWriteBarrierEAX, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBX, JIT_CheckedWriteBarrierEBX, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ECX, JIT_CheckedWriteBarrierECX, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ESI, JIT_CheckedWriteBarrierESI, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EDI, JIT_CheckedWriteBarrierEDI, METHOD__NIL) - JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBP, JIT_CheckedWriteBarrierEBP, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, RhpAssignRefEAX, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, RhpAssignRefEBX, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ECX, RhpAssignRefECX, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ESI, RhpAssignRefESI, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EDI, RhpAssignRefEDI, METHOD__NIL) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBP, RhpAssignRefEBP, METHOD__NIL) + + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EAX, RhpCheckedAssignRefEAX, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBX, RhpCheckedAssignRefEBX, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ECX, RhpCheckedAssignRefECX, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ESI, RhpCheckedAssignRefESI, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EDI, RhpCheckedAssignRefEDI, METHOD__NIL) + JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBP, RhpCheckedAssignRefEBP, METHOD__NIL) #else JITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, NULL, METHOD__NIL) JITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, NULL, METHOD__NIL) diff --git a/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.inc b/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.inc new file mode 100644 index 00000000000000..956d4d22e38313 --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.inc @@ -0,0 +1,6 @@ +; Licensed to the .NET Foundation under one or more agreements. +; The .NET Foundation licenses this file to you under the MIT license. + +; This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +include AsmMacros.inc \ No newline at end of file diff --git a/src/coreclr/runtime/i386/StubDispatch.asm b/src/coreclr/runtime/i386/StubDispatch.asm index 9b70dd727f5874..b3672d4e1eb045 100644 --- a/src/coreclr/runtime/i386/StubDispatch.asm +++ b/src/coreclr/runtime/i386/StubDispatch.asm @@ -7,7 +7,7 @@ .code -include AsmMacros.inc +include AsmMacros_Shared.inc ifdef FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/runtime/i386/WriteBarriers.asm b/src/coreclr/runtime/i386/WriteBarriers.asm index 133081bee83198..a0ea69e425212b 100644 --- a/src/coreclr/runtime/i386/WriteBarriers.asm +++ b/src/coreclr/runtime/i386/WriteBarriers.asm @@ -13,7 +13,7 @@ option casemap:none .code -include AsmMacros.inc +include AsmMacros_Shared.inc ;; Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used ;; during garbage collections to verify that object references where never written to the heap without using a @@ -104,6 +104,7 @@ FASTCALL_FUNC RhpAssignRef&REFREG&, 8 ;; Export the canonical write barrier under unqualified name as well ifidni , ALTERNATE_ENTRY RhpAssignRef + ALTERNATE_ENTRY @RhpAssignRef@8 ALTERNATE_ENTRY _RhpAssignRefAVLocation endif @@ -202,6 +203,7 @@ FASTCALL_FUNC RhpCheckedAssignRef&REFREG&, 8 ;; Export the canonical write barrier under unqualified name as well ifidni , ALTERNATE_ENTRY RhpCheckedAssignRef + ALTERNATE_ENTRY @RhpCheckedAssignRef@8 ALTERNATE_ENTRY _RhpCheckedAssignRefAVLocation endif diff --git a/src/coreclr/vm/i386/AsmMacros.inc b/src/coreclr/vm/i386/AsmMacros.inc index 79345b6d18b56d..5acea99ddf6b7c 100644 --- a/src/coreclr/vm/i386/AsmMacros.inc +++ b/src/coreclr/vm/i386/AsmMacros.inc @@ -5,6 +5,24 @@ ; Define macros to build unwind data for prologues. ; +FASTCALL_FUNC macro FuncName,cbArgs + FuncNameReal EQU @&FuncName&@&cbArgs + FuncNameReal proc public + FuncName label proc + PUBLIC FuncName + +endm + +FASTCALL_ENDFUNC macro + FuncNameReal endp +endm + +ALTERNATE_ENTRY macro Name + +Name label proc +PUBLIC Name + endm + __tls_array equ 2Ch ;; offsetof(TEB, ThreadLocalStoragePointer) t_CurrentThreadInfo TEXTEQU diff --git a/src/coreclr/vm/i386/AsmMacros_Shared.inc b/src/coreclr/vm/i386/AsmMacros_Shared.inc new file mode 100644 index 00000000000000..968bcf2674ca5e --- /dev/null +++ b/src/coreclr/vm/i386/AsmMacros_Shared.inc @@ -0,0 +1,19 @@ +; Licensed to the .NET Foundation under one or more agreements. +; The .NET Foundation licenses this file to you under the MIT license. + +; This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +include asmconstants.inc +include asmmacros.inc + +G_LOWEST_ADDRESS equ _g_lowest_address +G_HIGHEST_ADDRESS equ _g_highest_address +G_EPHEMERAL_LOW equ _g_ephemeral_low +G_EPHEMERAL_HIGH equ _g_ephemeral_high +G_CARD_TABLE equ _g_card_table + +EXTERN G_LOWEST_ADDRESS : DWORD +EXTERN G_HIGHEST_ADDRESS : DWORD +EXTERN G_EPHEMERAL_LOW : DWORD +EXTERN G_EPHEMERAL_HIGH : DWORD +EXTERN G_CARD_TABLE : DWORD diff --git a/src/coreclr/vm/i386/jitinterfacex86.cpp b/src/coreclr/vm/i386/jitinterfacex86.cpp index 63d603e722513d..28b82e15db88bd 100644 --- a/src/coreclr/vm/i386/jitinterfacex86.cpp +++ b/src/coreclr/vm/i386/jitinterfacex86.cpp @@ -846,61 +846,61 @@ void InitJITHelpers1() _ASSERTE_ALL_BUILDS((BYTE*)JIT_PatchedWriteBarrierGroup_End - (BYTE*)JIT_PatchedWriteBarrierGroup < (ptrdiff_t)GetOsPageSize()); // Copy the write barriers to their final resting place. - for (int iBarrier = 0; iBarrier < NUM_WRITE_BARRIERS; iBarrier++) + if (IsWriteBarrierCopyEnabled()) { - BYTE * pfunc = (BYTE *) JIT_WriteBarrierReg_PreGrow; + for (int iBarrier = 0; iBarrier < NUM_WRITE_BARRIERS; iBarrier++) + { + BYTE * pfunc = (BYTE *) JIT_WriteBarrierReg_PreGrow; - BYTE * pBuf = GetWriteBarrierCodeLocation((BYTE *)c_rgWriteBarriers[iBarrier]); - int reg = c_rgWriteBarrierRegs[iBarrier]; + BYTE * pBuf = GetWriteBarrierCodeLocation((BYTE *)c_rgWriteBarriers[iBarrier]); + int reg = c_rgWriteBarrierRegs[iBarrier]; - BYTE * pBufRW = pBuf; - ExecutableWriterHolderNoLog barrierWriterHolder; - if (IsWriteBarrierCopyEnabled()) - { + BYTE * pBufRW = pBuf; + ExecutableWriterHolderNoLog barrierWriterHolder; barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 34); pBufRW = barrierWriterHolder.GetRW(); - } - memcpy(pBufRW, pfunc, 34); + memcpy(pBufRW, pfunc, 34); - // assert the copied code ends in a ret to make sure we got the right length - _ASSERTE(pBuf[33] == 0xC3); + // assert the copied code ends in a ret to make sure we got the right length + _ASSERTE(pBuf[33] == 0xC3); - // We need to adjust registers in a couple of instructions - // It would be nice to have the template contain all zeroes for - // the register fields (corresponding to EAX), but that doesn't - // work because then we get a smaller encoding for the compares - // that only works for EAX but not the other registers. - // So we always have to clear the register fields before updating them. + // We need to adjust registers in a couple of instructions + // It would be nice to have the template contain all zeroes for + // the register fields (corresponding to EAX), but that doesn't + // work because then we get a smaller encoding for the compares + // that only works for EAX but not the other registers. + // So we always have to clear the register fields before updating them. - // First instruction to patch is a mov [edx], reg + // First instruction to patch is a mov [edx], reg - _ASSERTE(pBuf[0] == 0x89); - // Update the reg field (bits 3..5) of the ModR/M byte of this instruction - pBufRW[1] &= 0xc7; - pBufRW[1] |= reg << 3; + _ASSERTE(pBuf[0] == 0x89); + // Update the reg field (bits 3..5) of the ModR/M byte of this instruction + pBufRW[1] &= 0xc7; + pBufRW[1] |= reg << 3; - // Second instruction to patch is cmp reg, imm32 (low bound) + // Second instruction to patch is cmp reg, imm32 (low bound) - _ASSERTE(pBuf[2] == 0x81); - // Here the lowest three bits in ModR/M field are the register - pBufRW[3] &= 0xf8; - pBufRW[3] |= reg; + _ASSERTE(pBuf[2] == 0x81); + // Here the lowest three bits in ModR/M field are the register + pBufRW[3] &= 0xf8; + pBufRW[3] |= reg; #ifdef WRITE_BARRIER_CHECK - // Don't do the fancy optimization just jump to the old one - // Use the slow one for write barrier checks build because it has some good asserts - if (g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK) { - pfunc = &pBufRW[0]; - *pfunc++ = 0xE9; // JMP c_rgDebugWriteBarriers[iBarrier] - *((DWORD*) pfunc) = (BYTE*) c_rgDebugWriteBarriers[iBarrier] - (&pBuf[1] + sizeof(DWORD)); - } + // Don't do the fancy optimization just jump to the old one + // Use the slow one for write barrier checks build because it has some good asserts + if (g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK) { + pfunc = &pBufRW[0]; + *pfunc++ = 0xE9; // JMP c_rgDebugWriteBarriers[iBarrier] + *((DWORD*) pfunc) = (BYTE*) c_rgDebugWriteBarriers[iBarrier] - (&pBuf[1] + sizeof(DWORD)); + } #endif // WRITE_BARRIER_CHECK - } + } #ifndef CODECOVERAGE - ValidateWriteBarrierHelpers(); + ValidateWriteBarrierHelpers(); #endif + } // Leave the patched region writable for StompWriteBarrierEphemeral(), StompWriteBarrierResize() } @@ -988,6 +988,12 @@ int StompWriteBarrierEphemeral(bool /* isRuntimeSuspended */) int stompWBCompleteActions = SWB_PASS; + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + #ifdef WRITE_BARRIER_CHECK // Don't do the fancy optimization if we are checking write barrier if ((GetWriteBarrierCodeLocation((BYTE *)JIT_WriteBarrierEAX))[0] == 0xE9) // we are using slow write barrier @@ -1001,11 +1007,8 @@ int StompWriteBarrierEphemeral(bool /* isRuntimeSuspended */) BYTE * pBufRW = pBuf; ExecutableWriterHolderNoLog barrierWriterHolder; - if (IsWriteBarrierCopyEnabled()) - { - barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 42); - pBufRW = barrierWriterHolder.GetRW(); - } + barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 42); + pBufRW = barrierWriterHolder.GetRW(); // assert there is in fact a cmp r/m32, imm32 there _ASSERTE(pBuf[2] == 0x81); @@ -1052,6 +1055,12 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) int stompWBCompleteActions = SWB_PASS; + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + #ifdef WRITE_BARRIER_CHECK // Don't do the fancy optimization if we are checking write barrier if ((GetWriteBarrierCodeLocation((BYTE *)JIT_WriteBarrierEAX))[0] == 0xE9) // we are using slow write barrier @@ -1070,11 +1079,8 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) BYTE * pBufRW = pBuf; ExecutableWriterHolderNoLog barrierWriterHolder; - if (IsWriteBarrierCopyEnabled()) - { - barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 42); - pBufRW = barrierWriterHolder.GetRW(); - } + barrierWriterHolder.AssignExecutableWriterHolder(pBuf, 42); + pBufRW = barrierWriterHolder.GetRW(); // Check if we are still using the pre-grow version of the write barrier. if (bWriteBarrierIsPreGrow) diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 2ad312d43eb21e..d893d96145ecb6 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -291,9 +291,10 @@ extern "C" // JIThelp.asm/JIThelp.s #define X86_WRITE_BARRIER_REGISTER(reg) \ - void STDCALL JIT_CheckedWriteBarrier##reg(); \ void STDCALL JIT_DebugWriteBarrier##reg(); \ - void STDCALL JIT_WriteBarrier##reg(); + void STDCALL JIT_WriteBarrier##reg(); \ + void FASTCALL RhpAssignRef##reg(Object**, Object*); \ + void FASTCALL RhpCheckedAssignRef##reg(Object**, Object*); ENUM_X86_WRITE_BARRIER_REGISTERS() #undef X86_WRITE_BARRIER_REGISTER diff --git a/src/coreclr/vm/threads.cpp b/src/coreclr/vm/threads.cpp index ab2e86215cc369..aaca7ca6193935 100644 --- a/src/coreclr/vm/threads.cpp +++ b/src/coreclr/vm/threads.cpp @@ -1128,11 +1128,18 @@ void InitThreadManager() ETW::MethodLog::StubInitialized((ULONGLONG)GetWriteBarrierCodeLocation((void*)JIT_ByRefWriteBarrier), W("@ByRefWriteBarrier")); #endif // TARGET_ARM64 || TARGET_ARM || TARGET_LOONGARCH64 || TARGET_RISCV64 +#if defined(TARGET_AMD64) + // On AMD64 the Checked/ByRef variants of the helpers jump througn an indirection + // to the patched barrier, but are not part of the patched set of helpers. + SetJitHelperFunction(CORINFO_HELP_CHECKED_ASSIGN_REF, (void*)JIT_CheckedWriteBarrier); + SetJitHelperFunction(CORINFO_HELP_ASSIGN_BYREF, (void*)JIT_ByRefWriteBarrier); +#endif // TARGET_AMD64 + } else { #ifdef TARGET_X86 - JIT_WriteBarrierEAX_Loc = (void*)JIT_WriteBarrierEAX; + JIT_WriteBarrierEAX_Loc = (void*)RhpAssignRefEAX; #else JIT_WriteBarrier_Loc = (void*)RhpAssignRef; #endif From 9c2e247967d01c7714dde0e6b57f5ba8dcab7307 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Mon, 28 Apr 2025 12:00:17 -0700 Subject: [PATCH 11/34] Attempt to fix Windows Arm64 and Windows X86 issues --- src/coreclr/runtime/arm64/WriteBarriers.asm | 2 +- src/coreclr/vm/excep.cpp | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/coreclr/runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm index 7feb05c879c2bd..10fb789fa37fc1 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -206,7 +206,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1 ;; - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address - LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT + LEAF_ENTRY RhpByRefAssignRefArm64 ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ldr x15, [x13], 8 diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 728630b85ebbaa..1efe9d9baf46bb 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -6018,18 +6018,21 @@ AdjustContextForJITHelpers( if (pExceptionRecord == nullptr) { #if defined(TARGET_X86) - bool withinWriteBarrierGroup = ((ip >= (PCODE) JIT_WriteBarrierGroup) && (ip <= (PCODE) JIT_WriteBarrierGroup_End)); - bool withinPatchedWriteBarrierGroup = ((ip >= (PCODE) JIT_PatchedWriteBarrierGroup) && (ip <= (PCODE) JIT_PatchedWriteBarrierGroup_End)); - if (withinWriteBarrierGroup || withinPatchedWriteBarrierGroup) + if (IsIPInMarkedJitHelper((UINT_PTR)f_IP)) { DWORD* esp = (DWORD*)pContext->Esp; +#if defined(WRITE_BARRIER_CHECK) + // The assembly for patched write barriers carries the shadow gc heap writing code in such a way that the GC + // heap change is done after pushing some state and always happens in the non-patched region. + // (We don't check for this in the normal AV pathway, as in that case, we have an instruction which triggers + // the AV without writing to memory before adjusting the stack. + bool withinWriteBarrierGroup = ((ip >= (PCODE) JIT_WriteBarrierGroup) && (ip <= (PCODE) JIT_WriteBarrierGroup_End)); if (withinWriteBarrierGroup) { -#if defined(WRITE_BARRIER_CHECK) pContext->Ebp = *esp++; pContext->Ecx = *esp++; -#endif } +#endif pContext->Eip = *esp++; pContext->Esp = (DWORD)esp; return TRUE; @@ -6051,8 +6054,7 @@ AdjustContextForJITHelpers( #if defined(TARGET_X86) && !defined(TARGET_UNIX) void* f_IP = (void *)GetIP(pContext); - if (((f_IP >= (void *) JIT_WriteBarrierGroup) && (f_IP <= (void *) JIT_WriteBarrierGroup_End)) || - ((f_IP >= (void *) JIT_PatchedWriteBarrierGroup) && (f_IP <= (void *) JIT_PatchedWriteBarrierGroup_End))) + if (IsIPInMarkedJitHelper((UINT_PTR)f_IP)) { // set the exception IP to be the instruction that called the write barrier void* callsite = (void *)GetAdjustedCallAddress(*dac_cast(GetSP(pContext))); From 4601ff8fbc67d37376e4f92f26da43f62f420520 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Mon, 28 Apr 2025 12:10:54 -0700 Subject: [PATCH 12/34] Attempt to fix Linux Arm build --- src/coreclr/vm/arm/stubs.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/coreclr/vm/arm/stubs.cpp b/src/coreclr/vm/arm/stubs.cpp index 1c47a46a5ebf32..69472209a2e866 100644 --- a/src/coreclr/vm/arm/stubs.cpp +++ b/src/coreclr/vm/arm/stubs.cpp @@ -473,6 +473,12 @@ void UpdateGCWriteBarriers(bool postGrow = false) int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) { + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + // The runtime is not always suspended when this is called (unlike StompWriteBarrierEphemeral) but we have // no way to update the barrier code atomically on ARM since each 32-bit value we change is loaded over // two instructions. So we have to suspend the EE (which forces code out of the barrier functions) before @@ -498,6 +504,12 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) int StompWriteBarrierEphemeral(bool isRuntimeSuspended) { + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + UNREFERENCED_PARAMETER(isRuntimeSuspended); _ASSERTE(isRuntimeSuspended); UpdateGCWriteBarriers(); @@ -507,6 +519,12 @@ int StompWriteBarrierEphemeral(bool isRuntimeSuspended) #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) { + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + UNREFERENCED_PARAMETER(isRuntimeSuspended); _ASSERTE(isRuntimeSuspended); UpdateGCWriteBarriers(); @@ -515,6 +533,12 @@ int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) { + if (!IsWriteBarrierCopyEnabled()) + { + // If we didn't copy the write barriers, then don't update them. + return SWB_PASS; + } + UNREFERENCED_PARAMETER(isRuntimeSuspended); _ASSERTE(isRuntimeSuspended); UpdateGCWriteBarriers(); From ff8a695267f51982e368d9f07bcef6ae3a2bab54 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Apr 2025 04:02:34 +0000 Subject: [PATCH 13/34] Fix Linux Arm32 --- src/coreclr/vm/excep.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 1efe9d9baf46bb..107e0142b1684c 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5961,7 +5961,11 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) ASSERT(*(uint8_t*)writeBarrierAVLocations[i] != 0xE9); // jmp XXXXXXXX #endif +#ifdef TARGET_ARM + if ((writeBarrierAVLocations[i] | THUMB_CODE) == (uControlPc | THUMB_CODE)) +#else if (writeBarrierAVLocations[i] == uControlPc) +#endif return true; } From a882fbf80db778b333e6efefdd56b0685310abbd Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Apr 2025 04:18:17 +0000 Subject: [PATCH 14/34] Alternative approach to fixing the EH issue that may fix OSX as well. If it does... I'll likely do this logic for all architectures --- src/coreclr/runtime/arm/WriteBarriers.S | 8 +++++++ src/coreclr/runtime/arm64/WriteBarriers.S | 9 ++++++++ src/coreclr/runtime/arm64/WriteBarriers.asm | 8 +++++++ src/coreclr/runtime/i386/WriteBarriers.S | 3 ++- src/coreclr/vm/excep.cpp | 23 +++++++++++++-------- 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index 99cef3ad302794..d126e0e96e65e3 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -179,6 +179,10 @@ GLOBAL_LABEL RhpCheckedAssignRefAVLocation LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT .endm +LEAF_ENTRY RhpWriteBarriers, _TEXT + bx lr +LEAF_END RhpWriteBarriers, _TEXT + // One day we might have write barriers for all the possible argument registers but for now we have // just one write barrier that assumes the input register is RSI. DEFINE_CHECKED_WRITE_BARRIER r1, r1 @@ -302,3 +306,7 @@ LOCAL_LABEL(RhpByRefAssignRef_NotInHeap): add r1, #4 bx lr LEAF_END RhpByRefAssignRef, _TEXT + +LEAF_ENTRY RhpWriteBarriers_End, _TEXT + bx lr +LEAF_END RhpWriteBarriers_End, _TEXT diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index 2970f453dbc3f5..6a6937691ea2a3 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -179,6 +179,10 @@ // Exit label .endm +LEAF_ENTRY RhpWriteBarriers, _TEXT + ret +LEAF_END RhpWriteBarriers, _TEXT + // void JIT_ByRefWriteBarrier // On entry: // x13 : the source address (points to object reference to write) @@ -397,3 +401,8 @@ LOCAL_LABEL(NoBarrierXchg): .arch_extension nolse #endif #endif // FEATURE_NATIVEAOT + +LEAF_ENTRY RhpWriteBarriers_End, _TEXT + ret +LEAF_END RhpWriteBarriers_End, _TEXT + diff --git a/src/coreclr/runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm index 10fb789fa37fc1..f663cd6767dbc2 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -189,6 +189,10 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; Exit label MEND + LEAF_ENTRY RhpWriteBarriers + ret + LEAF_END RhpWriteBarriers + ;; void JIT_ByRefWriteBarrier ;; On entry: ;; x13 : the source address (points to object reference to write) @@ -391,4 +395,8 @@ NoBarrierXchg LEAF_END RhpCheckedXchg #endif // FEATURE_NATIVEAOT + LEAF_ENTRY RhpWriteBarriers_End + ret + LEAF_END RhpWriteBarriers_End + end diff --git a/src/coreclr/runtime/i386/WriteBarriers.S b/src/coreclr/runtime/i386/WriteBarriers.S index 876f2dfbcb80d6..8af80ee05c6961 100644 --- a/src/coreclr/runtime/i386/WriteBarriers.S +++ b/src/coreclr/runtime/i386/WriteBarriers.S @@ -1,4 +1,5 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// TODO: Implement +#include "AsmMacros_Shared.h" + diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 107e0142b1684c..eecf3b0f82453f 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5895,9 +5895,14 @@ BOOL IsIPinVirtualStub(PCODE f_IP) #endif // FEATURE_VIRTUAL_STUB_DISPATCH } +#if defined(TARGET_ARM64) || defined(TARGET_ARM) +EXTERN_C void RhpWriteBarriers(); +EXTERN_C void RhpWriteBarriers_End(); +#endif + typedef uint8_t CODE_LOCATION; EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; -#if defined(HOST_X86) +#if defined(TARGET_X86) EXTERN_C CODE_LOCATION RhpAssignRefEAXAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefECXAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefEBXAVLocation; @@ -5906,7 +5911,7 @@ EXTERN_C CODE_LOCATION RhpAssignRefEDIAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefEBPAVLocation; #endif EXTERN_C CODE_LOCATION RhpCheckedAssignRefAVLocation; -#if defined(HOST_X86) +#if defined(TARGET_X86) EXTERN_C CODE_LOCATION RhpCheckedAssignRefEAXAVLocation; EXTERN_C CODE_LOCATION RhpCheckedAssignRefECXAVLocation; EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBXAVLocation; @@ -5916,14 +5921,14 @@ EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBPAVLocation; #endif EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation1; -#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) +#if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation2; #endif static uintptr_t writeBarrierAVLocations[] = { (uintptr_t)&RhpAssignRefAVLocation, -#if defined(HOST_X86) +#if defined(TARGET_X86) (uintptr_t)&RhpAssignRefEAXAVLocation, (uintptr_t)&RhpAssignRefECXAVLocation, (uintptr_t)&RhpAssignRefEBXAVLocation, @@ -5941,7 +5946,7 @@ static uintptr_t writeBarrierAVLocations[] = (uintptr_t)&RhpCheckedAssignRefEBPAVLocation, #endif (uintptr_t)&RhpByRefAssignRefAVLocation1, -#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) +#if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) (uintptr_t)&RhpByRefAssignRefAVLocation2, #endif }; @@ -5961,17 +5966,17 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) ASSERT(*(uint8_t*)writeBarrierAVLocations[i] != 0xE9); // jmp XXXXXXXX #endif -#ifdef TARGET_ARM - if ((writeBarrierAVLocations[i] | THUMB_CODE) == (uControlPc | THUMB_CODE)) -#else if (writeBarrierAVLocations[i] == uControlPc) -#endif return true; } #define CHECK_RANGE(name) \ if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; +#if defined(TARGET_ARM64) || defined(TARGET_ARM) + CHECK_RANGE(RhpWriteBarriers) +#endif + #ifndef TARGET_X86 CHECK_RANGE(JIT_WriteBarrier) CHECK_RANGE(JIT_CheckedWriteBarrier) From f5a07a77735fd196e2d25fd16e78d2872a6a0d96 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Apr 2025 16:12:51 +0000 Subject: [PATCH 15/34] Revert "Alternative approach to fixing the EH issue that may fix OSX as well. If it does... I'll likely do this logic for all architectures" This reverts commit a882fbf80db778b333e6efefdd56b0685310abbd. --- src/coreclr/runtime/arm/WriteBarriers.S | 8 ------- src/coreclr/runtime/arm64/WriteBarriers.S | 9 -------- src/coreclr/runtime/arm64/WriteBarriers.asm | 8 ------- src/coreclr/runtime/i386/WriteBarriers.S | 3 +-- src/coreclr/vm/excep.cpp | 23 ++++++++------------- 5 files changed, 10 insertions(+), 41 deletions(-) diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index d126e0e96e65e3..99cef3ad302794 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -179,10 +179,6 @@ GLOBAL_LABEL RhpCheckedAssignRefAVLocation LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT .endm -LEAF_ENTRY RhpWriteBarriers, _TEXT - bx lr -LEAF_END RhpWriteBarriers, _TEXT - // One day we might have write barriers for all the possible argument registers but for now we have // just one write barrier that assumes the input register is RSI. DEFINE_CHECKED_WRITE_BARRIER r1, r1 @@ -306,7 +302,3 @@ LOCAL_LABEL(RhpByRefAssignRef_NotInHeap): add r1, #4 bx lr LEAF_END RhpByRefAssignRef, _TEXT - -LEAF_ENTRY RhpWriteBarriers_End, _TEXT - bx lr -LEAF_END RhpWriteBarriers_End, _TEXT diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index 6a6937691ea2a3..2970f453dbc3f5 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -179,10 +179,6 @@ // Exit label .endm -LEAF_ENTRY RhpWriteBarriers, _TEXT - ret -LEAF_END RhpWriteBarriers, _TEXT - // void JIT_ByRefWriteBarrier // On entry: // x13 : the source address (points to object reference to write) @@ -401,8 +397,3 @@ LOCAL_LABEL(NoBarrierXchg): .arch_extension nolse #endif #endif // FEATURE_NATIVEAOT - -LEAF_ENTRY RhpWriteBarriers_End, _TEXT - ret -LEAF_END RhpWriteBarriers_End, _TEXT - diff --git a/src/coreclr/runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm index f663cd6767dbc2..10fb789fa37fc1 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -189,10 +189,6 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; Exit label MEND - LEAF_ENTRY RhpWriteBarriers - ret - LEAF_END RhpWriteBarriers - ;; void JIT_ByRefWriteBarrier ;; On entry: ;; x13 : the source address (points to object reference to write) @@ -395,8 +391,4 @@ NoBarrierXchg LEAF_END RhpCheckedXchg #endif // FEATURE_NATIVEAOT - LEAF_ENTRY RhpWriteBarriers_End - ret - LEAF_END RhpWriteBarriers_End - end diff --git a/src/coreclr/runtime/i386/WriteBarriers.S b/src/coreclr/runtime/i386/WriteBarriers.S index 8af80ee05c6961..876f2dfbcb80d6 100644 --- a/src/coreclr/runtime/i386/WriteBarriers.S +++ b/src/coreclr/runtime/i386/WriteBarriers.S @@ -1,5 +1,4 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include "AsmMacros_Shared.h" - +// TODO: Implement diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index eecf3b0f82453f..107e0142b1684c 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5895,14 +5895,9 @@ BOOL IsIPinVirtualStub(PCODE f_IP) #endif // FEATURE_VIRTUAL_STUB_DISPATCH } -#if defined(TARGET_ARM64) || defined(TARGET_ARM) -EXTERN_C void RhpWriteBarriers(); -EXTERN_C void RhpWriteBarriers_End(); -#endif - typedef uint8_t CODE_LOCATION; EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; -#if defined(TARGET_X86) +#if defined(HOST_X86) EXTERN_C CODE_LOCATION RhpAssignRefEAXAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefECXAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefEBXAVLocation; @@ -5911,7 +5906,7 @@ EXTERN_C CODE_LOCATION RhpAssignRefEDIAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefEBPAVLocation; #endif EXTERN_C CODE_LOCATION RhpCheckedAssignRefAVLocation; -#if defined(TARGET_X86) +#if defined(HOST_X86) EXTERN_C CODE_LOCATION RhpCheckedAssignRefEAXAVLocation; EXTERN_C CODE_LOCATION RhpCheckedAssignRefECXAVLocation; EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBXAVLocation; @@ -5921,14 +5916,14 @@ EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBPAVLocation; #endif EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation1; -#if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) +#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation2; #endif static uintptr_t writeBarrierAVLocations[] = { (uintptr_t)&RhpAssignRefAVLocation, -#if defined(TARGET_X86) +#if defined(HOST_X86) (uintptr_t)&RhpAssignRefEAXAVLocation, (uintptr_t)&RhpAssignRefECXAVLocation, (uintptr_t)&RhpAssignRefEBXAVLocation, @@ -5946,7 +5941,7 @@ static uintptr_t writeBarrierAVLocations[] = (uintptr_t)&RhpCheckedAssignRefEBPAVLocation, #endif (uintptr_t)&RhpByRefAssignRefAVLocation1, -#if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) +#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) (uintptr_t)&RhpByRefAssignRefAVLocation2, #endif }; @@ -5966,17 +5961,17 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) ASSERT(*(uint8_t*)writeBarrierAVLocations[i] != 0xE9); // jmp XXXXXXXX #endif +#ifdef TARGET_ARM + if ((writeBarrierAVLocations[i] | THUMB_CODE) == (uControlPc | THUMB_CODE)) +#else if (writeBarrierAVLocations[i] == uControlPc) +#endif return true; } #define CHECK_RANGE(name) \ if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; -#if defined(TARGET_ARM64) || defined(TARGET_ARM) - CHECK_RANGE(RhpWriteBarriers) -#endif - #ifndef TARGET_X86 CHECK_RANGE(JIT_WriteBarrier) CHECK_RANGE(JIT_CheckedWriteBarrier) From 45a866bf0ab547908aecf55473abf4cc955481d7 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Apr 2025 16:16:55 +0000 Subject: [PATCH 16/34] Fix the Linux Arm32 Rhp write barrier to support FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP --- src/coreclr/runtime/arm/WriteBarriers.S | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index 99cef3ad302794..aa77666775d714 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -22,6 +22,26 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): #endif // WRITE_BARRIER_CHECK +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg + + PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, __wbScratch + cbz __wbScratch, 2f + add __wbScratch, __wbScratch, \ptrReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift + + ldrb \tmpReg, [__wbScratch] + cmp \tmpReg, #0xff + itt ne + movne \tmpReg, 0xff + strbne \tmpReg, [__wbScratch] + +2: +.endm +#else +.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg +.endm +#endif + // There are several different helpers used depending on which register holds the object reference. Since all // the helpers have identical structure we use a macro to define this structure. Two arguments are taken, the // name of the register that points to the location to be updated and the name of the register that holds the @@ -42,6 +62,8 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): cmp \REFREG, r12 bhs LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG) + UPDATE_WRITE_WATCH_TABLE r0, r12 + // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write @@ -279,6 +301,8 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2 add r1, #4 add r0, #4 + UPDATE_WRITE_WATCH_TABLE r2, r3 + // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write From 516656e90738a24b44c9a9685761cf7aaff0cb87 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 30 Apr 2025 14:44:04 -0700 Subject: [PATCH 17/34] Remove ALTERNATE_ENTRY AVLocations --- src/coreclr/runtime/amd64/StubDispatch.S | 1 - src/coreclr/runtime/amd64/StubDispatch.asm | 1 - src/coreclr/runtime/amd64/WriteBarriers.S | 4 ---- src/coreclr/runtime/amd64/WriteBarriers.asm | 4 ---- src/coreclr/runtime/arm64/StubDispatch.S | 1 - src/coreclr/runtime/arm64/StubDispatch.asm | 1 - src/coreclr/runtime/arm64/WriteBarriers.S | 3 --- src/coreclr/runtime/arm64/WriteBarriers.asm | 3 --- src/coreclr/runtime/i386/StubDispatch.asm | 1 - src/coreclr/runtime/i386/WriteBarriers.asm | 7 ------- src/coreclr/runtime/loongarch64/StubDispatch.S | 1 - src/coreclr/runtime/loongarch64/WriteBarriers.S | 3 --- src/coreclr/runtime/riscv64/StubDispatch.S | 1 - src/coreclr/runtime/riscv64/WriteBarriers.S | 3 --- src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S | 1 - src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm | 1 - src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S | 1 - src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm | 1 - 18 files changed, 38 deletions(-) diff --git a/src/coreclr/runtime/amd64/StubDispatch.S b/src/coreclr/runtime/amd64/StubDispatch.S index 3af2bc6ac019b6..f62b7c034ab7f1 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.S +++ b/src/coreclr/runtime/amd64/StubDispatch.S @@ -20,7 +20,6 @@ LEAF_ENTRY RhpInterfaceDispatch\entries, _TEXT mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] // Load the MethodTable from the object instance in rdi. - ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries mov rax, [rdi] CurrentOffset = OFFSETOF__InterfaceDispatchCache__m_rgEntries diff --git a/src/coreclr/runtime/amd64/StubDispatch.asm b/src/coreclr/runtime/amd64/StubDispatch.asm index 1863a43c14720d..9bda656525a77a 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.asm +++ b/src/coreclr/runtime/amd64/StubDispatch.asm @@ -34,7 +34,6 @@ LEAF_ENTRY StubName, _TEXT mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] ;; Load the MethodTable from the object instance in rcx. - ALTERNATE_ENTRY StubAVLocation mov rax, [rcx] CurrentEntry = 0 diff --git a/src/coreclr/runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S index f23d29a3f14af5..bdd7e1eee4f742 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -151,7 +151,6 @@ LEAF_ENTRY RhpAssignRef\EXPORT_REG_NAME, _TEXT // Export the canonical write barrier under unqualified name as well .ifc \REFREG, RSI ALTERNATE_ENTRY RhpAssignRef - ALTERNATE_ENTRY RhpAssignRefAVLocation .endif // Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -205,7 +204,6 @@ LEAF_ENTRY RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT // Export the canonical write barrier under unqualified name as well .ifc \REFREG, RSI ALTERNATE_ENTRY RhpCheckedAssignRef - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation .endif // Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -259,9 +257,7 @@ LEAF_END RhpCheckedXchg, _TEXT // - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1/2 // - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address LEAF_ENTRY RhpByRefAssignRef, _TEXT -ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 mov rcx, [rsi] -ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 mov [rdi], rcx // Check whether the writes were even into the heap. If not there's no card update required. diff --git a/src/coreclr/runtime/amd64/WriteBarriers.asm b/src/coreclr/runtime/amd64/WriteBarriers.asm index c08109d65ea62b..6505f2b8ee9e38 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.asm +++ b/src/coreclr/runtime/amd64/WriteBarriers.asm @@ -167,7 +167,6 @@ LEAF_ENTRY RhpAssignRef&EXPORT_REG_NAME&, _TEXT ;; Export the canonical write barrier under unqualified name as well ifidni , ALTERNATE_ENTRY RhpAssignRef - ALTERNATE_ENTRY RhpAssignRefAVLocation endif ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -221,7 +220,6 @@ LEAF_ENTRY RhpCheckedAssignRef&EXPORT_REG_NAME&, _TEXT ;; Export the canonical write barrier under unqualified name as well ifidni , ALTERNATE_ENTRY RhpCheckedAssignRef - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation endif ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -275,9 +273,7 @@ LEAF_END RhpCheckedXchg, _TEXT ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1/2 ;; - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address LEAF_ENTRY RhpByRefAssignRef, _TEXT -ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 mov rcx, [rsi] -ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 mov [rdi], rcx ;; Check whether the writes were even into the heap. If not there's no card update required. diff --git a/src/coreclr/runtime/arm64/StubDispatch.S b/src/coreclr/runtime/arm64/StubDispatch.S index 1155e6ac257a1a..4d39c8db3a4557 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.S +++ b/src/coreclr/runtime/arm64/StubDispatch.S @@ -31,7 +31,6 @@ ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] // Load the MethodTable from the object instance in x0. - ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries ldr x10, [x0] .global CurrentEntry diff --git a/src/coreclr/runtime/arm64/StubDispatch.asm b/src/coreclr/runtime/arm64/StubDispatch.asm index 697d3a10f52e01..545e271e7c735f 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.asm +++ b/src/coreclr/runtime/arm64/StubDispatch.asm @@ -38,7 +38,6 @@ ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] ;; Load the MethodTable from the object instance in x0. - ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation$entries ldr x10, [x0] GBLA CurrentEntry diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index 2970f453dbc3f5..c26dca6d3bf519 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -198,7 +198,6 @@ // - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT - ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ldr x15, [x13], 8 b C_FUNC(RhpCheckedAssignRefArm64) @@ -229,7 +228,6 @@ LEAF_END RhpByRefAssignRefArm64, _TEXT b C_FUNC(RhpAssignRefArm64) LOCAL_LABEL(NotInHeap): - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation str x15, [x14], 8 ret @@ -249,7 +247,6 @@ LEAF_END RhpCheckedAssignRefArm64, _TEXT // x14 : incremented by 8 LEAF_ENTRY RhpAssignRefArm64, _TEXT - ALTERNATE_ENTRY RhpAssignRefAVLocation stlr x15, [x14] INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15 diff --git a/src/coreclr/runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm index 10fb789fa37fc1..8115fc25e2d8ad 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -208,7 +208,6 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address LEAF_ENTRY RhpByRefAssignRefArm64 - ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ldr x15, [x13], 8 b RhpCheckedAssignRefArm64 @@ -238,7 +237,6 @@ INVALIDGCVALUE EQU 0xCCCCCCCD blo RhpAssignRefArm64 NotInHeap - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation str x15, [x14], 8 ret @@ -258,7 +256,6 @@ NotInHeap ;; x14 : incremented by 8 LEAF_ENTRY RhpAssignRefArm64 - ALTERNATE_ENTRY RhpAssignRefAVLocation stlr x15, [x14] INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15 diff --git a/src/coreclr/runtime/i386/StubDispatch.asm b/src/coreclr/runtime/i386/StubDispatch.asm index b3672d4e1eb045..85ca2a4811876e 100644 --- a/src/coreclr/runtime/i386/StubDispatch.asm +++ b/src/coreclr/runtime/i386/StubDispatch.asm @@ -40,7 +40,6 @@ StubAVLocation textequ @CatStr( _RhpInterfaceDispatchAVLocation, entries ) ;; mess up the stack trace. We also don't have a spare scratch register (eax holds the cache pointer ;; and the push of ebx below is precisely so we can access a second register to hold the MethodTable ;; pointer). - ALTERNATE_ENTRY StubAVLocation cmp dword ptr [ecx], ecx ;; eax currently contains the indirection cell address. We need to update it to point to the cache diff --git a/src/coreclr/runtime/i386/WriteBarriers.asm b/src/coreclr/runtime/i386/WriteBarriers.asm index a0ea69e425212b..ff26268bc45108 100644 --- a/src/coreclr/runtime/i386/WriteBarriers.asm +++ b/src/coreclr/runtime/i386/WriteBarriers.asm @@ -105,11 +105,8 @@ FASTCALL_FUNC RhpAssignRef&REFREG&, 8 ifidni , ALTERNATE_ENTRY RhpAssignRef ALTERNATE_ENTRY @RhpAssignRef@8 - ALTERNATE_ENTRY _RhpAssignRefAVLocation endif - ALTERNATE_ENTRY _RhpAssignRef&REFREG&AVLocation - ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here ;; and the card table update we may perform below. mov dword ptr [DESTREG], REFREG @@ -204,10 +201,8 @@ FASTCALL_FUNC RhpCheckedAssignRef&REFREG&, 8 ifidni , ALTERNATE_ENTRY RhpCheckedAssignRef ALTERNATE_ENTRY @RhpCheckedAssignRef@8 - ALTERNATE_ENTRY _RhpCheckedAssignRefAVLocation endif - ALTERNATE_ENTRY _RhpCheckedAssignRef&REFREG&AVLocation ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here ;; and the card table update we may perform below. @@ -271,9 +266,7 @@ FASTCALL_ENDFUNC ;; ecx: trashed ;; FASTCALL_FUNC RhpByRefAssignRef, 8 -ALTERNATE_ENTRY _RhpByRefAssignRefAVLocation1 mov ecx, [esi] -ALTERNATE_ENTRY _RhpByRefAssignRefAVLocation2 mov [edi], ecx ;; Check whether the writes were even into the heap. If not there's no card update required. diff --git a/src/coreclr/runtime/loongarch64/StubDispatch.S b/src/coreclr/runtime/loongarch64/StubDispatch.S index af0ef71273abdc..86d4e003fd7bfe 100644 --- a/src/coreclr/runtime/loongarch64/StubDispatch.S +++ b/src/coreclr/runtime/loongarch64/StubDispatch.S @@ -33,7 +33,6 @@ ld.d $t0, $t8, OFFSETOF__InterfaceDispatchCell__m_pCache // Load the MethodTable from the object instance in a0. - ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries ld.d $t1, $a0, 0 .global CurrentEntry diff --git a/src/coreclr/runtime/loongarch64/WriteBarriers.S b/src/coreclr/runtime/loongarch64/WriteBarriers.S index 20888056b04ea8..720ba8c8a98552 100644 --- a/src/coreclr/runtime/loongarch64/WriteBarriers.S +++ b/src/coreclr/runtime/loongarch64/WriteBarriers.S @@ -196,7 +196,6 @@ // - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and RA contains the return address LEAF_ENTRY RhpByRefAssignRef, _TEXT - ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ld.d $t7, $t8, 0 addi.d $t8, $t8, 8 b C_FUNC(RhpCheckedAssignRef) @@ -226,7 +225,6 @@ LEAF_END RhpByRefAssignRef, _TEXT b C_FUNC(RhpAssignRefLoongArch64) LOCAL_LABEL(NotInHeap): - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation st.d $t7, $t6, 0 addi.d $t6, $t6, 8 jirl $r0, $ra, 0 @@ -248,7 +246,6 @@ LEAF_END RhpCheckedAssignRef, _TEXT LEAF_ENTRY RhpAssignRefLoongArch64, _TEXT dbar 0 - ALTERNATE_ENTRY RhpAssignRefAVLocation st.d $t7, $t6, 0 INSERT_UNCHECKED_WRITE_BARRIER_CORE $t6, $t7 diff --git a/src/coreclr/runtime/riscv64/StubDispatch.S b/src/coreclr/runtime/riscv64/StubDispatch.S index cc4101a4a17470..00c2054c6a0557 100644 --- a/src/coreclr/runtime/riscv64/StubDispatch.S +++ b/src/coreclr/runtime/riscv64/StubDispatch.S @@ -36,7 +36,6 @@ ld t0, OFFSETOF__InterfaceDispatchCell__m_pCache(t5) // Using a1 as an alternative base register // Load the MethodTable from the object instance in a0. - ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries ld t1, 0(a0) .global CurrentEntry diff --git a/src/coreclr/runtime/riscv64/WriteBarriers.S b/src/coreclr/runtime/riscv64/WriteBarriers.S index 469908665b3533..cd6e7ea1d31706 100644 --- a/src/coreclr/runtime/riscv64/WriteBarriers.S +++ b/src/coreclr/runtime/riscv64/WriteBarriers.S @@ -206,7 +206,6 @@ // - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and RA contains the return address LEAF_ENTRY RhpByRefAssignRef, _TEXT - ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ld t4, 0(t5) addi t5, t5, 8 j C_FUNC(RhpCheckedAssignRef) @@ -240,7 +239,6 @@ LEAF_ENTRY RhpCheckedAssignRef, _TEXT j C_FUNC(RhpAssignRefRiscV64) LOCAL_LABEL(NotInHeap): - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation sd t4, 0(t3) addi t3, t3, 8 @@ -263,7 +261,6 @@ LEAF_END RhpCheckedAssignRef, _TEXT LEAF_ENTRY RhpAssignRefRiscV64, _TEXT fence rw, rw - ALTERNATE_ENTRY RhpAssignRefAVLocation sd t4, 0(t3) INSERT_UNCHECKED_WRITE_BARRIER_CORE t3, t4 diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S index a3a45be29ddb75..05e6e47f644c8b 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S @@ -20,7 +20,6 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // Load the MethodTable from the object instance in rdi, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference -ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation add rax, [rdi] // Load the target address of the vtable chunk into rax diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm index f5cacb3207e150..e29645c4b64703 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm @@ -22,7 +22,6 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT ;; Load the MethodTable from the object instance in rcx, and add it to the vtable offset ;; to get the address in the vtable chunk list of what we want to dereference -ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation add rax, [rcx] ;; Load the target address of the vtable chunk into rax diff --git a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S index f0d7f3bf433017..936f626748221f 100644 --- a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S @@ -21,7 +21,6 @@ // Load the MethodTable from the object instance in x0, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference - ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation ldr x9, [x0] add x9, x10, x9 diff --git a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm index 4b117a0336e6b2..50fc40fef26f70 100644 --- a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm +++ b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm @@ -27,7 +27,6 @@ ;; Load the MethodTable from the object instance in x0, and add it to the vtable offset ;; to get the address in the vtable chunk list of what we want to dereference - ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation ldr x9, [x0] add x9, x10, x9 From 2b791c6c14f4bd35f40b24b04432a06b13cbca5e Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 30 Apr 2025 14:44:16 -0700 Subject: [PATCH 18/34] Reapply "Alternative approach to fixing the EH issue that may fix OSX as well. If it does... I'll likely do this logic for all architectures" This reverts commit f5a07a77735fd196e2d25fd16e78d2872a6a0d96. --- src/coreclr/runtime/arm/WriteBarriers.S | 8 +++++++ src/coreclr/runtime/arm64/WriteBarriers.S | 9 ++++++++ src/coreclr/runtime/arm64/WriteBarriers.asm | 8 +++++++ src/coreclr/runtime/i386/WriteBarriers.S | 3 ++- src/coreclr/vm/excep.cpp | 23 +++++++++++++-------- 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index aa77666775d714..953b4a7d0aff0f 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -201,6 +201,10 @@ GLOBAL_LABEL RhpCheckedAssignRefAVLocation LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT .endm +LEAF_ENTRY RhpWriteBarriers, _TEXT + bx lr +LEAF_END RhpWriteBarriers, _TEXT + // One day we might have write barriers for all the possible argument registers but for now we have // just one write barrier that assumes the input register is RSI. DEFINE_CHECKED_WRITE_BARRIER r1, r1 @@ -326,3 +330,7 @@ LOCAL_LABEL(RhpByRefAssignRef_NotInHeap): add r1, #4 bx lr LEAF_END RhpByRefAssignRef, _TEXT + +LEAF_ENTRY RhpWriteBarriers_End, _TEXT + bx lr +LEAF_END RhpWriteBarriers_End, _TEXT diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index c26dca6d3bf519..cb823bccfbe9cb 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -179,6 +179,10 @@ // Exit label .endm +LEAF_ENTRY RhpWriteBarriers, _TEXT + ret +LEAF_END RhpWriteBarriers, _TEXT + // void JIT_ByRefWriteBarrier // On entry: // x13 : the source address (points to object reference to write) @@ -394,3 +398,8 @@ LOCAL_LABEL(NoBarrierXchg): .arch_extension nolse #endif #endif // FEATURE_NATIVEAOT + +LEAF_ENTRY RhpWriteBarriers_End, _TEXT + ret +LEAF_END RhpWriteBarriers_End, _TEXT + diff --git a/src/coreclr/runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm index 8115fc25e2d8ad..7ed7c44aef16b4 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -189,6 +189,10 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; Exit label MEND + LEAF_ENTRY RhpWriteBarriers + ret + LEAF_END RhpWriteBarriers + ;; void JIT_ByRefWriteBarrier ;; On entry: ;; x13 : the source address (points to object reference to write) @@ -388,4 +392,8 @@ NoBarrierXchg LEAF_END RhpCheckedXchg #endif // FEATURE_NATIVEAOT + LEAF_ENTRY RhpWriteBarriers_End + ret + LEAF_END RhpWriteBarriers_End + end diff --git a/src/coreclr/runtime/i386/WriteBarriers.S b/src/coreclr/runtime/i386/WriteBarriers.S index 876f2dfbcb80d6..8af80ee05c6961 100644 --- a/src/coreclr/runtime/i386/WriteBarriers.S +++ b/src/coreclr/runtime/i386/WriteBarriers.S @@ -1,4 +1,5 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// TODO: Implement +#include "AsmMacros_Shared.h" + diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 107e0142b1684c..eecf3b0f82453f 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5895,9 +5895,14 @@ BOOL IsIPinVirtualStub(PCODE f_IP) #endif // FEATURE_VIRTUAL_STUB_DISPATCH } +#if defined(TARGET_ARM64) || defined(TARGET_ARM) +EXTERN_C void RhpWriteBarriers(); +EXTERN_C void RhpWriteBarriers_End(); +#endif + typedef uint8_t CODE_LOCATION; EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; -#if defined(HOST_X86) +#if defined(TARGET_X86) EXTERN_C CODE_LOCATION RhpAssignRefEAXAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefECXAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefEBXAVLocation; @@ -5906,7 +5911,7 @@ EXTERN_C CODE_LOCATION RhpAssignRefEDIAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefEBPAVLocation; #endif EXTERN_C CODE_LOCATION RhpCheckedAssignRefAVLocation; -#if defined(HOST_X86) +#if defined(TARGET_X86) EXTERN_C CODE_LOCATION RhpCheckedAssignRefEAXAVLocation; EXTERN_C CODE_LOCATION RhpCheckedAssignRefECXAVLocation; EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBXAVLocation; @@ -5916,14 +5921,14 @@ EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBPAVLocation; #endif EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation1; -#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) +#if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation2; #endif static uintptr_t writeBarrierAVLocations[] = { (uintptr_t)&RhpAssignRefAVLocation, -#if defined(HOST_X86) +#if defined(TARGET_X86) (uintptr_t)&RhpAssignRefEAXAVLocation, (uintptr_t)&RhpAssignRefECXAVLocation, (uintptr_t)&RhpAssignRefEBXAVLocation, @@ -5941,7 +5946,7 @@ static uintptr_t writeBarrierAVLocations[] = (uintptr_t)&RhpCheckedAssignRefEBPAVLocation, #endif (uintptr_t)&RhpByRefAssignRefAVLocation1, -#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) +#if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) (uintptr_t)&RhpByRefAssignRefAVLocation2, #endif }; @@ -5961,17 +5966,17 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) ASSERT(*(uint8_t*)writeBarrierAVLocations[i] != 0xE9); // jmp XXXXXXXX #endif -#ifdef TARGET_ARM - if ((writeBarrierAVLocations[i] | THUMB_CODE) == (uControlPc | THUMB_CODE)) -#else if (writeBarrierAVLocations[i] == uControlPc) -#endif return true; } #define CHECK_RANGE(name) \ if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; +#if defined(TARGET_ARM64) || defined(TARGET_ARM) + CHECK_RANGE(RhpWriteBarriers) +#endif + #ifndef TARGET_X86 CHECK_RANGE(JIT_WriteBarrier) CHECK_RANGE(JIT_CheckedWriteBarrier) From ecc06a7e2b9425983b949568668457c1cb91d6f4 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 30 Apr 2025 16:32:38 -0700 Subject: [PATCH 19/34] Use the different style of lookup only on apple platforms on CoreCLR --- .../Runtime/unix/unixasmmacrosamd64.inc | 2 + .../Runtime/unix/unixasmmacrosarm64.inc | 2 + src/coreclr/pal/inc/unixasmmacrosamd64.inc | 6 +++ src/coreclr/pal/inc/unixasmmacrosarm64.inc | 6 +++ src/coreclr/runtime/amd64/StubDispatch.S | 9 ++++- src/coreclr/runtime/amd64/StubDispatch.asm | 1 + src/coreclr/runtime/amd64/WriteBarriers.S | 13 +++++++ src/coreclr/runtime/amd64/WriteBarriers.asm | 4 ++ src/coreclr/runtime/arm/WriteBarriers.S | 8 ---- src/coreclr/runtime/arm64/StubDispatch.S | 9 +++++ src/coreclr/runtime/arm64/StubDispatch.asm | 1 + src/coreclr/runtime/arm64/WriteBarriers.S | 3 ++ src/coreclr/runtime/arm64/WriteBarriers.asm | 11 ++---- src/coreclr/runtime/i386/StubDispatch.asm | 1 + src/coreclr/runtime/i386/WriteBarriers.asm | 7 ++++ .../runtime/loongarch64/StubDispatch.S | 1 + .../runtime/loongarch64/WriteBarriers.S | 3 ++ src/coreclr/runtime/riscv64/StubDispatch.S | 1 + src/coreclr/runtime/riscv64/WriteBarriers.S | 3 ++ .../vm/amd64/CachedInterfaceDispatchCoreCLR.S | 9 +++++ .../amd64/CachedInterfaceDispatchCoreCLR.asm | 1 + .../vm/arm64/CachedInterfaceDispatchCoreCLR.S | 10 +++++ .../arm64/CachedInterfaceDispatchCoreCLR.asm | 1 + src/coreclr/vm/excep.cpp | 26 ++++++++----- src/coreclr/vm/virtualcallstub.cpp | 39 ++++++++++++++++++- src/coreclr/vm/virtualcallstub.h | 17 ++++---- 26 files changed, 156 insertions(+), 38 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc b/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc index 735c17a904966c..185deba1ca95e1 100644 --- a/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc +++ b/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc @@ -3,6 +3,8 @@ #define C_VAR(Name) rip + C_FUNC(Name) +#define AVLOCATION_ALTERNATE_ENTRY(name) ALTERNATE_ENTRY name + .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosarm64.inc b/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosarm64.inc index 36698fece5050b..a5ba5fe9254831 100644 --- a/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosarm64.inc +++ b/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosarm64.inc @@ -3,6 +3,8 @@ #include "AsmOffsets.inc" +#define AVLOCATION_ALTERNATE_ENTRY(name) ALTERNATE_ENTRY name + .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/pal/inc/unixasmmacrosamd64.inc b/src/coreclr/pal/inc/unixasmmacrosamd64.inc index a70aa048938abf..49c77f2aa43371 100644 --- a/src/coreclr/pal/inc/unixasmmacrosamd64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosamd64.inc @@ -3,6 +3,12 @@ #define C_VAR(Name) rip + C_FUNC(Name) +#if defined(__APPLE__) +#define AVLOCATION_ALTERNATE_ENTRY(name) +#else +#define AVLOCATION_ALTERNATE_ENTRY(name) ALTERNATE_ENTRY name +#endif + .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/pal/inc/unixasmmacrosarm64.inc b/src/coreclr/pal/inc/unixasmmacrosarm64.inc index 640716f8058d5d..1b50c4631e1fd1 100644 --- a/src/coreclr/pal/inc/unixasmmacrosarm64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosarm64.inc @@ -1,6 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#ifdef TARGET_APPLE +#define AVLOCATION_ALTERNATE_ENTRY(name) +#else +#define AVLOCATION_ALTERNATE_ENTRY(name) ALTERNATE_ENTRY name +#endif + .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/runtime/amd64/StubDispatch.S b/src/coreclr/runtime/amd64/StubDispatch.S index f62b7c034ab7f1..40d9bae9c0f633 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.S +++ b/src/coreclr/runtime/amd64/StubDispatch.S @@ -20,6 +20,7 @@ LEAF_ENTRY RhpInterfaceDispatch\entries, _TEXT mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] // Load the MethodTable from the object instance in rdi. + AVLOCATION_ALTERNATE_ENTRY(RhpInterfaceDispatchAVLocation\entries) mov rax, [rdi] CurrentOffset = OFFSETOF__InterfaceDispatchCache__m_rgEntries @@ -42,6 +43,9 @@ LEAF_END RhpInterfaceDispatch\entries, _TEXT .endm // DEFINE_INTERFACE_DISPATCH_STUB +LEAF_ENTRY RhpInterfaceDispatchHelpers, _TEXT + ret +LEAF_END RhpInterfaceDispatchHelpers, _TEXT // Define all the stub routines we currently need. // @@ -62,7 +66,7 @@ DEFINE_INTERFACE_DISPATCH_STUB 64 // Initial dispatch on an interface when we don't have a cache yet. LEAF_ENTRY RhpInitialInterfaceDispatch, _TEXT -ALTERNATE_ENTRY RhpInitialDynamicInterfaceDispatch +AVLOCATION_ALTERNATE_ENTRY(RhpInitialDynamicInterfaceDispatch) // Trigger an AV if we're dispatching on a null this. // The exception handling infrastructure is aware of the fact that this is the first // instruction of RhpInitialInterfaceDispatch and uses it to translate an AV here @@ -74,5 +78,8 @@ ALTERNATE_ENTRY RhpInitialDynamicInterfaceDispatch LEAF_END RhpInitialInterfaceDispatch, _TEXT +LEAF_ENTRY RhpInterfaceDispatchHelpers_End, _TEXT + ret +LEAF_END RhpInterfaceDispatchHelpers_End, _TEXT #endif // FEATURE_CACHED_INTERFACE_DISPATCH \ No newline at end of file diff --git a/src/coreclr/runtime/amd64/StubDispatch.asm b/src/coreclr/runtime/amd64/StubDispatch.asm index 9bda656525a77a..1863a43c14720d 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.asm +++ b/src/coreclr/runtime/amd64/StubDispatch.asm @@ -34,6 +34,7 @@ LEAF_ENTRY StubName, _TEXT mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] ;; Load the MethodTable from the object instance in rcx. + ALTERNATE_ENTRY StubAVLocation mov rax, [rcx] CurrentEntry = 0 diff --git a/src/coreclr/runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S index bdd7e1eee4f742..823eaa20e8d0c2 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -151,6 +151,7 @@ LEAF_ENTRY RhpAssignRef\EXPORT_REG_NAME, _TEXT // Export the canonical write barrier under unqualified name as well .ifc \REFREG, RSI ALTERNATE_ENTRY RhpAssignRef + AVLOCATION_ALTERNATE_ENTRY(RhpAssignRefAVLocation) .endif // Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -162,6 +163,10 @@ LEAF_ENTRY RhpAssignRef\EXPORT_REG_NAME, _TEXT LEAF_END RhpAssignRef\EXPORT_REG_NAME, _TEXT .endm +LEAF_ENTRY RhpWriteBarriers, _TEXT + ret +LEAF_END RhpWriteBarriers, _TEXT + // One day we might have write barriers for all the possible argument registers but for now we have // just one write barrier that assumes the input register is RSI. DEFINE_UNCHECKED_WRITE_BARRIER RSI, ESI @@ -204,6 +209,7 @@ LEAF_ENTRY RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT // Export the canonical write barrier under unqualified name as well .ifc \REFREG, RSI ALTERNATE_ENTRY RhpCheckedAssignRef + AVLOCATION_ALTERNATE_ENTRY(RhpCheckedAssignRefAVLocation) .endif // Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -257,7 +263,9 @@ LEAF_END RhpCheckedXchg, _TEXT // - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1/2 // - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address LEAF_ENTRY RhpByRefAssignRef, _TEXT + AVLOCATION_ALTERNATE_ENTRY(RhpByRefAssignRefAVLocation1) mov rcx, [rsi] + AVLOCATION_ALTERNATE_ENTRY(RhpByRefAssignRefAVLocation2) mov [rdi], rcx // Check whether the writes were even into the heap. If not there's no card update required. @@ -322,3 +330,8 @@ LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired): add rsi, 0x8 ret LEAF_END RhpByRefAssignRef, _TEXT + +LEAF_ENTRY RhpWriteBarriers_End, _TEXT + ret +LEAF_END RhpWriteBarriers_End, _TEXT + diff --git a/src/coreclr/runtime/amd64/WriteBarriers.asm b/src/coreclr/runtime/amd64/WriteBarriers.asm index 6505f2b8ee9e38..c08109d65ea62b 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.asm +++ b/src/coreclr/runtime/amd64/WriteBarriers.asm @@ -167,6 +167,7 @@ LEAF_ENTRY RhpAssignRef&EXPORT_REG_NAME&, _TEXT ;; Export the canonical write barrier under unqualified name as well ifidni , ALTERNATE_ENTRY RhpAssignRef + ALTERNATE_ENTRY RhpAssignRefAVLocation endif ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -220,6 +221,7 @@ LEAF_ENTRY RhpCheckedAssignRef&EXPORT_REG_NAME&, _TEXT ;; Export the canonical write barrier under unqualified name as well ifidni , ALTERNATE_ENTRY RhpCheckedAssignRef + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation endif ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -273,7 +275,9 @@ LEAF_END RhpCheckedXchg, _TEXT ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1/2 ;; - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address LEAF_ENTRY RhpByRefAssignRef, _TEXT +ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 mov rcx, [rsi] +ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 mov [rdi], rcx ;; Check whether the writes were even into the heap. If not there's no card update required. diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index 953b4a7d0aff0f..aa77666775d714 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -201,10 +201,6 @@ GLOBAL_LABEL RhpCheckedAssignRefAVLocation LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT .endm -LEAF_ENTRY RhpWriteBarriers, _TEXT - bx lr -LEAF_END RhpWriteBarriers, _TEXT - // One day we might have write barriers for all the possible argument registers but for now we have // just one write barrier that assumes the input register is RSI. DEFINE_CHECKED_WRITE_BARRIER r1, r1 @@ -330,7 +326,3 @@ LOCAL_LABEL(RhpByRefAssignRef_NotInHeap): add r1, #4 bx lr LEAF_END RhpByRefAssignRef, _TEXT - -LEAF_ENTRY RhpWriteBarriers_End, _TEXT - bx lr -LEAF_END RhpWriteBarriers_End, _TEXT diff --git a/src/coreclr/runtime/arm64/StubDispatch.S b/src/coreclr/runtime/arm64/StubDispatch.S index 4d39c8db3a4557..876ba30cffd1c0 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.S +++ b/src/coreclr/runtime/arm64/StubDispatch.S @@ -31,6 +31,7 @@ ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] // Load the MethodTable from the object instance in x0. + AVLOCATION_ALTERNATE_ENTRY(RhpInterfaceDispatchAVLocation\entries) ldr x10, [x0] .global CurrentEntry @@ -48,6 +49,10 @@ .endm +LEAF_ENTRY RhpInterfaceDispatchHelpers, _TEXT + ret +LEAF_END RhpInterfaceDispatchHelpers, _TEXT + // // Define all the stub routines we currently need. // @@ -78,4 +83,8 @@ b C_FUNC(RhpInterfaceDispatchSlow) LEAF_END RhpInitialInterfaceDispatch, _TEXT +LEAF_ENTRY RhpInterfaceDispatchHelpers_End, _TEXT + ret +LEAF_END RhpInterfaceDispatchHelpers_End, _TEXT + #endif // FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/runtime/arm64/StubDispatch.asm b/src/coreclr/runtime/arm64/StubDispatch.asm index 545e271e7c735f..697d3a10f52e01 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.asm +++ b/src/coreclr/runtime/arm64/StubDispatch.asm @@ -38,6 +38,7 @@ ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] ;; Load the MethodTable from the object instance in x0. + ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation$entries ldr x10, [x0] GBLA CurrentEntry diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index cb823bccfbe9cb..a7a9dc3c14ea64 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -202,6 +202,7 @@ LEAF_END RhpWriteBarriers, _TEXT // - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT + AVLOCATION_ALTERNATE_ENTRY(RhpByRefAssignRefAVLocation1) ldr x15, [x13], 8 b C_FUNC(RhpCheckedAssignRefArm64) @@ -232,6 +233,7 @@ LEAF_END RhpByRefAssignRefArm64, _TEXT b C_FUNC(RhpAssignRefArm64) LOCAL_LABEL(NotInHeap): + AVLOCATION_ALTERNATE_ENTRY(RhpCheckedAssignRefAVLocation) str x15, [x14], 8 ret @@ -251,6 +253,7 @@ LEAF_END RhpCheckedAssignRefArm64, _TEXT // x14 : incremented by 8 LEAF_ENTRY RhpAssignRefArm64, _TEXT + AVLOCATION_ALTERNATE_ENTRY(RhpAssignRefAVLocation) stlr x15, [x14] INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15 diff --git a/src/coreclr/runtime/arm64/WriteBarriers.asm b/src/coreclr/runtime/arm64/WriteBarriers.asm index 7ed7c44aef16b4..10fb789fa37fc1 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/runtime/arm64/WriteBarriers.asm @@ -189,10 +189,6 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; Exit label MEND - LEAF_ENTRY RhpWriteBarriers - ret - LEAF_END RhpWriteBarriers - ;; void JIT_ByRefWriteBarrier ;; On entry: ;; x13 : the source address (points to object reference to write) @@ -212,6 +208,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address LEAF_ENTRY RhpByRefAssignRefArm64 + ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ldr x15, [x13], 8 b RhpCheckedAssignRefArm64 @@ -241,6 +238,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD blo RhpAssignRefArm64 NotInHeap + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation str x15, [x14], 8 ret @@ -260,6 +258,7 @@ NotInHeap ;; x14 : incremented by 8 LEAF_ENTRY RhpAssignRefArm64 + ALTERNATE_ENTRY RhpAssignRefAVLocation stlr x15, [x14] INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15 @@ -392,8 +391,4 @@ NoBarrierXchg LEAF_END RhpCheckedXchg #endif // FEATURE_NATIVEAOT - LEAF_ENTRY RhpWriteBarriers_End - ret - LEAF_END RhpWriteBarriers_End - end diff --git a/src/coreclr/runtime/i386/StubDispatch.asm b/src/coreclr/runtime/i386/StubDispatch.asm index 85ca2a4811876e..b3672d4e1eb045 100644 --- a/src/coreclr/runtime/i386/StubDispatch.asm +++ b/src/coreclr/runtime/i386/StubDispatch.asm @@ -40,6 +40,7 @@ StubAVLocation textequ @CatStr( _RhpInterfaceDispatchAVLocation, entries ) ;; mess up the stack trace. We also don't have a spare scratch register (eax holds the cache pointer ;; and the push of ebx below is precisely so we can access a second register to hold the MethodTable ;; pointer). + ALTERNATE_ENTRY StubAVLocation cmp dword ptr [ecx], ecx ;; eax currently contains the indirection cell address. We need to update it to point to the cache diff --git a/src/coreclr/runtime/i386/WriteBarriers.asm b/src/coreclr/runtime/i386/WriteBarriers.asm index ff26268bc45108..a0ea69e425212b 100644 --- a/src/coreclr/runtime/i386/WriteBarriers.asm +++ b/src/coreclr/runtime/i386/WriteBarriers.asm @@ -105,8 +105,11 @@ FASTCALL_FUNC RhpAssignRef&REFREG&, 8 ifidni , ALTERNATE_ENTRY RhpAssignRef ALTERNATE_ENTRY @RhpAssignRef@8 + ALTERNATE_ENTRY _RhpAssignRefAVLocation endif + ALTERNATE_ENTRY _RhpAssignRef&REFREG&AVLocation + ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here ;; and the card table update we may perform below. mov dword ptr [DESTREG], REFREG @@ -201,8 +204,10 @@ FASTCALL_FUNC RhpCheckedAssignRef&REFREG&, 8 ifidni , ALTERNATE_ENTRY RhpCheckedAssignRef ALTERNATE_ENTRY @RhpCheckedAssignRef@8 + ALTERNATE_ENTRY _RhpCheckedAssignRefAVLocation endif + ALTERNATE_ENTRY _RhpCheckedAssignRef&REFREG&AVLocation ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here ;; and the card table update we may perform below. @@ -266,7 +271,9 @@ FASTCALL_ENDFUNC ;; ecx: trashed ;; FASTCALL_FUNC RhpByRefAssignRef, 8 +ALTERNATE_ENTRY _RhpByRefAssignRefAVLocation1 mov ecx, [esi] +ALTERNATE_ENTRY _RhpByRefAssignRefAVLocation2 mov [edi], ecx ;; Check whether the writes were even into the heap. If not there's no card update required. diff --git a/src/coreclr/runtime/loongarch64/StubDispatch.S b/src/coreclr/runtime/loongarch64/StubDispatch.S index 86d4e003fd7bfe..af0ef71273abdc 100644 --- a/src/coreclr/runtime/loongarch64/StubDispatch.S +++ b/src/coreclr/runtime/loongarch64/StubDispatch.S @@ -33,6 +33,7 @@ ld.d $t0, $t8, OFFSETOF__InterfaceDispatchCell__m_pCache // Load the MethodTable from the object instance in a0. + ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries ld.d $t1, $a0, 0 .global CurrentEntry diff --git a/src/coreclr/runtime/loongarch64/WriteBarriers.S b/src/coreclr/runtime/loongarch64/WriteBarriers.S index 720ba8c8a98552..20888056b04ea8 100644 --- a/src/coreclr/runtime/loongarch64/WriteBarriers.S +++ b/src/coreclr/runtime/loongarch64/WriteBarriers.S @@ -196,6 +196,7 @@ // - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and RA contains the return address LEAF_ENTRY RhpByRefAssignRef, _TEXT + ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ld.d $t7, $t8, 0 addi.d $t8, $t8, 8 b C_FUNC(RhpCheckedAssignRef) @@ -225,6 +226,7 @@ LEAF_END RhpByRefAssignRef, _TEXT b C_FUNC(RhpAssignRefLoongArch64) LOCAL_LABEL(NotInHeap): + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation st.d $t7, $t6, 0 addi.d $t6, $t6, 8 jirl $r0, $ra, 0 @@ -246,6 +248,7 @@ LEAF_END RhpCheckedAssignRef, _TEXT LEAF_ENTRY RhpAssignRefLoongArch64, _TEXT dbar 0 + ALTERNATE_ENTRY RhpAssignRefAVLocation st.d $t7, $t6, 0 INSERT_UNCHECKED_WRITE_BARRIER_CORE $t6, $t7 diff --git a/src/coreclr/runtime/riscv64/StubDispatch.S b/src/coreclr/runtime/riscv64/StubDispatch.S index 00c2054c6a0557..cc4101a4a17470 100644 --- a/src/coreclr/runtime/riscv64/StubDispatch.S +++ b/src/coreclr/runtime/riscv64/StubDispatch.S @@ -36,6 +36,7 @@ ld t0, OFFSETOF__InterfaceDispatchCell__m_pCache(t5) // Using a1 as an alternative base register // Load the MethodTable from the object instance in a0. + ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries ld t1, 0(a0) .global CurrentEntry diff --git a/src/coreclr/runtime/riscv64/WriteBarriers.S b/src/coreclr/runtime/riscv64/WriteBarriers.S index cd6e7ea1d31706..469908665b3533 100644 --- a/src/coreclr/runtime/riscv64/WriteBarriers.S +++ b/src/coreclr/runtime/riscv64/WriteBarriers.S @@ -206,6 +206,7 @@ // - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and RA contains the return address LEAF_ENTRY RhpByRefAssignRef, _TEXT + ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ld t4, 0(t5) addi t5, t5, 8 j C_FUNC(RhpCheckedAssignRef) @@ -239,6 +240,7 @@ LEAF_ENTRY RhpCheckedAssignRef, _TEXT j C_FUNC(RhpAssignRefRiscV64) LOCAL_LABEL(NotInHeap): + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation sd t4, 0(t3) addi t3, t3, 8 @@ -261,6 +263,7 @@ LEAF_END RhpCheckedAssignRef, _TEXT LEAF_ENTRY RhpAssignRefRiscV64, _TEXT fence rw, rw + ALTERNATE_ENTRY RhpAssignRefAVLocation sd t4, 0(t3) INSERT_UNCHECKED_WRITE_BARRIER_CORE t3, t4 diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S index 05e6e47f644c8b..9012f06966ae30 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S @@ -7,6 +7,10 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH +LEAF_ENTRY RhpVirtualDispatchHelpers, _TEXT + ret +LEAF_END RhpVirtualDispatchHelpers, _TEXT + // Stub dispatch routine for dispatch to a vtable slot LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // r11 currently contains the indirection cell address. @@ -20,6 +24,7 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // Load the MethodTable from the object instance in rdi, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference + AVLOCATON_ALTERNATE_ENTRY(RhpVTableOffsetDispatchAVLocation) add rax, [rdi] // Load the target address of the vtable chunk into rax @@ -34,6 +39,10 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT TAILJMP_RAX LEAF_END RhpVTableOffsetDispatch, _TEXT +LEAF_ENTRY RhpVirtualDispatchHelpers_End, _TEXT + ret +LEAF_END RhpVirtualDispatchHelpers_End, _TEXT + // On Input: // r11 contains the address of the indirection cell // [rsp+0] m_ReturnAddress: contains the return address of caller to stub diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm index e29645c4b64703..f5cacb3207e150 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.asm @@ -22,6 +22,7 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT ;; Load the MethodTable from the object instance in rcx, and add it to the vtable offset ;; to get the address in the vtable chunk list of what we want to dereference +ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation add rax, [rcx] ;; Load the target address of the vtable chunk into rax diff --git a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S index 936f626748221f..14e1ed15fc32cb 100644 --- a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S @@ -6,6 +6,10 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH +LEAF_ENTRY RhpVirtualDispatchHelpers, _TEXT + ret +LEAF_END RhpVirtualDispatchHelpers, _TEXT + // // Stub dispatch routine for dispatch to a vtable slot // @@ -21,6 +25,7 @@ // Load the MethodTable from the object instance in x0, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference + AVLOCATON_ALTERNATE_ENTRY(RhpVTableOffsetDispatchAVLocation) ldr x9, [x0] add x9, x10, x9 @@ -36,6 +41,11 @@ EPILOG_BRANCH_REG x9 LEAF_END RhpVTableOffsetDispatch, _TEXT +LEAF_ENTRY RhpVirtualDispatchHelpers_End, _TEXT + ret +LEAF_END RhpVirtualDispatchHelpers_End, _TEXT + + // // Cache miss case, call the runtime to resolve the target and update the cache. // x11 contains the interface dispatch cell address. diff --git a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm index 50fc40fef26f70..4b117a0336e6b2 100644 --- a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm +++ b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.asm @@ -27,6 +27,7 @@ ;; Load the MethodTable from the object instance in x0, and add it to the vtable offset ;; to get the address in the vtable chunk list of what we want to dereference + ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation ldr x9, [x0] add x9, x10, x9 diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index eecf3b0f82453f..734a3c58f0dea9 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5895,14 +5895,13 @@ BOOL IsIPinVirtualStub(PCODE f_IP) #endif // FEATURE_VIRTUAL_STUB_DISPATCH } -#if defined(TARGET_ARM64) || defined(TARGET_ARM) +#if defined(TARGET_APPLE) EXTERN_C void RhpWriteBarriers(); EXTERN_C void RhpWriteBarriers_End(); -#endif - +#else typedef uint8_t CODE_LOCATION; EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; -#if defined(TARGET_X86) +#if defined(HOST_X86) EXTERN_C CODE_LOCATION RhpAssignRefEAXAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefECXAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefEBXAVLocation; @@ -5911,7 +5910,7 @@ EXTERN_C CODE_LOCATION RhpAssignRefEDIAVLocation; EXTERN_C CODE_LOCATION RhpAssignRefEBPAVLocation; #endif EXTERN_C CODE_LOCATION RhpCheckedAssignRefAVLocation; -#if defined(TARGET_X86) +#if defined(HOST_X86) EXTERN_C CODE_LOCATION RhpCheckedAssignRefEAXAVLocation; EXTERN_C CODE_LOCATION RhpCheckedAssignRefECXAVLocation; EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBXAVLocation; @@ -5921,14 +5920,14 @@ EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBPAVLocation; #endif EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation1; -#if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) +#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) EXTERN_C CODE_LOCATION RhpByRefAssignRefAVLocation2; #endif static uintptr_t writeBarrierAVLocations[] = { (uintptr_t)&RhpAssignRefAVLocation, -#if defined(TARGET_X86) +#if defined(HOST_X86) (uintptr_t)&RhpAssignRefEAXAVLocation, (uintptr_t)&RhpAssignRefECXAVLocation, (uintptr_t)&RhpAssignRefEBXAVLocation, @@ -5946,10 +5945,11 @@ static uintptr_t writeBarrierAVLocations[] = (uintptr_t)&RhpCheckedAssignRefEBPAVLocation, #endif (uintptr_t)&RhpByRefAssignRefAVLocation1, -#if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) +#if !defined(HOST_ARM64) && !defined(HOST_LOONGARCH64) && !defined(HOST_RISCV64) (uintptr_t)&RhpByRefAssignRefAVLocation2, #endif }; +#endif // Check if the passed in instruction pointer is in one of the // JIT helper functions. @@ -5957,6 +5957,7 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) { LIMITED_METHOD_CONTRACT; +#ifndef TARGET_APPLE // compare the IP against the list of known possible AV locations in the write barrier helpers for (size_t i = 0; i < sizeof(writeBarrierAVLocations)/sizeof(writeBarrierAVLocations[0]); i++) { @@ -5966,14 +5967,19 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) ASSERT(*(uint8_t*)writeBarrierAVLocations[i] != 0xE9); // jmp XXXXXXXX #endif +#ifdef TARGET_ARM + if ((writeBarrierAVLocations[i] | THUMB_CODE) == (uControlPc | THUMB_CODE)) +#else if (writeBarrierAVLocations[i] == uControlPc) +#endif return true; } - +#endif // !TARGET_APPLE + #define CHECK_RANGE(name) \ if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; -#if defined(TARGET_ARM64) || defined(TARGET_ARM) +#ifdef TARGET_APPLE CHECK_RANGE(RhpWriteBarriers) #endif diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index 70fb83c007eb99..90124971d54643 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -4123,6 +4123,12 @@ extern "C" void RhpInterfaceDispatch64(); extern "C" void RhpVTableOffsetDispatch(); +#ifdef TARGET_APPLE +extern "C" void RhpVirtualDispatchHelpers(); +extern "C" void RhpVirtualDispatchHelpers_End(); +extern "C" void RhpInterfaceDispatchHelpers(); +extern "C" void RhpInterfaceDispatchHelpers_End(); +#else extern "C" void RhpInterfaceDispatchAVLocation1(); extern "C" void RhpInterfaceDispatchAVLocation2(); extern "C" void RhpInterfaceDispatchAVLocation4(); @@ -4131,7 +4137,7 @@ extern "C" void RhpInterfaceDispatchAVLocation16(); extern "C" void RhpInterfaceDispatchAVLocation32(); extern "C" void RhpInterfaceDispatchAVLocation64(); extern "C" void RhpVTableOffsetDispatchAVLocation(); - +#endif // TARGET_APPLE #endif // FEATURE_CACHED_INTERFACE_DISPATCH ///////////////////////////////////////////////////////////////////////////////////////////// @@ -4162,6 +4168,10 @@ VirtualCallStubManagerManager::VirtualCallStubManagerManager() _ASSERTE(helperCount == CACHED_INTERFACE_DISPATCH_HELPER_COUNT); helperCount = 0; +#ifdef TARGET_APPLE + pCachedInterfaceDispatchHelpersAVLocation = 0; // On Apple platforms, we don't use AV locations for cached interface dispatch helpers + countCachedInterfaceDispatchHelpers = helperCount; +#else pCachedInterfaceDispatchHelpersAVLocation = new PCODE[CACHED_INTERFACE_DISPATCH_HELPER_COUNT]; RECORD_CACHED_INTERFACE_DISPATCH_HELPER_AVLOCATION(RhpInterfaceDispatchAVLocation1); RECORD_CACHED_INTERFACE_DISPATCH_HELPER_AVLOCATION(RhpInterfaceDispatchAVLocation2); @@ -4175,9 +4185,33 @@ VirtualCallStubManagerManager::VirtualCallStubManagerManager() _ASSERTE(helperCount == CACHED_INTERFACE_DISPATCH_HELPER_COUNT); countCachedInterfaceDispatchHelpers = helperCount; +#endif // TARGET_APPLE + #endif // FEATURE_CACHED_INTERFACE_DISPATCH } +#if defined(FEATURE_CACHED_INTERFACE_DISPATCH) && !defined(DACCESS_COMPILE) +bool VirtualCallStubManagerManager::isCachedInterfaceDispatchStubAVLocation(PCODE addr) +{ + LIMITED_METHOD_CONTRACT; + + #define CHECK_RANGE(name) \ + if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; + +#ifdef TARGET_APPLE + CHECK_RANGE(RhpVirtualDispatchHelpers); + CHECK_RANGE(RhpInterfaceDispatch); +#else + for (size_t i = 0; i < countCachedInterfaceDispatchHelpers; i++) + { + if (pCachedInterfaceDispatchHelpersAVLocation[i] == addr) + return true; + } +#endif + return false; +} +#endif + ///////////////////////////////////////////////////////////////////////////////////////////// /* static */ void VirtualCallStubManagerManager::InitStatic() @@ -4340,6 +4374,7 @@ bool VirtualCallStubManager::isCachedInterfaceDispatchStub(PCODE addr) return pGlobalManager->isCachedInterfaceDispatchStub(addr); } +#ifndef DACCESS_COMPILE bool VirtualCallStubManager::isCachedInterfaceDispatchStubAVLocation(PCODE addr) { LIMITED_METHOD_DAC_CONTRACT; @@ -4350,4 +4385,6 @@ bool VirtualCallStubManager::isCachedInterfaceDispatchStubAVLocation(PCODE addr) return false; return pGlobalManager->isCachedInterfaceDispatchStubAVLocation(addr); } +#endif DACCESS_COMPILE + #endif \ No newline at end of file diff --git a/src/coreclr/vm/virtualcallstub.h b/src/coreclr/vm/virtualcallstub.h index 016afcdc632c19..4e89b000c5243f 100644 --- a/src/coreclr/vm/virtualcallstub.h +++ b/src/coreclr/vm/virtualcallstub.h @@ -316,7 +316,11 @@ class VirtualCallStubManager : public StubManager #endif // !DACCESS_COMPILE static bool isCachedInterfaceDispatchStub(PCODE addr); +#ifdef FEATURE_CACHED_INTERFACE_DISPATCH +#ifndef DACCESS_COMPILE static bool isCachedInterfaceDispatchStubAVLocation(PCODE addr); +#endif // !DACCESS_COMPILE +#endif static BOOL isStubStatic(PCODE addr) { @@ -848,16 +852,9 @@ class VirtualCallStubManagerManager : public StubManager return false; } - bool isCachedInterfaceDispatchStubAVLocation(PCODE addr) - { - LIMITED_METHOD_DAC_CONTRACT; - for (size_t i = 0; i < countCachedInterfaceDispatchHelpers; i++) - { - if (pCachedInterfaceDispatchHelpersAVLocation[i] == addr) - return true; - } - return false; - } +#ifndef DACCESS_COMPILE + bool isCachedInterfaceDispatchStubAVLocation(PCODE addr); +#endif // DACCESS_COMPILE #endif // FEATURE_CACHED_INTERFACE_DISPATCH #ifdef _DEBUG From e8c9a345078f24dd8bfa91fc0eac469069680ffd Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 30 Apr 2025 17:36:58 -0700 Subject: [PATCH 20/34] Fix build breaks --- src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S | 2 +- src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S | 2 +- src/coreclr/vm/virtualcallstub.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S index 9012f06966ae30..d0eadb4a2e44a5 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S @@ -24,7 +24,7 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // Load the MethodTable from the object instance in rdi, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference - AVLOCATON_ALTERNATE_ENTRY(RhpVTableOffsetDispatchAVLocation) + AVLOCATION_ALTERNATE_ENTRY(RhpVTableOffsetDispatchAVLocation) add rax, [rdi] // Load the target address of the vtable chunk into rax diff --git a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S index 14e1ed15fc32cb..03e45b3305c1c4 100644 --- a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S @@ -25,7 +25,7 @@ LEAF_END RhpVirtualDispatchHelpers, _TEXT // Load the MethodTable from the object instance in x0, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference - AVLOCATON_ALTERNATE_ENTRY(RhpVTableOffsetDispatchAVLocation) + AVLOCATION_ALTERNATE_ENTRY(RhpVTableOffsetDispatchAVLocation) ldr x9, [x0] add x9, x10, x9 diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index 90124971d54643..b744b64deff5b0 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -4196,11 +4196,11 @@ bool VirtualCallStubManagerManager::isCachedInterfaceDispatchStubAVLocation(PCOD LIMITED_METHOD_CONTRACT; #define CHECK_RANGE(name) \ - if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; + if (GetEEFuncEntryPoint(name) <= addr && addr < GetEEFuncEntryPoint(name##_End)) return true; #ifdef TARGET_APPLE CHECK_RANGE(RhpVirtualDispatchHelpers); - CHECK_RANGE(RhpInterfaceDispatch); + CHECK_RANGE(RhpInterfaceDispatchHelpers); #else for (size_t i = 0; i < countCachedInterfaceDispatchHelpers; i++) { From e992b86471cef2d1e586fcf1f7543dc4cadb65ae Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 10:36:13 -0700 Subject: [PATCH 21/34] Go back to using AVLocation alternate entries (The stae of Fix th Linux RhpWriteBarrier to support FEATURE_USE_SORTWARE_WRITE_WATCH --- .../Runtime/unix/unixasmmacrosamd64.inc | 2 - .../Runtime/unix/unixasmmacrosarm64.inc | 2 - src/coreclr/pal/inc/unixasmmacrosamd64.inc | 6 --- src/coreclr/pal/inc/unixasmmacrosarm64.inc | 6 --- src/coreclr/runtime/amd64/StubDispatch.S | 10 +---- src/coreclr/runtime/amd64/WriteBarriers.S | 17 ++------ src/coreclr/runtime/arm64/StubDispatch.S | 10 +---- src/coreclr/runtime/arm64/WriteBarriers.S | 15 ++----- src/coreclr/runtime/i386/WriteBarriers.S | 3 +- .../vm/amd64/CachedInterfaceDispatchCoreCLR.S | 10 +---- .../vm/arm64/CachedInterfaceDispatchCoreCLR.S | 11 +----- src/coreclr/vm/excep.cpp | 13 +------ src/coreclr/vm/virtualcallstub.cpp | 39 +------------------ src/coreclr/vm/virtualcallstub.h | 17 ++++---- 14 files changed, 25 insertions(+), 136 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc b/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc index 185deba1ca95e1..735c17a904966c 100644 --- a/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc +++ b/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc @@ -3,8 +3,6 @@ #define C_VAR(Name) rip + C_FUNC(Name) -#define AVLOCATION_ALTERNATE_ENTRY(name) ALTERNATE_ENTRY name - .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosarm64.inc b/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosarm64.inc index a5ba5fe9254831..36698fece5050b 100644 --- a/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosarm64.inc +++ b/src/coreclr/nativeaot/Runtime/unix/unixasmmacrosarm64.inc @@ -3,8 +3,6 @@ #include "AsmOffsets.inc" -#define AVLOCATION_ALTERNATE_ENTRY(name) ALTERNATE_ENTRY name - .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/pal/inc/unixasmmacrosamd64.inc b/src/coreclr/pal/inc/unixasmmacrosamd64.inc index 49c77f2aa43371..a70aa048938abf 100644 --- a/src/coreclr/pal/inc/unixasmmacrosamd64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosamd64.inc @@ -3,12 +3,6 @@ #define C_VAR(Name) rip + C_FUNC(Name) -#if defined(__APPLE__) -#define AVLOCATION_ALTERNATE_ENTRY(name) -#else -#define AVLOCATION_ALTERNATE_ENTRY(name) ALTERNATE_ENTRY name -#endif - .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/pal/inc/unixasmmacrosarm64.inc b/src/coreclr/pal/inc/unixasmmacrosarm64.inc index 1b50c4631e1fd1..640716f8058d5d 100644 --- a/src/coreclr/pal/inc/unixasmmacrosarm64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosarm64.inc @@ -1,12 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#ifdef TARGET_APPLE -#define AVLOCATION_ALTERNATE_ENTRY(name) -#else -#define AVLOCATION_ALTERNATE_ENTRY(name) ALTERNATE_ENTRY name -#endif - .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/runtime/amd64/StubDispatch.S b/src/coreclr/runtime/amd64/StubDispatch.S index 40d9bae9c0f633..3af2bc6ac019b6 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.S +++ b/src/coreclr/runtime/amd64/StubDispatch.S @@ -20,7 +20,7 @@ LEAF_ENTRY RhpInterfaceDispatch\entries, _TEXT mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] // Load the MethodTable from the object instance in rdi. - AVLOCATION_ALTERNATE_ENTRY(RhpInterfaceDispatchAVLocation\entries) + ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries mov rax, [rdi] CurrentOffset = OFFSETOF__InterfaceDispatchCache__m_rgEntries @@ -43,9 +43,6 @@ LEAF_END RhpInterfaceDispatch\entries, _TEXT .endm // DEFINE_INTERFACE_DISPATCH_STUB -LEAF_ENTRY RhpInterfaceDispatchHelpers, _TEXT - ret -LEAF_END RhpInterfaceDispatchHelpers, _TEXT // Define all the stub routines we currently need. // @@ -66,7 +63,7 @@ DEFINE_INTERFACE_DISPATCH_STUB 64 // Initial dispatch on an interface when we don't have a cache yet. LEAF_ENTRY RhpInitialInterfaceDispatch, _TEXT -AVLOCATION_ALTERNATE_ENTRY(RhpInitialDynamicInterfaceDispatch) +ALTERNATE_ENTRY RhpInitialDynamicInterfaceDispatch // Trigger an AV if we're dispatching on a null this. // The exception handling infrastructure is aware of the fact that this is the first // instruction of RhpInitialInterfaceDispatch and uses it to translate an AV here @@ -78,8 +75,5 @@ AVLOCATION_ALTERNATE_ENTRY(RhpInitialDynamicInterfaceDispatch) LEAF_END RhpInitialInterfaceDispatch, _TEXT -LEAF_ENTRY RhpInterfaceDispatchHelpers_End, _TEXT - ret -LEAF_END RhpInterfaceDispatchHelpers_End, _TEXT #endif // FEATURE_CACHED_INTERFACE_DISPATCH \ No newline at end of file diff --git a/src/coreclr/runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S index 823eaa20e8d0c2..f23d29a3f14af5 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -151,7 +151,7 @@ LEAF_ENTRY RhpAssignRef\EXPORT_REG_NAME, _TEXT // Export the canonical write barrier under unqualified name as well .ifc \REFREG, RSI ALTERNATE_ENTRY RhpAssignRef - AVLOCATION_ALTERNATE_ENTRY(RhpAssignRefAVLocation) + ALTERNATE_ENTRY RhpAssignRefAVLocation .endif // Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -163,10 +163,6 @@ LEAF_ENTRY RhpAssignRef\EXPORT_REG_NAME, _TEXT LEAF_END RhpAssignRef\EXPORT_REG_NAME, _TEXT .endm -LEAF_ENTRY RhpWriteBarriers, _TEXT - ret -LEAF_END RhpWriteBarriers, _TEXT - // One day we might have write barriers for all the possible argument registers but for now we have // just one write barrier that assumes the input register is RSI. DEFINE_UNCHECKED_WRITE_BARRIER RSI, ESI @@ -209,7 +205,7 @@ LEAF_ENTRY RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT // Export the canonical write barrier under unqualified name as well .ifc \REFREG, RSI ALTERNATE_ENTRY RhpCheckedAssignRef - AVLOCATION_ALTERNATE_ENTRY(RhpCheckedAssignRefAVLocation) + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation .endif // Write the reference into the location. Note that we rely on the fact that no GC can occur between here @@ -263,9 +259,9 @@ LEAF_END RhpCheckedXchg, _TEXT // - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1/2 // - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address LEAF_ENTRY RhpByRefAssignRef, _TEXT - AVLOCATION_ALTERNATE_ENTRY(RhpByRefAssignRefAVLocation1) +ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 mov rcx, [rsi] - AVLOCATION_ALTERNATE_ENTRY(RhpByRefAssignRefAVLocation2) +ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 mov [rdi], rcx // Check whether the writes were even into the heap. If not there's no card update required. @@ -330,8 +326,3 @@ LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired): add rsi, 0x8 ret LEAF_END RhpByRefAssignRef, _TEXT - -LEAF_ENTRY RhpWriteBarriers_End, _TEXT - ret -LEAF_END RhpWriteBarriers_End, _TEXT - diff --git a/src/coreclr/runtime/arm64/StubDispatch.S b/src/coreclr/runtime/arm64/StubDispatch.S index 876ba30cffd1c0..1155e6ac257a1a 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.S +++ b/src/coreclr/runtime/arm64/StubDispatch.S @@ -31,7 +31,7 @@ ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] // Load the MethodTable from the object instance in x0. - AVLOCATION_ALTERNATE_ENTRY(RhpInterfaceDispatchAVLocation\entries) + ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries ldr x10, [x0] .global CurrentEntry @@ -49,10 +49,6 @@ .endm -LEAF_ENTRY RhpInterfaceDispatchHelpers, _TEXT - ret -LEAF_END RhpInterfaceDispatchHelpers, _TEXT - // // Define all the stub routines we currently need. // @@ -83,8 +79,4 @@ LEAF_END RhpInterfaceDispatchHelpers, _TEXT b C_FUNC(RhpInterfaceDispatchSlow) LEAF_END RhpInitialInterfaceDispatch, _TEXT -LEAF_ENTRY RhpInterfaceDispatchHelpers_End, _TEXT - ret -LEAF_END RhpInterfaceDispatchHelpers_End, _TEXT - #endif // FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index a7a9dc3c14ea64..2970f453dbc3f5 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -179,10 +179,6 @@ // Exit label .endm -LEAF_ENTRY RhpWriteBarriers, _TEXT - ret -LEAF_END RhpWriteBarriers, _TEXT - // void JIT_ByRefWriteBarrier // On entry: // x13 : the source address (points to object reference to write) @@ -202,7 +198,7 @@ LEAF_END RhpWriteBarriers, _TEXT // - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT - AVLOCATION_ALTERNATE_ENTRY(RhpByRefAssignRefAVLocation1) + ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 ldr x15, [x13], 8 b C_FUNC(RhpCheckedAssignRefArm64) @@ -233,7 +229,7 @@ LEAF_END RhpByRefAssignRefArm64, _TEXT b C_FUNC(RhpAssignRefArm64) LOCAL_LABEL(NotInHeap): - AVLOCATION_ALTERNATE_ENTRY(RhpCheckedAssignRefAVLocation) + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation str x15, [x14], 8 ret @@ -253,7 +249,7 @@ LEAF_END RhpCheckedAssignRefArm64, _TEXT // x14 : incremented by 8 LEAF_ENTRY RhpAssignRefArm64, _TEXT - AVLOCATION_ALTERNATE_ENTRY(RhpAssignRefAVLocation) + ALTERNATE_ENTRY RhpAssignRefAVLocation stlr x15, [x14] INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15 @@ -401,8 +397,3 @@ LOCAL_LABEL(NoBarrierXchg): .arch_extension nolse #endif #endif // FEATURE_NATIVEAOT - -LEAF_ENTRY RhpWriteBarriers_End, _TEXT - ret -LEAF_END RhpWriteBarriers_End, _TEXT - diff --git a/src/coreclr/runtime/i386/WriteBarriers.S b/src/coreclr/runtime/i386/WriteBarriers.S index 8af80ee05c6961..876f2dfbcb80d6 100644 --- a/src/coreclr/runtime/i386/WriteBarriers.S +++ b/src/coreclr/runtime/i386/WriteBarriers.S @@ -1,5 +1,4 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include "AsmMacros_Shared.h" - +// TODO: Implement diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S index d0eadb4a2e44a5..a3a45be29ddb75 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S @@ -7,10 +7,6 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH -LEAF_ENTRY RhpVirtualDispatchHelpers, _TEXT - ret -LEAF_END RhpVirtualDispatchHelpers, _TEXT - // Stub dispatch routine for dispatch to a vtable slot LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // r11 currently contains the indirection cell address. @@ -24,7 +20,7 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // Load the MethodTable from the object instance in rdi, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference - AVLOCATION_ALTERNATE_ENTRY(RhpVTableOffsetDispatchAVLocation) +ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation add rax, [rdi] // Load the target address of the vtable chunk into rax @@ -39,10 +35,6 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT TAILJMP_RAX LEAF_END RhpVTableOffsetDispatch, _TEXT -LEAF_ENTRY RhpVirtualDispatchHelpers_End, _TEXT - ret -LEAF_END RhpVirtualDispatchHelpers_End, _TEXT - // On Input: // r11 contains the address of the indirection cell // [rsp+0] m_ReturnAddress: contains the return address of caller to stub diff --git a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S index 03e45b3305c1c4..f0d7f3bf433017 100644 --- a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S @@ -6,10 +6,6 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH -LEAF_ENTRY RhpVirtualDispatchHelpers, _TEXT - ret -LEAF_END RhpVirtualDispatchHelpers, _TEXT - // // Stub dispatch routine for dispatch to a vtable slot // @@ -25,7 +21,7 @@ LEAF_END RhpVirtualDispatchHelpers, _TEXT // Load the MethodTable from the object instance in x0, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference - AVLOCATION_ALTERNATE_ENTRY(RhpVTableOffsetDispatchAVLocation) + ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation ldr x9, [x0] add x9, x10, x9 @@ -41,11 +37,6 @@ LEAF_END RhpVirtualDispatchHelpers, _TEXT EPILOG_BRANCH_REG x9 LEAF_END RhpVTableOffsetDispatch, _TEXT -LEAF_ENTRY RhpVirtualDispatchHelpers_End, _TEXT - ret -LEAF_END RhpVirtualDispatchHelpers_End, _TEXT - - // // Cache miss case, call the runtime to resolve the target and update the cache. // x11 contains the interface dispatch cell address. diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 734a3c58f0dea9..107e0142b1684c 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5895,10 +5895,6 @@ BOOL IsIPinVirtualStub(PCODE f_IP) #endif // FEATURE_VIRTUAL_STUB_DISPATCH } -#if defined(TARGET_APPLE) -EXTERN_C void RhpWriteBarriers(); -EXTERN_C void RhpWriteBarriers_End(); -#else typedef uint8_t CODE_LOCATION; EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; #if defined(HOST_X86) @@ -5949,7 +5945,6 @@ static uintptr_t writeBarrierAVLocations[] = (uintptr_t)&RhpByRefAssignRefAVLocation2, #endif }; -#endif // Check if the passed in instruction pointer is in one of the // JIT helper functions. @@ -5957,7 +5952,6 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) { LIMITED_METHOD_CONTRACT; -#ifndef TARGET_APPLE // compare the IP against the list of known possible AV locations in the write barrier helpers for (size_t i = 0; i < sizeof(writeBarrierAVLocations)/sizeof(writeBarrierAVLocations[0]); i++) { @@ -5974,15 +5968,10 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) #endif return true; } -#endif // !TARGET_APPLE - + #define CHECK_RANGE(name) \ if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; -#ifdef TARGET_APPLE - CHECK_RANGE(RhpWriteBarriers) -#endif - #ifndef TARGET_X86 CHECK_RANGE(JIT_WriteBarrier) CHECK_RANGE(JIT_CheckedWriteBarrier) diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index b744b64deff5b0..70fb83c007eb99 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -4123,12 +4123,6 @@ extern "C" void RhpInterfaceDispatch64(); extern "C" void RhpVTableOffsetDispatch(); -#ifdef TARGET_APPLE -extern "C" void RhpVirtualDispatchHelpers(); -extern "C" void RhpVirtualDispatchHelpers_End(); -extern "C" void RhpInterfaceDispatchHelpers(); -extern "C" void RhpInterfaceDispatchHelpers_End(); -#else extern "C" void RhpInterfaceDispatchAVLocation1(); extern "C" void RhpInterfaceDispatchAVLocation2(); extern "C" void RhpInterfaceDispatchAVLocation4(); @@ -4137,7 +4131,7 @@ extern "C" void RhpInterfaceDispatchAVLocation16(); extern "C" void RhpInterfaceDispatchAVLocation32(); extern "C" void RhpInterfaceDispatchAVLocation64(); extern "C" void RhpVTableOffsetDispatchAVLocation(); -#endif // TARGET_APPLE + #endif // FEATURE_CACHED_INTERFACE_DISPATCH ///////////////////////////////////////////////////////////////////////////////////////////// @@ -4168,10 +4162,6 @@ VirtualCallStubManagerManager::VirtualCallStubManagerManager() _ASSERTE(helperCount == CACHED_INTERFACE_DISPATCH_HELPER_COUNT); helperCount = 0; -#ifdef TARGET_APPLE - pCachedInterfaceDispatchHelpersAVLocation = 0; // On Apple platforms, we don't use AV locations for cached interface dispatch helpers - countCachedInterfaceDispatchHelpers = helperCount; -#else pCachedInterfaceDispatchHelpersAVLocation = new PCODE[CACHED_INTERFACE_DISPATCH_HELPER_COUNT]; RECORD_CACHED_INTERFACE_DISPATCH_HELPER_AVLOCATION(RhpInterfaceDispatchAVLocation1); RECORD_CACHED_INTERFACE_DISPATCH_HELPER_AVLOCATION(RhpInterfaceDispatchAVLocation2); @@ -4185,33 +4175,9 @@ VirtualCallStubManagerManager::VirtualCallStubManagerManager() _ASSERTE(helperCount == CACHED_INTERFACE_DISPATCH_HELPER_COUNT); countCachedInterfaceDispatchHelpers = helperCount; -#endif // TARGET_APPLE - #endif // FEATURE_CACHED_INTERFACE_DISPATCH } -#if defined(FEATURE_CACHED_INTERFACE_DISPATCH) && !defined(DACCESS_COMPILE) -bool VirtualCallStubManagerManager::isCachedInterfaceDispatchStubAVLocation(PCODE addr) -{ - LIMITED_METHOD_CONTRACT; - - #define CHECK_RANGE(name) \ - if (GetEEFuncEntryPoint(name) <= addr && addr < GetEEFuncEntryPoint(name##_End)) return true; - -#ifdef TARGET_APPLE - CHECK_RANGE(RhpVirtualDispatchHelpers); - CHECK_RANGE(RhpInterfaceDispatchHelpers); -#else - for (size_t i = 0; i < countCachedInterfaceDispatchHelpers; i++) - { - if (pCachedInterfaceDispatchHelpersAVLocation[i] == addr) - return true; - } -#endif - return false; -} -#endif - ///////////////////////////////////////////////////////////////////////////////////////////// /* static */ void VirtualCallStubManagerManager::InitStatic() @@ -4374,7 +4340,6 @@ bool VirtualCallStubManager::isCachedInterfaceDispatchStub(PCODE addr) return pGlobalManager->isCachedInterfaceDispatchStub(addr); } -#ifndef DACCESS_COMPILE bool VirtualCallStubManager::isCachedInterfaceDispatchStubAVLocation(PCODE addr) { LIMITED_METHOD_DAC_CONTRACT; @@ -4385,6 +4350,4 @@ bool VirtualCallStubManager::isCachedInterfaceDispatchStubAVLocation(PCODE addr) return false; return pGlobalManager->isCachedInterfaceDispatchStubAVLocation(addr); } -#endif DACCESS_COMPILE - #endif \ No newline at end of file diff --git a/src/coreclr/vm/virtualcallstub.h b/src/coreclr/vm/virtualcallstub.h index 4e89b000c5243f..016afcdc632c19 100644 --- a/src/coreclr/vm/virtualcallstub.h +++ b/src/coreclr/vm/virtualcallstub.h @@ -316,11 +316,7 @@ class VirtualCallStubManager : public StubManager #endif // !DACCESS_COMPILE static bool isCachedInterfaceDispatchStub(PCODE addr); -#ifdef FEATURE_CACHED_INTERFACE_DISPATCH -#ifndef DACCESS_COMPILE static bool isCachedInterfaceDispatchStubAVLocation(PCODE addr); -#endif // !DACCESS_COMPILE -#endif static BOOL isStubStatic(PCODE addr) { @@ -852,9 +848,16 @@ class VirtualCallStubManagerManager : public StubManager return false; } -#ifndef DACCESS_COMPILE - bool isCachedInterfaceDispatchStubAVLocation(PCODE addr); -#endif // DACCESS_COMPILE + bool isCachedInterfaceDispatchStubAVLocation(PCODE addr) + { + LIMITED_METHOD_DAC_CONTRACT; + for (size_t i = 0; i < countCachedInterfaceDispatchHelpers; i++) + { + if (pCachedInterfaceDispatchHelpersAVLocation[i] == addr) + return true; + } + return false; + } #endif // FEATURE_CACHED_INTERFACE_DISPATCH #ifdef _DEBUG From db5d24f4ef77400414b0411e0761d161c5193dc1 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 11:42:42 -0700 Subject: [PATCH 22/34] Attempt to fix arm32 build issue --- src/coreclr/jit/targetarm.h | 4 +++- src/coreclr/runtime/arm/WriteBarriers.S | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 95cb19a2291a49..1e1cee90c53e90 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -138,7 +138,8 @@ // On exit: // r0: trashed // r3: trashed - // CORINFO_HELP_ASSIGN_BYREF (JIT_ByRefWriteBarrier): + // r12: trashed +// CORINFO_HELP_ASSIGN_BYREF (JIT_ByRefWriteBarrier): // On entry: // r0: the destination address (object reference written here) // r1: the source address (points to object reference to write) @@ -147,6 +148,7 @@ // r1: incremented by 4 // r2: trashed // r3: trashed + // r12: trashed #define REG_WRITE_BARRIER_DST REG_ARG_0 #define RBM_WRITE_BARRIER_DST RBM_ARG_0 diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index aa77666775d714..60504685050059 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -23,7 +23,7 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): #endif // WRITE_BARRIER_CHECK #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg +.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg, __wbScratch PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, __wbScratch cbz __wbScratch, 2f @@ -38,7 +38,7 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): 2: .endm #else -.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg +.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg, __wbScratch .endm #endif @@ -46,7 +46,7 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): // the helpers have identical structure we use a macro to define this structure. Two arguments are taken, the // name of the register that points to the location to be updated and the name of the register that holds the // object reference (this should be in upper case as it's used in the definition of the name of the helper). -.macro DEFINE_UNCHECKED_WRITE_BARRIER_CORE BASENAME, REFREG +.macro DEFINE_UNCHECKED_WRITE_BARRIER_CORE BASENAME, REFREG, TMPREG // Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless // we're in a debug build and write barrier checking has been enabled). @@ -62,7 +62,7 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): cmp \REFREG, r12 bhs LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG) - UPDATE_WRITE_WATCH_TABLE r0, r12 + UPDATE_WRITE_WATCH_TABLE r0, r12, TMPREG // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set @@ -125,7 +125,7 @@ GLOBAL_LABEL RhpAssignRefAVLocation .endif str \REFREG, [r0] - DEFINE_UNCHECKED_WRITE_BARRIER_CORE RhpAssignRef, \REFREG + DEFINE_UNCHECKED_WRITE_BARRIER_CORE RhpAssignRef, \REFREG, r3 bx lr LEAF_END RhpAssignRef\EXPORT_REG_NAME, _TEXT @@ -142,7 +142,7 @@ DEFINE_UNCHECKED_WRITE_BARRIER r1, r1 // collection. // -.macro DEFINE_CHECKED_WRITE_BARRIER_CORE BASENAME, REFREG +.macro DEFINE_CHECKED_WRITE_BARRIER_CORE BASENAME, REFREG, TMPREG // The location being updated might not even lie in the GC heap (a handle or stack location for instance), // in which case no write barrier is required. @@ -153,7 +153,7 @@ DEFINE_UNCHECKED_WRITE_BARRIER r1, r1 cmp r0, r12 bhs LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG) - DEFINE_UNCHECKED_WRITE_BARRIER_CORE \BASENAME, \REFREG + DEFINE_UNCHECKED_WRITE_BARRIER_CORE \BASENAME, \REFREG, TMPREG .endm @@ -195,7 +195,7 @@ GLOBAL_LABEL RhpCheckedAssignRefAVLocation .endif str \REFREG, [r0] - DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedAssignRef, \REFREG + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedAssignRef, \REFREG, r3 bx lr LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT @@ -224,7 +224,7 @@ LOCAL_LABEL(RhpCheckedLockCmpXchgRetry): bne LOCAL_LABEL(RhpCheckedLockCmpXchgRetry) mov r3, r2 - DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, r1 + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, r1, r2 mov r0, r3 bx lr @@ -242,7 +242,7 @@ LOCAL_LABEL(RhpCheckedXchgRetry): cmp r3, #0 bne LOCAL_LABEL(RhpCheckedXchgRetry) - DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedXchg, r1 + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedXchg, r1, r3 // The original value is currently in r2. We need to return it in r0. mov r0, r2 @@ -301,7 +301,7 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2 add r1, #4 add r0, #4 - UPDATE_WRITE_WATCH_TABLE r2, r3 + UPDATE_WRITE_WATCH_TABLE r2, r3, r12 // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set From f3f93209e4e46f5ac0947053e01b41962411968a Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 12:41:23 -0700 Subject: [PATCH 23/34] Attempt to fix NullRefernce problem by moving ALTERNATE_ENTRY to start of all functions that use them on Apple platforms. Experiments indicate this should work without messing with any of the general purpose code. --- src/coreclr/runtime/amd64/StubDispatch.S | 8 ++++---- src/coreclr/runtime/amd64/WriteBarriers.S | 12 ++++++++++++ src/coreclr/runtime/arm64/StubDispatch.S | 6 +++--- src/coreclr/runtime/arm64/WriteBarriers.S | 15 ++++++++++++++- .../vm/amd64/CachedInterfaceDispatchCoreCLR.S | 9 ++++++++- .../vm/arm64/CachedInterfaceDispatchCoreCLR.S | 8 +++++--- 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/coreclr/runtime/amd64/StubDispatch.S b/src/coreclr/runtime/amd64/StubDispatch.S index 3af2bc6ac019b6..85bc078dcffa99 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.S +++ b/src/coreclr/runtime/amd64/StubDispatch.S @@ -15,14 +15,14 @@ LEAF_ENTRY RhpInterfaceDispatch\entries, _TEXT - // r11 currently contains the indirection cell address. - // load r10 to point to the cache block. - mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] - // Load the MethodTable from the object instance in rdi. ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries mov rax, [rdi] + // r11 currently contains the indirection cell address. + // load r10 to point to the cache block. + mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] + CurrentOffset = OFFSETOF__InterfaceDispatchCache__m_rgEntries // For each entry in the cache, see if its MethodTable type matches the MethodTable in rax. diff --git a/src/coreclr/runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S index f23d29a3f14af5..93658c4ae0681d 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -261,6 +261,14 @@ LEAF_END RhpCheckedXchg, _TEXT LEAF_ENTRY RhpByRefAssignRef, _TEXT ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 mov rcx, [rsi] +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function + jmp C_FUNC(RhpByRefAssignRef2) +LEAF_END RhpByRefAssignRef, _TEXT + +LEAF_ENTRY RhpByRefAssignRef2, _TEXT +#endif ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 mov [rdi], rcx @@ -325,4 +333,8 @@ LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired): add rdi, 0x8 add rsi, 0x8 ret +#ifdef TARGET_APPLE +LEAF_END RhpByRefAssignRef2, _TEXT +#else LEAF_END RhpByRefAssignRef, _TEXT +#endif diff --git a/src/coreclr/runtime/arm64/StubDispatch.S b/src/coreclr/runtime/arm64/StubDispatch.S index 1155e6ac257a1a..198a93b2456c0c 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.S +++ b/src/coreclr/runtime/arm64/StubDispatch.S @@ -27,13 +27,13 @@ NESTED_ENTRY "RhpInterfaceDispatch\entries", _TEXT, NoHandler - // x11 holds the indirection cell address. Load the cache pointer. - ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] - // Load the MethodTable from the object instance in x0. ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries ldr x10, [x0] + // x11 holds the indirection cell address. Load the cache pointer. + ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] + .global CurrentEntry .set CurrentEntry, 0 diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index 2970f453dbc3f5..dd3d2bb6f78f67 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -229,12 +229,25 @@ LEAF_END RhpByRefAssignRefArm64, _TEXT b C_FUNC(RhpAssignRefArm64) LOCAL_LABEL(NotInHeap): +#ifdef TARGET_APPLE + b C_FUNC(RhpCheckedAssignRefArm64_NotInHeap) +#else ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation str x15, [x14], 8 ret - +#endif LEAF_END RhpCheckedAssignRefArm64, _TEXT +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function +LEAF_ENTRY RhpCheckedAssignRefArm64_NotInHeap, _TEXT + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation + str x15, [x14], 8 + ret +LEAF_END RhpCheckedAssignRefArm64_NotInHeap, _TEXT +#endif + // JIT_WriteBarrier(Object** dst, Object* src) // // Write barrier for writes to objects that are known to diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S index a3a45be29ddb75..bae65e1c659cff 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S @@ -9,6 +9,11 @@ // Stub dispatch routine for dispatch to a vtable slot LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT + +#ifdef TARGET_APPLE // place all ALTERNATE_ENTRY for arm64 at the beginning of a method to make unwinding work + ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation + cmp [rdi], 0 +#endif // r11 currently contains the indirection cell address. // load r11 to point to the vtable offset (which is stored in the m_pCache field). mov r11, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] @@ -20,7 +25,9 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // Load the MethodTable from the object instance in rdi, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference -ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation +#ifndef TARGET_APPLE + ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation +#endif add rax, [rdi] // Load the target address of the vtable chunk into rax diff --git a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S index f0d7f3bf433017..068936ef5235e2 100644 --- a/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/arm64/CachedInterfaceDispatchCoreCLR.S @@ -11,6 +11,10 @@ // LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT + // Load the MethodTable from the object instance in x0 + ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation + ldr x9, [x0] + // x11 currently contains the indirection cell address. // load x11 to point to the vtable offset (which is stored in the m_pCache field). ldr x11, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] @@ -19,10 +23,8 @@ // to get to the VTable chunk lsr x10, x11, #32 - // Load the MethodTable from the object instance in x0, and add it to the vtable offset + // Add the MethodTable to the vtable offset // to get the address in the vtable chunk list of what we want to dereference - ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation - ldr x9, [x0] add x9, x10, x9 // Load the target address of the vtable chunk into x9 From c35cbbd984808e9c0fe1b3f6c559309b160a1a16 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 14:14:56 -0700 Subject: [PATCH 24/34] Fix Windows X86 build --- src/coreclr/vm/excep.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 107e0142b1684c..22578e7709b717 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -6022,7 +6022,7 @@ AdjustContextForJITHelpers( if (pExceptionRecord == nullptr) { #if defined(TARGET_X86) - if (IsIPInMarkedJitHelper((UINT_PTR)f_IP)) + if (IsIPInMarkedJitHelper((UINT_PTR)ip)) { DWORD* esp = (DWORD*)pContext->Esp; #if defined(WRITE_BARRIER_CHECK) From 0fa2885940e86c9f962ce60328271416925939c5 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 16:09:02 -0700 Subject: [PATCH 25/34] Linux X86 builds, and probably works --- .../nativeaot/Runtime/i386/AsmMacros_Shared.h | 6 + src/coreclr/pal/inc/unixasmmacrosx86.inc | 6 + src/coreclr/runtime/i386/WriteBarriers.S | 301 +++++++++++++++++- src/coreclr/vm/i386/AsmMacros_Shared.h | 8 + 4 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.h create mode 100644 src/coreclr/vm/i386/AsmMacros_Shared.h diff --git a/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.h b/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.h new file mode 100644 index 00000000000000..51951864f299ad --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.h @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +// TODO: Implement \ No newline at end of file diff --git a/src/coreclr/pal/inc/unixasmmacrosx86.inc b/src/coreclr/pal/inc/unixasmmacrosx86.inc index 7bc994a779c356..0de1d04a2437c5 100644 --- a/src/coreclr/pal/inc/unixasmmacrosx86.inc +++ b/src/coreclr/pal/inc/unixasmmacrosx86.inc @@ -24,6 +24,12 @@ C_FUNC(\Name): C_FUNC(\Name): .endm +.macro ALTERNATE_ENTRY Name + .global C_FUNC(\Name) + .type \Name, %function +C_FUNC(\Name): +.endm + .macro LEAF_END Name, Section .size \Name, .-\Name .cfi_endproc diff --git a/src/coreclr/runtime/i386/WriteBarriers.S b/src/coreclr/runtime/i386/WriteBarriers.S index 876f2dfbcb80d6..f26590a88ab462 100644 --- a/src/coreclr/runtime/i386/WriteBarriers.S +++ b/src/coreclr/runtime/i386/WriteBarriers.S @@ -1,4 +1,303 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// TODO: Implement +.intel_syntax noprefix +#include "AsmMacros_Shared.h" + +// TODO! This is implemented, but not tested. + +#ifdef WRITE_BARRIER_CHECK + +.macro UPDATE_GC_SHADOW BASENAME, DESTREG, REFREG, TEMPREG + + // If g_GCShadow is 0, don't perform the check. + PREPARE_EXTERNAL_VAR g_GCShadow, \TEMPREG + cmp dword ptr [\TEMPREG], 0 + je LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Done_\DESTREG\()_\REFREG) + + // Save DESTREG since we're about to modify it (and we need the original value both within the macro and + // once we exit the macro). + push \DESTREG + + // Transform DESTREG into the equivalent address in the shadow heap. + PREPARE_EXTERNAL_VAR g_lowest_address, \TEMPREG + sub \DESTREG, [\TEMPREG] + jb LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_PopThenDone_\DESTREG\()_\REFREG) + PREPARE_EXTERNAL_VAR g_GCShadow, \TEMPREG + add \DESTREG, [\TEMPREG] + PREPARE_EXTERNAL_VAR g_GCShadowEnd, \TEMPREG + cmp \DESTREG, [\TEMPREG] + jae LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_PopThenDone_\DESTREG\()_\REFREG) + + // Update the shadow heap. + mov [\DESTREG], \REFREG + + // Now check that the real heap location still contains the value we just wrote into the shadow heap. This + // read must be strongly ordered wrt to the previous write to prevent race conditions. We also need to + // recover the old value of DESTREG for the comparison so use an xchg instruction (which has an implicit lock + // prefix). + xchg [esp], \DESTREG + cmp [\DESTREG], \REFREG + jne LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Invalidate_\DESTREG\()_\REFREG) + + // The original DESTREG value is now restored but the stack has a value (the shadow version of the + // location) pushed. Need to discard this push before we are done. + add esp, 4 + jmp LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Done_\DESTREG\()_\REFREG) + +LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Invalidate_\DESTREG\()_\REFREG): + // Someone went and updated the real heap. We need to invalidate the shadow location since we can't + // guarantee whose shadow update won. + + // Retrieve shadow location from the stack and restore original DESTREG to the stack. This is an + // additional memory barrier we don't require but it's on the rare path and x86 doesn't have an xchg + // variant that doesn't implicitly specify the lock prefix. + xchg [esp], \DESTREG + mov dword ptr [\DESTREG], INVALIDGCVALUE + +LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_PopThenDone_\DESTREG\()_\REFREG): + // Restore original DESTREG value from the stack. + pop \DESTREG + +LOCAL_LABEL(\BASENAME\()UpdateShadowHeap_Done_\DESTREG\()_\REFREG): +.endm + +#else // WRITE_BARRIER_CHECK + +.macro UPDATE_GC_SHADOW BASENAME, DESTREG, REFREG, TEMPREG +.endm +#endif + +// There are several different helpers used depending on which register holds the object reference. Since all +// the helpers have identical structure we use a macro to define this structure. Two arguments are taken, the +// name of the register that points to the location to be updated and the name of the register that holds the +// object reference (this should be in upper case as it's used in the definition of the name of the helper). +.macro DEFINE_WRITE_BARRIER DESTREG, REFREG, TEMPREG + +// Define a helper with a name of the form RhpAssignRefEAX etc. (along with suitable calling standard +// decoration). The location to be updated is in DESTREG. The object reference that will be assigned into that +// location is in one of the other general registers determined by the value of REFREG. +LEAF_ENTRY RhpAssignRef\REFREG, _TEXT + + // Export the canonical write barrier under unqualified name as well + .ifc \REFREG, EDX + ALTERNATE_ENTRY RhpAssignRef + ALTERNATE_ENTRY RhpAssignRefAVLocation + .endif + + ALTERNATE_ENTRY RhpAssignRef\REFREG\()AVLocation + + // Write the reference into the location. Note that we rely on the fact that no GC can occur between here + // and the card table update we may perform below. + mov dword ptr [\DESTREG], \REFREG + + // Save a register so that we have an available register as a temporary for PREPARE_EXTERNAL_VAR + push \TEMPREG + + // Update the shadow copy of the heap with the same value (if enabled). + UPDATE_GC_SHADOW RhpAssignRef, \DESTREG, \REFREG, \TEMPREG + + // If the reference is to an object that's not in an ephemeral generation we have no need to track it + // (since the object won't be collected or moved by an ephemeral collection). + PREPARE_EXTERNAL_VAR g_ephemeral_low, \TEMPREG + cmp \REFREG, [\TEMPREG] + jb LOCAL_LABEL(WriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + PREPARE_EXTERNAL_VAR g_ephemeral_high, \TEMPREG + cmp \REFREG, [\TEMPREG] + jae LOCAL_LABEL(WriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + + // We have a location on the GC heap being updated with a reference to an ephemeral object so we must + // track this write. The location address is translated into an offset in the card table bitmap. We set + // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write + // the byte if it hasn't already been done since writes are expensive and impact scaling. + shr \DESTREG, 10 + PREPARE_EXTERNAL_VAR g_card_table, \TEMPREG + add \DESTREG, [\TEMPREG] + cmp byte ptr [\DESTREG], 0xFF + jne LOCAL_LABEL(WriteBarrier_UpdateCardTable_\DESTREG\()_\REFREG) + +LOCAL_LABEL(WriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG): + pop \TEMPREG + ret + +// We get here if it's necessary to update the card table. +LOCAL_LABEL(WriteBarrier_UpdateCardTable_\DESTREG\()_\REFREG): + pop \TEMPREG + mov byte ptr [\DESTREG], 0xFF + ret +LEAF_END RhpAssignRef\REFREG, _TEXT +.endm + +.macro DEFINE_CHECKED_WRITE_BARRIER_CORE BASENAME, DESTREG, REFREG, TEMPREG + + // The location being updated might not even lie in the GC heap (a handle or stack location for instance), + // in which case no write barrier is required. + PREPARE_EXTERNAL_VAR g_lowest_address, \TEMPREG + cmp \DESTREG, [\TEMPREG] + jb LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + PREPARE_EXTERNAL_VAR g_highest_address, \TEMPREG + cmp \DESTREG, [\TEMPREG] + jae LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + + // Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless + // we're in a debug build and write barrier checking has been enabled). + UPDATE_GC_SHADOW \BASENAME, \DESTREG, \REFREG, \TEMPREG + + // If the reference is to an object that's not in an ephemeral generation we have no need to track it + // (since the object won't be collected or moved by an ephemeral collection). + PREPARE_EXTERNAL_VAR g_ephemeral_low, \TEMPREG + cmp \REFREG, [\TEMPREG] + jb LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + PREPARE_EXTERNAL_VAR g_ephemeral_high, \TEMPREG + cmp \REFREG, [\TEMPREG] + jae LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + + // We have a location on the GC heap being updated with a reference to an ephemeral object so we must + // track this write. The location address is translated into an offset in the card table bitmap. We set + // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write + // the byte if it hasn't already been done since writes are expensive and impact scaling. + shr \DESTREG, 10 + PREPARE_EXTERNAL_VAR g_card_table, \TEMPREG + add \DESTREG, [\TEMPREG] + cmp byte ptr [\DESTREG], 0xFF + je LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG) + // We get here if it's necessary to update the card table. + mov byte ptr [\DESTREG], 0xFF +LOCAL_LABEL(\BASENAME\()CheckedWriteBarrier_NoBarrierRequired_\DESTREG\()_\REFREG): +.endm + +// This macro is very much like the one above except that it generates a variant of the function which also +// checks whether the destination is actually somewhere within the GC heap. +.macro DEFINE_CHECKED_WRITE_BARRIER DESTREG, REFREG, TEMPREG + +// Define a helper with a name of the form RhpCheckedAssignRefEAX etc. (along with suitable calling standard +// decoration). The location to be updated is in DESTREG. The object reference that will be assigned into +// that location is in one of the other general registers determined by the value of REFREG. + +// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: +// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen on the first instruction +// - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address +LEAF_ENTRY RhpCheckedAssignRef\REFREG, _TEXT + + // Export the canonical write barrier under unqualified name as well + .ifc \REFREG, EDX + ALTERNATE_ENTRY RhpCheckedAssignRef + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation + .endif + + ALTERNATE_ENTRY RhpCheckedAssignRef\REFREG\()AVLocation + + // Write the reference into the location. Note that we rely on the fact that no GC can occur between here + // and the card table update we may perform below. + mov dword ptr [\DESTREG], \REFREG + + // Save a register so that we have an available register as a temporary for PREPARE_EXTERNAL_VAR + push \TEMPREG + + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedAssignRef, \DESTREG, \REFREG, \TEMPREG + pop \TEMPREG + ret + +LEAF_END RhpCheckedAssignRef\REFREG, _TEXT + +.endm + +// One day we might have write barriers for all the possible argument registers but for now we have +// just one write barrier that assumes the input register is EDX. +DEFINE_CHECKED_WRITE_BARRIER ECX, EDX, EAX +DEFINE_WRITE_BARRIER ECX, EDX, EAX + +// Need some more write barriers to run CLR compiled MDIL on Redhawk - commented out for now +DEFINE_WRITE_BARRIER EDX, EAX, ECX +DEFINE_WRITE_BARRIER EDX, ECX, EAX +DEFINE_WRITE_BARRIER EDX, EBX, EAX +DEFINE_WRITE_BARRIER EDX, ESI, EAX +DEFINE_WRITE_BARRIER EDX, EDI, EAX +DEFINE_WRITE_BARRIER EDX, EBP, EAX + +DEFINE_CHECKED_WRITE_BARRIER EDX, EAX, ECX +DEFINE_CHECKED_WRITE_BARRIER EDX, ECX, EAX +DEFINE_CHECKED_WRITE_BARRIER EDX, EBX, EAX +DEFINE_CHECKED_WRITE_BARRIER EDX, ESI, EAX +DEFINE_CHECKED_WRITE_BARRIER EDX, EDI, EAX +DEFINE_CHECKED_WRITE_BARRIER EDX, EBP, EAX + +LEAF_ENTRY RhpCheckedLockCmpXchg, _TEXT + mov eax, [esp+4] + lock cmpxchg [ecx], edx + jne LOCAL_LABEL(RhpCheckedLockCmpXchg_NoWrite) + push eax + + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, ECX, EDX, EAX + pop eax +LOCAL_LABEL(RhpCheckedLockCmpXchg_NoWrite): + ret 4 +LEAF_END RhpCheckedLockCmpXchg, _TEXT + +LEAF_ENTRY RhpCheckedXchg, _TEXT + + // Setup eax with the new object for the exchange, that way it will automatically hold the correct result + // afterwards and we can leave edx unaltered ready for the GC write barrier below. + mov eax, edx + xchg [ecx], eax + push eax + + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedXchg, ECX, EDX, EAX + pop eax + ret +LEAF_END RhpCheckedXchg, _TEXT + +// +// RhpByRefAssignRef simulates movs instruction for object references. +// +// On entry: +// edi: address of ref-field (assigned to) +// esi: address of the data (source) +// +// On exit: +// edi, esi are incremented by 4, +// ecx: trashed +// +LEAF_ENTRY RhpByRefAssignRef, _TEXT +ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 + mov ecx, [esi] +ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 + mov [edi], ecx + + push eax + + // Check whether the writes were even into the heap. If not there's no card update required. + PREPARE_EXTERNAL_VAR g_lowest_address, eax + cmp edi, [eax] + jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + PREPARE_EXTERNAL_VAR g_highest_address, eax + cmp edi, [eax] + jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + + UPDATE_GC_SHADOW RhpByRefAssignRef, ecx, edi, eax + + // If the reference is to an object that's not in an ephemeral generation we have no need to track it + // (since the object won't be collected or moved by an ephemeral collection). + PREPARE_EXTERNAL_VAR g_ephemeral_low, eax + cmp ecx, [eax] + jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + PREPARE_EXTERNAL_VAR g_ephemeral_high, eax + cmp ecx, [eax] + jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + + mov ecx, edi + shr ecx, 10 + PREPARE_EXTERNAL_VAR g_card_table, eax + add ecx, [eax] + cmp byte ptr [ecx], 0xFF + je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) + + mov byte ptr [ecx], 0xFF + +LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired): + // Increment the pointers before leaving + add esi,4 + add edi,4 + pop eax + ret +LEAF_END RhpByRefAssignRef, _TEXT diff --git a/src/coreclr/vm/i386/AsmMacros_Shared.h b/src/coreclr/vm/i386/AsmMacros_Shared.h new file mode 100644 index 00000000000000..87920d58b2ac65 --- /dev/null +++ b/src/coreclr/vm/i386/AsmMacros_Shared.h @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible + +#include "unixasmmacros.inc" +#include "asmconstants.h" + From 89422b634b6a03950cc40cec3b6b014430102783 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 17:01:52 -0700 Subject: [PATCH 26/34] Fix arm build --- src/coreclr/runtime/arm/WriteBarriers.S | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index 60504685050059..869aa40f0437cf 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -25,15 +25,15 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP .macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg, __wbScratch - PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, __wbScratch - cbz __wbScratch, 2f - add __wbScratch, __wbScratch, \ptrReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift + PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, \__wbScratch + cbz \__wbScratch, 2f + add \__wbScratch, \__wbScratch, \ptrReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift - ldrb \tmpReg, [__wbScratch] + ldrb \tmpReg, [\__wbScratch] cmp \tmpReg, #0xff itt ne movne \tmpReg, 0xff - strbne \tmpReg, [__wbScratch] + strbne \tmpReg, [\__wbScratch] 2: .endm @@ -62,7 +62,7 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): cmp \REFREG, r12 bhs LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG) - UPDATE_WRITE_WATCH_TABLE r0, r12, TMPREG + UPDATE_WRITE_WATCH_TABLE r0, r12, \TMPREG // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set @@ -153,7 +153,7 @@ DEFINE_UNCHECKED_WRITE_BARRIER r1, r1 cmp r0, r12 bhs LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG) - DEFINE_UNCHECKED_WRITE_BARRIER_CORE \BASENAME, \REFREG, TMPREG + DEFINE_UNCHECKED_WRITE_BARRIER_CORE \BASENAME, \REFREG, \TMPREG .endm @@ -301,7 +301,11 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2 add r1, #4 add r0, #4 - UPDATE_WRITE_WATCH_TABLE r2, r3, r12 +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + mov r12, r0 + UPDATE_WRITE_WATCH_TABLE r2, r3, r0 + mov r0, r12 +#endif // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set From fd2cc83972a0f9aa46ebd3b3a80723fd169527b2 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 17:10:22 -0700 Subject: [PATCH 27/34] Attempt to workaround build failures on apple platforms --- src/coreclr/runtime/amd64/StubDispatch.S | 9 +++++++++ src/coreclr/runtime/amd64/WriteBarriers.S | 8 ++++++++ src/coreclr/runtime/arm64/StubDispatch.S | 8 ++++++++ src/coreclr/runtime/arm64/WriteBarriers.S | 8 ++++++++ 4 files changed, 33 insertions(+) diff --git a/src/coreclr/runtime/amd64/StubDispatch.S b/src/coreclr/runtime/amd64/StubDispatch.S index 85bc078dcffa99..2fbbcf80ec6a8e 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.S +++ b/src/coreclr/runtime/amd64/StubDispatch.S @@ -6,6 +6,15 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH +#ifdef defined(__APPLE__) + // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. + // Fix, but adding an empty, unused method + LEAF_ENTRY RhpStubDispatchDoNotFailToBuild, _TEXT + ret + LEAF_END RhpStubDispatchDoNotFailToBuild, _TEXT +#endif + + // trick to avoid PLT relocation at runtime which corrupts registers #define REL_C_FUNC(name) C_FUNC(name)@gotpcrel diff --git a/src/coreclr/runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S index 93658c4ae0681d..124701059dc167 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -4,6 +4,14 @@ .intel_syntax noprefix #include "AsmMacros_Shared.h" +#ifdef defined(__APPLE__) + // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. + // Fix, but adding an empty, unused method + LEAF_ENTRY RhpWriteBarriersDoNotFailToBuild, _TEXT + ret + LEAF_END RhpWriteBarriersDoNotFailToBuild, _TEXT +#endif + #ifdef WRITE_BARRIER_CHECK .macro UPDATE_GC_SHADOW BASENAME, REFREG, DESTREG diff --git a/src/coreclr/runtime/arm64/StubDispatch.S b/src/coreclr/runtime/arm64/StubDispatch.S index 198a93b2456c0c..766212d1321cef 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.S +++ b/src/coreclr/runtime/arm64/StubDispatch.S @@ -5,6 +5,14 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH +#ifdef defined(__APPLE__) + // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. + // Fix, but adding an empty, unused method + LEAF_ENTRY RhpStubDispatchDoNotFailToBuild, _TEXT + ret + LEAF_END RhpStubDispatchDoNotFailToBuild, _TEXT +#endif + // Macro that generates code to check a single cache entry. .macro CHECK_CACHE_ENTRY entry // Check a single entry in the cache. diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index dd3d2bb6f78f67..f938e3053c5a35 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -3,6 +3,14 @@ #include "AsmMacros_Shared.h" +#ifdef defined(__APPLE__) + // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. + // Fix, but adding an empty, unused method + LEAF_ENTRY RhpWriteBarriersDoNotFailToBuild, _TEXT + ret + LEAF_END RhpWriteBarriersDoNotFailToBuild, _TEXT +#endif + // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used // during garbage collections to verify that object references where never written to the heap without using a // write barrier. Note that we are potentially racing to update the shadow heap while other threads are writing From 4dd7420826b8eaf5adaf4ac685e87ff0a93d15fc Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 17:31:15 -0700 Subject: [PATCH 28/34] Sigh... --- src/coreclr/runtime/amd64/StubDispatch.S | 2 +- src/coreclr/runtime/amd64/WriteBarriers.S | 2 +- src/coreclr/runtime/arm64/StubDispatch.S | 2 +- src/coreclr/runtime/arm64/WriteBarriers.S | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/runtime/amd64/StubDispatch.S b/src/coreclr/runtime/amd64/StubDispatch.S index 2fbbcf80ec6a8e..771ad879bd2113 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.S +++ b/src/coreclr/runtime/amd64/StubDispatch.S @@ -6,7 +6,7 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH -#ifdef defined(__APPLE__) +#if defined(__APPLE__) // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. // Fix, but adding an empty, unused method LEAF_ENTRY RhpStubDispatchDoNotFailToBuild, _TEXT diff --git a/src/coreclr/runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S index 124701059dc167..8f61c9ce261995 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -4,7 +4,7 @@ .intel_syntax noprefix #include "AsmMacros_Shared.h" -#ifdef defined(__APPLE__) +#if defined(__APPLE__) // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. // Fix, but adding an empty, unused method LEAF_ENTRY RhpWriteBarriersDoNotFailToBuild, _TEXT diff --git a/src/coreclr/runtime/arm64/StubDispatch.S b/src/coreclr/runtime/arm64/StubDispatch.S index 766212d1321cef..bb935b9f602ca2 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.S +++ b/src/coreclr/runtime/arm64/StubDispatch.S @@ -5,7 +5,7 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH -#ifdef defined(__APPLE__) +#if defined(__APPLE__) // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. // Fix, but adding an empty, unused method LEAF_ENTRY RhpStubDispatchDoNotFailToBuild, _TEXT diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index f938e3053c5a35..bdc6f2306ad2cd 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -3,7 +3,7 @@ #include "AsmMacros_Shared.h" -#ifdef defined(__APPLE__) +#if defined(__APPLE__) // Currently the build is failing without this due to an issue if the first method in the assembly file has an alternate entry at the start of the file. // Fix, but adding an empty, unused method LEAF_ENTRY RhpWriteBarriersDoNotFailToBuild, _TEXT From 5ab9e76770b9e1b27a87bf531ee16c359da04a43 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 May 2025 21:00:35 -0700 Subject: [PATCH 29/34] Fix more build breaks --- src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S | 2 +- src/coreclr/vm/excep.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S index bae65e1c659cff..b9df18790bb5ae 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S @@ -12,7 +12,7 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT #ifdef TARGET_APPLE // place all ALTERNATE_ENTRY for arm64 at the beginning of a method to make unwinding work ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation - cmp [rdi], 0 + cmp qword ptr [rdi], 0 #endif // r11 currently contains the indirection cell address. // load r11 to point to the vtable offset (which is stored in the m_pCache field). diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 22578e7709b717..6d6f0b43c5627d 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5894,6 +5894,7 @@ BOOL IsIPinVirtualStub(PCODE f_IP) return FALSE; #endif // FEATURE_VIRTUAL_STUB_DISPATCH } +#endif // FEATURE_EH_FUNCLETS typedef uint8_t CODE_LOCATION; EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; @@ -5992,7 +5993,6 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) return false; } -#endif // FEATURE_EH_FUNCLETS // Returns TRUE if caller should resume execution. BOOL From 9467e257b2c11ecb25cd86f52f5a4a6930f5cfef Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 2 May 2025 20:26:40 +0000 Subject: [PATCH 30/34] Put the write watch table update in the right spot --- src/coreclr/runtime/arm/WriteBarriers.S | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/coreclr/runtime/arm/WriteBarriers.S b/src/coreclr/runtime/arm/WriteBarriers.S index 869aa40f0437cf..30afebc760d0a1 100644 --- a/src/coreclr/runtime/arm/WriteBarriers.S +++ b/src/coreclr/runtime/arm/WriteBarriers.S @@ -52,6 +52,8 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): // we're in a debug build and write barrier checking has been enabled). UPDATE_GC_SHADOW \BASENAME, \REFREG, r0 + UPDATE_WRITE_WATCH_TABLE r0, r12, \TMPREG + // If the reference is to an object that's not in an ephemeral generation we have no need to track it // (since the object won't be collected or moved by an ephemeral collection). PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, r12 @@ -62,7 +64,6 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG): cmp \REFREG, r12 bhs LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG) - UPDATE_WRITE_WATCH_TABLE r0, r12, \TMPREG // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set @@ -287,6 +288,8 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2 // we're in a debug build and write barrier checking has been enabled). UPDATE_GC_SHADOW BASENAME, r2, r0 + UPDATE_WRITE_WATCH_TABLE r0, r12, r3 + // If the reference is to an object that's not in an ephemeral generation we have no need to track it // (since the object won't be collected or moved by an ephemeral collection). PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, r3 @@ -301,12 +304,6 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2 add r1, #4 add r0, #4 -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - mov r12, r0 - UPDATE_WRITE_WATCH_TABLE r2, r3, r0 - mov r0, r12 -#endif - // We have a location on the GC heap being updated with a reference to an ephemeral object so we must // track this write. The location address is translated into an offset in the card table bitmap. We set // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write From 4e1ee538bd1a42f2507a0b33d1ade66f77ceddee Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Mon, 5 May 2025 11:28:39 -0700 Subject: [PATCH 31/34] Set UseGCWriteBarrierCopy to its production value --- src/coreclr/inc/clrconfigvalues.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 0ca1675efaf771..2002292dc1b2e0 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -254,7 +254,7 @@ CONFIG_DWORD_INFO(INTERNAL_FastGCCheckStack, W("FastGCCheckStack"), 0, "") CONFIG_DWORD_INFO(INTERNAL_FastGCStress, W("FastGCStress"), 0, "Reduce the number of GCs done by enabling GCStress") RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_GCBreakOnOOM, W("GCBreakOnOOM"), 0, "Does a DebugBreak at the soonest time we detect an OOM") RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConcurrent, W("gcConcurrent"), (DWORD)-1, "Enables/Disables concurrent GC") -RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_UseGCWriteBarrierCopy, W("UseGCWriteBarrierCopy"), 0, "Use a copy of the write barrier for the GC. This is somewhat faster and for optimizations where the barrier is mutated as the program runs.") +RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_UseGCWriteBarrierCopy, W("UseGCWriteBarrierCopy"), 1, "Use a copy of the write barrier for the GC. This is somewhat faster and for optimizations where the barrier is mutated as the program runs. Setting this to 0 removes scenarios where the write barrier is ever mutable.") #ifdef FEATURE_CONSERVATIVE_GC RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConservative, W("gcConservative"), 0, "Enables/Disables conservative GC") From 6ca24797cb62a130d16f448a4032f51d1b4d1348 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Mon, 5 May 2025 15:56:14 -0700 Subject: [PATCH 32/34] Fix Windows X86 issue where the we failed to stack walk out of a write barrier when using the old write barriers --- src/coreclr/vm/excep.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 9808d0bf72c02e..d4ac41cf5a28c5 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5981,10 +5981,8 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) CHECK_RANGE(JIT_StackProbe) #endif // !TARGET_ARM64 && !TARGET_LOONGARCH64 && !TARGET_RISCV64 #else -#ifdef TARGET_UNIX CHECK_RANGE(JIT_WriteBarrierGroup) CHECK_RANGE(JIT_PatchedWriteBarrierGroup) -#endif // TARGET_UNIX #endif // TARGET_X86 #if defined(TARGET_AMD64) && defined(_DEBUG) From 435c220628f92d0d7b1712d0f2f08569a196c67f Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 6 May 2025 13:44:27 -0700 Subject: [PATCH 33/34] Code review feedback --- src/coreclr/vm/excep.cpp | 22 +++++++++------------- src/coreclr/vm/excep.h | 2 +- src/coreclr/vm/threads.cpp | 2 +- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index d4ac41cf5a28c5..2ae2a21a942f3e 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -5949,7 +5949,7 @@ static uintptr_t writeBarrierAVLocations[] = // Check if the passed in instruction pointer is in one of the // JIT helper functions. -bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) +bool IsIPInMarkedJitHelper(PCODE uControlPc) { LIMITED_METHOD_CONTRACT; @@ -5962,11 +5962,7 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) ASSERT(*(uint8_t*)writeBarrierAVLocations[i] != 0xE9); // jmp XXXXXXXX #endif -#ifdef TARGET_ARM - if ((writeBarrierAVLocations[i] | THUMB_CODE) == (uControlPc | THUMB_CODE)) -#else - if (writeBarrierAVLocations[i] == uControlPc) -#endif + if (writeBarrierAVLocations[i] == PCODEToPINSTR(uControlPc)) return true; } @@ -6020,7 +6016,7 @@ AdjustContextForJITHelpers( if (pExceptionRecord == nullptr) { #if defined(TARGET_X86) - if (IsIPInMarkedJitHelper((UINT_PTR)ip)) + if (IsIPInMarkedJitHelper(ip)) { DWORD* esp = (DWORD*)pContext->Esp; #if defined(WRITE_BARRIER_CHECK) @@ -6040,7 +6036,7 @@ AdjustContextForJITHelpers( return TRUE; } #elif defined(TARGET_AMD64) - if (IsIPInMarkedJitHelper((UINT_PTR)ip)) + if (IsIPInMarkedJitHelper(ip)) { Thread::VirtualUnwindToFirstManagedCallFrame(pContext); return TRUE; @@ -6054,9 +6050,9 @@ AdjustContextForJITHelpers( #endif // FEATURE_DATABREAKPOINT #if defined(TARGET_X86) && !defined(TARGET_UNIX) - void* f_IP = (void *)GetIP(pContext); + PCODE f_IP = GetIP(pContext); - if (IsIPInMarkedJitHelper((UINT_PTR)f_IP)) + if (IsIPInMarkedJitHelper(ip)) { // set the exception IP to be the instruction that called the write barrier void* callsite = (void *)GetAdjustedCallAddress(*dac_cast(GetSP(pContext))); @@ -6067,7 +6063,7 @@ AdjustContextForJITHelpers( SetSP(pContext, PCODE((BYTE*)GetSP(pContext) + sizeof(void*))); } - if ((f_IP >= (void *) JIT_StackProbe) && (f_IP <= (void *) JIT_StackProbe_End)) + if ((f_IP >= (PCODE) JIT_StackProbe) && (f_IP <= (PCODE) JIT_StackProbe_End)) { TADDR ebp = GetFP(pContext); void* callsite = (void *)*dac_cast(ebp + 4); @@ -6081,12 +6077,12 @@ AdjustContextForJITHelpers( return FALSE; #elif defined(FEATURE_EH_FUNCLETS) // TARGET_X86 && !TARGET_UNIX - void* f_IP = dac_cast(GetIP(pContext)); + PCODE f_IP = GetIP(pContext); CONTEXT tempContext; CONTEXT* pExceptionContext = pContext; - BOOL fExcluded = IsIPInMarkedJitHelper((UINT_PTR)f_IP); + BOOL fExcluded = IsIPInMarkedJitHelper(f_IP); if (fExcluded) { diff --git a/src/coreclr/vm/excep.h b/src/coreclr/vm/excep.h index d3235b8c2e5c98..e1d57765128b77 100644 --- a/src/coreclr/vm/excep.h +++ b/src/coreclr/vm/excep.h @@ -23,7 +23,7 @@ class Thread; BOOL IsExceptionFromManagedCode(const EXCEPTION_RECORD * pExceptionRecord); BOOL IsIPinVirtualStub(PCODE f_IP); -bool IsIPInMarkedJitHelper(UINT_PTR uControlPc); +bool IsIPInMarkedJitHelper(PCODE uControlPc); BOOL IsProcessCorruptedStateException(DWORD dwExceptionCode, OBJECTREF throwable); diff --git a/src/coreclr/vm/threads.cpp b/src/coreclr/vm/threads.cpp index 00cbbd42cff273..f183b0c90b76fb 100644 --- a/src/coreclr/vm/threads.cpp +++ b/src/coreclr/vm/threads.cpp @@ -1133,7 +1133,7 @@ void InitThreadManager() #endif // TARGET_ARM64 || TARGET_ARM || TARGET_LOONGARCH64 || TARGET_RISCV64 #if defined(TARGET_AMD64) - // On AMD64 the Checked/ByRef variants of the helpers jump througn an indirection + // On AMD64 the Checked/ByRef variants of the helpers jump through an indirection // to the patched barrier, but are not part of the patched set of helpers. SetJitHelperFunction(CORINFO_HELP_CHECKED_ASSIGN_REF, (void*)JIT_CheckedWriteBarrier); SetJitHelperFunction(CORINFO_HELP_ASSIGN_BYREF, (void*)JIT_ByRefWriteBarrier); From 2950b1f10e1e497be665e611448471b9310bb8ef Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 13 May 2025 09:59:22 -0700 Subject: [PATCH 34/34] Try the cfi_startproc/cfi_endproc wrappers for ALTERNATE_ENTRY to see if the Apple linker bug can be fixed that way --- src/coreclr/runtime/amd64/StubDispatch.S | 16 +++++++++++---- src/coreclr/runtime/amd64/WriteBarriers.S | 14 +++++-------- src/coreclr/runtime/arm64/StubDispatch.S | 14 ++++++++++--- src/coreclr/runtime/arm64/WriteBarriers.S | 20 +++++++------------ .../vm/amd64/CachedInterfaceDispatchCoreCLR.S | 12 ++++++----- 5 files changed, 42 insertions(+), 34 deletions(-) diff --git a/src/coreclr/runtime/amd64/StubDispatch.S b/src/coreclr/runtime/amd64/StubDispatch.S index 771ad879bd2113..0ae32980c5a9b2 100644 --- a/src/coreclr/runtime/amd64/StubDispatch.S +++ b/src/coreclr/runtime/amd64/StubDispatch.S @@ -24,14 +24,22 @@ LEAF_ENTRY RhpInterfaceDispatch\entries, _TEXT - // Load the MethodTable from the object instance in rdi. - ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries - mov rax, [rdi] - // r11 currently contains the indirection cell address. // load r10 to point to the cache block. mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] + // Load the MethodTable from the object instance in rdi. +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif + ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries +#ifdef TARGET_APPLE +.cfi_startproc +#endif + mov rax, [rdi] + CurrentOffset = OFFSETOF__InterfaceDispatchCache__m_rgEntries // For each entry in the cache, see if its MethodTable type matches the MethodTable in rax. diff --git a/src/coreclr/runtime/amd64/WriteBarriers.S b/src/coreclr/runtime/amd64/WriteBarriers.S index 8f61c9ce261995..c15ee30e5dffea 100644 --- a/src/coreclr/runtime/amd64/WriteBarriers.S +++ b/src/coreclr/runtime/amd64/WriteBarriers.S @@ -271,13 +271,13 @@ ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1 mov rcx, [rsi] #ifdef TARGET_APPLE // Apple's linker has issues which break unwind info if -// an ALTERNATE_ENTRY is present in the middle of a function - jmp C_FUNC(RhpByRefAssignRef2) -LEAF_END RhpByRefAssignRef, _TEXT - -LEAF_ENTRY RhpByRefAssignRef2, _TEXT +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc #endif ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 +#ifdef TARGET_APPLE +.cfi_startproc +#endif mov [rdi], rcx // Check whether the writes were even into the heap. If not there's no card update required. @@ -341,8 +341,4 @@ LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired): add rdi, 0x8 add rsi, 0x8 ret -#ifdef TARGET_APPLE -LEAF_END RhpByRefAssignRef2, _TEXT -#else LEAF_END RhpByRefAssignRef, _TEXT -#endif diff --git a/src/coreclr/runtime/arm64/StubDispatch.S b/src/coreclr/runtime/arm64/StubDispatch.S index bb935b9f602ca2..750a99db42638a 100644 --- a/src/coreclr/runtime/arm64/StubDispatch.S +++ b/src/coreclr/runtime/arm64/StubDispatch.S @@ -35,13 +35,21 @@ NESTED_ENTRY "RhpInterfaceDispatch\entries", _TEXT, NoHandler + // x11 holds the indirection cell address. Load the cache pointer. + ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] + // Load the MethodTable from the object instance in x0. +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif ALTERNATE_ENTRY RhpInterfaceDispatchAVLocation\entries +#ifdef TARGET_APPLE +.cfi_startproc +#endif ldr x10, [x0] - // x11 holds the indirection cell address. Load the cache pointer. - ldr x9, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] - .global CurrentEntry .set CurrentEntry, 0 diff --git a/src/coreclr/runtime/arm64/WriteBarriers.S b/src/coreclr/runtime/arm64/WriteBarriers.S index bdc6f2306ad2cd..8087b290f00272 100644 --- a/src/coreclr/runtime/arm64/WriteBarriers.S +++ b/src/coreclr/runtime/arm64/WriteBarriers.S @@ -237,24 +237,18 @@ LEAF_END RhpByRefAssignRefArm64, _TEXT b C_FUNC(RhpAssignRefArm64) LOCAL_LABEL(NotInHeap): -#ifdef TARGET_APPLE - b C_FUNC(RhpCheckedAssignRefArm64_NotInHeap) -#else - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation - str x15, [x14], 8 - ret -#endif -LEAF_END RhpCheckedAssignRefArm64, _TEXT - #ifdef TARGET_APPLE // Apple's linker has issues which break unwind info if -// an ALTERNATE_ENTRY is present in the middle of a function -LEAF_ENTRY RhpCheckedAssignRefArm64_NotInHeap, _TEXT +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation +#ifdef TARGET_APPLE +.cfi_startproc +#endif str x15, [x14], 8 ret -LEAF_END RhpCheckedAssignRefArm64_NotInHeap, _TEXT -#endif +LEAF_END RhpCheckedAssignRefArm64, _TEXT // JIT_WriteBarrier(Object** dst, Object* src) // diff --git a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S index b9df18790bb5ae..bc0edaccea0567 100644 --- a/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S +++ b/src/coreclr/vm/amd64/CachedInterfaceDispatchCoreCLR.S @@ -10,10 +10,6 @@ // Stub dispatch routine for dispatch to a vtable slot LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT -#ifdef TARGET_APPLE // place all ALTERNATE_ENTRY for arm64 at the beginning of a method to make unwinding work - ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation - cmp qword ptr [rdi], 0 -#endif // r11 currently contains the indirection cell address. // load r11 to point to the vtable offset (which is stored in the m_pCache field). mov r11, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] @@ -25,8 +21,14 @@ LEAF_ENTRY RhpVTableOffsetDispatch, _TEXT // Load the MethodTable from the object instance in rdi, and add it to the vtable offset // to get the address in the vtable chunk list of what we want to dereference -#ifndef TARGET_APPLE +#ifdef TARGET_APPLE +// Apple's linker has issues which break unwind info if +// an ALTERNATE_ENTRY is present in the middle of a function see https://github.com/dotnet/runtime/pull/114982#discussion_r2083272768 +.cfi_endproc +#endif ALTERNATE_ENTRY RhpVTableOffsetDispatchAVLocation +#ifdef TARGET_APPLE +.cfi_startproc #endif add rax, [rdi]