Skip to content
This repository has been archived by the owner on Jan 23, 2023. It is now read-only.

Commit

Permalink
StompWriteBarrier initialization path refactoring (#14105)
Browse files Browse the repository at this point in the history 
* refactored arm, arm64, amd64 and x86 to signal about icache flush and ee restarts

* refactored gc init stage to stomp write barrier (hence flush icache) only once

* review fixes, care taken of icache invalidation during StompResize

* fixed heap boundaries initialization bug introduced after refactoring gc.cpp

* stylistic review fixe

* global variable rename

* global variable rename once more
  • Loading branch information
@Rattenkrieg @swgillespie
Rattenkrieg authored and swgillespie committed Sep 29, 2017
1 parent 32c4673 commit 5b10f0e
Show file tree
Hide file tree
Showing 8 changed files with 169 additions and 122 deletions.
21 changes: 15 additions & 6 deletions src/gc/gc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2327,7 +2327,7 @@ void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_hi
GCToEEInterface::StompWriteBarrier(&args);
}

void stomp_write_barrier_initialize()
void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
{
WriteBarrierParameters args = {};
args.operation = WriteBarrierOp::Initialize;
Expand All @@ -2341,8 +2341,8 @@ void stomp_write_barrier_initialize()

args.lowest_address = g_gc_lowest_address;
args.highest_address = g_gc_highest_address;
args.ephemeral_low = reinterpret_cast<uint8_t*>(1);
args.ephemeral_high = reinterpret_cast<uint8_t*>(~0);
args.ephemeral_low = ephemeral_low;
args.ephemeral_high = ephemeral_high;
GCToEEInterface::StompWriteBarrier(&args);
}

Expand Down Expand Up @@ -10602,7 +10602,18 @@ gc_heap::init_gc_heap (int h_number)
make_background_mark_stack (b_arr);
#endif //BACKGROUND_GC

adjust_ephemeral_limits();
ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
if (heap_number == 0)
{
stomp_write_barrier_initialize(
#ifdef MULTIPLE_HEAPS
reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
#else
ephemeral_low, ephemeral_high
#endif //!MULTIPLE_HEAPS
);
}

#ifdef MARK_ARRAY
// why would we clear the mark array for this page? it should be cleared..
Expand Down Expand Up @@ -33575,8 +33586,6 @@ HRESULT GCHeap::Initialize ()
return E_FAIL;
}

stomp_write_barrier_initialize();

#ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
#if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
if (GCStress<cfg_any>::IsEnabled()) {
Expand Down
92 changes: 43 additions & 49 deletions src/vm/amd64/jitinterfaceamd64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,14 +278,14 @@ PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int
return ((LPBYTE)GetEEFuncEntryPoint(JIT_WriteBarrier) + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base) + offset));
}

void WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended)
int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended)
{
GCX_MAYBE_COOP_NO_THREAD_BROKEN((!isRuntimeSuspended && GetThread() != NULL));
BOOL bEESuspendedHere = FALSE;
if(!isRuntimeSuspended && m_currentWriteBarrier != WRITE_BARRIER_UNINITIALIZED)
int stompWBCompleteActions = SWB_PASS;
if (!isRuntimeSuspended && m_currentWriteBarrier != WRITE_BARRIER_UNINITIALIZED)
{
ThreadSuspend::SuspendEE(ThreadSuspend::SUSPEND_FOR_GC_PREP);
bEESuspendedHere = TRUE;
stompWBCompleteActions |= SWB_EE_RESTART;
}

_ASSERTE(m_currentWriteBarrier != newWriteBarrier);
Expand Down Expand Up @@ -409,13 +409,10 @@ void WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier,
UNREACHABLE_MSG("unexpected write barrier type!");
}

UpdateEphemeralBounds(true);
UpdateWriteWatchAndCardTableLocations(true, false);
stompWBCompleteActions |= UpdateEphemeralBounds(true);
stompWBCompleteActions |= UpdateWriteWatchAndCardTableLocations(true, false);

if(bEESuspendedHere)
{
ThreadSuspend::RestartEE(FALSE, TRUE);
}
return stompWBCompleteActions;
}

#undef CALC_PATCH_LOCATION
Expand Down Expand Up @@ -521,21 +518,20 @@ bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, W
return m_currentWriteBarrier != writeBarrierType;
}

void WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
{
bool needToFlushCache = false;

WriteBarrierType newType;
if (NeedDifferentWriteBarrier(false, &newType))
{
ChangeWriteBarrierTo(newType, isRuntimeSuspended);
return;
return ChangeWriteBarrierTo(newType, isRuntimeSuspended);
}

int stompWBCompleteActions = SWB_PASS;

#ifdef _DEBUG
// Using debug-only write barrier?
if (m_currentWriteBarrier == WRITE_BARRIER_UNINITIALIZED)
return;
return stompWBCompleteActions;
#endif

switch (m_currentWriteBarrier)
Expand All @@ -549,7 +545,7 @@ void WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
if (*(UINT64*)m_pUpperBoundImmediate != (size_t)g_ephemeral_high)
{
*(UINT64*)m_pUpperBoundImmediate = (size_t)g_ephemeral_high;
needToFlushCache = true;
stompWBCompleteActions |= SWB_ICACHE_FLUSH;
}
}
//
Expand All @@ -564,7 +560,7 @@ void WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
if (*(UINT64*)m_pLowerBoundImmediate != (size_t)g_ephemeral_low)
{
*(UINT64*)m_pLowerBoundImmediate = (size_t)g_ephemeral_low;
needToFlushCache = true;
stompWBCompleteActions |= SWB_ICACHE_FLUSH;
}
break;
}
Expand All @@ -583,31 +579,27 @@ void WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
UNREACHABLE_MSG("unexpected m_currentWriteBarrier in UpdateEphemeralBounds");
}

if (needToFlushCache)
{
FlushInstructionCache(GetCurrentProcess(), (PVOID)JIT_WriteBarrier, GetCurrentWriteBarrierSize());
}
return stompWBCompleteActions;
}

void WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
{
// If we are told that we require an upper bounds check (GC did some heap reshuffling),
// we need to switch to the WriteBarrier_PostGrow function for good.

WriteBarrierType newType;
if (NeedDifferentWriteBarrier(bReqUpperBoundsCheck, &newType))
{
ChangeWriteBarrierTo(newType, isRuntimeSuspended);
return;
return ChangeWriteBarrierTo(newType, isRuntimeSuspended);
}

int stompWBCompleteActions = SWB_PASS;

#ifdef _DEBUG
// Using debug-only write barrier?
if (m_currentWriteBarrier == WRITE_BARRIER_UNINITIALIZED)
return;
return stompWBCompleteActions;
#endif

bool fFlushCache = false;

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
switch (m_currentWriteBarrier)
Expand All @@ -620,7 +612,7 @@ void WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSu
if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table)
{
*(UINT64*)m_pWriteWatchTableImmediate = (size_t)g_sw_ww_table;
fFlushCache = true;
stompWBCompleteActions |= SWB_ICACHE_FLUSH;
}
break;

Expand All @@ -632,32 +624,29 @@ void WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSu
if (*(UINT64*)m_pCardTableImmediate != (size_t)g_card_table)
{
*(UINT64*)m_pCardTableImmediate = (size_t)g_card_table;
fFlushCache = true;
stompWBCompleteActions |= SWB_ICACHE_FLUSH;
}

#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
if (*(UINT64*)m_pCardBundleTableImmediate != (size_t)g_card_bundle_table)
{
*(UINT64*)m_pCardBundleTableImmediate = (size_t)g_card_bundle_table;
fFlushCache = true;
stompWBCompleteActions |= SWB_ICACHE_FLUSH;
}
#endif

if (fFlushCache)
{
FlushInstructionCache(GetCurrentProcess(), (LPVOID)JIT_WriteBarrier, GetCurrentWriteBarrierSize());
}
return stompWBCompleteActions;
}

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
void WriteBarrierManager::SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
int WriteBarrierManager::SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
{
WriteBarrierType newWriteBarrierType;
switch (m_currentWriteBarrier)
{
case WRITE_BARRIER_UNINITIALIZED:
// Using the debug-only write barrier
return;
return SWB_PASS;

case WRITE_BARRIER_PREGROW64:
newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_PREGROW64;
Expand All @@ -677,17 +666,17 @@ void WriteBarrierManager::SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
UNREACHABLE();
}

ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended);
return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended);
}

void WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
{
WriteBarrierType newWriteBarrierType;
switch (m_currentWriteBarrier)
{
case WRITE_BARRIER_UNINITIALIZED:
// Using the debug-only write barrier
return;
return SWB_PASS;

case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
newWriteBarrierType = WRITE_BARRIER_PREGROW64;
Expand All @@ -707,42 +696,47 @@ void WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
UNREACHABLE();
}

ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended);
return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended);
}
#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP

// This function bashes the super fast amd64 version of the JIT_WriteBarrier
// helper. It should be called by the GC whenever the ephermeral region
// bounds get changed, but still remain on the top of the GC Heap.
void StompWriteBarrierEphemeral(bool isRuntimeSuspended)
int StompWriteBarrierEphemeral(bool isRuntimeSuspended)
{
WRAPPER_NO_CONTRACT;

g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended);
return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended);
}

// This function bashes the super fast amd64 versions of the JIT_WriteBarrier
// helpers. It should be called by the GC whenever the ephermeral region gets moved
// from being at the top of the GC Heap, and/or when the cards table gets moved.
void StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
{
WRAPPER_NO_CONTRACT;

g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck);
return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck);
}

void FlushWriteBarrierInstructionCache()
{
FlushInstructionCache(GetCurrentProcess(), (PVOID)JIT_WriteBarrier, g_WriteBarrierManager.GetCurrentWriteBarrierSize());
}

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
void SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
int SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
{
WRAPPER_NO_CONTRACT;

g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended);
return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended);
}

void SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
{
WRAPPER_NO_CONTRACT;

g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended);
return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended);
}
#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
28 changes: 16 additions & 12 deletions src/vm/arm/stubs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -442,15 +442,9 @@ void UpdateGCWriteBarriers(bool postGrow = false)

pDesc++;
}

// We've changed code so we must flush the instruction cache.
BYTE *pbAlteredRange;
DWORD cbAlteredRange;
ComputeWriteBarrierRange(&pbAlteredRange, &cbAlteredRange);
FlushInstructionCache(GetCurrentProcess(), pbAlteredRange, cbAlteredRange);
}

void StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
{
// The runtime is not always suspended when this is called (unlike StompWriteBarrierEphemeral) but we have
// no way to update the barrier code atomically on ARM since each 32-bit value we change is loaded over
Expand All @@ -462,26 +456,36 @@ void StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
// suspend/resuming the EE under GC stress will trigger a GC and if we're holding the
// GC lock due to allocating a LOH segment it will cause a deadlock so disable it here.
GCStressPolicy::InhibitHolder iholder;
int stompWBCompleteActions = SWB_ICACHE_FLUSH;

bool fSuspended = false;
if (!isRuntimeSuspended)
{
ThreadSuspend::SuspendEE(ThreadSuspend::SUSPEND_OTHER);
fSuspended = true;
stompWBCompleteActions |= SWB_EE_RESTART;
}

UpdateGCWriteBarriers(bReqUpperBoundsCheck);

if (fSuspended)
ThreadSuspend::RestartEE(FALSE, TRUE);
return stompWBCompleteActions;
}

void StompWriteBarrierEphemeral(bool isRuntimeSuspended)
int StompWriteBarrierEphemeral(bool isRuntimeSuspended)
{
UNREFERENCED_PARAMETER(isRuntimeSuspended);
_ASSERTE(isRuntimeSuspended);
UpdateGCWriteBarriers();
return SWB_ICACHE_FLUSH;
}

void FlushWriteBarrierInstructionCache()
{
// We've changed code so we must flush the instruction cache.
BYTE *pbAlteredRange;
DWORD cbAlteredRange;
ComputeWriteBarrierRange(&pbAlteredRange, &cbAlteredRange);
FlushInstructionCache(GetCurrentProcess(), pbAlteredRange, cbAlteredRange);
}

#endif // CROSSGEN_COMPILE

#endif // !DACCESS_COMPILE
Expand Down
17 changes: 13 additions & 4 deletions src/vm/arm64/stubs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1348,26 +1348,35 @@ LONG CLRNoCatchHandler(EXCEPTION_POINTERS* pExceptionInfo, PVOID pv)
return EXCEPTION_CONTINUE_SEARCH;
}

void FlushWriteBarrierInstructionCache()
{
// this wouldn't be called in arm64, just to comply with gchelpers.h
}

#ifndef CROSSGEN_COMPILE
void StompWriteBarrierEphemeral(bool isRuntimeSuspended)
int StompWriteBarrierEphemeral(bool isRuntimeSuspended)
{
JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
return SWB_PASS;
}

void StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
{
JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
return SWB_PASS;
}

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
void SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
int SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
{
JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
return SWB_PASS;
}

void SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
{
JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
return SWB_PASS;
}
#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
#endif // CROSSGEN_COMPILE
Expand Down
Loading

0 comments on commit 5b10f0e

Please sign in to comment.