[Local GC] Move Software Write Watch's write barrier updates to GCToE…

…EInterface::StompWriteBarrier (#8605)

* Move Software Write Watch's write barrier updates to use the new
GCToEEInterface::StompWriteBarrier to stomp the EE's write barrier.

* Address code review feedback, move SetCardsAfterBulkCopy to EE side of the interface

src/classlibnative/bcltype/arraynative.cpp

@@ -961,7 +961,7 @@ void memmoveGCRefs(void *dest, const void *src, size_t len)
         }
     }
 
-    GCHeapUtilities::GetGCHeap()->SetCardsAfterBulkCopy((Object**)dest, len);
+    SetCardsAfterBulkCopy((Object**)dest, len);
 }
 
 void ArrayNative::ArrayCopyNoTypeCheck(BASEARRAYREF pSrc, unsigned int srcIndex, BASEARRAYREF pDest, unsigned int destIndex, unsigned int length)

src/gc/gc.cpp

@@ -1402,9 +1402,6 @@ int mark_time, plan_time, sweep_time, reloc_time, compact_time;
 
 #ifndef MULTIPLE_HEAPS
 
-#define ephemeral_low           g_gc_ephemeral_low
-#define ephemeral_high          g_gc_ephemeral_high
-
 #endif // MULTIPLE_HEAPS
 
 #ifdef TRACE_GC
@@ -2187,27 +2184,22 @@ void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_b
     args.card_table = g_gc_card_table;
     args.lowest_address = g_gc_lowest_address;
     args.highest_address = g_gc_highest_address;
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+    if (SoftwareWriteWatch::IsEnabledForGCHeap())
+    {
+        args.write_watch_table = g_gc_sw_ww_table;
+    }
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
     GCToEEInterface::StompWriteBarrier(&args);
 }
 
-void stomp_write_barrier_ephemeral(bool is_runtime_suspended, uint8_t* ephemeral_lo, uint8_t* ephemeral_hi)
+void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
 {
     WriteBarrierParameters args = {};
     args.operation = WriteBarrierOp::StompEphemeral;
-    args.is_runtime_suspended = is_runtime_suspended;
-    args.ephemeral_lo = g_gc_ephemeral_low;
-    args.ephemeral_hi = g_gc_ephemeral_high;
-#ifdef MULTIPLE_HEAPS
-    // It is not correct to update the EE's g_ephemeral_low and g_ephemeral_high
-    // to anything other than their default values when using Server GC, since
-    // there is no single ephemeral generation across all of the heaps.
-    // Server GC write barriers do not reference these two globals, but ErectWriteBarrier does.
-    //
-    // When MULTIPLE_HEAPS is defined, g_gc_ephemeral_low and g_gc_ephemeral_high should
-    // always have their default values.
-    assert(args.ephemeral_lo == (uint8_t*)1);
-    assert(args.ephemeral_hi == (uint8_t*)~0);
-#endif // MULTIPLE_HEAPS
+    args.is_runtime_suspended = true;
+    args.ephemeral_low = ephemeral_low;
+    args.ephemeral_high = ephemeral_high;
     GCToEEInterface::StompWriteBarrier(&args);
 }
 
@@ -2220,6 +2212,8 @@ void stomp_write_barrier_initialize()
     args.card_table = g_gc_card_table;
     args.lowest_address = g_gc_lowest_address;
     args.highest_address = g_gc_highest_address;
+    args.ephemeral_low = reinterpret_cast<uint8_t*>(1);
+    args.ephemeral_high = reinterpret_cast<uint8_t*>(~0);
     GCToEEInterface::StompWriteBarrier(&args);
 }
 
@@ -2430,6 +2424,10 @@ BOOL        gc_heap::ro_segments_in_range;
 
 size_t      gc_heap::gen0_big_free_spaces = 0;
 
+uint8_t*    gc_heap::ephemeral_low;
+
+uint8_t*    gc_heap::ephemeral_high;
+
 uint8_t*    gc_heap::lowest_address;
 
 uint8_t*    gc_heap::highest_address;
@@ -7277,9 +7275,6 @@ int gc_heap::grow_brick_card_tables (uint8_t* start,
             }
 
             g_gc_card_table = translated_ct;
-            g_gc_lowest_address = saved_g_lowest_address;
-            g_gc_highest_address = saved_g_highest_address;
-
             SoftwareWriteWatch::SetResizedUntranslatedTable(
                 mem + sw_ww_table_offset,
                 saved_g_lowest_address,
@@ -7290,6 +7285,8 @@ int gc_heap::grow_brick_card_tables (uint8_t* start,
             // grow version of the write barrier.  This test tells us if the new
             // segment was allocated at a lower address than the old, requiring
             // that we start doing an upper bounds check in the write barrier.
+            g_gc_lowest_address = saved_g_lowest_address;
+            g_gc_highest_address = saved_g_highest_address;
             stomp_write_barrier_resize(true, la != saved_g_lowest_address);
             write_barrier_updated = true;
 
@@ -9662,16 +9659,18 @@ void gc_heap::make_generation (generation& gen, heap_segment* seg, uint8_t* star
 #endif //FREE_USAGE_STATS
 }
 
-void gc_heap::adjust_ephemeral_limits (bool is_runtime_suspended)
+void gc_heap::adjust_ephemeral_limits ()
 {
     ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
     ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
 
     dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
                  (size_t)ephemeral_low, (size_t)ephemeral_high))
 
+#ifndef MULTIPLE_HEAPS
     // This updates the write barrier helpers with the new info.
-    stomp_write_barrier_ephemeral(is_runtime_suspended, ephemeral_low, ephemeral_high);
+    stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
+#endif // MULTIPLE_HEAPS
 }
 
 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
@@ -10466,7 +10465,7 @@ gc_heap::init_gc_heap (int  h_number)
     make_background_mark_stack (b_arr);
 #endif //BACKGROUND_GC
 
-    adjust_ephemeral_limits(true);
+    adjust_ephemeral_limits();
 
 #ifdef MARK_ARRAY
     // why would we clear the mark array for this page? it should be cleared..
@@ -15364,7 +15363,8 @@ void gc_heap::gc1()
     if (!settings.concurrent)
 #endif //BACKGROUND_GC
     {
-        adjust_ephemeral_limits(!!IsGCThread());
+        assert(!!IsGCThread());
+        adjust_ephemeral_limits();
     }
 
 #ifdef BACKGROUND_GC
@@ -16204,7 +16204,8 @@ BOOL gc_heap::expand_soh_with_minimal_gc()
         dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
         dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
 
-        adjust_ephemeral_limits(!!IsGCThread());
+        assert(!!IsGCThread());
+        adjust_ephemeral_limits();
         return TRUE;
     }
     else
@@ -32778,8 +32779,8 @@ gc_heap::verify_heap (BOOL begin_gc_p)
 #endif //BACKGROUND_GC 
 
 #ifndef MULTIPLE_HEAPS
-    if ((g_gc_ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
-        (g_gc_ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
+    if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
+        (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
     {
         FATAL_GC_ERROR();
     }
@@ -35681,85 +35682,6 @@ void GCHeap::SetFinalizationRun (Object* obj)
 
 #endif // FEATURE_PREMORTEM_FINALIZATION
 
-//----------------------------------------------------------------------------
-//
-// Write Barrier Support for bulk copy ("Clone") operations
-//
-// StartPoint is the target bulk copy start point
-// len is the length of the bulk copy (in bytes)
-//
-//
-// Performance Note:
-//
-// This is implemented somewhat "conservatively", that is we
-// assume that all the contents of the bulk copy are object
-// references.  If they are not, and the value lies in the
-// ephemeral range, we will set false positives in the card table.
-//
-// We could use the pointer maps and do this more accurately if necessary
-
-#if defined(_MSC_VER) && defined(_TARGET_X86_)
-#pragma optimize("y", on)        // Small critical routines, don't put in EBP frame 
-#endif //_MSC_VER && _TARGET_X86_
-
-void
-GCHeap::SetCardsAfterBulkCopy( Object **StartPoint, size_t len )
-{
-    Object **rover;
-    Object **end;
-
-    // Target should aligned
-    assert(Aligned ((size_t)StartPoint));
-
-
-    // Don't optimize the Generation 0 case if we are checking for write barrier voilations
-    // since we need to update the shadow heap even in the generation 0 case.
-#if defined (WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
-    if (g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK)
-        for(unsigned i=0; i < len / sizeof(Object*); i++)
-            updateGCShadow(&StartPoint[i], StartPoint[i]);
-#endif //WRITE_BARRIER_CHECK && !SERVER_GC
-
-#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-    if (SoftwareWriteWatch::IsEnabledForGCHeap())
-    {
-        SoftwareWriteWatch::SetDirtyRegion(StartPoint, len);
-    }
-#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-
-    // If destination is in Gen 0 don't bother
-    if (
-#ifdef BACKGROUND_GC
-        (!gc_heap::settings.concurrent) &&
-#endif //BACKGROUND_GC
-        (g_theGCHeap->WhichGeneration( (Object*) StartPoint ) == 0))
-        return;
-
-    rover = StartPoint;
-    end = StartPoint + (len/sizeof(Object*));
-    while (rover < end)
-    {
-        if ( (((uint8_t*)*rover) >= g_gc_ephemeral_low) && (((uint8_t*)*rover) < g_gc_ephemeral_high) )
-        {
-            // Set Bit For Card and advance to next card
-            size_t card = gcard_of ((uint8_t*)rover);
-
-            Interlocked::Or (&g_gc_card_table[card/card_word_width], (1U << (card % card_word_width)));
-            // Skip to next card for the object
-            rover = (Object**)align_on_card ((uint8_t*)(rover+1));
-        }
-        else
-        {
-            rover++;
-        }
-    }
-}
-
-#if defined(_MSC_VER) && defined(_TARGET_X86_)
-#pragma optimize("", on)        // Go back to command line default optimizations
-#endif //_MSC_VER && _TARGET_X86_
-
-
 #ifdef FEATURE_PREMORTEM_FINALIZATION
 
 //--------------------------------------------------------------------

src/gc/gc.h

@@ -140,8 +140,6 @@ class DacHeapWalker;
 extern "C" uint32_t* g_gc_card_table;
 extern "C" uint8_t* g_gc_lowest_address;
 extern "C" uint8_t* g_gc_highest_address;
-extern "C" uint8_t* g_gc_ephemeral_low;
-extern "C" uint8_t* g_gc_ephemeral_high;
 
 namespace WKS {
     ::IGCHeapInternal* CreateGCHeap();

src/gc/gccommon.cpp

@@ -41,8 +41,6 @@ uint8_t* g_shadow_lowest_address = NULL;
 uint32_t* g_gc_card_table;
 uint8_t* g_gc_lowest_address  = 0;
 uint8_t* g_gc_highest_address = 0;
-uint8_t* g_gc_ephemeral_low   = (uint8_t*)1;
-uint8_t* g_gc_ephemeral_high  = (uint8_t*)~0;
 
 VOLATILE(int32_t) m_GCLock = -1;
 

src/gc/gcimpl.h

@@ -198,7 +198,6 @@ class GCHeap : public IGCHeapInternal
     BOOL FinalizeAppDomain(AppDomain *pDomain, BOOL fRunFinalizers);
     BOOL ShouldRestartFinalizerWatchDog();
 
-    void SetCardsAfterBulkCopy( Object**, size_t);
     void DiagWalkObject (Object* obj, walk_fn fn, void* context);
 
 public:	// FIX 

src/gc/gcinterface.h

@@ -46,7 +46,9 @@ enum class WriteBarrierOp
 {
     StompResize,
     StompEphemeral,
-    Initialize
+    Initialize,
+    SwitchToWriteWatch,
+    SwitchToNonWriteWatch
 };
 
 // Arguments to GCToEEInterface::StompWriteBarrier
@@ -85,11 +87,15 @@ struct WriteBarrierParameters
 
     // The new start of the ephemeral generation. 
     // Used for WriteBarrierOp::StompEphemeral.
-    uint8_t* ephemeral_lo;
+    uint8_t* ephemeral_low;
 
     // The new end of the ephemeral generation.
     // Used for WriteBarrierOp::StompEphemeral.
-    uint8_t* ephemeral_hi;
+    uint8_t* ephemeral_high;
+
+    // The new write watch table, if we are using our own write watch
+    // implementation. Used for WriteBarrierOp::SwitchToWriteWatch only.
+    uint8_t* write_watch_table;
 };
 
 #include "gcinterface.ee.h"
@@ -148,6 +154,10 @@ struct segment_info
 
 #define max_generation 2
 
+// The bit shift used to convert a memory address into an index into the
+// Software Write Watch table.
+#define SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift 0xc
+
 class Object;
 class IGCHeap;
 
@@ -398,9 +408,6 @@ class IGCHeap {
     // sanity checks asserting that a GC has not occured.
     virtual unsigned GetGcCount() = 0;
 
-    // Sets cards after an object has been memmoved. 
-    virtual void SetCardsAfterBulkCopy(Object** obj, size_t length) = 0;
-
     // Gets whether or not the home heap of this alloc context matches the heap
     // associated with this thread.
     virtual bool IsThreadUsingAllocationContextHeap(gc_alloc_context* acontext, int thread_number) = 0;

src/gc/gcpriv.h

@@ -1671,7 +1671,7 @@ class gc_heap
     PER_HEAP
     void reset_write_watch (BOOL concurrent_p);
     PER_HEAP
-    void adjust_ephemeral_limits (bool is_runtime_suspended);
+    void adjust_ephemeral_limits ();
     PER_HEAP
     void make_generation (generation& gen, heap_segment* seg,
                           uint8_t* start, uint8_t* pointer);
@@ -2802,13 +2802,11 @@ class gc_heap
     PER_HEAP
     void exit_gc_done_event_lock();
 
-#ifdef MULTIPLE_HEAPS
     PER_HEAP
     uint8_t*  ephemeral_low;      //lowest ephemeral address
 
     PER_HEAP
     uint8_t*  ephemeral_high;     //highest ephemeral address
-#endif //MULTIPLE_HEAPS
 
     PER_HEAP
     uint32_t* card_table;

src/gc/gcsvr.cpp

@@ -13,6 +13,7 @@
 #include "gc.h"
 #include "gcscan.h"
 #include "gcdesc.h"
+#include "softwarewritewatch.h"
 
 #define SERVER_GC 1
 

src/gc/gcwks.cpp

@@ -11,6 +11,7 @@
 #include "gc.h"
 #include "gcscan.h"
 #include "gcdesc.h"
+#include "softwarewritewatch.h"
 
 #ifdef SERVER_GC
 #undef SERVER_GC

src/gc/sample/GCSample.cpp

@@ -94,14 +94,11 @@ inline void ErectWriteBarrier(Object ** dst, Object * ref)
     if (((uint8_t*)dst < g_gc_lowest_address) || ((uint8_t*)dst >= g_gc_highest_address))
         return;
 
-    if((uint8_t*)ref >= g_gc_ephemeral_low && (uint8_t*)ref < g_gc_ephemeral_high)
-    {
-        // volatile is used here to prevent fetch of g_card_table from being reordered 
-        // with g_lowest/highest_address check above. See comment in code:gc_heap::grow_brick_card_tables.
-        uint8_t* pCardByte = (uint8_t *)*(volatile uint8_t **)(&g_gc_card_table) + card_byte((uint8_t *)dst);
-        if(*pCardByte != 0xFF)
-            *pCardByte = 0xFF;
-    }
+    // volatile is used here to prevent fetch of g_card_table from being reordered 
+    // with g_lowest/highest_address check above. See comment in code:gc_heap::grow_brick_card_tables.
+    uint8_t* pCardByte = (uint8_t *)*(volatile uint8_t **)(&g_gc_card_table) + card_byte((uint8_t *)dst);
+    if(*pCardByte != 0xFF)
+        *pCardByte = 0xFF;
 }
 
 void WriteBarrier(Object ** dst, Object * ref)