8347335: ZGC: Use limitless mark stack memory

src/hotspot/share/gc/z/zAddressSpaceLimit.cpp

@@ -39,12 +39,6 @@ static size_t address_space_limit() {
   return SIZE_MAX;
 }
 
-size_t ZAddressSpaceLimit::mark_stack() {
-  // Allow mark stacks to occupy 10% of the address space
-  const size_t limit = address_space_limit() / 10;
-  return align_up(limit, ZMarkStackSpaceExpandSize);
-}
-
 size_t ZAddressSpaceLimit::heap() {
   // Allow the heap to occupy 50% of the address space
   const size_t limit = address_space_limit() / MaxVirtMemFraction;

src/hotspot/share/gc/z/zAddressSpaceLimit.hpp

@@ -29,7 +29,6 @@
 
 class ZAddressSpaceLimit : public AllStatic {
 public:
-  static size_t mark_stack();
   static size_t heap();
 };
 

src/hotspot/share/gc/z/zArguments.cpp

@@ -120,15 +120,6 @@ void ZArguments::select_max_gc_threads() {
 void ZArguments::initialize() {
   GCArguments::initialize();
 
-  // Check mark stack size
-  const size_t mark_stack_space_limit = ZAddressSpaceLimit::mark_stack();
-  if (ZMarkStackSpaceLimit > mark_stack_space_limit) {
-    if (!FLAG_IS_DEFAULT(ZMarkStackSpaceLimit)) {
-      vm_exit_during_initialization("ZMarkStackSpaceLimit too large for limited address space");
-    }
-    FLAG_SET_DEFAULT(ZMarkStackSpaceLimit, mark_stack_space_limit);
-  }
-
   // Enable NUMA by default
   if (FLAG_IS_DEFAULT(UseNUMA)) {
     FLAG_SET_DEFAULT(UseNUMA, true);

src/hotspot/share/gc/z/zBarrierSet.cpp

@@ -101,7 +101,7 @@ void ZBarrierSet::on_thread_attach(Thread* thread) {
 
 void ZBarrierSet::on_thread_detach(Thread* thread) {
   // Flush and free any remaining mark stacks
-  ZHeap::heap()->mark_flush_and_free(thread);
+  ZHeap::heap()->mark_flush(thread);
 }
 
 static void deoptimize_allocation(JavaThread* thread) {

src/hotspot/share/gc/z/zGeneration.cpp

@@ -131,10 +131,6 @@ ZGeneration::ZGeneration(ZGenerationId id, ZPageTable* page_table, ZPageAllocato
     _stat_relocation(),
     _gc_timer(nullptr) {}
 
-bool ZGeneration::is_initialized() const {
-  return _mark.is_initialized();
-}
-
 ZWorkers* ZGeneration::workers() {
   return &_workers;
 }
@@ -151,8 +147,8 @@ void ZGeneration::threads_do(ThreadClosure* tc) const {
   _workers.threads_do(tc);
 }
 
-void ZGeneration::mark_flush_and_free(Thread* thread) {
-  _mark.flush_and_free(thread);
+void ZGeneration::mark_flush(Thread* thread) {
+  _mark.flush(thread);
 }
 
 void ZGeneration::mark_free() {

src/hotspot/share/gc/z/zGeneration.hpp

@@ -99,8 +99,6 @@ class ZGeneration {
   void log_phase_switch(Phase from, Phase to);
 
 public:
-  bool is_initialized() const;
-
   // GC phases
   void set_phase(Phase new_phase);
   bool is_phase_relocate() const;
@@ -161,7 +159,7 @@ class ZGeneration {
   void mark_object(zaddress addr);
   template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
   void mark_object_if_active(zaddress addr);
-  void mark_flush_and_free(Thread* thread);
+  void mark_flush(Thread* thread);
 
   // Relocation
   void synchronize_relocation();

src/hotspot/share/gc/z/zGlobals.hpp

@@ -67,17 +67,6 @@ const int         ZObjectAlignmentLarge         = 1 << ZObjectAlignmentLargeShif
 const size_t      ZCacheLineSize                = ZPlatformCacheLineSize;
 #define           ZCACHE_ALIGNED                ATTRIBUTE_ALIGNED(ZCacheLineSize)
 
-// Mark stack space
-const size_t      ZMarkStackSpaceExpandSize     = (size_t)1 << 25; // 32M
-
-// Mark stack and magazine sizes
-const size_t      ZMarkStackSizeShift           = 11; // 2K
-const size_t      ZMarkStackSize                = (size_t)1 << ZMarkStackSizeShift;
-const size_t      ZMarkStackHeaderSize          = (size_t)1 << 4; // 16B
-const size_t      ZMarkStackSlots               = (ZMarkStackSize - ZMarkStackHeaderSize) / sizeof(uintptr_t);
-const size_t      ZMarkStackMagazineSize        = (size_t)1 << 15; // 32K
-const size_t      ZMarkStackMagazineSlots       = (ZMarkStackMagazineSize / ZMarkStackSize) - 1;
-
 // Mark stripe size
 const size_t      ZMarkStripeShift              = ZGranuleSizeShift;
 

src/hotspot/share/gc/z/zHeap.cpp

@@ -68,7 +68,7 @@ ZHeap::ZHeap()
   assert(_heap == nullptr, "Already initialized");
   _heap = this;
 
-  if (!_page_allocator.is_initialized() || !_young.is_initialized() || !_old.is_initialized()) {
+  if (!_page_allocator.is_initialized()) {
     return;
   }
 
@@ -271,9 +271,9 @@ void ZHeap::keep_alive(oop obj) {
   ZBarrier::mark<ZMark::Resurrect, ZMark::AnyThread, ZMark::Follow, ZMark::Strong>(addr);
 }
 
-void ZHeap::mark_flush_and_free(Thread* thread) {
-  _young.mark_flush_and_free(thread);
-  _old.mark_flush_and_free(thread);
+void ZHeap::mark_flush(Thread* thread) {
+  _young.mark_flush(thread);
+  _old.mark_flush(thread);
 }
 
 bool ZHeap::is_allocating(zaddress addr) const {

src/hotspot/share/gc/z/zHeap.hpp

@@ -99,7 +99,7 @@ class ZHeap {
   bool is_object_live(zaddress addr) const;
   bool is_object_strongly_live(zaddress addr) const;
   void keep_alive(oop obj);
-  void mark_flush_and_free(Thread* thread);
+  void mark_flush(Thread* thread);
 
   // Page allocation
   ZPage* alloc_page(ZPageType type, size_t size, ZAllocationFlags flags, ZPageAge age);

src/hotspot/share/gc/z/zInitialize.cpp

@@ -31,7 +31,6 @@
 #include "gc/z/zInitialize.hpp"
 #include "gc/z/zJNICritical.hpp"
 #include "gc/z/zLargePages.hpp"
-#include "gc/z/zMarkStackAllocator.hpp"
 #include "gc/z/zNMT.hpp"
 #include "gc/z/zNUMA.hpp"
 #include "gc/z/zStat.hpp"

src/hotspot/share/gc/z/zMark.cpp

@@ -81,8 +81,8 @@ static const ZStatSubPhase ZSubPhaseConcurrentMarkRootColoredOld("Concurrent Mar
 ZMark::ZMark(ZGeneration* generation, ZPageTable* page_table)
   : _generation(generation),
     _page_table(page_table),
-    _allocator(),
-    _stripes(_allocator.start()),
+    _marking_smr(),
+    _stripes(),
     _terminate(),
     _work_nproactiveflush(0),
     _work_nterminateflush(0),
@@ -92,10 +92,6 @@ ZMark::ZMark(ZGeneration* generation, ZPageTable* page_table)
     _ncontinue(0),
     _nworkers(0) {}
 
-bool ZMark::is_initialized() const {
-  return _allocator.is_initialized();
-}
-
 size_t ZMark::calculate_nstripes(uint nworkers) const {
   // Calculate the number of stripes from the number of workers we use,
   // where the number of stripes must be a power of two and we want to
@@ -196,7 +192,7 @@ void ZMark::push_partial_array(zpointer* addr, size_t length, bool finalizable)
   log_develop_trace(gc, marking)("Array push partial: " PTR_FORMAT " (%zu), stripe: %zu",
                                  p2i(addr), length, _stripes.stripe_id(stripe));
 
-  stacks->push(&_allocator, &_stripes, stripe, &_terminate, entry, false /* publish */);
+  stacks->push(&_stripes, stripe, &_terminate, entry, false /* publish */);
 }
 
 static void mark_barrier_on_oop_array(volatile zpointer* p, size_t length, bool finalizable, bool young) {
@@ -471,21 +467,26 @@ bool ZMark::rebalance_work(ZMarkContext* context) {
   const size_t nstripes = _stripes.nstripes();
 
   if (assumed_nstripes != nstripes) {
+    // The number of stripes has changed; reflect that change locally
     context->set_nstripes(nstripes);
-  } else if (nstripes < calculate_nstripes(_nworkers) && _allocator.clear_and_get_expanded_recently()) {
+  } else if (nstripes < calculate_nstripes(_nworkers) && _stripes.is_crowded()) {
+    // We are running on a reduced number of threads to minimize the amount of work
+    // hidden in local stacks when the stripes are less well balanced. When this situation
+    // starts getting crowded, we bump the number of stripes again.
     const size_t new_nstripes = nstripes << 1;
-    _stripes.set_nstripes(new_nstripes);
-    context->set_nstripes(new_nstripes);
+    if (_stripes.try_set_nstripes(nstripes, new_nstripes)) {
+      context->set_nstripes(new_nstripes);
+    }
   }
 
   ZMarkStripe* stripe = _stripes.stripe_for_worker(_nworkers, WorkerThread::worker_id());
   if (context->stripe() != stripe) {
     // Need to switch stripe
     context->set_stripe(stripe);
-    flush_and_free();
+    flush(Thread::current());
   } else if (!_terminate.saturated()) {
     // Work imbalance detected; striped marking is likely going to be in the way
-    flush_and_free();
+    flush(Thread::current());
   }
 
   SuspendibleThreadSet::yield();
@@ -502,7 +503,7 @@ bool ZMark::drain(ZMarkContext* context) {
   context->set_nstripes(_stripes.nstripes());
 
   // Drain stripe stacks
-  while (stacks->pop(&_allocator, &_stripes, context->stripe(), entry)) {
+  while (stacks->pop(&_marking_smr, &_stripes, context->stripe(), &entry)) {
     mark_and_follow(context, entry);
 
     if ((processed++ & 31) == 0 && rebalance_work(context)) {
@@ -541,7 +542,7 @@ bool ZMark::try_steal_global(ZMarkContext* context) {
   for (ZMarkStripe* victim_stripe = _stripes.stripe_next(stripe);
        victim_stripe != stripe;
        victim_stripe = _stripes.stripe_next(victim_stripe)) {
-    ZMarkStack* const stack = victim_stripe->steal_stack();
+    ZMarkStack* const stack = victim_stripe->steal_stack(&_marking_smr);
     if (stack != nullptr) {
       // Success, install the stolen stack
       stacks->install(&_stripes, stripe, stack);
@@ -557,19 +558,19 @@ bool ZMark::try_steal(ZMarkContext* context) {
   return try_steal_local(context) || try_steal_global(context);
 }
 
-class ZMarkFlushAndFreeStacksClosure : public HandshakeClosure {
+class ZMarkFlushStacksClosure : public HandshakeClosure {
 private:
   ZMark* const _mark;
   bool         _flushed;
 
 public:
-  ZMarkFlushAndFreeStacksClosure(ZMark* mark)
-    : HandshakeClosure("ZMarkFlushAndFreeStacks"),
+  ZMarkFlushStacksClosure(ZMark* mark)
+    : HandshakeClosure("ZMarkFlushStacks"),
       _mark(mark),
       _flushed(false) {}
 
   void do_thread(Thread* thread) {
-    if (_mark->flush_and_free(thread)) {
+    if (_mark->flush(thread)) {
       _flushed = true;
       if (SafepointSynchronize::is_at_safepoint()) {
         log_debug(gc, marking)("Thread broke mark termination %s", thread->name());
@@ -606,7 +607,7 @@ class VM_ZMarkFlushOperation : public VM_Operation {
 };
 
 bool ZMark::flush() {
-  ZMarkFlushAndFreeStacksClosure cl(this);
+  ZMarkFlushStacksClosure cl(this);
   VM_ZMarkFlushOperation vm_cl(&cl);
   Handshake::execute(&cl);
   VMThread::execute(&vm_cl);
@@ -623,8 +624,7 @@ bool ZMark::try_terminate_flush() {
     verify_worker_stacks_empty();
   }
 
-  return flush() ||
-         _terminate.resurrected();
+  return flush() || _terminate.resurrected();
 }
 
 bool ZMark::try_proactive_flush() {
@@ -850,7 +850,7 @@ class ZMarkOldRootsTask : public ZTask {
     // the set of workers executing during root scanning
     // can be different from the set of workers executing
     // during mark.
-    ZHeap::heap()->mark_flush_and_free(Thread::current());
+    ZHeap::heap()->mark_flush(Thread::current());
   }
 };
 
@@ -908,7 +908,7 @@ class ZMarkYoungRootsTask : public ZTask {
     // the set of workers executing during root scanning
     // can be different from the set of workers executing
     // during mark.
-    ZHeap::heap()->mark_flush_and_free(Thread::current());
+    ZHeap::heap()->mark_flush(Thread::current());
   }
 };
 
@@ -934,7 +934,7 @@ class ZMarkTask : public ZRestartableTask {
     // publish such marking stacks to prevent that generation from getting a mark continue.
     // We also flush in case of a resize where a new worker thread continues the marking
     // work, causing a mark continue for the collected generation.
-    ZHeap::heap()->mark_flush_and_free(Thread::current());
+    ZHeap::heap()->mark_flush(Thread::current());
   }
 
   virtual void resize_workers(uint nworkers) {
@@ -978,7 +978,7 @@ bool ZMark::try_end() {
   }
 
   // Try end marking
-  ZMarkFlushAndFreeStacksClosure cl(this);
+  ZMarkFlushStacksClosure cl(this);
   Threads::non_java_threads_do(&cl);
 
   // Check if non-java threads have any pending marking
@@ -1012,25 +1012,15 @@ bool ZMark::end() {
 
 void ZMark::free() {
   // Free any unused mark stack space
-  _allocator.free();
-
-  // Update statistics
-  _generation->stat_mark()->at_mark_free(_allocator.size());
-}
-
-void ZMark::flush_and_free() {
-  Thread* const thread = Thread::current();
-  flush_and_free(thread);
+  _marking_smr.free();
 }
 
-bool ZMark::flush_and_free(Thread* thread) {
+bool ZMark::flush(Thread* thread) {
   if (thread->is_Java_thread()) {
     ZThreadLocalData::store_barrier_buffer(thread)->flush();
   }
   ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(thread, _generation->id());
-  const bool flushed = stacks->flush(&_allocator, &_stripes, &_terminate);
-  stacks->free(&_allocator);
-  return flushed;
+  return stacks->flush(&_stripes, &_terminate);
 }
 
 class ZVerifyMarkStacksEmptyClosure : public ThreadClosure {

src/hotspot/share/gc/z/zMark.hpp

@@ -25,8 +25,8 @@
 #define SHARE_GC_Z_ZMARK_HPP
 
 #include "gc/z/zAddress.hpp"
+#include "gc/z/zMarkingSMR.hpp"
 #include "gc/z/zMarkStack.hpp"
-#include "gc/z/zMarkStackAllocator.hpp"
 #include "gc/z/zMarkStackEntry.hpp"
 #include "gc/z/zMarkTerminate.hpp"
 #include "oops/oopsHierarchy.hpp"
@@ -55,18 +55,18 @@ class ZMark {
   static const bool Finalizable   = true;
 
 private:
-  ZGeneration* const  _generation;
-  ZPageTable* const   _page_table;
-  ZMarkStackAllocator _allocator;
-  ZMarkStripeSet      _stripes;
-  ZMarkTerminate      _terminate;
-  volatile size_t     _work_nproactiveflush;
-  volatile size_t     _work_nterminateflush;
-  size_t              _nproactiveflush;
-  size_t              _nterminateflush;
-  size_t              _ntrycomplete;
-  size_t              _ncontinue;
-  uint                _nworkers;
+  ZGeneration* const _generation;
+  ZPageTable* const  _page_table;
+  ZMarkingSMR        _marking_smr;
+  ZMarkStripeSet     _stripes;
+  ZMarkTerminate     _terminate;
+  volatile size_t    _work_nproactiveflush;
+  volatile size_t    _work_nterminateflush;
+  size_t             _nproactiveflush;
+  size_t             _nterminateflush;
+  size_t             _ntrycomplete;
+  size_t             _ncontinue;
+  uint               _nworkers;
 
   size_t calculate_nstripes(uint nworkers) const;
 
@@ -101,8 +101,6 @@ class ZMark {
 public:
   ZMark(ZGeneration* generation, ZPageTable* page_table);
 
-  bool is_initialized() const;
-
   template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
   void mark_object(zaddress addr);
 
@@ -113,8 +111,7 @@ class ZMark {
   bool end();
   void free();
 
-  void flush_and_free();
-  bool flush_and_free(Thread* thread);
+  bool flush(Thread* thread);
 
   // Following work
   void prepare_work();