8246097: Shenandoah: limit parallelism in CLDG root handling

Reviewed-by: zgu

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1660,7 +1660,7 @@ class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots Task"),
     _vm_roots(phase),
-    _cld_roots(phase) {}
+    _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()) {}
 
   void work(uint worker_id) {
     ShenandoahConcurrentWorkerSession worker_session(worker_id);
@@ -1781,7 +1781,7 @@ class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
     _string_table_roots(OopStorageSet::string_table_weak(), phase, ShenandoahPhaseTimings::StringTableRoots),
     _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), phase, ShenandoahPhaseTimings::ResolvedMethodTableRoots),
     _vm_roots(OopStorageSet::vm_weak(), phase, ShenandoahPhaseTimings::VMWeakRoots),
-    _cld_roots(phase),
+    _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()),
     _nmethod_itr(ShenandoahCodeRoots::table()),
     _concurrent_class_unloading(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
     StringTable::reset_dead_counter();

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp

@@ -202,7 +202,7 @@ ShenandoahRootScanner::ShenandoahRootScanner(uint n_workers, ShenandoahPhaseTimi
   _code_roots(phase),
   _vm_roots(phase),
   _dedup_roots(phase),
-  _cld_roots(phase) {
+  _cld_roots(phase, n_workers) {
 }
 
 void ShenandoahRootScanner::roots_do(uint worker_id, OopClosure* oops) {
@@ -234,9 +234,9 @@ void ShenandoahRootScanner::roots_do(uint worker_id, OopClosure* oops, CLDClosur
    // Process light-weight/limited parallel roots then
   _vm_roots.oops_do(oops, worker_id);
   _dedup_roots.oops_do(&always_true, oops, worker_id);
+  _cld_roots.cld_do(clds, worker_id);
 
   // Process heavy-weight/fully parallel roots the last
-  _cld_roots.cld_do(clds, worker_id);
   _thread_roots.threads_do(&tc_cl, worker_id);
 }
 
@@ -251,9 +251,9 @@ void ShenandoahRootScanner::strong_roots_do(uint worker_id, OopClosure* oops, CL
 
   // Process light-weight/limited parallel roots then
   _vm_roots.oops_do(oops, worker_id);
+  _cld_roots.always_strong_cld_do(clds, worker_id);
 
   // Process heavy-weight/fully parallel roots the last
-  _cld_roots.always_strong_cld_do(clds, worker_id);
   _thread_roots.threads_do(&tc_cl, worker_id);
 }
 
@@ -264,7 +264,7 @@ ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers,
   ShenandoahRootProcessor(phase),
   _serial_roots(phase),
   _vm_roots(phase),
-  _cld_roots(phase),
+  _cld_roots(phase, n_workers),
   _thread_roots(phase, n_workers > 1),
   _serial_weak_roots(phase),
   _weak_roots(phase),
@@ -292,11 +292,13 @@ void ShenandoahRootEvacuator::roots_do(uint worker_id, OopClosure* oops) {
     _weak_roots.oops_do<OopClosure>(oops, worker_id);
     _dedup_roots.oops_do(&always_true, oops, worker_id);
   }
-
-  // Process heavy-weight/fully parallel roots the last
   if (_stw_class_unloading) {
     CLDToOopClosure clds(oops, ClassLoaderData::_claim_strong);
     _cld_roots.cld_do(&clds, worker_id);
+  }
+
+  // Process heavy-weight/fully parallel roots the last
+  if (_stw_class_unloading) {
     _code_roots.code_blobs_do(codes_cl, worker_id);
     _thread_roots.oops_do(oops, NULL, worker_id);
   } else {
@@ -308,7 +310,7 @@ ShenandoahRootUpdater::ShenandoahRootUpdater(uint n_workers, ShenandoahPhaseTimi
   ShenandoahRootProcessor(phase),
   _serial_roots(phase),
   _vm_roots(phase),
-  _cld_roots(phase),
+  _cld_roots(phase, n_workers),
   _thread_roots(phase, n_workers > 1),
   _serial_weak_roots(phase),
   _weak_roots(phase),
@@ -320,7 +322,7 @@ ShenandoahRootAdjuster::ShenandoahRootAdjuster(uint n_workers, ShenandoahPhaseTi
   ShenandoahRootProcessor(phase),
   _serial_roots(phase),
   _vm_roots(phase),
-  _cld_roots(phase),
+  _cld_roots(phase, n_workers),
   _thread_roots(phase, n_workers > 1),
   _serial_weak_roots(phase),
   _weak_roots(phase),
@@ -346,9 +348,9 @@ void ShenandoahRootAdjuster::roots_do(uint worker_id, OopClosure* oops) {
   _vm_roots.oops_do(oops, worker_id);
   _weak_roots.oops_do<OopClosure>(oops, worker_id);
   _dedup_roots.oops_do(&always_true, oops, worker_id);
+  _cld_roots.cld_do(&adjust_cld_closure, worker_id);
 
   // Process heavy-weight/fully parallel roots the last
-  _cld_roots.cld_do(&adjust_cld_closure, worker_id);
   _code_roots.code_blobs_do(adjust_code_closure, worker_id);
   _thread_roots.oops_do(oops, NULL, worker_id);
 }
@@ -358,7 +360,7 @@ ShenandoahHeapIterationRootScanner::ShenandoahHeapIterationRootScanner() :
    _serial_roots(ShenandoahPhaseTimings::heap_iteration_roots),
    _thread_roots(ShenandoahPhaseTimings::heap_iteration_roots, false /*is par*/),
    _vm_roots(ShenandoahPhaseTimings::heap_iteration_roots),
-   _cld_roots(ShenandoahPhaseTimings::heap_iteration_roots),
+   _cld_roots(ShenandoahPhaseTimings::heap_iteration_roots, 1),
    _serial_weak_roots(ShenandoahPhaseTimings::heap_iteration_roots),
    _weak_roots(ShenandoahPhaseTimings::heap_iteration_roots),
    _code_roots(ShenandoahPhaseTimings::heap_iteration_roots) {
@@ -382,9 +384,9 @@ ShenandoahHeapIterationRootScanner::ShenandoahHeapIterationRootScanner() :
    _vm_roots.oops_do(oops, 0);
    _weak_roots.oops_do<OopClosure>(oops, 0);
    _dedup_roots.oops_do(&always_true, oops, 0);
+   _cld_roots.cld_do(&clds, 0);
 
    // Process heavy-weight/fully parallel roots the last
-   _cld_roots.cld_do(&clds, 0);
    _code_roots.code_blobs_do(&code, 0);
    _thread_roots.threads_do(&tc_cl, 0);
  }

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp

@@ -226,9 +226,18 @@ class ShenandoahCodeCacheRoots {
 template <bool CONCURRENT, bool SINGLE_THREADED>
 class ShenandoahClassLoaderDataRoots {
 private:
+  ShenandoahSharedSemaphore     _semaphore;
   ShenandoahPhaseTimings::Phase _phase;
+
+  static uint worker_count(uint n_workers) {
+    // Limit concurrency a bit, otherwise it wastes resources when workers are tripping
+    // over each other. This also leaves free workers to process other parts of the root
+    // set, while admitted workers are busy with doing the CLDG walk.
+    return MAX2(1u, MIN2(ShenandoahSharedSemaphore::max_tokens(), n_workers / 2));
+  }
+
 public:
-  ShenandoahClassLoaderDataRoots(ShenandoahPhaseTimings::Phase phase);
+  ShenandoahClassLoaderDataRoots(ShenandoahPhaseTimings::Phase phase, uint n_workers);
   ~ShenandoahClassLoaderDataRoots();
 
   void always_strong_cld_do(CLDClosure* clds, uint worker_id);

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp

@@ -122,7 +122,8 @@ void ShenandoahVMRoots<CONCURRENT>::oops_do(T* cl, uint worker_id) {
 }
 
 template <bool CONCURRENT, bool SINGLE_THREADED>
-ShenandoahClassLoaderDataRoots<CONCURRENT, SINGLE_THREADED>::ShenandoahClassLoaderDataRoots(ShenandoahPhaseTimings::Phase phase) :
+ShenandoahClassLoaderDataRoots<CONCURRENT, SINGLE_THREADED>::ShenandoahClassLoaderDataRoots(ShenandoahPhaseTimings::Phase phase, uint n_workers) :
+  _semaphore(worker_count(n_workers)),
   _phase(phase) {
   if (!SINGLE_THREADED) {
     ClassLoaderDataGraph::clear_claimed_marks();
@@ -146,9 +147,10 @@ void ShenandoahClassLoaderDataRoots<CONCURRENT, SINGLE_THREADED>::always_strong_
     assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
     assert(Thread::current()->is_VM_thread(), "Single threaded CLDG iteration can only be done by VM thread");
     ClassLoaderDataGraph::always_strong_cld_do(clds);
-  } else {
+  } else if (_semaphore.try_acquire()) {
     ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CLDGRoots, worker_id);
     ClassLoaderDataGraph::always_strong_cld_do(clds);
+    _semaphore.claim_all();
   }
 }
 
@@ -158,9 +160,10 @@ void ShenandoahClassLoaderDataRoots<CONCURRENT, SINGLE_THREADED>::cld_do(CLDClos
     assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
     assert(Thread::current()->is_VM_thread(), "Single threaded CLDG iteration can only be done by VM thread");
     ClassLoaderDataGraph::cld_do(clds);
-  } else {
+  } else if (_semaphore.try_acquire()) {
     ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CLDGRoots, worker_id);
     ClassLoaderDataGraph::cld_do(clds);
+    _semaphore.claim_all();
   }
 }
 

src/hotspot/share/gc/shenandoah/shenandoahSharedVariables.hpp

@@ -245,4 +245,38 @@ struct ShenandoahSharedEnumFlag {
 
 };
 
+typedef struct ShenandoahSharedSemaphore {
+  shenandoah_padding(0);
+  volatile ShenandoahSharedValue value;
+  shenandoah_padding(1);
+
+  static uint max_tokens() {
+    return sizeof(ShenandoahSharedValue) * CHAR_MAX;
+  }
+
+  ShenandoahSharedSemaphore(uint tokens) {
+    assert(tokens <= max_tokens(), "sanity");
+    Atomic::release_store_fence(&value, (ShenandoahSharedValue)tokens);
+  }
+
+  bool try_acquire() {
+    while (true) {
+      ShenandoahSharedValue ov = Atomic::load_acquire(&value);
+      if (ov == 0) {
+        return false;
+      }
+      ShenandoahSharedValue nv = ov - 1;
+      if (Atomic::cmpxchg(&value, ov, nv) == ov) {
+        // successfully set
+        return true;
+      }
+    }
+  }
+
+  void claim_all() {
+    Atomic::release_store_fence(&value, (ShenandoahSharedValue)0);
+  }
+
+} ShenandoahSharedSemaphore;
+
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSHAREDVARIABLES_HPP