8263427: Shenandoah: Trigger weak-LRB even when heap is stable

Reviewed-by: shade, zgu

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

@@ -342,7 +342,11 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
 #endif
 
   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
-  __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
+  int flags = ShenandoahHeap::HAS_FORWARDED;
+  if (!is_strong) {
+    flags |= ShenandoahHeap::WEAK_ROOTS;
+  }
+  __ testb(gc_state, flags);
   __ jcc(Assembler::zero, heap_stable);
 
   Register tmp1 = noreg, tmp2 = noreg;

src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp

@@ -138,7 +138,11 @@ LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, L
   // Read and check the gc-state-flag.
   LIR_Opr flag_val = gen->new_register(T_INT);
   __ load(active_flag_addr, flag_val);
-  LIR_Opr mask = LIR_OprFact::intConst(ShenandoahHeap::HAS_FORWARDED);
+  int flags = ShenandoahHeap::HAS_FORWARDED;
+  if (!ShenandoahBarrierSet::is_strong_access(decorators)) {
+    flags |= ShenandoahHeap::WEAK_ROOTS;
+  }
+  LIR_Opr mask = LIR_OprFact::intConst(flags);
   LIR_Opr mask_reg = gen->new_register(T_INT);
   __ move(mask, mask_reg);
 

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

@@ -1362,7 +1362,11 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
 
     // Stable path.
-    test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED);
+    int flags = ShenandoahHeap::HAS_FORWARDED;
+    if (!ShenandoahBarrierSet::is_strong_access(lrb->decorators())) {
+      flags |= ShenandoahHeap::WEAK_ROOTS;
+    }
+    test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, flags);
     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
 
     // Heap stable case

src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp

@@ -140,12 +140,6 @@ class ShenandoahCodeBlobAndDisarmClosure: public CodeBlobToOopClosure {
   inline void do_code_blob(CodeBlob* cb);
 };
 
-class ShenandoahRendezvousClosure : public HandshakeClosure {
-public:
-  inline ShenandoahRendezvousClosure();
-  inline void do_thread(Thread* thread);
-};
-
 #ifdef ASSERT
 class ShenandoahAssertNotForwardedClosure : public OopClosure {
 private:

src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp

@@ -233,13 +233,6 @@ void ShenandoahCodeBlobAndDisarmClosure::do_code_blob(CodeBlob* cb) {
   }
 }
 
-ShenandoahRendezvousClosure::ShenandoahRendezvousClosure() :
-  HandshakeClosure("ShenandoahRendezvous") {
-}
-
-void ShenandoahRendezvousClosure::do_thread(Thread* thread) {
-}
-
 #ifdef ASSERT
 template <class T>
 void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) {

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -166,11 +166,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
     // Update references freed up collection set, kick the cleanup to reclaim the space.
     entry_cleanup_complete();
   } else {
-    // Concurrent weak/strong root flags are unset concurrently. We depend on updateref GC safepoints
-    // to ensure the changes are visible to all mutators before gc cycle is completed.
-    // In case of no evacuation, updateref GC safepoints are skipped. Therefore, we will need
-    // to perform thread handshake to ensure their consistences.
-    entry_rendezvous_roots();
+    vmop_entry_final_roots();
   }
 
   return true;
@@ -216,6 +212,17 @@ void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() {
   VMThread::execute(&op);
 }
 
+void ShenandoahConcurrentGC::vmop_entry_final_roots() {
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+  TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
+  ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
+
+  // This phase does not use workers, no need for setup
+  heap->try_inject_alloc_failure();
+  VM_ShenandoahFinalRoots op(this);
+  VMThread::execute(&op);
+}
+
 void ShenandoahConcurrentGC::entry_init_mark() {
   const char* msg = init_mark_event_message();
   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
@@ -261,6 +268,14 @@ void ShenandoahConcurrentGC::entry_final_updaterefs() {
   op_final_updaterefs();
 }
 
+void ShenandoahConcurrentGC::entry_final_roots() {
+  static const char* msg = "Pause Final Roots";
+  ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
+  EventMark em("%s", msg);
+
+  op_final_roots();
+}
+
 void ShenandoahConcurrentGC::entry_reset() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
@@ -393,18 +408,6 @@ void ShenandoahConcurrentGC::entry_cleanup_early() {
   op_cleanup_early();
 }
 
-void ShenandoahConcurrentGC::entry_rendezvous_roots() {
-  ShenandoahHeap* const heap = ShenandoahHeap::heap();
-  TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
-  static const char* msg = "Rendezvous roots";
-  ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_rendezvous_roots);
-  EventMark em("%s", msg);
-
-  // This phase does not use workers, no need for setup
-  heap->try_inject_alloc_failure();
-  op_rendezvous_roots();
-}
-
 void ShenandoahConcurrentGC::entry_evacuate() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
@@ -804,16 +807,6 @@ void ShenandoahConcurrentGC::op_weak_roots() {
     ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
     heap->workers()->run_task(&task);
   }
-
-  // Perform handshake to flush out dead oops
-  {
-    ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
-    heap->rendezvous_threads();
-  }
-
-  if (!ShenandoahHeap::heap()->unload_classes()) {
-    heap->set_concurrent_weak_root_in_progress(false);
-  }
 }
 
 void ShenandoahConcurrentGC::op_class_unloading() {
@@ -822,7 +815,6 @@ void ShenandoahConcurrentGC::op_class_unloading() {
           heap->unload_classes(),
           "Checked by caller");
   heap->do_class_unloading();
-  heap->set_concurrent_weak_root_in_progress(false);
 }
 
 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
@@ -913,17 +905,14 @@ void ShenandoahConcurrentGC::op_cleanup_early() {
   ShenandoahHeap::heap()->free_set()->recycle_trash();
 }
 
-void ShenandoahConcurrentGC::op_rendezvous_roots() {
-  ShenandoahHeap::heap()->rendezvous_threads();
-}
-
 void ShenandoahConcurrentGC::op_evacuate() {
   ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/);
 }
 
 void ShenandoahConcurrentGC::op_init_updaterefs() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   heap->set_evacuation_in_progress(false);
+  heap->set_concurrent_weak_root_in_progress(false);
   heap->prepare_update_heap_references(true /*concurrent*/);
   heap->set_update_refs_in_progress(true);
 
@@ -995,6 +984,10 @@ void ShenandoahConcurrentGC::op_final_updaterefs() {
   heap->rebuild_free_set(true /*concurrent*/);
 }
 
+void ShenandoahConcurrentGC::op_final_roots() {
+  ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false);
+}
+
 void ShenandoahConcurrentGC::op_cleanup_complete() {
   ShenandoahHeap::heap()->free_set()->recycle_trash();
 }

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp

@@ -40,6 +40,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
   friend class VM_ShenandoahFinalMarkStartEvac;
   friend class VM_ShenandoahInitUpdateRefs;
   friend class VM_ShenandoahFinalUpdateRefs;
+  friend class VM_ShenandoahFinalRoots;
 
 private:
   ShenandoahConcurrentMark  _mark;
@@ -59,13 +60,15 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
   void vmop_entry_final_mark();
   void vmop_entry_init_updaterefs();
   void vmop_entry_final_updaterefs();
+  void vmop_entry_final_roots();
 
   // Entry methods to normally STW GC operations. These set up logging, monitoring
   // and workers for net VM operation
   void entry_init_mark();
   void entry_final_mark();
   void entry_init_updaterefs();
   void entry_final_updaterefs();
+  void entry_final_roots();
 
   // Entry methods to normally concurrent GC operations. These set up logging, monitoring
   // for concurrent operation.
@@ -78,7 +81,6 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
   void entry_class_unloading();
   void entry_strong_roots();
   void entry_cleanup_early();
-  void entry_rendezvous_roots();
   void entry_evacuate();
   void entry_update_thread_roots();
   void entry_updaterefs();
@@ -96,12 +98,12 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
   void op_class_unloading();
   void op_strong_roots();
   void op_cleanup_early();
-  void op_rendezvous_roots();
   void op_evacuate();
   void op_init_updaterefs();
   void op_updaterefs();
   void op_update_thread_roots();
   void op_final_updaterefs();
+  void op_final_roots();
   void op_cleanup_complete();
 
   // Messages for GC trace events, they have to be immortal for

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

@@ -124,10 +124,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
 
   // Degenerated GC may carry concurrent root flags when upgrading to
   // full GC. We need to reset it before mutators resume.
-  if (ClassUnloading) {
-    heap->set_concurrent_strong_root_in_progress(false);
-    heap->set_concurrent_weak_root_in_progress(false);
-  }
+  heap->set_concurrent_strong_root_in_progress(false);
+  heap->set_concurrent_weak_root_in_progress(false);
 
   heap->set_full_gc_in_progress(true);
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1545,11 +1545,6 @@ class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionCl
   bool is_thread_safe() { return true; }
 };
 
-void ShenandoahHeap::rendezvous_threads() {
-  ShenandoahRendezvousClosure cl;
-  Handshake::execute(&cl);
-}
-
 void ShenandoahHeap::recycle_trash() {
   free_set()->recycle_trash();
 }
@@ -1655,7 +1650,6 @@ void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
 
 void ShenandoahHeap::do_class_unloading() {
   _unloader.unload();
-  set_concurrent_weak_root_in_progress(false);
 }
 
 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
@@ -1714,12 +1708,8 @@ void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
   }
 }
 
-void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
-  if (in_progress) {
-    _concurrent_weak_root_in_progress.set();
-  } else {
-    _concurrent_weak_root_in_progress.unset();
-  }
+void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
+  set_gc_state_mask(WEAK_ROOTS, cond);
 }
 
 GCTracer* ShenandoahHeap::tracer() {

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -263,6 +263,9 @@ class ShenandoahHeap : public CollectedHeap {
 
     // Heap is under updating: needs no additional barriers.
     UPDATEREFS_BITPOS = 3,
+
+    // Heap is under weak-reference/roots processing: needs weak-LRB barriers.
+    WEAK_ROOTS_BITPOS  = 4,
   };
 
   enum GCState {
@@ -271,6 +274,7 @@ class ShenandoahHeap : public CollectedHeap {
     MARKING       = 1 << MARKING_BITPOS,
     EVACUATION    = 1 << EVACUATION_BITPOS,
     UPDATEREFS    = 1 << UPDATEREFS_BITPOS,
+    WEAK_ROOTS    = 1 << WEAK_ROOTS_BITPOS,
   };
 
 private:
@@ -280,7 +284,6 @@ class ShenandoahHeap : public CollectedHeap {
   ShenandoahSharedFlag   _full_gc_move_in_progress;
   ShenandoahSharedFlag   _progress_last_gc;
   ShenandoahSharedFlag   _concurrent_strong_root_in_progress;
-  ShenandoahSharedFlag   _concurrent_weak_root_in_progress;
 
   void set_gc_state_all_threads(char state);
   void set_gc_state_mask(uint mask, bool value);
@@ -367,7 +370,6 @@ class ShenandoahHeap : public CollectedHeap {
   void update_heap_region_states(bool concurrent);
   void rebuild_free_set(bool concurrent);
 
-  void rendezvous_threads();
   void recycle_trash();
 
 public:

src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp

@@ -340,7 +340,7 @@ inline bool ShenandoahHeap::is_concurrent_strong_root_in_progress() const {
 }
 
 inline bool ShenandoahHeap::is_concurrent_weak_root_in_progress() const {
-  return _concurrent_weak_root_in_progress.is_set();
+  return _gc_state.is_set(WEAK_ROOTS);
 }
 
 template<class T>

src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp

@@ -28,6 +28,7 @@
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shenandoah/shenandoahNMethod.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
 
 nmethod* ShenandoahNMethod::nm() const {
   return _nm;

src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp

@@ -82,7 +82,6 @@ class outputStream;
   f(conc_weak_roots,                                "Concurrent Weak Roots")           \
   f(conc_weak_roots_work,                           "  Roots")                         \
   SHENANDOAH_PAR_PHASE_DO(conc_weak_roots_work_,    "    CWR: ", f)                    \
-  f(conc_weak_roots_rendezvous,                     "  Rendezvous")                    \
   f(conc_cleanup_early,                             "Concurrent Cleanup")              \
   f(conc_class_unload,                              "Concurrent Class Unloading")      \
   f(conc_class_unload_unlink,                       "  Unlink Stale")                  \
@@ -96,9 +95,11 @@ class outputStream;
   f(conc_class_unload_purge_ec,                     "    Exception Caches")            \
   f(conc_strong_roots,                              "Concurrent Strong Roots")         \
   SHENANDOAH_PAR_PHASE_DO(conc_strong_roots_,       "  CSR: ", f)                      \
-  f(conc_rendezvous_roots,                          "Rendezvous")                      \
   f(conc_evac,                                      "Concurrent Evacuation")           \
                                                                                        \
+  f(final_roots_gross,                              "Pause Final Roots (G)")           \
+  f(final_roots,                                    "Pause Final Roots (N)")           \
+                                                                                       \
   f(init_update_refs_gross,                         "Pause Init  Update Refs (G)")     \
   f(init_update_refs,                               "Pause Init  Update Refs (N)")     \
   f(init_update_refs_manage_gclabs,                 "  Manage GCLABs")                 \

src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp

@@ -46,13 +46,11 @@ ShenandoahGCStateResetter::ShenandoahGCStateResetter() :
   _gc_state(_heap->gc_state()),
   _concurrent_weak_root_in_progress(ShenandoahHeap::heap()->is_concurrent_weak_root_in_progress()) {
   _heap->_gc_state.clear();
-  _heap->set_concurrent_weak_root_in_progress(false);
 }
 
 ShenandoahGCStateResetter::~ShenandoahGCStateResetter() {
   _heap->_gc_state.set(_gc_state);
   assert(_heap->gc_state() == _gc_state, "Should be restored");
-  _heap->set_concurrent_weak_root_in_progress(_concurrent_weak_root_in_progress);
 }
 
 void ShenandoahRootVerifier::roots_do(OopClosure* oops) {

src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp

@@ -31,8 +31,6 @@
 #include "code/dependencyContext.hpp"
 #include "gc/shared/gcBehaviours.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
-#include "gc/shenandoah/shenandoahClosures.inline.hpp"
-#include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 #include "gc/shenandoah/shenandoahLock.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
@@ -135,6 +133,12 @@ void ShenandoahUnload::prepare() {
   DependencyContext::cleaning_start();
 }
 
+class ShenandoahRendezvousClosure : public HandshakeClosure {
+public:
+  inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
+  inline void do_thread(Thread* thread) {}
+};
+
 void ShenandoahUnload::unload() {
   ShenandoahHeap* heap = ShenandoahHeap::heap();
   assert(ClassUnloading, "Filtered by caller");

src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp

@@ -165,6 +165,7 @@ class ShenandoahSafepoint : public AllStatic {
            type == VM_Operation::VMOp_ShenandoahFinalMarkStartEvac ||
            type == VM_Operation::VMOp_ShenandoahInitUpdateRefs ||
            type == VM_Operation::VMOp_ShenandoahFinalUpdateRefs ||
+           type == VM_Operation::VMOp_ShenandoahFinalRoots ||
            type == VM_Operation::VMOp_ShenandoahFullGC ||
            type == VM_Operation::VMOp_ShenandoahDegeneratedGC;
   }