8344049: Shenandoah: Eliminate init-update-refs safepoint

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -67,8 +67,7 @@ inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load
 
   oop fwd = resolve_forwarded_not_null_mutator(obj);
   if (obj == fwd) {
-    assert(_heap->is_evacuation_in_progress(),
-           "evac should be in progress");
+    assert(_heap->is_evacuation_in_progress(), "evac should be in progress");
     Thread* const t = Thread::current();
     ShenandoahEvacOOMScope scope(t);
     fwd = _heap->evacuate_object(obj, t);
@@ -86,8 +85,8 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj) {
   if (!ShenandoahLoadRefBarrier) {
     return obj;
   }
-  if (_heap->has_forwarded_objects() &&
-      _heap->in_collection_set(obj)) { // Subsumes null-check
+  if (_heap->has_forwarded_objects() && _heap->in_collection_set(obj)) {
+    // Subsumes null-check
     assert(obj != nullptr, "cset check must have subsumed null-check");
     oop fwd = resolve_forwarded_not_null(obj);
     if (obj == fwd && _heap->is_evacuation_in_progress()) {
@@ -381,7 +380,7 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
   // this barrier will be called with ENQUEUE=true and HAS_FWD=false, even though the young generation
   // may have forwarded objects. In this case, the `arraycopy_work` is first called with HAS_FWD=true and
   // ENQUEUE=false.
-  assert(HAS_FWD == _heap->has_forwarded_objects() || (_heap->gc_state() & ShenandoahHeap::OLD_MARKING) != 0,
+  assert(HAS_FWD == _heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(),
          "Forwarded object status is sane");
   // This function cannot be called to handle marking and evacuation at the same time (they operate on
   // different sides of the copy).
@@ -418,7 +417,7 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
     return;
   }
 
-  int gc_state = _heap->gc_state();
+  char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
   if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
     arraycopy_evacuation(src, count);
   } else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) {

src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp

@@ -95,8 +95,7 @@ void ShenandoahBarrierSet::clone_barrier(oop obj) {
   assert(ShenandoahCloneBarrier, "only get here with clone barriers enabled");
   shenandoah_assert_correct(nullptr, obj);
 
-  int gc_state = _heap->gc_state();
-  if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
+  if (_heap->is_evacuation_in_progress()) {
     clone_evacuation(obj);
   } else {
     clone_update(obj);

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -192,8 +192,14 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
       return false;
     }
 
+    // Evacuation is complete, retire gc labs
+    heap->concurrent_prepare_for_update_refs();
+
     // Perform update-refs phase.
-    vmop_entry_init_updaterefs();
+    if (ShenandoahVerify || ShenandoahPacing) {
+      vmop_entry_init_updaterefs();
+    }
+
     entry_updaterefs();
     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) {
       return false;
@@ -748,10 +754,6 @@ void ShenandoahConcurrentGC::op_final_mark() {
           heap->verifier()->verify_after_concmark();
         }
       }
-
-      if (VerifyAfterGC) {
-        Universe::verify();
-      }
     }
   }
 }
@@ -920,8 +922,8 @@ class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
     }
 
     // If we are going to perform concurrent class unloading later on, we need to
-    // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
-    // can cleanup immediate garbage sooner.
+    // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
+    // can clean up immediate garbage sooner.
     if (ShenandoahHeap::heap()->unload_classes()) {
       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
       // CLD's holder or evacuate it.
@@ -947,21 +949,10 @@ void ShenandoahConcurrentGC::op_weak_roots() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
   // Concurrent weak root processing
-  {
-    ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
-    ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
-    ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
-    heap->workers()->run_task(&task);
-  }
-
-  // Perform handshake to flush out dead oops
-  {
-    ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
-    heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
-  }
-  // We can only toggle concurrent_weak_root_in_progress flag
-  // at a safepoint, so that mutators see a consistent
-  // value. The flag will be cleared at the next safepoint.
+  ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
+  ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
+  ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
+  heap->workers()->run_task(&task);
 }
 
 void ShenandoahConcurrentGC::op_class_unloading() {
@@ -1058,10 +1049,6 @@ void ShenandoahConcurrentGC::op_evacuate() {
 
 void ShenandoahConcurrentGC::op_init_updaterefs() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
-  heap->set_evacuation_in_progress(false);
-  heap->set_concurrent_weak_root_in_progress(false);
-  heap->prepare_update_heap_references(true /*concurrent*/);
-  heap->set_update_refs_in_progress(true);
   if (ShenandoahVerify) {
     heap->verifier()->verify_before_updaterefs();
   }
@@ -1178,6 +1165,10 @@ void ShenandoahConcurrentGC::op_final_roots() {
       ShenandoahGenerationalHeap::heap()->update_region_ages(_generation->complete_marking_context());
     }
   }
+
+  if (VerifyAfterGC) {
+    Universe::verify();
+  }
 }
 
 void ShenandoahConcurrentGC::op_cleanup_complete() {

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -998,6 +998,8 @@ HeapWord* ShenandoahFreeSet::allocate_aligned_plab(size_t size, ShenandoahAllocR
 HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) {
   assert (has_alloc_capacity(r), "Performance: should avoid full regions on this path: " SIZE_FORMAT, r->index());
   if (_heap->is_concurrent_weak_root_in_progress() && r->is_trash()) {
+    // We cannot use this region for allocation when weak roots are in progress because the collector may need
+    // to reference unmarked oops during concurrent classunloading.
     return nullptr;
   }
   HeapWord* result = nullptr;