8256298: Shenandoah: Enable concurrent stack processing

Reviewed-by: rkennke, shade

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -24,14 +24,13 @@
 
 #include "precompiled.hpp"
 #include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetClone.inline.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetNMethod.hpp"
-#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
-#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
-#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahStackWatermark.hpp"
 #include "memory/iterator.inline.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #ifdef COMPILER1
@@ -112,6 +111,10 @@ void ShenandoahBarrierSet::on_thread_attach(Thread *thread) {
     ShenandoahThreadLocalData::set_gc_state(thread, _heap->gc_state());
     ShenandoahThreadLocalData::initialize_gclab(thread);
     ShenandoahThreadLocalData::set_disarmed_value(thread, ShenandoahCodeRoots::disarmed_value());
+
+    JavaThread* const jt = thread->as_Java_thread();
+    StackWatermark* const watermark = new ShenandoahStackWatermark(jt);
+    StackWatermarkSet::add_watermark(jt, watermark);
   }
 }
 
@@ -123,6 +126,16 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
     if (gclab != NULL) {
       gclab->retire();
     }
+
+    // SATB protocol requires to keep alive reacheable oops from roots at the beginning of GC
+    ShenandoahHeap* const heap = ShenandoahHeap::heap();
+    if (heap->is_concurrent_mark_in_progress()) {
+      ShenandoahKeepAliveClosure oops;
+      StackWatermarkSet::finish_processing(thread->as_Java_thread(), &oops, StackWatermarkKind::gc);
+    } else if (heap->is_concurrent_weak_root_in_progress() && heap->is_evacuation_in_progress()) {
+      ShenandoahContextEvacuateUpdateRootsClosure oops;
+      StackWatermarkSet::finish_processing(thread->as_Java_thread(), &oops, StackWatermarkKind::gc);
+    }
   }
 }
 
@@ -131,4 +144,3 @@ void ShenandoahBarrierSet::clone_barrier_runtime(oop src) {
     clone_barrier(src);
   }
 }
-

src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp

@@ -28,6 +28,7 @@
 #include "oops/accessDecorators.hpp"
 #include "runtime/handshake.hpp"
 
+class ShenandoahBarrierSet;
 class ShenandoahHeap;
 class ShenandoahMarkingContext;
 class ShenandoahHeapRegionSet;
@@ -57,6 +58,18 @@ class ShenandoahIsAliveSelector : public StackObj {
   inline BoolObjectClosure* is_alive_closure();
 };
 
+class ShenandoahKeepAliveClosure : public OopClosure {
+private:
+  ShenandoahBarrierSet* const _bs;
+public:
+  inline ShenandoahKeepAliveClosure();
+  inline void do_oop(oop* p);
+  inline void do_oop(narrowOop* p);
+private:
+  template <typename T>
+  void do_oop_work(T* p);
+};
+
 class ShenandoahUpdateRefsClosure: public OopClosure {
 private:
   ShenandoahHeap* _heap;
@@ -70,12 +83,12 @@ class ShenandoahUpdateRefsClosure: public OopClosure {
 };
 
 template <DecoratorSet MO = MO_UNORDERED>
-class ShenandoahEvacuateUpdateRootsClosure: public BasicOopIterateClosure {
+class ShenandoahEvacuateUpdateMetadataClosure: public BasicOopIterateClosure {
 private:
-  ShenandoahHeap* _heap;
-  Thread* _thread;
+  ShenandoahHeap* const _heap;
+  Thread* const         _thread;
 public:
-  inline ShenandoahEvacuateUpdateRootsClosure();
+  inline ShenandoahEvacuateUpdateMetadataClosure();
   inline void do_oop(oop* p);
   inline void do_oop(narrowOop* p);
 
@@ -84,12 +97,24 @@ class ShenandoahEvacuateUpdateRootsClosure: public BasicOopIterateClosure {
   inline void do_oop_work(T* p);
 };
 
-class ShenandoahEvacUpdateOopStorageRootsClosure : public BasicOopIterateClosure {
+// Context free version, cannot cache calling thread
+class ShenandoahEvacuateUpdateRootsClosure : public BasicOopIterateClosure {
 private:
-  ShenandoahHeap* _heap;
-  Thread* _thread;
+  ShenandoahHeap* const _heap;
+public:
+  inline ShenandoahEvacuateUpdateRootsClosure();
+  inline void do_oop(oop* p);
+  inline void do_oop(narrowOop* p);
+protected:
+  template <typename T>
+  inline void do_oop_work(T* p, Thread* thr);
+};
+
+class ShenandoahContextEvacuateUpdateRootsClosure : public ShenandoahEvacuateUpdateRootsClosure {
+private:
+  Thread* const _thread;
 public:
-  inline ShenandoahEvacUpdateOopStorageRootsClosure();
+  inline ShenandoahContextEvacuateUpdateRootsClosure();
   inline void do_oop(oop* p);
   inline void do_oop(narrowOop* p);
 };

src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp

@@ -26,7 +26,9 @@
 
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahClosures.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
@@ -64,6 +66,30 @@ BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
          reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
 }
 
+ShenandoahKeepAliveClosure::ShenandoahKeepAliveClosure() :
+  _bs(static_cast<ShenandoahBarrierSet*>(BarrierSet::barrier_set())) {
+}
+
+void ShenandoahKeepAliveClosure::do_oop(oop* p) {
+  do_oop_work(p);
+}
+
+void ShenandoahKeepAliveClosure::do_oop(narrowOop* p) {
+  do_oop_work(p);
+}
+
+template <typename T>
+void ShenandoahKeepAliveClosure::do_oop_work(T* p) {
+  assert(ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Only for concurrent marking phase");
+  assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
+
+  T o = RawAccess<>::oop_load(p);
+  if (!CompressedOops::is_null(o)) {
+    oop obj = CompressedOops::decode_not_null(o);
+    _bs->enqueue(obj);
+  }
+}
+
 ShenandoahUpdateRefsClosure::ShenandoahUpdateRefsClosure() :
   _heap(ShenandoahHeap::heap()) {
 }
@@ -81,16 +107,17 @@ void ShenandoahUpdateRefsClosure::do_oop(oop* p)       { do_oop_work(p); }
 void ShenandoahUpdateRefsClosure::do_oop(narrowOop* p) { do_oop_work(p); }
 
 template <DecoratorSet MO>
-ShenandoahEvacuateUpdateRootsClosure<MO>::ShenandoahEvacuateUpdateRootsClosure() :
+ShenandoahEvacuateUpdateMetadataClosure<MO>::ShenandoahEvacuateUpdateMetadataClosure() :
   _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
 }
 
 template <DecoratorSet MO>
 template <class T>
-void ShenandoahEvacuateUpdateRootsClosure<MO>::do_oop_work(T* p) {
+void ShenandoahEvacuateUpdateMetadataClosure<MO>::do_oop_work(T* p) {
   assert(_heap->is_concurrent_weak_root_in_progress() ||
          _heap->is_concurrent_strong_root_in_progress(),
          "Only do this in root processing phase");
+  assert(_thread == Thread::current(), "Wrong thread");
 
   T o = RawAccess<>::oop_load(p);
   if (! CompressedOops::is_null(o)) {
@@ -107,41 +134,64 @@ void ShenandoahEvacuateUpdateRootsClosure<MO>::do_oop_work(T* p) {
   }
 }
 template <DecoratorSet MO>
-void ShenandoahEvacuateUpdateRootsClosure<MO>::do_oop(oop* p) {
+void ShenandoahEvacuateUpdateMetadataClosure<MO>::do_oop(oop* p) {
   do_oop_work(p);
 }
 
 template <DecoratorSet MO>
-void ShenandoahEvacuateUpdateRootsClosure<MO>::do_oop(narrowOop* p) {
+void ShenandoahEvacuateUpdateMetadataClosure<MO>::do_oop(narrowOop* p) {
   do_oop_work(p);
 }
 
-ShenandoahEvacUpdateOopStorageRootsClosure::ShenandoahEvacUpdateOopStorageRootsClosure() :
-  _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
+ShenandoahEvacuateUpdateRootsClosure::ShenandoahEvacuateUpdateRootsClosure() :
+  _heap(ShenandoahHeap::heap()) {
 }
 
-void ShenandoahEvacUpdateOopStorageRootsClosure::do_oop(oop* p) {
+template <typename T>
+void ShenandoahEvacuateUpdateRootsClosure::do_oop_work(T* p, Thread* t) {
   assert(_heap->is_concurrent_weak_root_in_progress() ||
          _heap->is_concurrent_strong_root_in_progress(),
          "Only do this in root processing phase");
+  assert(t == Thread::current(), "Wrong thread");
 
-  oop obj = RawAccess<>::oop_load(p);
-  if (! CompressedOops::is_null(obj)) {
+  T o = RawAccess<>::oop_load(p);
+  if (!CompressedOops::is_null(o)) {
+    oop obj = CompressedOops::decode_not_null(o);
     if (_heap->in_collection_set(obj)) {
       assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
       shenandoah_assert_marked(p, obj);
       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
       if (resolved == obj) {
-        resolved = _heap->evacuate_object(obj, _thread);
+        resolved = _heap->evacuate_object(obj, t);
       }
-
-      Atomic::cmpxchg(p, obj, resolved);
+      _heap->cas_oop(resolved, p, o);
     }
   }
 }
 
-void ShenandoahEvacUpdateOopStorageRootsClosure::do_oop(narrowOop* p) {
-  ShouldNotReachHere();
+void ShenandoahEvacuateUpdateRootsClosure::do_oop(oop* p) {
+  ShenandoahEvacOOMScope scope;
+  do_oop_work(p, Thread::current());
+}
+
+void ShenandoahEvacuateUpdateRootsClosure::do_oop(narrowOop* p) {
+  ShenandoahEvacOOMScope scope;
+  do_oop_work(p, Thread::current());
+}
+
+ShenandoahContextEvacuateUpdateRootsClosure::ShenandoahContextEvacuateUpdateRootsClosure() :
+  ShenandoahEvacuateUpdateRootsClosure(),
+  _thread(Thread::current()) {
+}
+
+void ShenandoahContextEvacuateUpdateRootsClosure::do_oop(oop* p) {
+  ShenandoahEvacOOMScope scope;
+  do_oop_work(p, _thread);
+}
+
+void ShenandoahContextEvacuateUpdateRootsClosure::do_oop(narrowOop* p) {
+  ShenandoahEvacOOMScope scope;
+  do_oop_work(p, _thread);
 }
 
 template <bool CONCURRENT, typename IsAlive, typename KeepAlive>