8286829: Shenandoah: fix Shenandoah Loom support

Reviewed-by: shade

Author: zhengyu123

src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -60,8 +60,13 @@ bool ShenandoahBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
     return false;
   }
 
-  // Heal oops and disarm
+  // Heal oops
   ShenandoahNMethod::heal_nmethod(nm);
+
+  // CodeCache sweeper support
+  nm->mark_as_maybe_on_continuation();
+
+  // Disarm
   ShenandoahNMethod::disarm_nmethod(nm);
   return true;
 }

src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP
 
+#include "code/nmethod.hpp"
 #include "memory/iterator.hpp"
 #include "oops/accessDecorators.hpp"
 #include "runtime/handshake.hpp"
@@ -71,7 +72,12 @@ class ShenandoahKeepAliveClosure : public OopClosure {
   void do_oop_work(T* p);
 };
 
-class ShenandoahUpdateRefsClosure: public OopClosure {
+class ShenandoahOopClosureBase : public MetadataVisitingOopIterateClosure {
+public:
+  inline void do_nmethod(nmethod* nm);
+};
+
+class ShenandoahUpdateRefsClosure: public ShenandoahOopClosureBase {
 private:
   ShenandoahHeap* _heap;
 public:
@@ -84,7 +90,7 @@ class ShenandoahUpdateRefsClosure: public OopClosure {
 };
 
 template <DecoratorSet MO = MO_UNORDERED>
-class ShenandoahEvacuateUpdateMetadataClosure: public BasicOopIterateClosure {
+class ShenandoahEvacuateUpdateMetadataClosure: public ShenandoahOopClosureBase {
 private:
   ShenandoahHeap* const _heap;
   Thread* const         _thread;
@@ -99,7 +105,7 @@ class ShenandoahEvacuateUpdateMetadataClosure: public BasicOopIterateClosure {
 };
 
 // Context free version, cannot cache calling thread
-class ShenandoahEvacuateUpdateRootsClosure : public BasicOopIterateClosure {
+class ShenandoahEvacuateUpdateRootsClosure : public ShenandoahOopClosureBase {
 private:
   ShenandoahHeap* const _heap;
 public:

src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp

@@ -30,6 +30,7 @@
 #include "gc/shenandoah/shenandoahAsserts.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
+#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
@@ -67,8 +68,12 @@ BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
          reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
 }
 
+void ShenandoahOopClosureBase::do_nmethod(nmethod* nm) {
+  nm->run_nmethod_entry_barrier();
+}
+
 ShenandoahKeepAliveClosure::ShenandoahKeepAliveClosure() :
-  _bs(static_cast<ShenandoahBarrierSet*>(BarrierSet::barrier_set())) {
+  _bs(ShenandoahBarrierSet::barrier_set()) {
 }
 
 void ShenandoahKeepAliveClosure::do_oop(oop* p) {

src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2017, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -244,7 +244,13 @@ class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
     if (_bs->is_armed(nm)) {
       ShenandoahEvacOOMScope oom_evac_scope;
       ShenandoahNMethod::heal_nmethod_metadata(nm_data);
-      _bs->disarm(nm);
+      if (Continuations::enabled()) {
+        // Loom needs to know about visited nmethods. Arm the nmethods to get
+        // mark_as_maybe_on_continuation() callbacks when they are used again.
+        _bs->arm(nm, 0);
+      } else {
+        _bs->disarm(nm);
+      }
     }
 
     // Clear compiled ICs and exception caches

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shared/collectorCounters.hpp"
+#include "gc/shared/continuationGCSupport.inline.hpp"
 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
@@ -506,6 +507,10 @@ class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionCl
   bool is_thread_safe() { return true; }
 };
 
+void ShenandoahConcurrentGC::start_mark() {
+  _mark.start_mark();
+}
+
 void ShenandoahConcurrentGC::op_init_mark() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
@@ -525,6 +530,8 @@ void ShenandoahConcurrentGC::op_init_mark() {
 
   heap->set_concurrent_mark_in_progress(true);
 
+  start_mark();
+
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
     ShenandoahInitMarkUpdateRegionStateClosure cl;

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp

@@ -106,6 +106,8 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
   void op_final_roots();
   void op_cleanup_complete();
 
+  void start_mark();
+
   // Messages for GC trace events, they have to be immortal for
   // passing around the logging/tracing systems
   const char* init_mark_event_message() const;

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp

@@ -41,6 +41,7 @@
 #include "gc/shenandoah/shenandoahUtils.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/resourceArea.hpp"
+#include "runtime/continuation.hpp"
 
 class ShenandoahConcurrentMarkingTask : public WorkerTask {
 private:
@@ -238,6 +239,8 @@ void ShenandoahConcurrentMark::finish_mark() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   heap->set_concurrent_mark_in_progress(false);
   heap->mark_complete_marking_context();
+
+  end_mark();
 }
 
 void ShenandoahConcurrentMark::finish_mark_work() {

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2312,16 +2312,15 @@ bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
 
   // Objects allocated after marking start are implicitly alive, don't need any barriers during
   // marking phase.
-  if (is_concurrent_mark_in_progress() && marking_context()->allocated_after_mark_start(obj)) {
-    return false;
+  if (is_concurrent_mark_in_progress() &&
+     !marking_context()->allocated_after_mark_start(obj)) {
+    return true;
   }
 
-  // Objects allocated after evacuation start are guaranteed in to-space, don't need any barriers
-  // during evacuation/update references phases.
-  if (has_forwarded_objects() &&
-      cast_from_oop<HeapWord*>(obj) >= heap_region_containing(obj)->get_update_watermark()) {
-    return false;
+  // Can not guarantee obj is deeply good.
+  if (has_forwarded_objects()) {
+    return true;
   }
 
-  return true;
+  return false;
 }

src/hotspot/share/gc/shenandoah/shenandoahMark.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,7 @@
 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
 #include "gc/shenandoah/shenandoahVerifier.hpp"
+#include "runtime/continuation.hpp"
 
 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q,  ShenandoahReferenceProcessor* rp) :
   MetadataVisitingOopIterateClosure(rp),
@@ -45,6 +46,20 @@ ShenandoahMark::ShenandoahMark() :
   _task_queues(ShenandoahHeap::heap()->marking_context()->task_queues()) {
 }
 
+void ShenandoahMark::start_mark() {
+  // Tell the sweeper that we start a marking cycle.
+  if (!Continuations::is_gc_marking_cycle_active()) {
+    Continuations::on_gc_marking_cycle_start();
+  }
+}
+
+void ShenandoahMark::end_mark() {
+  // Tell the sweeper that we finished a marking cycle.
+  // Unlike other GCs, we do not arm the nmethods
+  // when marking terminates.
+  Continuations::on_gc_marking_cycle_finish();
+}
+
 void ShenandoahMark::clear() {
   // Clean up marking stacks.
   ShenandoahObjToScanQueueSet* queues = ShenandoahHeap::heap()->marking_context()->task_queues();

src/hotspot/share/gc/shenandoah/shenandoahMark.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -50,6 +50,10 @@ class ShenandoahMark: public StackObj {
 
   static void clear();
 
+  // Loom support
+  void start_mark();
+  void end_mark();
+
   // Helpers
   inline ShenandoahObjToScanQueueSet* task_queues() const;
   inline ShenandoahObjToScanQueue* get_queue(uint index) const;