Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
8263427: Shenandoah: Trigger weak-LRB even when heap is stable
Reviewed-by: shade, zgu
  • Loading branch information
rkennke committed Mar 16, 2021
1 parent 4517d72 commit 75ef6f580e830f506c4e2f93105f111e6d0a5bd0
Showing with 226 additions and 77 deletions.
  1. +5 −1 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
  2. +5 −1 src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
  3. +5 −1 src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
  4. +0 −6 src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp
  5. +0 −7 src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
  6. +25 −32 src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
  7. +4 −2 src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp
  8. +2 −4 src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
  9. +2 −12 src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
  10. +4 −2 src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
  11. +1 −1 src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
  12. +1 −0 src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp
  13. +3 −2 src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp
  14. +0 −2 src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp
  15. +6 −2 src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp
  16. +1 −0 src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp
  17. +5 −0 src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp
  18. +11 −0 src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp
  19. +14 −2 src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
  20. +3 −0 src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp
  21. +1 −0 src/hotspot/share/runtime/vmOperation.hpp
  22. +26 −0 test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java
  23. +102 −0 test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java
@@ -342,7 +342,11 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
#endif

Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
__ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
int flags = ShenandoahHeap::HAS_FORWARDED;
if (!is_strong) {
flags |= ShenandoahHeap::WEAK_ROOTS;
}
__ testb(gc_state, flags);
__ jcc(Assembler::zero, heap_stable);

Register tmp1 = noreg, tmp2 = noreg;
@@ -138,7 +138,11 @@ LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, L
// Read and check the gc-state-flag.
LIR_Opr flag_val = gen->new_register(T_INT);
__ load(active_flag_addr, flag_val);
LIR_Opr mask = LIR_OprFact::intConst(ShenandoahHeap::HAS_FORWARDED);
int flags = ShenandoahHeap::HAS_FORWARDED;
if (!ShenandoahBarrierSet::is_strong_access(decorators)) {
flags |= ShenandoahHeap::WEAK_ROOTS;
}
LIR_Opr mask = LIR_OprFact::intConst(flags);
LIR_Opr mask_reg = gen->new_register(T_INT);
__ move(mask, mask_reg);

@@ -1362,7 +1362,11 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);

// Stable path.
test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED);
int flags = ShenandoahHeap::HAS_FORWARDED;
if (!ShenandoahBarrierSet::is_strong_access(lrb->decorators())) {
flags |= ShenandoahHeap::WEAK_ROOTS;
}
test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, flags);
IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();

// Heap stable case
@@ -140,12 +140,6 @@ class ShenandoahCodeBlobAndDisarmClosure: public CodeBlobToOopClosure {
inline void do_code_blob(CodeBlob* cb);
};

class ShenandoahRendezvousClosure : public HandshakeClosure {
public:
inline ShenandoahRendezvousClosure();
inline void do_thread(Thread* thread);
};

#ifdef ASSERT
class ShenandoahAssertNotForwardedClosure : public OopClosure {
private:
@@ -233,13 +233,6 @@ void ShenandoahCodeBlobAndDisarmClosure::do_code_blob(CodeBlob* cb) {
}
}

ShenandoahRendezvousClosure::ShenandoahRendezvousClosure() :
HandshakeClosure("ShenandoahRendezvous") {
}

void ShenandoahRendezvousClosure::do_thread(Thread* thread) {
}

#ifdef ASSERT
template <class T>
void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) {
@@ -166,11 +166,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
// Update references freed up collection set, kick the cleanup to reclaim the space.
entry_cleanup_complete();
} else {
// Concurrent weak/strong root flags are unset concurrently. We depend on updateref GC safepoints
// to ensure the changes are visible to all mutators before gc cycle is completed.
// In case of no evacuation, updateref GC safepoints are skipped. Therefore, we will need
// to perform thread handshake to ensure their consistences.
entry_rendezvous_roots();
vmop_entry_final_roots();
}

return true;
@@ -216,6 +212,17 @@ void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() {
VMThread::execute(&op);
}

void ShenandoahConcurrentGC::vmop_entry_final_roots() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);

// This phase does not use workers, no need for setup
heap->try_inject_alloc_failure();
VM_ShenandoahFinalRoots op(this);
VMThread::execute(&op);
}

void ShenandoahConcurrentGC::entry_init_mark() {
const char* msg = init_mark_event_message();
ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
@@ -261,6 +268,14 @@ void ShenandoahConcurrentGC::entry_final_updaterefs() {
op_final_updaterefs();
}

void ShenandoahConcurrentGC::entry_final_roots() {
static const char* msg = "Pause Final Roots";
ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
EventMark em("%s", msg);

op_final_roots();
}

void ShenandoahConcurrentGC::entry_reset() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
@@ -393,18 +408,6 @@ void ShenandoahConcurrentGC::entry_cleanup_early() {
op_cleanup_early();
}

void ShenandoahConcurrentGC::entry_rendezvous_roots() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
static const char* msg = "Rendezvous roots";
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_rendezvous_roots);
EventMark em("%s", msg);

// This phase does not use workers, no need for setup
heap->try_inject_alloc_failure();
op_rendezvous_roots();
}

void ShenandoahConcurrentGC::entry_evacuate() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
@@ -804,16 +807,6 @@ void ShenandoahConcurrentGC::op_weak_roots() {
ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
heap->workers()->run_task(&task);
}

// Perform handshake to flush out dead oops
{
ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
heap->rendezvous_threads();
}

if (!ShenandoahHeap::heap()->unload_classes()) {
heap->set_concurrent_weak_root_in_progress(false);
}
}

void ShenandoahConcurrentGC::op_class_unloading() {
@@ -822,7 +815,6 @@ void ShenandoahConcurrentGC::op_class_unloading() {
heap->unload_classes(),
"Checked by caller");
heap->do_class_unloading();
heap->set_concurrent_weak_root_in_progress(false);
}

class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
@@ -913,17 +905,14 @@ void ShenandoahConcurrentGC::op_cleanup_early() {
ShenandoahHeap::heap()->free_set()->recycle_trash();
}

void ShenandoahConcurrentGC::op_rendezvous_roots() {
ShenandoahHeap::heap()->rendezvous_threads();
}

void ShenandoahConcurrentGC::op_evacuate() {
ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/);
}

void ShenandoahConcurrentGC::op_init_updaterefs() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
heap->set_evacuation_in_progress(false);
heap->set_concurrent_weak_root_in_progress(false);
heap->prepare_update_heap_references(true /*concurrent*/);
heap->set_update_refs_in_progress(true);

@@ -995,6 +984,10 @@ void ShenandoahConcurrentGC::op_final_updaterefs() {
heap->rebuild_free_set(true /*concurrent*/);
}

void ShenandoahConcurrentGC::op_final_roots() {
ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false);
}

void ShenandoahConcurrentGC::op_cleanup_complete() {
ShenandoahHeap::heap()->free_set()->recycle_trash();
}
@@ -40,6 +40,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
friend class VM_ShenandoahFinalMarkStartEvac;
friend class VM_ShenandoahInitUpdateRefs;
friend class VM_ShenandoahFinalUpdateRefs;
friend class VM_ShenandoahFinalRoots;

private:
ShenandoahConcurrentMark _mark;
@@ -59,13 +60,15 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
void vmop_entry_final_mark();
void vmop_entry_init_updaterefs();
void vmop_entry_final_updaterefs();
void vmop_entry_final_roots();

// Entry methods to normally STW GC operations. These set up logging, monitoring
// and workers for net VM operation
void entry_init_mark();
void entry_final_mark();
void entry_init_updaterefs();
void entry_final_updaterefs();
void entry_final_roots();

// Entry methods to normally concurrent GC operations. These set up logging, monitoring
// for concurrent operation.
@@ -78,7 +81,6 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
void entry_class_unloading();
void entry_strong_roots();
void entry_cleanup_early();
void entry_rendezvous_roots();
void entry_evacuate();
void entry_update_thread_roots();
void entry_updaterefs();
@@ -96,12 +98,12 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
void op_class_unloading();
void op_strong_roots();
void op_cleanup_early();
void op_rendezvous_roots();
void op_evacuate();
void op_init_updaterefs();
void op_updaterefs();
void op_update_thread_roots();
void op_final_updaterefs();
void op_final_roots();
void op_cleanup_complete();

// Messages for GC trace events, they have to be immortal for
@@ -124,10 +124,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {

// Degenerated GC may carry concurrent root flags when upgrading to
// full GC. We need to reset it before mutators resume.
if (ClassUnloading) {
heap->set_concurrent_strong_root_in_progress(false);
heap->set_concurrent_weak_root_in_progress(false);
}
heap->set_concurrent_strong_root_in_progress(false);
heap->set_concurrent_weak_root_in_progress(false);

heap->set_full_gc_in_progress(true);

@@ -1545,11 +1545,6 @@ class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionCl
bool is_thread_safe() { return true; }
};

void ShenandoahHeap::rendezvous_threads() {
ShenandoahRendezvousClosure cl;
Handshake::execute(&cl);
}

void ShenandoahHeap::recycle_trash() {
free_set()->recycle_trash();
}
@@ -1655,7 +1650,6 @@ void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {

void ShenandoahHeap::do_class_unloading() {
_unloader.unload();
set_concurrent_weak_root_in_progress(false);
}

void ShenandoahHeap::stw_weak_refs(bool full_gc) {
@@ -1714,12 +1708,8 @@ void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
}
}

void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
if (in_progress) {
_concurrent_weak_root_in_progress.set();
} else {
_concurrent_weak_root_in_progress.unset();
}
void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
set_gc_state_mask(WEAK_ROOTS, cond);
}

GCTracer* ShenandoahHeap::tracer() {
@@ -263,6 +263,9 @@ class ShenandoahHeap : public CollectedHeap {

// Heap is under updating: needs no additional barriers.
UPDATEREFS_BITPOS = 3,

// Heap is under weak-reference/roots processing: needs weak-LRB barriers.
WEAK_ROOTS_BITPOS = 4,
};

enum GCState {
@@ -271,6 +274,7 @@ class ShenandoahHeap : public CollectedHeap {
MARKING = 1 << MARKING_BITPOS,
EVACUATION = 1 << EVACUATION_BITPOS,
UPDATEREFS = 1 << UPDATEREFS_BITPOS,
WEAK_ROOTS = 1 << WEAK_ROOTS_BITPOS,
};

private:
@@ -280,7 +284,6 @@ class ShenandoahHeap : public CollectedHeap {
ShenandoahSharedFlag _full_gc_move_in_progress;
ShenandoahSharedFlag _progress_last_gc;
ShenandoahSharedFlag _concurrent_strong_root_in_progress;
ShenandoahSharedFlag _concurrent_weak_root_in_progress;

void set_gc_state_all_threads(char state);
void set_gc_state_mask(uint mask, bool value);
@@ -367,7 +370,6 @@ class ShenandoahHeap : public CollectedHeap {
void update_heap_region_states(bool concurrent);
void rebuild_free_set(bool concurrent);

void rendezvous_threads();
void recycle_trash();

public:
@@ -340,7 +340,7 @@ inline bool ShenandoahHeap::is_concurrent_strong_root_in_progress() const {
}

inline bool ShenandoahHeap::is_concurrent_weak_root_in_progress() const {
return _concurrent_weak_root_in_progress.is_set();
return _gc_state.is_set(WEAK_ROOTS);
}

template<class T>
@@ -28,6 +28,7 @@
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shenandoah/shenandoahNMethod.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"

nmethod* ShenandoahNMethod::nm() const {
return _nm;
@@ -82,7 +82,6 @@ class outputStream;
f(conc_weak_roots, "Concurrent Weak Roots") \
f(conc_weak_roots_work, " Roots") \
SHENANDOAH_PAR_PHASE_DO(conc_weak_roots_work_, " CWR: ", f) \
f(conc_weak_roots_rendezvous, " Rendezvous") \
f(conc_cleanup_early, "Concurrent Cleanup") \
f(conc_class_unload, "Concurrent Class Unloading") \
f(conc_class_unload_unlink, " Unlink Stale") \
@@ -96,9 +95,11 @@ class outputStream;
f(conc_class_unload_purge_ec, " Exception Caches") \
f(conc_strong_roots, "Concurrent Strong Roots") \
SHENANDOAH_PAR_PHASE_DO(conc_strong_roots_, " CSR: ", f) \
f(conc_rendezvous_roots, "Rendezvous") \
f(conc_evac, "Concurrent Evacuation") \
\
f(final_roots_gross, "Pause Final Roots (G)") \
f(final_roots, "Pause Final Roots (N)") \
\
f(init_update_refs_gross, "Pause Init Update Refs (G)") \
f(init_update_refs, "Pause Init Update Refs (N)") \
f(init_update_refs_manage_gclabs, " Manage GCLABs") \
@@ -46,13 +46,11 @@ ShenandoahGCStateResetter::ShenandoahGCStateResetter() :
_gc_state(_heap->gc_state()),
_concurrent_weak_root_in_progress(ShenandoahHeap::heap()->is_concurrent_weak_root_in_progress()) {
_heap->_gc_state.clear();
_heap->set_concurrent_weak_root_in_progress(false);
}

ShenandoahGCStateResetter::~ShenandoahGCStateResetter() {
_heap->_gc_state.set(_gc_state);
assert(_heap->gc_state() == _gc_state, "Should be restored");
_heap->set_concurrent_weak_root_in_progress(_concurrent_weak_root_in_progress);
}

void ShenandoahRootVerifier::roots_do(OopClosure* oops) {
@@ -31,8 +31,6 @@
#include "code/dependencyContext.hpp"
#include "gc/shared/gcBehaviours.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahNMethod.inline.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
@@ -135,6 +133,12 @@ void ShenandoahUnload::prepare() {
DependencyContext::cleaning_start();
}

class ShenandoahRendezvousClosure : public HandshakeClosure {
public:
inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
inline void do_thread(Thread* thread) {}
};

void ShenandoahUnload::unload() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
assert(ClassUnloading, "Filtered by caller");
@@ -165,6 +165,7 @@ class ShenandoahSafepoint : public AllStatic {
type == VM_Operation::VMOp_ShenandoahFinalMarkStartEvac ||
type == VM_Operation::VMOp_ShenandoahInitUpdateRefs ||
type == VM_Operation::VMOp_ShenandoahFinalUpdateRefs ||
type == VM_Operation::VMOp_ShenandoahFinalRoots ||
type == VM_Operation::VMOp_ShenandoahFullGC ||
type == VM_Operation::VMOp_ShenandoahDegeneratedGC;
}

0 comments on commit 75ef6f5

Please sign in to comment.