Skip to content
Permalink
Browse files
8242054: Shenandoah: New incremental-update mode
Reviewed-by: shade
  • Loading branch information
rkennke committed Apr 6, 2020
1 parent ba26538 commit c4d01f8c8bfbd0af82fb80009d23000e344ca04c
Show file tree
Hide file tree
Showing 49 changed files with 819 additions and 78 deletions.
@@ -49,7 +49,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
Register src, Register dst, Register count, RegSet saved_regs) {
if (is_oop) {
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {

Label done;

@@ -59,7 +59,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
// Is GC active?
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
__ ldrb(rscratch1, gc_state);
if (dest_uninitialized) {
if (ShenandoahSATBBarrier && dest_uninitialized) {
__ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
} else {
__ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
@@ -53,7 +53,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec

if (is_reference_type(type)) {

if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {
#ifdef _LP64
Register thread = r15_thread;
#else
@@ -79,9 +79,11 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec

// Avoid runtime call when not active.
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
int flags = ShenandoahHeap::HAS_FORWARDED;
if (!dest_uninitialized) {
flags |= ShenandoahHeap::MARKING;
int flags;
if (ShenandoahSATBBarrier && dest_uninitialized) {
flags = ShenandoahHeap::HAS_FORWARDED;
} else {
flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
}
__ testb(gc_state, flags);
__ jcc(Assembler::zero, done);
@@ -774,7 +774,7 @@ bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_couple
if (!is_oop) {
return false;
}
if (tightly_coupled_alloc) {
if (ShenandoahSATBBarrier && tightly_coupled_alloc) {
if (phase == Optimization) {
return false;
}
@@ -841,7 +841,11 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo
debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));

Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
int flags = ShenandoahHeap::HAS_FORWARDED;
if (ShenandoahStoreValEnqueueBarrier) {
flags |= ShenandoahHeap::MARKING;
}
Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));

@@ -860,8 +860,8 @@ static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNod
inner->clear_strip_mined();
}

void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase) {
void ShenandoahBarrierC2Support::test_heap_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase, int flags) {
IdealLoopTree* loop = phase->get_loop(ctrl);
Node* thread = new ThreadLocalNode();
phase->register_new_node(thread, ctrl);
@@ -875,7 +875,7 @@ void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, No

Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
phase->register_new_node(gc_state, ctrl);
Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(flags));
phase->register_new_node(heap_stable_and, ctrl);
Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
phase->register_new_node(heap_stable_cmp, ctrl);
@@ -889,7 +889,7 @@ void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, No
ctrl = new IfTrueNode(heap_stable_iff);
phase->register_control(ctrl, loop, heap_stable_iff);

assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
assert(is_heap_state_test(heap_stable_iff, flags), "Should match the shape");
}

void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
@@ -1437,7 +1437,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);

// Stable path.
test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
test_heap_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED);
IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();

// Heap stable case
@@ -1608,7 +1608,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);

// Stable path.
test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
test_heap_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING);
region->init_req(_heap_stable, heap_stable_ctrl);
phi->init_req(_heap_stable, raw_mem);

@@ -58,8 +58,8 @@ class ShenandoahBarrierC2Support : public AllStatic {
static Node* find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase);
static void follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase);
static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase);
static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase);
static void test_heap_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase, int flags);
static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase);
static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase);
static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses,
@@ -46,7 +46,7 @@ ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics() : ShenandoahHeu

// Final configuration checks
SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier || ShenandoahStoreValEnqueueBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
}
@@ -44,7 +44,7 @@ ShenandoahCompactHeuristics::ShenandoahCompactHeuristics() : ShenandoahHeuristic

// Final configuration checks
SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier || ShenandoahStoreValEnqueueBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
}
@@ -37,7 +37,7 @@ ShenandoahStaticHeuristics::ShenandoahStaticHeuristics() : ShenandoahHeuristics(

// Final configuration checks
SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier || ShenandoahStoreValEnqueueBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
}
@@ -241,7 +241,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_native_impl(oop obj, T* load_ad
}

void ShenandoahBarrierSet::clone_barrier_runtime(oop src) {
if (_heap->has_forwarded_objects()) {
if (_heap->has_forwarded_objects() || (ShenandoahStoreValEnqueueBarrier && _heap->is_concurrent_mark_in_progress())) {
clone_barrier(src);
}
}
@@ -107,12 +107,16 @@ class ShenandoahBarrierSet: public BarrierSet {

private:
template <class T>
inline void arraycopy_marking(T* ary, size_t count);
inline void arraycopy_marking(T* src, T* dst, size_t count);
template <class T>
inline void arraycopy_evacuation(T* src, size_t count);
template <class T>
inline void arraycopy_update(T* src, size_t count);

inline void clone_marking(oop src);
inline void clone_evacuation(oop src);
inline void clone_update(oop src);

template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
inline void arraycopy_work(T* src, size_t count);

@@ -86,7 +86,7 @@ inline void ShenandoahBarrierSet::satb_enqueue(oop value) {
}

inline void ShenandoahBarrierSet::storeval_barrier(oop obj) {
if (obj != NULL && ShenandoahStoreValEnqueueBarrier) {
if (ShenandoahStoreValEnqueueBarrier && obj != NULL && _heap->is_concurrent_mark_in_progress()) {
enqueue(obj);
}
}
@@ -291,7 +291,7 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
}
int gc_state = _heap->gc_state();
if ((gc_state & ShenandoahHeap::MARKING) != 0) {
arraycopy_marking(dst, count);
arraycopy_marking(src, dst, count);
} else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
arraycopy_evacuation(src, count);
} else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) {
@@ -300,8 +300,9 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
}

template <class T>
void ShenandoahBarrierSet::arraycopy_marking(T* array, size_t count) {
void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count) {
assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
T* array = ShenandoahSATBBarrier ? dst : src;
if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(array))) {
arraycopy_work<T, false, false, true>(array, count);
}
@@ -33,7 +33,7 @@
#include "oops/access.hpp"
#include "oops/compressedOops.hpp"

template <bool EVAC, bool ENQUEUE>
template <bool HAS_FWD, bool EVAC, bool ENQUEUE>
class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure {
private:
ShenandoahHeap* const _heap;
@@ -46,18 +46,18 @@ class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
if (_cset->is_in(obj)) {
if (HAS_FWD && _cset->is_in(obj)) {
oop fwd = _bs->resolve_forwarded_not_null(obj);
if (EVAC && obj == fwd) {
fwd = _heap->evacuate_object(obj, _thread);
}
if (ENQUEUE) {
_bs->enqueue(fwd);
}
assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded");
ShenandoahHeap::cas_oop(fwd, p, o);
obj = fwd;
}
if (ENQUEUE) {
_bs->enqueue(obj);
}

}
}
public:
@@ -72,24 +72,43 @@ class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure {
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
};

void ShenandoahBarrierSet::clone_barrier(oop obj) {
assert(ShenandoahCloneBarrier, "only get here with clone barriers enabled");
assert(_heap->has_forwarded_objects(), "only when heap is unstable");
void ShenandoahBarrierSet::clone_marking(oop obj) {
assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
assert(ShenandoahStoreValEnqueueBarrier, "only with incremental-update");
if (!_heap->marking_context()->allocated_after_mark_start(obj)) {
ShenandoahUpdateRefsForOopClosure</* has_fwd = */ false, /* evac = */ false, /* enqueue */ true> cl;
obj->oop_iterate(&cl);
}
}

void ShenandoahBarrierSet::clone_evacuation(oop obj) {
assert(_heap->is_evacuation_in_progress(), "only during evacuation");
if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
ShenandoahEvacOOMScope oom_evac_scope;
ShenandoahUpdateRefsForOopClosure</* has_fwd = */ true, /* evac = */ true, /* enqueue */ false> cl;
obj->oop_iterate(&cl);
}
}

// This is called for cloning an object (see jvm.cpp) after the clone
// has been made. We are not interested in any 'previous value' because
// it would be NULL in any case. But we *are* interested in any oop*
// that potentially need to be updated.
void ShenandoahBarrierSet::clone_update(oop obj) {
assert(_heap->is_update_refs_in_progress(), "only during update-refs");
if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
ShenandoahUpdateRefsForOopClosure</* has_fwd = */ true, /* evac = */ false, /* enqueue */ false> cl;
obj->oop_iterate(&cl);
}
}

void ShenandoahBarrierSet::clone_barrier(oop obj) {
assert(ShenandoahCloneBarrier, "only get here with clone barriers enabled");
shenandoah_assert_correct(NULL, obj);
if (!need_bulk_update(cast_from_oop<HeapWord*>(obj))) return;
if (_heap->is_evacuation_in_progress()) {
ShenandoahEvacOOMScope evac_scope;
ShenandoahUpdateRefsForOopClosure</* evac = */ true, /* enqueue */ false> cl;
obj->oop_iterate(&cl);

int gc_state = _heap->gc_state();
if ((gc_state & ShenandoahHeap::MARKING) != 0) {
clone_marking(obj);
} else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
clone_evacuation(obj);
} else {
ShenandoahUpdateRefsForOopClosure</* evac = */ false, /* enqueue */ false> cl;
obj->oop_iterate(&cl);
clone_update(obj);
}
}

@@ -183,26 +183,32 @@ class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
class ShenandoahSATBAndRemarkCodeRootsThreadsClosure : public ThreadClosure {
private:
ShenandoahSATBBufferClosure* _satb_cl;
OopClosure* const _cl;
MarkingCodeBlobClosure* _code_cl;
uintx _claim_token;

public:
ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, MarkingCodeBlobClosure* code_cl) :
_satb_cl(satb_cl), _code_cl(code_cl),
ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, OopClosure* cl, MarkingCodeBlobClosure* code_cl) :
_satb_cl(satb_cl), _cl(cl), _code_cl(code_cl),
_claim_token(Threads::thread_claim_token()) {}

void do_thread(Thread* thread) {
if (thread->claim_threads_do(true, _claim_token)) {
ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
if (_code_cl != NULL && thread->is_Java_thread()) {
// In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
// however the liveness of oops reachable from nmethods have very complex lifecycles:
// * Alive if on the stack of an executing method
// * Weakly reachable otherwise
// Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
// live by the SATB invariant but other oops recorded in nmethods may behave differently.
JavaThread* jt = (JavaThread*)thread;
jt->nmethods_do(_code_cl);
if (thread->is_Java_thread()) {
if (_cl != NULL) {
ResourceMark rm;
thread->oops_do(_cl, _code_cl);
} else if (_code_cl != NULL) {
// In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
// however the liveness of oops reachable from nmethods have very complex lifecycles:
// * Alive if on the stack of an executing method
// * Weakly reachable otherwise
// Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
// live by the SATB invariant but other oops recorded in nmethods may behave differently.
JavaThread* jt = (JavaThread*)thread;
jt->nmethods_do(_code_cl);
}
}
}
}
@@ -242,21 +248,20 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask {
ShenandoahSATBBufferClosure cl(q);
SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
while (satb_mq_set.apply_closure_to_completed_buffer(&cl));

if (heap->unload_classes() && !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
if (heap->has_forwarded_objects()) {
ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp);
MarkingCodeBlobClosure blobsCl(&resolve_mark_cl, !CodeBlobToOopClosure::FixRelocations);
ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, &blobsCl);
Threads::threads_do(&tc);
} else {
ShenandoahMarkRefsClosure mark_cl(q, rp);
MarkingCodeBlobClosure blobsCl(&mark_cl, !CodeBlobToOopClosure::FixRelocations);
ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, &blobsCl);
Threads::threads_do(&tc);
}
bool do_nmethods = heap->unload_classes() && !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading();
if (heap->has_forwarded_objects()) {
ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp);
MarkingCodeBlobClosure blobsCl(&resolve_mark_cl, !CodeBlobToOopClosure::FixRelocations);
ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
ShenandoahStoreValEnqueueBarrier ? &resolve_mark_cl : NULL,
do_nmethods ? &blobsCl : NULL);
Threads::threads_do(&tc);
} else {
ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, NULL);
ShenandoahMarkRefsClosure mark_cl(q, rp);
MarkingCodeBlobClosure blobsCl(&mark_cl, !CodeBlobToOopClosure::FixRelocations);
ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
ShenandoahStoreValEnqueueBarrier ? &mark_cl : NULL,
do_nmethods ? &blobsCl : NULL);
Threads::threads_do(&tc);
}
}
@@ -46,6 +46,7 @@
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc/shenandoah/shenandoahIUMode.hpp"
#include "gc/shenandoah/shenandoahMarkCompact.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahMemoryPool.hpp"
@@ -397,6 +398,8 @@ void ShenandoahHeap::initialize_heuristics() {
if (ShenandoahGCMode != NULL) {
if (strcmp(ShenandoahGCMode, "normal") == 0) {
_gc_mode = new ShenandoahNormalMode();
} else if (strcmp(ShenandoahGCMode, "iu") == 0) {
_gc_mode = new ShenandoahIUMode();
} else if (strcmp(ShenandoahGCMode, "passive") == 0) {
_gc_mode = new ShenandoahPassiveMode();
} else {

0 comments on commit c4d01f8

Please sign in to comment.