Skip to content
Permalink
Browse files

8242130: Shenandoah: Simplify arraycopy-barrier dispatching

Reviewed-by: shade
  • Loading branch information
rkennke committed Apr 6, 2020
1 parent 08b9028 commit ec5ccb8de31621ea9d7ea66f6e548608dc0e8c3d
@@ -56,7 +56,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
// Avoid calling runtime if count == 0
__ cbz(count, done);

// Is marking active?
// Is GC active?
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
__ ldrb(rscratch1, gc_state);
if (dest_uninitialized) {
@@ -69,17 +69,9 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec

__ push(saved_regs, sp);
if (UseCompressedOops) {
if (dest_uninitialized) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
}
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
} else {
if (dest_uninitialized) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
}
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
}
__ pop(saved_regs, sp);
__ bind(done);
@@ -77,7 +77,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
__ testptr(count, count);
__ jcc(Assembler::zero, done);

// Avoid runtime call when not marking.
// Avoid runtime call when not active.
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
int flags = ShenandoahHeap::HAS_FORWARDED;
if (!dest_uninitialized) {
@@ -87,25 +87,21 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
__ jcc(Assembler::zero, done);

__ pusha(); // push registers

#ifdef _LP64
assert(src == rdi, "expected");
assert(dst == rsi, "expected");
assert(count == rdx, "expected");
if (UseCompressedOops) {
if (dest_uninitialized) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
}
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
src, dst, count);
} else
#endif
{
if (dest_uninitialized) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
}
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
src, dst, count);
}

__ popa();
__ bind(done);
NOT_LP64(__ pop(thread);)
@@ -68,13 +68,8 @@ class ShenandoahBarrierSet: public BarrierSet {

bool is_aligned(HeapWord* hw);

template <class T> void
write_ref_array_pre_work(T* src, T* dst, size_t count, bool dest_uninitialized);

inline void arraycopy_pre(oop* src, oop* dst, size_t count);
inline void arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count);
inline void arraycopy_update(oop* src, size_t count);
inline void arraycopy_update(narrowOop* src, size_t count);
template <class T>
inline void arraycopy_barrier(T* src, T* dst, size_t count);
inline void clone_barrier(oop src);
void clone_barrier_runtime(oop src);

@@ -112,18 +107,21 @@ class ShenandoahBarrierSet: public BarrierSet {

private:
template <class T>
inline void arraycopy_pre_work(T* src, T* dst, size_t count);
inline void arraycopy_marking(T* ary, size_t count);
template <class T>
inline void arraycopy_evacuation(T* src, size_t count);
template <class T>
inline void arraycopy_update(T* src, size_t count);

template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
inline void arraycopy_work(T* src, size_t count);
template <class T>
inline void arraycopy_update_impl(T* src, size_t count);

oop load_reference_barrier_impl(oop obj);

template <class T>
oop load_reference_barrier_native_impl(oop obj, T* load_addr);

inline bool skip_bulk_update(HeapWord* dst);
inline bool need_bulk_update(HeapWord* dst);
public:
// Callbacks for runtime accesses.
template <DecoratorSet decorators, typename BarrierSetT = ShenandoahBarrierSet>
@@ -249,9 +249,9 @@ bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
bs->arraycopy_pre(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw),
arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw),
length);
bs->arraycopy_barrier(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw),
arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw),
length);
return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
}

@@ -285,46 +285,47 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
}

template <class T>
void ShenandoahBarrierSet::arraycopy_pre_work(T* src, T* dst, size_t count) {
if (_heap->is_concurrent_mark_in_progress() &&
!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(dst))) {
arraycopy_work<T, false, false, true>(dst, count);
void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
if (count == 0) {
return;
}

if (_heap->has_forwarded_objects()) {
arraycopy_update_impl(src, count);
int gc_state = _heap->gc_state();
if ((gc_state & ShenandoahHeap::MARKING) != 0) {
arraycopy_marking(dst, count);
} else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
arraycopy_evacuation(src, count);
} else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) {
arraycopy_update(src, count);
}
}

void ShenandoahBarrierSet::arraycopy_pre(oop* src, oop* dst, size_t count) {
arraycopy_pre_work(src, dst, count);
}

void ShenandoahBarrierSet::arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count) {
arraycopy_pre_work(src, dst, count);
template <class T>
void ShenandoahBarrierSet::arraycopy_marking(T* array, size_t count) {
assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(array))) {
arraycopy_work<T, false, false, true>(array, count);
}
}

inline bool ShenandoahBarrierSet::skip_bulk_update(HeapWord* dst) {
return dst >= _heap->heap_region_containing(dst)->get_update_watermark();
inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) {
return ary < _heap->heap_region_containing(ary)->get_update_watermark();
}

template <class T>
void ShenandoahBarrierSet::arraycopy_update_impl(T* src, size_t count) {
if (skip_bulk_update(reinterpret_cast<HeapWord*>(src))) return;
if (_heap->is_evacuation_in_progress()) {
void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) {
assert(_heap->is_evacuation_in_progress(), "only during evacuation");
if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) {
ShenandoahEvacOOMScope oom_evac;
arraycopy_work<T, true, true, false>(src, count);
} else if (_heap->has_forwarded_objects()) {
arraycopy_work<T, true, false, false>(src, count);
}
}

void ShenandoahBarrierSet::arraycopy_update(oop* src, size_t count) {
arraycopy_update_impl(src, count);
}

void ShenandoahBarrierSet::arraycopy_update(narrowOop* src, size_t count) {
arraycopy_update_impl(src, count);
template <class T>
void ShenandoahBarrierSet::arraycopy_update(T* src, size_t count) {
assert(_heap->is_update_refs_in_progress(), "only during update-refs");
if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) {
arraycopy_work<T, true, false, false>(src, count);
}
}

#endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
@@ -82,7 +82,7 @@ void ShenandoahBarrierSet::clone_barrier(oop obj) {
// that potentially need to be updated.

shenandoah_assert_correct(NULL, obj);
if (skip_bulk_update(cast_from_oop<HeapWord*>(obj))) return;
if (!need_bulk_update(cast_from_oop<HeapWord*>(obj))) return;
if (_heap->is_evacuation_in_progress()) {
ShenandoahEvacOOMScope evac_scope;
ShenandoahUpdateRefsForOopClosure</* evac = */ true, /* enqueue */ false> cl;
@@ -31,24 +31,14 @@
#include "oops/oop.inline.hpp"
#include "utilities/copy.hpp"

void ShenandoahRuntime::write_ref_array_pre_oop_entry(oop* src, oop* dst, size_t length) {
void ShenandoahRuntime::arraycopy_barrier_oop_entry(oop* src, oop* dst, size_t length) {
ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set();
bs->arraycopy_pre(src, dst, length);
bs->arraycopy_barrier(src, dst, length);
}

void ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length) {
void ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length) {
ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set();
bs->arraycopy_pre(src, dst, length);
}

void ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry(oop* src, oop* dst, size_t length) {
ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set();
bs->arraycopy_update(src, length);
}

void ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length) {
ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set();
bs->arraycopy_update(src, length);
bs->arraycopy_barrier(src, dst, length);
}

// Shenandoah pre write barrier slowpath
@@ -33,10 +33,9 @@ class oopDesc;

class ShenandoahRuntime : public AllStatic {
public:
static void write_ref_array_pre_oop_entry(oop* src, oop* dst, size_t length);
static void write_ref_array_pre_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length);
static void write_ref_array_pre_duinit_oop_entry(oop* src, oop* dst, size_t length);
static void write_ref_array_pre_duinit_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length);
static void arraycopy_barrier_oop_entry(oop* src, oop* dst, size_t length);
static void arraycopy_barrier_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length);

static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread);

static oopDesc* load_reference_barrier(oopDesc* src, oop* load_addr);

0 comments on commit ec5ccb8

Please sign in to comment.