Skip to content

Commit 9510e7f

Browse files
rkennkerkennke
committed
8231086: Shenandoah: Stronger invariant for object-arraycopy
Reviewed-by: shade
1 parent 5eb6431 commit 9510e7f

13 files changed

+251
-518
lines changed

‎src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp‎

Lines changed: 17 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
4747
Register src, Register dst, Register count, RegSet saved_regs) {
4848
if (is_oop) {
4949
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
50-
if (ShenandoahSATBBarrier && !dest_uninitialized) {
50+
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
5151

5252
Label done;
5353

@@ -57,59 +57,34 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
5757
// Is marking active?
5858
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
5959
__ ldrb(rscratch1, gc_state);
60-
__ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, done);
60+
if (dest_uninitialized) {
61+
__ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
62+
} else {
63+
__ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
64+
__ tst(rscratch1, rscratch2);
65+
__ br(Assembler::EQ, done);
66+
}
6167

6268
__ push(saved_regs, sp);
63-
if (count == c_rarg0) {
64-
if (dst == c_rarg1) {
65-
// exactly backwards!!
66-
__ mov(rscratch1, c_rarg0);
67-
__ mov(c_rarg0, c_rarg1);
68-
__ mov(c_rarg1, rscratch1);
69+
if (UseCompressedOops) {
70+
if (dest_uninitialized) {
71+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
6972
} else {
70-
__ mov(c_rarg1, count);
71-
__ mov(c_rarg0, dst);
73+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
7274
}
7375
} else {
74-
__ mov(c_rarg0, dst);
75-
__ mov(c_rarg1, count);
76-
}
77-
if (UseCompressedOops) {
78-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
79-
} else {
80-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
76+
if (dest_uninitialized) {
77+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
78+
} else {
79+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
80+
}
8181
}
8282
__ pop(saved_regs, sp);
8383
__ bind(done);
8484
}
8585
}
8686
}
8787

88-
void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
89-
Register start, Register count, Register scratch, RegSet saved_regs) {
90-
if (is_oop) {
91-
Label done;
92-
93-
// Avoid calling runtime if count == 0
94-
__ cbz(count, done);
95-
96-
// Is updating references?
97-
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
98-
__ ldrb(rscratch1, gc_state);
99-
__ tbz(rscratch1, ShenandoahHeap::UPDATEREFS_BITPOS, done);
100-
101-
__ push(saved_regs, sp);
102-
assert_different_registers(start, count, scratch);
103-
assert_different_registers(c_rarg0, count);
104-
__ mov(c_rarg0, start);
105-
__ mov(c_rarg1, count);
106-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
107-
__ pop(saved_regs, sp);
108-
109-
__ bind(done);
110-
}
111-
}
112-
11388
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
11489
Register obj,
11590
Register pre_val,

‎src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,6 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
7676

7777
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
7878
Register src, Register dst, Register count, RegSet saved_regs);
79-
virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
80-
Register start, Register count, Register tmp, RegSet saved_regs);
8179
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
8280
Register dst, Address src, Register tmp1, Register tmp_thread);
8381
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,

‎src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp‎

Lines changed: 34 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -47,35 +47,28 @@ address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
4747
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
4848
Register src, Register dst, Register count) {
4949

50-
bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
51-
bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
52-
bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
5350
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
5451

5552
if (type == T_OBJECT || type == T_ARRAY) {
53+
54+
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
5655
#ifdef _LP64
57-
if (!checkcast) {
58-
if (!obj_int) {
59-
// Save count for barrier
60-
__ movptr(r11, count);
61-
} else if (disjoint) {
62-
// Save dst in r11 in the disjoint case
63-
__ movq(r11, dst);
64-
}
65-
}
56+
Register thread = r15_thread;
6657
#else
67-
if (disjoint) {
68-
__ mov(rdx, dst); // save 'to'
69-
}
70-
#endif
71-
72-
if (ShenandoahSATBBarrier && !dest_uninitialized) {
73-
Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
74-
assert_different_registers(dst, count, thread); // we don't care about src here?
75-
#ifndef _LP64
58+
Register thread = rax;
59+
if (thread == src || thread == dst || thread == count) {
60+
thread = rbx;
61+
}
62+
if (thread == src || thread == dst || thread == count) {
63+
thread = rcx;
64+
}
65+
if (thread == src || thread == dst || thread == count) {
66+
thread = rdx;
67+
}
7668
__ push(thread);
7769
__ get_thread(thread);
7870
#endif
71+
assert_different_registers(src, dst, count, thread);
7972

8073
Label done;
8174
// Short-circuit if count == 0.
@@ -84,32 +77,33 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
8477

8578
// Avoid runtime call when not marking.
8679
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
87-
__ testb(gc_state, ShenandoahHeap::MARKING);
80+
int flags = ShenandoahHeap::HAS_FORWARDED;
81+
if (!dest_uninitialized) {
82+
flags |= ShenandoahHeap::MARKING;
83+
}
84+
__ testb(gc_state, flags);
8885
__ jcc(Assembler::zero, done);
8986

9087
__ pusha(); // push registers
9188
#ifdef _LP64
92-
if (count == c_rarg0) {
93-
if (dst == c_rarg1) {
94-
// exactly backwards!!
95-
__ xchgptr(c_rarg1, c_rarg0);
89+
assert(src == rdi, "expected");
90+
assert(dst == rsi, "expected");
91+
assert(count == rdx, "expected");
92+
if (UseCompressedOops) {
93+
if (dest_uninitialized) {
94+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
9695
} else {
97-
__ movptr(c_rarg1, count);
98-
__ movptr(c_rarg0, dst);
96+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
9997
}
100-
} else {
101-
__ movptr(c_rarg0, dst);
102-
__ movptr(c_rarg1, count);
103-
}
104-
if (UseCompressedOops) {
105-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
106-
} else {
107-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
108-
}
109-
#else
110-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
111-
dst, count);
98+
} else
11299
#endif
100+
{
101+
if (dest_uninitialized) {
102+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
103+
} else {
104+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
105+
}
106+
}
113107
__ popa();
114108
__ bind(done);
115109
NOT_LP64(__ pop(thread);)
@@ -118,73 +112,6 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
118112

119113
}
120114

121-
void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
122-
Register src, Register dst, Register count) {
123-
bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
124-
bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
125-
bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
126-
Register tmp = rax;
127-
128-
if (type == T_OBJECT || type == T_ARRAY) {
129-
#ifdef _LP64
130-
if (!checkcast) {
131-
if (!obj_int) {
132-
// Save count for barrier
133-
count = r11;
134-
} else if (disjoint && obj_int) {
135-
// Use the saved dst in the disjoint case
136-
dst = r11;
137-
}
138-
} else {
139-
tmp = rscratch1;
140-
}
141-
#else
142-
if (disjoint) {
143-
__ mov(dst, rdx); // restore 'to'
144-
}
145-
#endif
146-
147-
Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
148-
assert_different_registers(dst, thread); // do we care about src at all here?
149-
150-
#ifndef _LP64
151-
__ push(thread);
152-
__ get_thread(thread);
153-
#endif
154-
155-
// Short-circuit if count == 0.
156-
Label done;
157-
__ testptr(count, count);
158-
__ jcc(Assembler::zero, done);
159-
160-
// Skip runtime call if no forwarded objects.
161-
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
162-
__ testb(gc_state, ShenandoahHeap::UPDATEREFS);
163-
__ jcc(Assembler::zero, done);
164-
165-
__ pusha(); // push registers (overkill)
166-
#ifdef _LP64
167-
if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
168-
assert_different_registers(c_rarg1, dst);
169-
__ mov(c_rarg1, count);
170-
__ mov(c_rarg0, dst);
171-
} else {
172-
assert_different_registers(c_rarg0, count);
173-
__ mov(c_rarg0, dst);
174-
__ mov(c_rarg1, count);
175-
}
176-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
177-
#else
178-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry),
179-
dst, count);
180-
#endif
181-
__ popa();
182-
183-
__ bind(done);
184-
NOT_LP64(__ pop(thread);)
185-
}
186-
}
187-
188115
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
189116
Register obj,
190117
Register pre_val,

‎src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,6 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
8383
bool exchange, Register tmp1, Register tmp2);
8484
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
8585
Register src, Register dst, Register count);
86-
virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
87-
Register src, Register dst, Register count);
8886
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
8987
Register dst, Address src, Register tmp1, Register tmp_thread);
9088
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,

‎src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp‎

Lines changed: 29 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -461,9 +461,11 @@ const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
461461
}
462462

463463
const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
464-
const Type **fields = TypeTuple::fields(1);
465-
fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
466-
const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
464+
const Type **fields = TypeTuple::fields(3);
465+
fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // src
466+
fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // dst
467+
fields[TypeFunc::Parms+2] = TypeInt::INT; // length
468+
const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
467469

468470
// create result type (range)
469471
fields = TypeTuple::fields(0);
@@ -705,11 +707,6 @@ Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& acces
705707
return result;
706708
}
707709

708-
void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
709-
assert(!src->is_AddP(), "unexpected input");
710-
BarrierSetC2::clone(kit, src, dst, size, is_array);
711-
}
712-
713710
// Support for GC barriers emitted during parsing
714711
bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
715712
if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
@@ -771,17 +768,16 @@ bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_couple
771768
return true;
772769
}
773770

774-
bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) {
775-
Node* src = ac->in(ArrayCopyNode::Src);
776-
const TypeOopPtr* src_type = igvn.type(src)->is_oopptr();
771+
bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
772+
const TypeOopPtr* src_type = gvn.type(src)->is_oopptr();
777773
if (src_type->isa_instptr() != NULL) {
778774
ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
779775
if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
780776
if (ik->has_object_fields()) {
781777
return true;
782778
} else {
783779
if (!src_type->klass_is_exact()) {
784-
igvn.C->dependencies()->assert_leaf_type(ik);
780+
Compile::current()->dependencies()->assert_leaf_type(ik);
785781
}
786782
}
787783
} else {
@@ -798,42 +794,29 @@ bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIte
798794
return false;
799795
}
800796

801-
void ShenandoahBarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
802-
assert(ac->is_clonebasic(), "no other kind of arraycopy here");
803-
804-
if (!clone_needs_postbarrier(ac, igvn)) {
805-
BarrierSetC2::clone_barrier_at_expansion(ac, call, igvn);
806-
return;
807-
}
808-
809-
const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
810-
Node* c = new ProjNode(call,TypeFunc::Control);
811-
c = igvn.transform(c);
812-
Node* m = new ProjNode(call, TypeFunc::Memory);
813-
m = igvn.transform(m);
797+
#define XTOP LP64_ONLY(COMMA phase->top())
814798

799+
void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
800+
Node* ctrl = ac->in(TypeFunc::Control);
801+
Node* mem = ac->in(TypeFunc::Memory);
802+
Node* src = ac->in(ArrayCopyNode::Src);
803+
Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
815804
Node* dest = ac->in(ArrayCopyNode::Dest);
816-
assert(dest->is_AddP(), "bad input");
817-
Node* barrier_call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
818-
CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
819-
"shenandoah_clone_barrier", raw_adr_type);
820-
barrier_call->init_req(TypeFunc::Control, c);
821-
barrier_call->init_req(TypeFunc::I_O , igvn.C->top());
822-
barrier_call->init_req(TypeFunc::Memory , m);
823-
barrier_call->init_req(TypeFunc::ReturnAdr, igvn.C->top());
824-
barrier_call->init_req(TypeFunc::FramePtr, igvn.C->top());
825-
barrier_call->init_req(TypeFunc::Parms+0, dest->in(AddPNode::Base));
826-
827-
barrier_call = igvn.transform(barrier_call);
828-
c = new ProjNode(barrier_call,TypeFunc::Control);
829-
c = igvn.transform(c);
830-
m = new ProjNode(barrier_call, TypeFunc::Memory);
831-
m = igvn.transform(m);
832-
833-
Node* out_c = ac->proj_out(TypeFunc::Control);
834-
Node* out_m = ac->proj_out(TypeFunc::Memory);
835-
igvn.replace_node(out_c, c);
836-
igvn.replace_node(out_m, m);
805+
Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
806+
Node* length = ac->in(ArrayCopyNode::Length);
807+
assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
808+
if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
809+
Node* call = phase->make_leaf_call(ctrl, mem,
810+
ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
811+
CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
812+
"shenandoah_clone",
813+
TypeRawPtr::BOTTOM,
814+
src, dest, length);
815+
call = phase->transform_later(call);
816+
phase->igvn().replace_node(ac, call);
817+
} else {
818+
BarrierSetC2::clone_at_expansion(phase, ac);
819+
}
837820
}
838821

839822

0 commit comments

Comments
 (0)