Skip to content

Commit

Permalink
8293351: Add second tmp register to aarch64 BarrierSetAssembler::load_at
Browse files Browse the repository at this point in the history 
Reviewed-by: aph, tschatzl, fyang
  • Loading branch information
@xmas92 @stefank
xmas92 authored and stefank committed Sep 13, 2022
1 parent 37234c8 commit 725f41f
Show file tree
Hide file tree
Showing 14 changed files with 102 additions and 97 deletions.
53 changes: 28 additions & 25 deletions src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
Register thread,
Register tmp,
Register tmp1,
Register tmp2,
bool tosca_live,
bool expand_call) {
// If expand_call is true then we expand the call_VM_leaf macro
Expand All @@ -111,21 +112,21 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
Label done;
Label runtime;

assert_different_registers(obj, pre_val, tmp, rscratch1);
assert(pre_val != noreg && tmp != noreg, "expecting a register");
assert_different_registers(obj, pre_val, tmp1, tmp2);
assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");

Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));

// Is marking active?
if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
__ ldrw(tmp, in_progress);
__ ldrw(tmp1, in_progress);
} else {
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
__ ldrb(tmp, in_progress);
__ ldrb(tmp1, in_progress);
}
__ cbzw(tmp, done);
__ cbzw(tmp1, done);

// Do we need to load the previous value?
if (obj != noreg) {
Expand All @@ -139,17 +140,17 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
// Is index == 0?
// (The index field is typed as size_t.)

__ ldr(tmp, index); // tmp := *index_adr
__ cbz(tmp, runtime); // tmp == 0?
__ ldr(tmp1, index); // tmp := *index_adr
__ cbz(tmp1, runtime); // tmp == 0?
// If yes, goto runtime

__ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize
__ str(tmp, index); // *index_adr := tmp
__ ldr(rscratch1, buffer);
__ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr
__ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
__ str(tmp1, index); // *index_adr := tmp
__ ldr(tmp2, buffer);
__ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr

// Record the previous value
__ str(pre_val, Address(tmp, 0));
__ str(pre_val, Address(tmp1, 0));
__ b(done);

__ bind(runtime);
Expand Down Expand Up @@ -185,12 +186,12 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
Register thread,
Register tmp,
Register tmp1,
Register tmp2) {
assert(thread == rthread, "must be");
assert_different_registers(store_addr, new_val, thread, tmp, tmp2,
assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
rscratch1);
assert(store_addr != noreg && new_val != noreg && tmp != noreg
assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
&& tmp2 != noreg, "expecting a register");

Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
Expand All @@ -205,17 +206,17 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,

// Does store cross heap regions?

__ eor(tmp, store_addr, new_val);
__ lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes);
__ cbz(tmp, done);
__ eor(tmp1, store_addr, new_val);
__ lsr(tmp1, tmp1, HeapRegion::LogOfHRGrainBytes);
__ cbz(tmp1, done);

// crosses regions, storing NULL?

__ cbz(new_val, done);

// storing region crossing non-NULL, is card already dirty?

const Register card_addr = tmp;
const Register card_addr = tmp1;

__ lsr(card_addr, store_addr, CardTable::card_shift());

Expand Down Expand Up @@ -258,12 +259,12 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
}

void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
Register dst, Address src, Register tmp1, Register tmp2) {
bool on_oop = is_reference_type(type);
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool on_reference = on_weak || on_phantom;
ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
if (on_oop && on_reference) {
// LR is live. It must be saved around calls.
__ enter(/*strip_ret_addr*/true); // barrier may call runtime
Expand All @@ -273,7 +274,8 @@ void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorator
noreg /* obj */,
dst /* pre_val */,
rthread /* thread */,
tmp1 /* tmp */,
tmp1 /* tmp1 */,
tmp2 /* tmp2 */,
true /* tosca_live */,
true /* expand_call */);
__ leave();
Expand All @@ -295,7 +297,8 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco
tmp3 /* obj */,
tmp2 /* pre_val */,
rthread /* thread */,
tmp1 /* tmp */,
tmp1 /* tmp1 */,
rscratch2 /* tmp2 */,
val != noreg /* tosca_live */,
false /* expand_call */);

Expand All @@ -313,7 +316,7 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco
tmp3 /* store_adr */,
new_val /* new_val */,
rthread /* thread */,
tmp1 /* tmp */,
tmp1 /* tmp1 */,
tmp2 /* tmp2 */);
}

Expand Down
7 changes: 4 additions & 3 deletions src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,16 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
Register obj,
Register pre_val,
Register thread,
Register tmp,
Register tmp1,
Register tmp2,
bool tosca_live,
bool expand_call);

void g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
Register thread,
Register tmp,
Register tmp1,
Register tmp2);

virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Expand All @@ -69,7 +70,7 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
#endif

void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread);
Register dst, Address src, Register tmp1, Register tmp2);
};

#endif // CPU_AARCH64_GC_G1_G1BARRIERSETASSEMBLER_AARCH64_HPP
9 changes: 4 additions & 5 deletions src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
#define __ masm->

void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
Register dst, Address src, Register tmp1, Register tmp2) {

// LR is live. It must be saved around calls.

Expand Down Expand Up @@ -285,13 +285,12 @@ void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
__ cbnz(rscratch2, method_live);

// Is it a weak but alive CLD?
__ stp(r10, r11, Address(__ pre(sp, -2 * wordSize)));
__ push(RegSet::of(r10), sp);
__ ldr(r10, Address(rscratch1, ClassLoaderData::holder_offset()));

// Uses rscratch1 & rscratch2, so we must pass new temporaries.
__ resolve_weak_handle(r10, r11);
__ resolve_weak_handle(r10, rscratch1, rscratch2);
__ mov(rscratch1, r10);
__ ldp(r10, r11, Address(__ post(sp, 2 * wordSize)));
__ pop(RegSet::of(r10), sp);
__ cbnz(rscratch1, method_live);

__ bind(bad_call);
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class BarrierSetAssembler: public CHeapObj<mtGC> {
virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register start, Register end, Register tmp, RegSet saved_regs) {}
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread);
Register dst, Address src, Register tmp1, Register tmp2);
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);

Expand Down
42 changes: 22 additions & 20 deletions src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,15 +85,16 @@ void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler*
bool tosca_live,
bool expand_call) {
if (ShenandoahSATBBarrier) {
satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, rscratch1, tosca_live, expand_call);
}
}

void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
Register thread,
Register tmp,
Register tmp1,
Register tmp2,
bool tosca_live,
bool expand_call) {
// If expand_call is true then we expand the call_VM_leaf macro
Expand All @@ -105,21 +106,21 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
Label done;
Label runtime;

assert_different_registers(obj, pre_val, tmp, rscratch1);
assert(pre_val != noreg && tmp != noreg, "expecting a register");
assert_different_registers(obj, pre_val, tmp1, tmp2);
assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");

Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));

// Is marking active?
if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
__ ldrw(tmp, in_progress);
__ ldrw(tmp1, in_progress);
} else {
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
__ ldrb(tmp, in_progress);
__ ldrb(tmp1, in_progress);
}
__ cbzw(tmp, done);
__ cbzw(tmp1, done);

// Do we need to load the previous value?
if (obj != noreg) {
Expand All @@ -133,17 +134,17 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
// Is index == 0?
// (The index field is typed as size_t.)

__ ldr(tmp, index); // tmp := *index_adr
__ cbz(tmp, runtime); // tmp == 0?
__ ldr(tmp1, index); // tmp := *index_adr
__ cbz(tmp1, runtime); // tmp == 0?
// If yes, goto runtime

__ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize
__ str(tmp, index); // *index_adr := tmp
__ ldr(rscratch1, buffer);
__ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr
__ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
__ str(tmp1, index); // *index_adr := tmp
__ ldr(tmp2, buffer);
__ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr

// Record the previous value
__ str(pre_val, Address(tmp, 0));
__ str(pre_val, Address(tmp1, 0));
__ b(done);

__ bind(runtime);
Expand Down Expand Up @@ -307,7 +308,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler* masm, Register dst, Register tmp) {
if (ShenandoahIUBarrier) {
__ push_call_clobbered_registers();
satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, rscratch1, true, false);
__ pop_call_clobbered_registers();
}
}
Expand All @@ -328,10 +329,10 @@ void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler* masm, Register ds
// dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
//
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
Register dst, Address src, Register tmp1, Register tmp2) {
// 1: non-reference load, no additional barrier is needed
if (!is_reference_type(type)) {
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
return;
}

Expand All @@ -345,7 +346,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
}
assert_different_registers(dst, src.base(), src.index());

BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);

load_reference_barrier(masm, dst, src, decorators);

Expand All @@ -354,7 +355,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
dst = result_dst;
}
} else {
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
}

// 3: apply keep-alive barrier if needed
Expand All @@ -365,7 +366,8 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
noreg /* obj */,
dst /* pre_val */,
rthread /* thread */,
tmp1 /* tmp */,
tmp1 /* tmp1 */,
tmp2 /* tmp2 */,
true /* tosca_live */,
true /* expand_call */);
__ pop_call_clobbered_registers();
Expand Down
5 changes: 3 additions & 2 deletions src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
Register obj,
Register pre_val,
Register thread,
Register tmp,
Register tmp1,
Register tmp2,
bool tosca_live,
bool expand_call);
void shenandoah_write_barrier_pre(MacroAssembler* masm,
Expand Down Expand Up @@ -74,7 +75,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register src, Register dst, Register count, RegSet saved_regs);
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread);
Register dst, Address src, Register tmp1, Register tmp2);
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ void InterpreterMacroAssembler::load_resolved_reference_at_index(
// load pointer for resolved_references[] objArray
ldr(result, Address(result, ConstantPool::cache_offset_in_bytes()));
ldr(result, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes()));
resolve_oop_handle(result, tmp);
resolve_oop_handle(result, tmp, rscratch2);
// Add in the index
add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)));
Expand Down
Loading

1 comment on commit 725f41f

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.