Skip to content

Commit

Permalink
8316880: AArch64: "stop: Header is not fast-locked" with -XX:-UseLSE …
Browse files Browse the repository at this point in the history
…since JDK-8315880

Reviewed-by: aph
Backport-of: b6a97c078043862b20bd8e1d1b8ccb8699995515
  • Loading branch information
shipilev committed Oct 19, 2023
1 parent abfe950 commit fd329f2
Show file tree
Hide file tree
Showing 10 changed files with 41 additions and 26 deletions.
6 changes: 3 additions & 3 deletions src/hotspot/cpu/aarch64/aarch64.ad
Original file line number Diff line number Diff line change
Expand Up @@ -16413,18 +16413,18 @@ instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
// ============================================================================
// inlined locking and unlocking

instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
%{
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP tmp2);
effect(TEMP tmp, TEMP tmp2, TEMP tmp3);

// TODO
// identify correct cost
ins_cost(5 * INSN_COST);
format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}

ins_encode %{
__ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
__ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
%}

ins_pipe(pipe_serial);
Expand Down
7 changes: 4 additions & 3 deletions src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ int LIR_Assembler::emit_unwind_handler() {
if (LockingMode == LM_MONITOR) {
__ b(*stub->entry());
} else {
__ unlock_object(r5, r4, r0, *stub->entry());
__ unlock_object(r5, r4, r0, r6, *stub->entry());
}
__ bind(*stub->continuation());
}
Expand Down Expand Up @@ -2558,6 +2558,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register obj = op->obj_opr()->as_register(); // may not be an oop
Register hdr = op->hdr_opr()->as_register();
Register lock = op->lock_opr()->as_register();
Register temp = op->scratch_opr()->as_register();
if (LockingMode == LM_MONITOR) {
if (op->info() != nullptr) {
add_debug_info_for_null_check_here(op->info());
Expand All @@ -2567,14 +2568,14 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
} else if (op->code() == lir_lock) {
assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
// add debug info for NullPointerException only if one is possible
int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
if (op->info() != nullptr) {
add_debug_info_for_null_check(null_check_offset, op->info());
}
// done
} else if (op->code() == lir_unlock) {
assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
__ unlock_object(hdr, obj, lock, *op->stub()->entry());
__ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
} else {
Unimplemented();
}
Expand Down
6 changes: 4 additions & 2 deletions src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {

// "lock" stores the address of the monitor stack slot, so this is not an oop
LIR_Opr lock = new_register(T_INT);
LIR_Opr scratch = new_register(T_INT);

CodeEmitInfo* info_for_exception = nullptr;
if (x->needs_null_check()) {
Expand All @@ -322,7 +323,7 @@ void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
// this CodeEmitInfo must not have the xhandlers because here the
// object is already locked (xhandlers expect object to be unlocked)
CodeEmitInfo* info = state_for(x, x->state(), true);
monitor_enter(obj.result(), lock, syncTempOpr(), LIR_OprFact::illegalOpr,
monitor_enter(obj.result(), lock, syncTempOpr(), scratch,
x->monitor_no(), info_for_exception, info);
}

Expand All @@ -335,8 +336,9 @@ void LIRGenerator::do_MonitorExit(MonitorExit* x) {

LIR_Opr lock = new_register(T_INT);
LIR_Opr obj_temp = new_register(T_INT);
LIR_Opr scratch = new_register(T_INT);
set_no_result(x);
monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no());
monitor_exit(obj_temp, lock, syncTempOpr(), scratch, x->monitor_no());
}

void LIRGenerator::do_NegateOp(NegateOp* x) {
Expand Down
12 changes: 6 additions & 6 deletions src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,10 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
}
}

int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert_different_registers(hdr, obj, disp_hdr);
assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
int null_check_offset = -1;

verify_oop(obj);
Expand All @@ -83,7 +83,7 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
// Load object header
ldr(hdr, Address(obj, hdr_offset));
if (LockingMode == LM_LIGHTWEIGHT) {
lightweight_lock(obj, hdr, rscratch1, rscratch2, slow_case);
lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
} else if (LockingMode == LM_LEGACY) {
Label done;
// and mark it as unlocked
Expand Down Expand Up @@ -125,10 +125,10 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
}


void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
Label done;

if (LockingMode != LM_LIGHTWEIGHT) {
Expand All @@ -149,7 +149,7 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
// be encoded.
tst(hdr, markWord::monitor_value);
br(Assembler::NE, slow_case);
lightweight_unlock(obj, hdr, rscratch1, rscratch2, slow_case);
lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
} else if (LockingMode == LM_LEGACY) {
// test if object header is pointing to the displaced header, and if so, restore
// the displaced header in the object - if the object header is not pointing to
Expand Down
6 changes: 4 additions & 2 deletions src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,16 @@ using MacroAssembler::null_check;
// hdr : must be r0, contents destroyed
// obj : must point to the object to lock, contents preserved
// disp_hdr: must point to the displaced header location, contents preserved
// temp : temporary register, must not be rscratch1 or rscratch2
// returns code offset at which to add null check debug information
int lock_object (Register swap, Register obj, Register disp_hdr, Label& slow_case);
int lock_object (Register swap, Register obj, Register disp_hdr, Register temp, Label& slow_case);

// unlocking
// hdr : contents destroyed
// obj : must point to the object to lock, contents preserved
// disp_hdr: must be r0 & must point to the displaced header location, contents destroyed
void unlock_object(Register swap, Register obj, Register lock, Label& slow_case);
// temp : temporary register, must not be rscratch1 or rscratch2
void unlock_object(Register swap, Register obj, Register lock, Register temp, Label& slow_case);

void initialize_object(
Register obj, // result: pointer to object after successful allocation
Expand Down
4 changes: 2 additions & 2 deletions src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);

void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmpReg,
Register tmp2Reg) {
Register tmp2Reg, Register tmp3Reg) {
Register oop = objectReg;
Register box = boxReg;
Register disp_hdr = tmpReg;
Expand Down Expand Up @@ -104,7 +104,7 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
b(cont);
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
lightweight_lock(oop, disp_hdr, tmp, rscratch1, no_count);
lightweight_lock(oop, disp_hdr, tmp, tmp3Reg, no_count);
b(count);
}

Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
public:
// Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
// See full description in macroAssembler_aarch64.cpp.
void fast_lock(Register object, Register box, Register tmp, Register tmp2);
void fast_lock(Register object, Register box, Register tmp, Register tmp2, Register tmp3);
void fast_unlock(Register object, Register box, Register tmp, Register tmp2);

void string_compare(Register str1, Register str2,
Expand Down
9 changes: 6 additions & 3 deletions src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,7 @@ void InterpreterMacroAssembler::remove_activation(
//
// Kills:
// r0
// c_rarg0, c_rarg1, c_rarg2, c_rarg3, .. (param regs)
// c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, .. (param regs)
// rscratch1, rscratch2 (scratch regs)
void InterpreterMacroAssembler::lock_object(Register lock_reg)
{
Expand All @@ -740,6 +740,8 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
const Register swap_reg = r0;
const Register tmp = c_rarg2;
const Register obj_reg = c_rarg3; // Will contain the oop
const Register tmp2 = c_rarg4;
const Register tmp3 = c_rarg5;

const int obj_offset = in_bytes(BasicObjectLock::obj_offset());
const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
Expand All @@ -760,7 +762,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)

if (LockingMode == LM_LIGHTWEIGHT) {
ldr(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
lightweight_lock(obj_reg, tmp, rscratch1, rscratch2, slow_case);
lightweight_lock(obj_reg, tmp, tmp2, tmp3, slow_case);
b(count);
} else if (LockingMode == LM_LEGACY) {
// Load (object->mark() | 1) into swap_reg
Expand Down Expand Up @@ -858,6 +860,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
const Register swap_reg = r0;
const Register header_reg = c_rarg2; // Will contain the old oopMark
const Register obj_reg = c_rarg3; // Will contain the oop
const Register tmp_reg = c_rarg4; // Temporary used by lightweight_unlock

save_bcp(); // Save in case of exception

Expand Down Expand Up @@ -891,7 +894,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)

ldr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
tbnz(header_reg, exact_log2(markWord::monitor_value), slow_case);
lightweight_unlock(obj_reg, header_reg, swap_reg, rscratch1, slow_case);
lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
b(count);
bind(slow_case);
} else if (LockingMode == LM_LEGACY) {
Expand Down
10 changes: 8 additions & 2 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2737,6 +2737,10 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
mov(result, expected);
lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true);
compare_eq(result, expected, size);
#ifdef ASSERT
// Poison rscratch1 which is written on !UseLSE branch
mov(rscratch1, 0x1f1f1f1f1f1f1f1f);
#endif
} else {
Label retry_load, done;
prfm(Address(addr), PSTL1STRM);
Expand Down Expand Up @@ -6221,7 +6225,7 @@ void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
// - t1, t2: temporary registers, will be destroyed
void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
assert_different_registers(obj, hdr, t1, t2);
assert_different_registers(obj, hdr, t1, t2, rscratch1);

// Check if we would have space on lock-stack for the object.
ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
Expand All @@ -6233,6 +6237,7 @@ void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, R
// Clear lock-bits, into t2
eor(t2, hdr, markWord::unlocked_value);
// Try to swing header from unlocked to locked
// Clobbers rscratch1 when UseLSE is false
cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
/*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
br(Assembler::NE, slow);
Expand All @@ -6253,7 +6258,7 @@ void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, R
// - t1, t2: temporary registers
void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
assert_different_registers(obj, hdr, t1, t2);
assert_different_registers(obj, hdr, t1, t2, rscratch1);

#ifdef ASSERT
{
Expand Down Expand Up @@ -6293,6 +6298,7 @@ void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register t1,
orr(t1, hdr, markWord::unlocked_value);

// Try to swing header from locked to unlocked
// Clobbers rscratch1 when UseLSE is false
cmpxchg(obj, hdr, t1, Assembler::xword,
/*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
br(Assembler::NE, slow);
Expand Down
5 changes: 3 additions & 2 deletions src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1759,6 +1759,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
const Register obj_reg = r19; // Will contain the oop
const Register lock_reg = r13; // Address of compiler lock object (BasicLock)
const Register old_hdr = r13; // value of old header at unlock time
const Register lock_tmp = r14; // Temporary used by lightweight_lock/unlock
const Register tmp = lr;

Label slow_path_lock;
Expand Down Expand Up @@ -1812,7 +1813,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
__ ldr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ lightweight_lock(obj_reg, swap_reg, tmp, rscratch1, slow_path_lock);
__ lightweight_lock(obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock);
}
__ bind(count);
__ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
Expand Down Expand Up @@ -1953,7 +1954,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
assert(LockingMode == LM_LIGHTWEIGHT, "");
__ ldr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ tbnz(old_hdr, exact_log2(markWord::monitor_value), slow_path_unlock);
__ lightweight_unlock(obj_reg, old_hdr, swap_reg, rscratch1, slow_path_unlock);
__ lightweight_unlock(obj_reg, old_hdr, swap_reg, lock_tmp, slow_path_unlock);
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
}

Expand Down

1 comment on commit fd329f2

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.