Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

8308469: [PPC64] Implement alternative fast-locking scheme #14069

Closed
wants to merge 5 commits into from
Closed

8308469: [PPC64] Implement alternative fast-locking scheme #14069

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fee3d1f
8308469: [PPC64] Implement alternative fast-locking scheme
TheRealMDoerr May 20, 2023
ba3e597
Make compiler_fast_lock/unlock a bit more generic and revert RTM chan…
TheRealMDoerr May 20, 2023
dca48f5
Support for flag = CR0 requires small addition.
TheRealMDoerr May 22, 2023
eae93f8
interp_masm: Rename displaced_header to header.
TheRealMDoerr May 25, 2023
8b4c837
Remove ObjectMonitor::ANONYMOUS_OWNER handling from fast path. We tak…
TheRealMDoerr May 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
101 changes: 57 additions & 44 deletions src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,41 +114,46 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox
bne(CCR0, slow_int);
}

// ... and mark it unlocked.
ori(Rmark, Rmark, markWord::unlocked_value);

// Save unlocked object header into the displaced header location on the stack.
std(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);

// Compare object markWord with Rmark and if equal exchange Rscratch with object markWord.
assert(oopDesc::mark_offset_in_bytes() == 0, "cas must take a zero displacement");
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/Rscratch,
/*compare_value=*/Rmark,
/*exchange_value=*/Rbox,
/*where=*/Roop/*+0==mark_offset_in_bytes*/,
MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
MacroAssembler::cmpxchgx_hint_acquire_lock(),
noreg,
&cas_failed,
/*check without membar and ldarx first*/true);
// If compare/exchange succeeded we found an unlocked object and we now have locked it
// hence we are done.
if (LockingMode == LM_LIGHTWEIGHT) {
fast_lock(Roop, Rmark, Rscratch, slow_int);
} else if (LockingMode == LM_LEGACY) {
// ... and mark it unlocked.
ori(Rmark, Rmark, markWord::unlocked_value);

// Save unlocked object header into the displaced header location on the stack.
std(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);

// Compare object markWord with Rmark and if equal exchange Rscratch with object markWord.
assert(oopDesc::mark_offset_in_bytes() == 0, "cas must take a zero displacement");
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/Rscratch,
/*compare_value=*/Rmark,
/*exchange_value=*/Rbox,
/*where=*/Roop/*+0==mark_offset_in_bytes*/,
MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
MacroAssembler::cmpxchgx_hint_acquire_lock(),
noreg,
&cas_failed,
/*check without membar and ldarx first*/true);
// If compare/exchange succeeded we found an unlocked object and we now have locked it
// hence we are done.
}
b(done);

bind(slow_int);
b(slow_case); // far

bind(cas_failed);
// We did not find an unlocked object so see if this is a recursive case.
sub(Rscratch, Rscratch, R1_SP);
load_const_optimized(R0, (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
and_(R0/*==0?*/, Rscratch, R0);
std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), Rbox);
bne(CCR0, slow_int);
if (LockingMode == LM_LEGACY) {
bind(cas_failed);
// We did not find an unlocked object so see if this is a recursive case.
sub(Rscratch, Rscratch, R1_SP);
load_const_optimized(R0, (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
and_(R0/*==0?*/, Rscratch, R0);
std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), Rbox);
bne(CCR0, slow_int);
}

bind(done);

inc_held_monitor_count(Rmark /*tmp*/);
}

Expand All @@ -161,33 +166,41 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb
Address mark_addr(Roop, oopDesc::mark_offset_in_bytes());
assert(mark_addr.disp() == 0, "cas must take a zero displacement");

// Test first if it is a fast recursive unlock.
ld(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
cmpdi(CCR0, Rmark, 0);
beq(CCR0, done);
if (LockingMode != LM_LIGHTWEIGHT) {
// Test first if it is a fast recursive unlock.
ld(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
cmpdi(CCR0, Rmark, 0);
beq(CCR0, done);
}

// Load object.
ld(Roop, in_bytes(BasicObjectLock::obj_offset()), Rbox);
verify_oop(Roop, FILE_AND_LINE);

// Check if it is still a light weight lock, this is is true if we see
// the stack address of the basicLock in the markWord of the object.
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/R0,
/*compare_value=*/Rbox,
/*exchange_value=*/Rmark,
/*where=*/Roop,
MacroAssembler::MemBarRel,
MacroAssembler::cmpxchgx_hint_release_lock(),
noreg,
&slow_int);
if (LockingMode == LM_LIGHTWEIGHT) {
ld(Rmark, oopDesc::mark_offset_in_bytes(), Roop);
andi_(R0, Rmark, markWord::monitor_value);
bne(CCR0, slow_int);
fast_unlock(Roop, Rmark, slow_int);
} else if (LockingMode == LM_LEGACY) {
// Check if it is still a light weight lock, this is is true if we see
// the stack address of the basicLock in the markWord of the object.
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/R0,
/*compare_value=*/Rbox,
/*exchange_value=*/Rmark,
/*where=*/Roop,
MacroAssembler::MemBarRel,
MacroAssembler::cmpxchgx_hint_release_lock(),
noreg,
&slow_int);
}
b(done);
bind(slow_int);
b(slow_case); // far

// Done
bind(done);

dec_held_monitor_count(Rmark /*tmp*/);
}

Expand Down
177 changes: 105 additions & 72 deletions src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -924,7 +924,7 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
if (LockingMode == LM_MONITOR) {
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
} else {
// template code:
// template code (for LM_LEGACY):
//
// markWord displaced_header = obj->mark().set_unlocked();
// monitor->lock()->set_displaced_header(displaced_header);
Expand All @@ -938,20 +938,20 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
// InterpreterRuntime::monitorenter(THREAD, monitor);
// }

const Register displaced_header = R7_ARG5;
const Register header = R7_ARG5;
const Register object_mark_addr = R8_ARG6;
const Register current_header = R9_ARG7;
const Register tmp = R10_ARG8;

Label count_locking, done;
Label cas_failed, slow_case;

assert_different_registers(displaced_header, object_mark_addr, current_header, tmp);
assert_different_registers(header, object_mark_addr, current_header, tmp);

// markWord displaced_header = obj->mark().set_unlocked();

// Load markWord from object into displaced_header.
ld(displaced_header, oopDesc::mark_offset_in_bytes(), object);
// Load markWord from object into header.
ld(header, oopDesc::mark_offset_in_bytes(), object);

if (DiagnoseSyncOnValueBasedClasses != 0) {
load_klass(tmp, object);
Expand All @@ -960,58 +960,64 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
bne(CCR0, slow_case);
}

// Set displaced_header to be (markWord of object | UNLOCK_VALUE).
ori(displaced_header, displaced_header, markWord::unlocked_value);
if (LockingMode == LM_LIGHTWEIGHT) {
fast_lock(object, /* mark word */ header, tmp, slow_case);
b(count_locking);
} else if (LockingMode == LM_LEGACY) {

// monitor->lock()->set_displaced_header(displaced_header);
const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
const int mark_offset = lock_offset +
BasicLock::displaced_header_offset_in_bytes();
// Set displaced_header to be (markWord of object | UNLOCK_VALUE).
ori(header, header, markWord::unlocked_value);

// Initialize the box (Must happen before we update the object mark!).
std(displaced_header, mark_offset, monitor);
// monitor->lock()->set_displaced_header(displaced_header);
const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
const int mark_offset = lock_offset +
BasicLock::displaced_header_offset_in_bytes();

// if (Atomic::cmpxchg(/*addr*/obj->mark_addr(), /*cmp*/displaced_header, /*ex=*/monitor) == displaced_header) {
// Initialize the box (Must happen before we update the object mark!).
std(header, mark_offset, monitor);

// Store stack address of the BasicObjectLock (this is monitor) into object.
addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());

// Must fence, otherwise, preceding store(s) may float below cmpxchg.
// CmpxchgX sets CCR0 to cmpX(current, displaced).
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/current_header,
/*compare_value=*/displaced_header, /*exchange_value=*/monitor,
/*where=*/object_mark_addr,
MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
MacroAssembler::cmpxchgx_hint_acquire_lock(),
noreg,
&cas_failed,
/*check without membar and ldarx first*/true);

// If the compare-and-exchange succeeded, then we found an unlocked
// object and we have now locked it.
b(count_locking);
bind(cas_failed);
// if (Atomic::cmpxchg(/*addr*/obj->mark_addr(), /*cmp*/displaced_header, /*ex=*/monitor) == displaced_header) {

// } else if (THREAD->is_lock_owned((address)displaced_header))
// // Simple recursive case.
// monitor->lock()->set_displaced_header(nullptr);
// Store stack address of the BasicObjectLock (this is monitor) into object.
addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());

// We did not see an unlocked object so try the fast recursive case.
// Must fence, otherwise, preceding store(s) may float below cmpxchg.
// CmpxchgX sets CCR0 to cmpX(current, displaced).
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/current_header,
/*compare_value=*/header, /*exchange_value=*/monitor,
/*where=*/object_mark_addr,
MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
MacroAssembler::cmpxchgx_hint_acquire_lock(),
noreg,
&cas_failed,
/*check without membar and ldarx first*/true);

// Check if owner is self by comparing the value in the markWord of object
// (current_header) with the stack pointer.
sub(current_header, current_header, R1_SP);
// If the compare-and-exchange succeeded, then we found an unlocked
// object and we have now locked it.
b(count_locking);
bind(cas_failed);

assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
load_const_optimized(tmp, ~(os::vm_page_size()-1) | markWord::lock_mask_in_place);
// } else if (THREAD->is_lock_owned((address)displaced_header))
// // Simple recursive case.
// monitor->lock()->set_displaced_header(nullptr);

and_(R0/*==0?*/, current_header, tmp);
// If condition is true we are done and hence we can store 0 in the displaced
// header indicating it is a recursive lock.
bne(CCR0, slow_case);
std(R0/*==0!*/, mark_offset, monitor);
b(count_locking);
// We did not see an unlocked object so try the fast recursive case.

// Check if owner is self by comparing the value in the markWord of object
// (current_header) with the stack pointer.
sub(current_header, current_header, R1_SP);

assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
load_const_optimized(tmp, ~(os::vm_page_size()-1) | markWord::lock_mask_in_place);

and_(R0/*==0?*/, current_header, tmp);
// If condition is true we are done and hence we can store 0 in the displaced
// header indicating it is a recursive lock.
bne(CCR0, slow_case);
std(R0/*==0!*/, mark_offset, monitor);
b(count_locking);
}

// } else {
// // Slow path.
Expand All @@ -1020,7 +1026,11 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
// None of the above fast optimizations worked so we have to get into the
// slow case of monitor enter.
bind(slow_case);
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
if (LockingMode == LM_LIGHTWEIGHT) {
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj), object);
} else {
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
}
b(done);
// }
align(32, 12);
Expand All @@ -1042,7 +1052,7 @@ void InterpreterMacroAssembler::unlock_object(Register monitor) {
call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), monitor);
} else {

// template code:
// template code (for LM_LEGACY):
//
// if ((displaced_header = monitor->displaced_header()) == nullptr) {
// // Recursive unlock. Mark the monitor unlocked by setting the object field to null.
Expand All @@ -1056,22 +1066,24 @@ void InterpreterMacroAssembler::unlock_object(Register monitor) {
// }

const Register object = R7_ARG5;
const Register displaced_header = R8_ARG6;
const Register header = R8_ARG6;
const Register object_mark_addr = R9_ARG7;
const Register current_header = R10_ARG8;

Label free_slot;
Label slow_case;

assert_different_registers(object, displaced_header, object_mark_addr, current_header);
assert_different_registers(object, header, object_mark_addr, current_header);

// Test first if we are in the fast recursive case.
ld(displaced_header, in_bytes(BasicObjectLock::lock_offset()) +
BasicLock::displaced_header_offset_in_bytes(), monitor);
if (LockingMode != LM_LIGHTWEIGHT) {
// Test first if we are in the fast recursive case.
ld(header, in_bytes(BasicObjectLock::lock_offset()) +
BasicLock::displaced_header_offset_in_bytes(), monitor);

// If the displaced header is zero, we have a recursive unlock.
cmpdi(CCR0, displaced_header, 0);
beq(CCR0, free_slot); // recursive unlock
// If the displaced header is zero, we have a recursive unlock.
cmpdi(CCR0, header, 0);
beq(CCR0, free_slot); // recursive unlock
}

// } else if (Atomic::cmpxchg(obj->mark_addr(), monitor, displaced_header) == monitor) {
// // We swapped the unlocked mark in displaced_header into the object's mark word.
Expand All @@ -1081,20 +1093,41 @@ void InterpreterMacroAssembler::unlock_object(Register monitor) {

// The object address from the monitor is in object.
ld(object, in_bytes(BasicObjectLock::obj_offset()), monitor);
addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());

// We have the displaced header in displaced_header. If the lock is still
// lightweight, it will contain the monitor address and we'll store the
// displaced header back into the object's mark word.
// CmpxchgX sets CCR0 to cmpX(current, monitor).
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/current_header,
/*compare_value=*/monitor, /*exchange_value=*/displaced_header,
/*where=*/object_mark_addr,
MacroAssembler::MemBarRel,
MacroAssembler::cmpxchgx_hint_release_lock(),
noreg,
&slow_case);

if (LockingMode == LM_LIGHTWEIGHT) {
// Check for non-symmetric locking. This is allowed by the spec and the interpreter
// must handle it.
Register tmp = current_header;
// First check for lock-stack underflow.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder why this is not done on x86? It's better to have that check.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Strange. The other platforms have it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rkennke: Do you know why this is different on x86?

lwz(tmp, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
cmplwi(CCR0, tmp, (unsigned)LockStack::start_offset());
ble(CCR0, slow_case);
// Then check if the top of the lock-stack matches the unlocked object.
addi(tmp, tmp, -oopSize);
ldx(tmp, tmp, R16_thread);
cmpd(CCR0, tmp, object);
bne(CCR0, slow_case);

ld(header, oopDesc::mark_offset_in_bytes(), object);
andi_(R0, header, markWord::monitor_value);
bne(CCR0, slow_case);
fast_unlock(object, header, slow_case);
} else {
addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());

// We have the displaced header in displaced_header. If the lock is still
// lightweight, it will contain the monitor address and we'll store the
// displaced header back into the object's mark word.
// CmpxchgX sets CCR0 to cmpX(current, monitor).
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/current_header,
/*compare_value=*/monitor, /*exchange_value=*/header,
/*where=*/object_mark_addr,
MacroAssembler::MemBarRel,
MacroAssembler::cmpxchgx_hint_release_lock(),
noreg,
&slow_case);
}
b(free_slot);

// } else {
Expand Down