Skip to content
Permalink
Browse files
8277180: Intrinsify recursive ObjectMonitor locking for C2 x64 and A64
Reviewed-by: aph, ngasson
  • Loading branch information
fisk committed Nov 18, 2021
1 parent 00c388b commit d93b238f9725727ae1e2e9f203943b5ddf778f35
Showing with 38 additions and 13 deletions.
  1. +22 −4 src/hotspot/cpu/aarch64/aarch64.ad
  2. +16 −9 src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -3945,7 +3945,7 @@ encode %{
// Try to CAS m->owner from NULL to current thread.
__ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
__ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
/*release*/ true, /*weak*/ false, noreg); // Sets flags for result
/*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result

// Store a non-null value into the box to avoid looking like a re-entrant
// lock. The fast-path monitor unlock code checks for
@@ -3954,6 +3954,15 @@ encode %{
__ mov(tmp, (address)markWord::unused_mark().value());
__ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));

__ br(Assembler::EQ, cont); // CAS success means locking succeeded

__ cmp(rscratch1, rthread);
__ br(Assembler::NE, cont); // Check for recursive locking

// Recursive lock case
__ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
// flag == EQ still from the cmp above, checking if this is a reentrant lock

__ bind(cont);
// flag == EQ indicates success
// flag == NE indicates failure
@@ -3997,11 +4006,20 @@ encode %{
__ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
__ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
__ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
__ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
__ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
__ cmp(rscratch1, zr); // Sets flags for result

Label notRecursive;
__ cmp(rscratch1, rthread);
__ br(Assembler::NE, cont);

__ cbz(disp_hdr, notRecursive);

// Recursive lock
__ sub(disp_hdr, disp_hdr, 1u);
__ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
// flag == EQ was set in the ownership check above
__ b(cont);

__ bind(notRecursive);
__ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
__ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
__ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
@@ -574,8 +574,13 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
// Unconditionally set box->_displaced_header = markWord::unused_mark().
// Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
// Intentional fall-through into DONE_LABEL ...
// Propagate ICC.ZF from CAS above into DONE_LABEL.
jcc(Assembler::equal, DONE_LABEL); // CAS above succeeded; propagate ZF = 1 (success)

cmpptr(r15_thread, rax); // Check if we are already the owner (recursive lock)
jcc(Assembler::notEqual, DONE_LABEL); // If not recursive, ZF = 0 at this point (fail)
incq(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
xorq(rax, rax); // Set ZF = 1 (success) for recursive lock, denoting locking success
#endif // _LP64
#if INCLUDE_RTM_OPT
} // use_rtm()
@@ -670,10 +675,6 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
// Refer to the comments in synchronizer.cpp for how we might encode extra
// state in _succ so we can avoid fetching EntryList|cxq.
//
// I'd like to add more cases in fast_lock() and fast_unlock() --
// such as recursive enter and exit -- but we have to be wary of
// I$ bloat, T$ effects and BP$ effects.
//
// If there's no contention try a 1-0 exit. That is, exit without
// a costly MEMBAR or CAS. See synchronizer.cpp for details on how
// we detect and recover from the race that the 1-0 exit admits.
@@ -721,9 +722,16 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
bind (CheckSucc);
#else // _LP64
// It's inflated
xorptr(boxReg, boxReg);
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
jccb (Assembler::notZero, DONE_LABEL);
Label LNotRecursive, LSuccess, LGoSlowPath;

cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 0);
jccb(Assembler::equal, LNotRecursive);

// Recursive inflated unlock
decq(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
jmpb(LSuccess);

bind(LNotRecursive);
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
jccb (Assembler::notZero, CheckSucc);
@@ -732,7 +740,6 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
jmpb (DONE_LABEL);

// Try to avoid passing control into the slow_path ...
Label LSuccess, LGoSlowPath ;
bind (CheckSucc);

// The following optional optimization can be elided if necessary

1 comment on commit d93b238

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on d93b238 Nov 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.