Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8276901: Implement UseHeavyMonitors consistently #6320

Closed
wants to merge 18 commits into from
Closed
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -3906,37 +3906,40 @@ encode %{
// Check for existing monitor
__ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);

// Set tmp to be (markWord of object | UNLOCK_VALUE).
__ orr(tmp, disp_hdr, markWord::unlocked_value);

// Initialize the box. (Must happen before we update the object mark!)
__ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));

// Compare object markWord with an unlocked value (tmp) and if
// equal exchange the stack address of our box with object markWord.
// On failure disp_hdr contains the possibly locked markWord.
__ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
/*release*/ true, /*weak*/ false, disp_hdr);
__ br(Assembler::EQ, cont);

assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");

// If the compare-and-exchange succeeded, then we found an unlocked
// object, will have now locked it will continue at label cont

__ bind(cas_failed);
// We did not see an unlocked object so try the fast recursive case.

// Check if the owner is self by comparing the value in the
// markWord of object (disp_hdr) with the stack pointer.
__ mov(rscratch1, sp);
__ sub(disp_hdr, disp_hdr, rscratch1);
__ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
// If condition is true we are cont and hence we can store 0 as the
// displaced header in the box, which indicates that it is a recursive lock.
__ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
__ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));

if (!UseHeavyMonitors) {
// Set tmp to be (markWord of object | UNLOCK_VALUE).
__ orr(tmp, disp_hdr, markWord::unlocked_value);

// Initialize the box. (Must happen before we update the object mark!)
__ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));

// Compare object markWord with an unlocked value (tmp) and if
// equal exchange the stack address of our box with object markWord.
// On failure disp_hdr contains the possibly locked markWord.
__ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
/*release*/ true, /*weak*/ false, disp_hdr);
__ br(Assembler::EQ, cont);

assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");

// If the compare-and-exchange succeeded, then we found an unlocked
// object, will have now locked it will continue at label cont

__ bind(cas_failed);
// We did not see an unlocked object so try the fast recursive case.

// Check if the owner is self by comparing the value in the
// markWord of object (disp_hdr) with the stack pointer.
__ mov(rscratch1, sp);
__ sub(disp_hdr, disp_hdr, rscratch1);
__ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
// If condition is true we are cont and hence we can store 0 as the
// displaced header in the box, which indicates that it is a recursive lock.
__ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
__ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
} else {
__ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
}
__ b(cont);

// Handle existing monitor.
@@ -3982,23 +3985,29 @@ encode %{

assert_different_registers(oop, box, tmp, disp_hdr);

// Find the lock address and load the displaced header from the stack.
__ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
if (!UseHeavyMonitors) {
// Find the lock address and load the displaced header from the stack.
__ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));

// If the displaced header is 0, we have a recursive unlock.
__ cmp(disp_hdr, zr);
__ br(Assembler::EQ, cont);
// If the displaced header is 0, we have a recursive unlock.
__ cmp(disp_hdr, zr);
__ br(Assembler::EQ, cont);
}

// Handle existing monitor.
__ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
__ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);

// Check if it is still a light weight lock, this is is true if we
// see the stack address of the basicLock in the markWord of the
// object.
if (!UseHeavyMonitors) {
// Check if it is still a light weight lock, this is is true if we
// see the stack address of the basicLock in the markWord of the
// object.

__ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
/*release*/ true, /*weak*/ false, tmp);
__ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
/*release*/ true, /*weak*/ false, tmp);
} else {
__ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
}
__ b(cont);

assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
@@ -439,7 +439,11 @@ int LIR_Assembler::emit_unwind_handler() {
if (method()->is_synchronized()) {
monitor_address(0, FrameMap::r0_opr);
stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
__ unlock_object(r5, r4, r0, *stub->entry());
if (UseHeavyMonitors) {
__ b(*stub->entry());
} else {
__ unlock_object(r5, r4, r0, *stub->entry());
}
__ bind(*stub->continuation());
}

@@ -2563,7 +2567,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register obj = op->obj_opr()->as_register(); // may not be an oop
Register hdr = op->hdr_opr()->as_register();
Register lock = op->lock_opr()->as_register();
if (!UseFastLocking) {
if (UseHeavyMonitors) {
__ b(*op->stub()->entry());
} else if (op->code() == lir_lock) {
assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
@@ -1642,39 +1642,42 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// Load the oop from the handle
__ ldr(obj_reg, Address(oop_handle_reg, 0));

// Load (object->mark() | 1) into swap_reg %r0
__ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ orr(swap_reg, rscratch1, 1);
if (!UseHeavyMonitors) {
// Load (object->mark() | 1) into swap_reg %r0
__ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ orr(swap_reg, rscratch1, 1);

// Save (object->mark() | 1) into BasicLock's displaced header
__ str(swap_reg, Address(lock_reg, mark_word_offset));
// Save (object->mark() | 1) into BasicLock's displaced header
__ str(swap_reg, Address(lock_reg, mark_word_offset));

// src -> dest iff dest == r0 else r0 <- dest
{ Label here;
__ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
}
// src -> dest iff dest == r0 else r0 <- dest
{ Label here;
__ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
}

// Hmm should this move to the slow path code area???
// Hmm should this move to the slow path code area???

// Test if the oopMark is an obvious stack pointer, i.e.,
// 1) (mark & 3) == 0, and
// 2) sp <= mark < mark + os::pagesize()
// These 3 tests can be done by evaluating the following
// expression: ((mark - sp) & (3 - os::vm_page_size())),
// assuming both stack pointer and pagesize have their
// least significant 2 bits clear.
// NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
// Test if the oopMark is an obvious stack pointer, i.e.,
// 1) (mark & 3) == 0, and
// 2) sp <= mark < mark + os::pagesize()
// These 3 tests can be done by evaluating the following
// expression: ((mark - sp) & (3 - os::vm_page_size())),
// assuming both stack pointer and pagesize have their
// least significant 2 bits clear.
// NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg

__ sub(swap_reg, sp, swap_reg);
__ neg(swap_reg, swap_reg);
__ ands(swap_reg, swap_reg, 3 - os::vm_page_size());
__ sub(swap_reg, sp, swap_reg);
__ neg(swap_reg, swap_reg);
__ ands(swap_reg, swap_reg, 3 - os::vm_page_size());

// Save the test result, for recursive case, the result is zero
__ str(swap_reg, Address(lock_reg, mark_word_offset));
__ br(Assembler::NE, slow_path_lock);
// Save the test result, for recursive case, the result is zero
__ str(swap_reg, Address(lock_reg, mark_word_offset));
__ br(Assembler::NE, slow_path_lock);
} else {
__ b(slow_path_lock);
}

// Slow path will re-enter here

__ bind(lock_done);
}

@@ -1775,26 +1778,30 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ ldr(obj_reg, Address(oop_handle_reg, 0));

Label done;
// Simple recursive lock?

__ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
__ cbz(rscratch1, done);
if (!UseHeavyMonitors) {
// Simple recursive lock?
__ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
__ cbz(rscratch1, done);

// Must save r0 if if it is live now because cmpxchg must use it
if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
save_native_result(masm, ret_type, stack_slots);
}
// Must save r0 if if it is live now because cmpxchg must use it
if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
save_native_result(masm, ret_type, stack_slots);
}


// get address of the stack lock
__ lea(r0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
// get old displaced header
__ ldr(old_hdr, Address(r0, 0));
// get address of the stack lock
__ lea(r0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
// get old displaced header
__ ldr(old_hdr, Address(r0, 0));

// Atomic swap old header if oop still contains the stack lock
Label succeed;
__ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, succeed, &slow_path_unlock);
__ bind(succeed);
// Atomic swap old header if oop still contains the stack lock
Label succeed;
__ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, succeed, &slow_path_unlock);
__ bind(succeed);
} else {
__ b(slow_path_unlock);
}

// slow path re-enters here
__ bind(unlock_done);
@@ -2425,7 +2425,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register hdr = op->hdr_opr()->as_pointer_register();
Register lock = op->lock_opr()->as_pointer_register();

if (!UseFastLocking) {
if (UseHeavyMonitors) {
__ b(*op->stub()->entry());
} else if (op->code() == lir_lock) {
assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
@@ -2689,7 +2689,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
// Obj may not be an oop.
if (op->code() == lir_lock) {
MonitorEnterStub* stub = (MonitorEnterStub*)op->stub();
if (UseFastLocking) {
if (!UseHeavyMonitors) {
assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
// Add debug info for NullPointerException only if one is possible.
if (op->info() != NULL) {
@@ -2711,7 +2711,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
}
} else {
assert (op->code() == lir_unlock, "Invalid code, expected lir_unlock");
if (UseFastLocking) {
if (!UseHeavyMonitors) {
assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
__ unlock_object(hdr, obj, lock, *op->stub()->entry());
} else {
@@ -2730,7 +2730,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register obj = op->obj_opr()->as_register(); // May not be an oop.
Register hdr = op->hdr_opr()->as_register();
Register lock = op->lock_opr()->as_register();
if (!UseFastLocking) {
if (UseHeavyMonitors) {
__ branch_optimized(Assembler::bcondAlways, *op->stub()->entry());
} else if (op->code() == lir_lock) {
assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
@@ -461,7 +461,11 @@ int LIR_Assembler::emit_unwind_handler() {
if (method()->is_synchronized()) {
monitor_address(0, FrameMap::rax_opr);
stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
__ unlock_object(rdi, rsi, rax, *stub->entry());
if (UseHeavyMonitors) {
__ jmp(*stub->entry());
} else {
__ unlock_object(rdi, rsi, rax, *stub->entry());
}
__ bind(*stub->continuation());
}

@@ -3498,7 +3502,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register obj = op->obj_opr()->as_register(); // may not be an oop
Register hdr = op->hdr_opr()->as_register();
Register lock = op->lock_opr()->as_register();
if (!UseFastLocking) {
if (UseHeavyMonitors) {
__ jmp(*op->stub()->entry());
} else if (op->code() == lir_lock) {
assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
@@ -485,6 +485,7 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp

#if INCLUDE_RTM_OPT
if (UseRTMForStackLocks && use_rtm) {
assert(!UseHeavyMonitors, "+UseHeavyMonitors and +UseRTMForStackLocks are mutually exclusive");
rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
stack_rtm_counters, method_data, profile_rtm,
DONE_LABEL, IsInflated);
@@ -495,20 +496,25 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
jccb(Assembler::notZero, IsInflated);

// Attempt stack-locking ...
orptr (tmpReg, markWord::unlocked_value);
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
lock();
cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
jcc(Assembler::equal, DONE_LABEL); // Success

// Recursive locking.
// The object is stack-locked: markword contains stack pointer to BasicLock.
// Locked by current thread if difference with current SP is less than one page.
subptr(tmpReg, rsp);
// Next instruction set ZFlag == 1 (Success) if difference is less then one page.
andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
movptr(Address(boxReg, 0), tmpReg);
if (!UseHeavyMonitors) {
// Attempt stack-locking ...
orptr (tmpReg, markWord::unlocked_value);
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
lock();
cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
jcc(Assembler::equal, DONE_LABEL); // Success

// Recursive locking.
// The object is stack-locked: markword contains stack pointer to BasicLock.
// Locked by current thread if difference with current SP is less than one page.
subptr(tmpReg, rsp);
// Next instruction set ZFlag == 1 (Success) if difference is less then one page.
andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
movptr(Address(boxReg, 0), tmpReg);
} else {
// Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
testptr(objReg, objReg);
}
jmp(DONE_LABEL);

bind(IsInflated);
@@ -638,6 +644,7 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t

#if INCLUDE_RTM_OPT
if (UseRTMForStackLocks && use_rtm) {
assert(!UseHeavyMonitors, "+UseHeavyMonitors and +UseRTMForStackLocks are mutually exclusive");
Label L_regular_unlock;
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // fetch markword
andptr(tmpReg, markWord::lock_mask_in_place); // look at 2 lock bits
@@ -649,11 +656,15 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
}
#endif

cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
if (!UseHeavyMonitors) {
cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
}
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
testptr(tmpReg, markWord::monitor_value); // Inflated?
jccb (Assembler::zero, Stacked);
if (!UseHeavyMonitors) {
testptr(tmpReg, markWord::monitor_value); // Inflated?
jccb (Assembler::zero, Stacked);
}

// It's inflated.
#if INCLUDE_RTM_OPT
@@ -795,11 +806,12 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
testl (boxReg, 0); // set ICC.ZF=1 to indicate success
jmpb (DONE_LABEL);

bind (Stacked);
movptr(tmpReg, Address (boxReg, 0)); // re-fetch
lock();
cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box

if (!UseHeavyMonitors) {
bind (Stacked);
movptr(tmpReg, Address (boxReg, 0)); // re-fetch
lock();
cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
}
#endif
bind(DONE_LABEL);
}