Skip to content

Commit

Permalink
8286957: Held monitor count
Browse files Browse the repository at this point in the history
Reviewed-by: rpressler, eosterlund
  • Loading branch information
robehn committed Jul 8, 2022
1 parent 1fec62f commit ac399e9
Show file tree
Hide file tree
Showing 44 changed files with 407 additions and 238 deletions.
3 changes: 2 additions & 1 deletion make/test/JtregNativeHotspot.gmk
Original file line number Diff line number Diff line change
Expand Up @@ -874,7 +874,7 @@ BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exesigtest := -ljvm

ifeq ($(call isTargetOs, windows), true)
BUILD_HOTSPOT_JTREG_EXECUTABLES_CFLAGS_exeFPRegs := -MT
BUILD_HOTSPOT_JTREG_EXCLUDE += exesigtest.c libterminatedThread.c libTestJNI.c
BUILD_HOTSPOT_JTREG_EXCLUDE += exesigtest.c libterminatedThread.c libTestJNI.c libCompleteExit.c
BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libatExit := jvm.lib
else
BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libbootclssearch_agent += -lpthread
Expand Down Expand Up @@ -1512,6 +1512,7 @@ else
BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libgetphase002 += -lpthread
BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libterminatedThread += -lpthread
BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libatExit += -ljvm
BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libCompleteExit += -lpthread
endif

# This evaluation is expensive and should only be done if this target was
Expand Down
16 changes: 12 additions & 4 deletions src/hotspot/cpu/aarch64/aarch64.ad
Original file line number Diff line number Diff line change
Expand Up @@ -3959,7 +3959,7 @@ encode %{
Register tmp = as_Register($tmp2$$reg);
Label cont;
Label object_has_monitor;
Label cas_failed;
Label no_count;

assert_different_registers(oop, box, tmp, disp_hdr);

Expand Down Expand Up @@ -3995,9 +3995,6 @@ encode %{
// If the compare-and-exchange succeeded, then we found an unlocked
// object, will have now locked it will continue at label cont

__ bind(cas_failed);
// We did not see an unlocked object so try the fast recursive case.

// Check if the owner is self by comparing the value in the
// markWord of object (disp_hdr) with the stack pointer.
__ mov(rscratch1, sp);
Expand Down Expand Up @@ -4042,6 +4039,11 @@ encode %{
__ bind(cont);
// flag == EQ indicates success
// flag == NE indicates failure
__ br(Assembler::NE, no_count);

__ increment(Address(rthread, JavaThread::held_monitor_count_offset()));

__ bind(no_count);
%}

enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
Expand All @@ -4052,6 +4054,7 @@ encode %{
Register tmp = as_Register($tmp2$$reg);
Label cont;
Label object_has_monitor;
Label no_count;

assert_different_registers(oop, box, tmp, disp_hdr);

Expand Down Expand Up @@ -4110,6 +4113,11 @@ encode %{
__ bind(cont);
// flag == EQ indicates success
// flag == NE indicates failure
__ br(Assembler::NE, no_count);

__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));

__ bind(no_count);
%}

%}
Expand Down
12 changes: 0 additions & 12 deletions src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,6 @@ int LIR_Assembler::emit_unwind_handler() {
__ unlock_object(r5, r4, r0, *stub->entry());
}
__ bind(*stub->continuation());
__ dec_held_monitor_count(rthread);
}

if (compilation()->env()->dtrace_method_probes()) {
Expand Down Expand Up @@ -2577,18 +2576,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
} else {
Unimplemented();
}
if (op->code() == lir_lock) {
// If deoptimization happens in Runtime1::monitorenter, inc_held_monitor_count after backing from slowpath
// will be skipped. Solution is:
// 1. Increase only in fastpath
// 2. Runtime1::monitorenter increase count after locking
__ inc_held_monitor_count(rthread);
}
__ bind(*op->stub()->continuation());
if (op->code() == lir_unlock) {
// unlock in slowpath is JRT_Leaf stub, no deoptimization can happen
__ dec_held_monitor_count(rthread);
}
}

void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
Expand Down
2 changes: 2 additions & 0 deletions src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
cbnz(hdr, slow_case);
// done
bind(done);
increment(Address(rthread, JavaThread::held_monitor_count_offset()));
return null_check_offset;
}

Expand Down Expand Up @@ -147,6 +148,7 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
}
// done
bind(done);
decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
}


Expand Down
2 changes: 2 additions & 0 deletions src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ const bool CCallingConventionRequiresIntsAsLongs = false;

#define SUPPORTS_NATIVE_CX8

#define SUPPORT_MONITOR_COUNT

// Aarch64 was not originally defined to be multi-copy-atomic, but now
// is. See: "Simplifying ARM Concurrency: Multicopy-atomic Axiomatic
// and Operational Models for ARMv8"
Expand Down
23 changes: 14 additions & 9 deletions src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,6 @@ void InterpreterMacroAssembler::remove_activation(

bind(unlock);
unlock_object(c_rarg1);
dec_held_monitor_count(rthread);
pop(state);

// Check that for block-structured locking (i.e., that all locked
Expand Down Expand Up @@ -649,7 +648,6 @@ void InterpreterMacroAssembler::remove_activation(

push(state);
unlock_object(c_rarg1);
dec_held_monitor_count(rthread);
pop(state);

if (install_monitor_exception) {
Expand Down Expand Up @@ -732,7 +730,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
lock_reg);
} else {
Label done;
Label count, done;

const Register swap_reg = r0;
const Register tmp = c_rarg2;
Expand Down Expand Up @@ -766,7 +764,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
"displached header must be first word in BasicObjectLock");

Label fail;
cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/NULL);

// Fast check for recursive lock.
//
Expand Down Expand Up @@ -803,14 +801,18 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)

// Save the test result, for recursive case, the result is zero
str(swap_reg, Address(lock_reg, mark_offset));
br(Assembler::EQ, done);
br(Assembler::EQ, count);

bind(slow_case);

// Call the runtime routine for slow case
call_VM(noreg,
CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
lock_reg);
b(done);

bind(count);
increment(Address(rthread, JavaThread::held_monitor_count_offset()));

bind(done);
}
Expand All @@ -835,7 +837,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
if (UseHeavyMonitors) {
call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
} else {
Label done;
Label count, done;

const Register swap_reg = r0;
const Register header_reg = c_rarg2; // Will contain the old oopMark
Expand All @@ -858,17 +860,20 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
BasicLock::displaced_header_offset_in_bytes()));

// Test for recursion
cbz(header_reg, done);
cbz(header_reg, count);

// Atomic swap back the old header
cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, /*fallthrough*/NULL);

// Call the runtime routine for slow case.
str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj
call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
b(done);

bind(done);
bind(count);
decrement(Address(rthread, JavaThread::held_monitor_count_offset()));

bind(done);
restore_bcp();
}
}
Expand Down
14 changes: 0 additions & 14 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -346,20 +346,6 @@ void MacroAssembler::pop_cont_fastpath(Register java_thread) {
bind(done);
}

void MacroAssembler::inc_held_monitor_count(Register java_thread) {
if (!Continuations::enabled()) return;
incrementw(Address(java_thread, JavaThread::held_monitor_count_offset()));
}

void MacroAssembler::dec_held_monitor_count(Register java_thread) {
if (!Continuations::enabled()) return;
decrementw(Address(java_thread, JavaThread::held_monitor_count_offset()));
}

void MacroAssembler::reset_held_monitor_count(Register java_thread) {
strw(zr, Address(java_thread, JavaThread::held_monitor_count_offset()));
}

void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
// we must set sp to zero to clear frame
str(zr, Address(rthread, JavaThread::last_Java_sp_offset()));
Expand Down
3 changes: 0 additions & 3 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -900,9 +900,6 @@ class MacroAssembler: public Assembler {

void push_cont_fastpath(Register java_thread);
void pop_cont_fastpath(Register java_thread);
void inc_held_monitor_count(Register java_thread);
void dec_held_monitor_count(Register java_thread);
void reset_held_monitor_count(Register java_thread);

// Round up to a power of two
void round_to(Register reg, int modulus);
Expand Down
23 changes: 14 additions & 9 deletions src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1609,7 +1609,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
Label lock_done;

if (method->is_synchronized()) {

Label count;
const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();

// Get the handle (the 2nd argument)
Expand All @@ -1631,9 +1631,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ str(swap_reg, Address(lock_reg, mark_word_offset));

// src -> dest iff dest == r0 else r0 <- dest
{ Label here;
__ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
}
__ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/NULL);

// Hmm should this move to the slow path code area???

Expand All @@ -1656,6 +1654,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
} else {
__ b(slow_path_lock);
}
__ bind(count);
__ increment(Address(rthread, JavaThread::held_monitor_count_offset()));

// Slow path will re-enter here
__ bind(lock_done);
Expand Down Expand Up @@ -1757,14 +1757,18 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// Get locked oop from the handle we passed to jni
__ ldr(obj_reg, Address(oop_handle_reg, 0));

Label done;
Label done, not_recursive;

if (!UseHeavyMonitors) {
// Simple recursive lock?
__ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
__ cbz(rscratch1, done);
__ cbnz(rscratch1, not_recursive);
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
__ b(done);
}

__ bind(not_recursive);

// Must save r0 if if it is live now because cmpxchg must use it
if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
save_native_result(masm, ret_type, stack_slots);
Expand All @@ -1777,9 +1781,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ ldr(old_hdr, Address(r0, 0));

// Atomic swap old header if oop still contains the stack lock
Label succeed;
__ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, succeed, &slow_path_unlock);
__ bind(succeed);
Label count;
__ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, count, &slow_path_unlock);
__ bind(count);
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
} else {
__ b(slow_path_unlock);
}
Expand Down
10 changes: 5 additions & 5 deletions src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8092,11 +8092,11 @@ void fill_continuation_entry(MacroAssembler* masm) {

__ ldr(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset()));
__ str(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
__ ldrw(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
__ strw(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
__ ldr(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
__ str(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));

__ str(zr, Address(rthread, JavaThread::cont_fastpath_offset()));
__ reset_held_monitor_count(rthread);
__ str(zr, Address(rthread, JavaThread::held_monitor_count_offset()));
}

// on entry, sp points to the ContinuationEntry
Expand All @@ -8113,8 +8113,8 @@ void continuation_enter_cleanup(MacroAssembler* masm) {

__ ldr(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
__ str(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset()));
__ ldrw(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
__ strw(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
__ ldr(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
__ str(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));

__ ldr(rscratch2, Address(sp, ContinuationEntry::parent_offset()));
__ str(rscratch2, Address(rthread, JavaThread::cont_entry_offset()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -777,7 +777,6 @@ void TemplateInterpreterGenerator::lock_method() {
__ str(r0, Address(esp, BasicObjectLock::obj_offset_in_bytes()));
__ mov(c_rarg1, esp); // object address
__ lock_object(c_rarg1);
__ inc_held_monitor_count(rthread);
}

// Generate a fixed interpreter frame. This is identical setup for
Expand Down Expand Up @@ -1496,7 +1495,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {

__ bind(unlock);
__ unlock_object(c_rarg1);
__ dec_held_monitor_count(rthread);
}
__ bind(L);
}
Expand Down
4 changes: 0 additions & 4 deletions src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3866,9 +3866,6 @@ void TemplateTable::monitorenter()
__ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
__ lock_object(c_rarg1);

// The object is stored so counter should be increased even if stackoverflow is generated
__ inc_held_monitor_count(rthread);

// check to make sure this monitor doesn't cause stack overflow after locking
__ save_bcp(); // in case of exception
__ generate_stack_overflow_check(0);
Expand Down Expand Up @@ -3927,7 +3924,6 @@ void TemplateTable::monitorexit()
__ bind(found);
__ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
__ unlock_object(c_rarg1);
__ dec_held_monitor_count(rthread);
__ pop_ptr(r0); // discard object
}

Expand Down
12 changes: 0 additions & 12 deletions src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,6 @@ int LIR_Assembler::emit_unwind_handler() {
__ unlock_object(rdi, rsi, rax, *stub->entry());
}
__ bind(*stub->continuation());
__ dec_held_monitor_count();
}

if (compilation()->env()->dtrace_method_probes()) {
Expand Down Expand Up @@ -3514,18 +3513,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
} else {
Unimplemented();
}
if (op->code() == lir_lock) {
// If deoptimization happens in Runtime1::monitorenter, inc_held_monitor_count after backing from slowpath
// will be skipped. Solution is
// 1. Increase only in fastpath
// 2. Runtime1::monitorenter increase count after locking
__ inc_held_monitor_count();
}
__ bind(*op->stub()->continuation());
if (op->code() == lir_unlock) {
// unlock in slowpath is JRT_Leaf stub, no deoptimization can happen
__ dec_held_monitor_count();
}
}

void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
Expand Down

1 comment on commit ac399e9

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.