diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 9ac3cf12623..d39991f33ad 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -7237,7 +7237,17 @@ instruct loadNKlass(iRegNNoSp dst, memory4 mem, rFlagsReg cr) ins_encode %{ assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset"); assert($mem$$index$$Register == noreg, "expect no index"); - __ load_nklass($dst$$Register, $mem$$base$$Register); + Register dst = $dst$$Register; + Register obj = $mem$$base$$Register; + C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst); + Compile::current()->output()->add_stub(stub); + __ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes())); + // NOTE: We can't use tbnz here, because the target is sometimes too far away + // and cannot be encoded. + __ tst(dst, markWord::monitor_value); + __ br(Assembler::NE, stub->entry()); + __ bind(stub->continuation()); + __ lsr(dst, dst, markWord::klass_shift); %} ins_pipe(pipe_slow); %} diff --git a/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp index aeb04e69bed..18b429c2420 100644 --- a/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp @@ -32,6 +32,7 @@ #include "c1/c1_Runtime1.hpp" #include "classfile/javaClasses.hpp" #include "nativeInst_aarch64.hpp" +#include "runtime/objectMonitor.hpp" #include "runtime/sharedRuntime.hpp" #include "vmreg_aarch64.inline.hpp" @@ -255,27 +256,8 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) { void LoadKlassStub::emit_code(LIR_Assembler* ce) { __ bind(_entry); - Register res = _result->as_register(); - ce->store_parameter(_obj->as_register(), 0); - if (res != r0) { - // Note: we cannot push/pop r0 around the call, because that - // would mess with the stack pointer sp, and we need that to - // remain intact for store_paramater/load_argument to work correctly. - // We swap r0 and res instead, which preserves current r0 in res. - // The preserved value is later saved and restored around the - // call in Runtime1::load_klass_id. - __ mov(rscratch1, r0); - __ mov(r0, res); - __ mov(res, rscratch1); - } - __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::load_klass_id))); - if (res != r0) { - // Swap back r0 and res. This brings the call return value - // from r0 into res, and the preserved value in res back into r0. - __ mov(rscratch1, r0); - __ mov(r0, res); - __ mov(res, rscratch1); - } + Register d = _result->as_register(); + __ ldr(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header))); __ b(_continuation); } diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 2733fa6252a..0eec1708f27 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -2562,7 +2562,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) { void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { Register obj = op->obj()->as_pointer_register(); Register result = op->result_opr()->as_pointer_register(); - Register tmp = rscratch1; CodeEmitInfo* info = op->info(); if (info != NULL) { @@ -2572,16 +2571,13 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { assert(UseCompressedClassPointers, "expects UseCompressedClassPointers"); // Check if we can take the (common) fast path, if obj is unlocked. - __ ldr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); - __ eor(tmp, tmp, markWord::unlocked_value); - __ tst(tmp, markWord::lock_mask_in_place); + __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes())); + __ tst(result, markWord::monitor_value); __ br(Assembler::NE, *op->stub()->entry()); + __ bind(*op->stub()->continuation()); - // Fast-path: shift and decode Klass*. - __ mov(result, tmp); + // Shift and decode Klass*. __ lsr(result, result, markWord::klass_shift); - - __ bind(*op->stub()->continuation()); __ decode_klass_not_null(result); } diff --git a/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp index 3deea520ed1..9f916be02d1 100644 --- a/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp @@ -675,16 +675,6 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { break; - case load_klass_id: - { - StubFrame f(sasm, "load_klass", dont_gc_arguments); - save_live_registers_no_oop_map(sasm, true); - f.load_argument(0, r0); // obj - __ call_VM_leaf(CAST_FROM_FN_PTR(address, oopDesc::load_nklass_runtime), r0); - restore_live_registers_except_r0(sasm, true); - } - break; - case counter_overflow_id: { Register bci = r0, method = r1; diff --git a/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp index f77d35cd3e3..5220757f1a9 100644 --- a/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "opto/c2_MacroAssembler.hpp" #include "opto/c2_CodeStubs.hpp" +#include "runtime/objectMonitor.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" @@ -74,4 +75,15 @@ void C2CheckLockStackStub::emit(C2_MacroAssembler& masm) { __ b(continuation()); } +int C2LoadNKlassStub::max_size() const { + return 8; +} + +void C2LoadNKlassStub::emit(C2_MacroAssembler& masm) { + __ bind(entry()); + Register d = dst(); + __ ldr(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header))); + __ b(continuation()); +} + #undef __ diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index db647529b39..e98c68fe8ef 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -4034,35 +4034,19 @@ void MacroAssembler::load_method_holder(Register holder, Register method) { void MacroAssembler::load_nklass(Register dst, Register src) { assert(UseCompressedClassPointers, "expects UseCompressedClassPointers"); - assert_different_registers(src, dst); - - Label slow, done; + Label fast; // Check if we can take the (common) fast path, if obj is unlocked. ldr(dst, Address(src, oopDesc::mark_offset_in_bytes())); - eor(dst, dst, markWord::unlocked_value); - tst(dst, markWord::lock_mask_in_place); - br(Assembler::NE, slow); + tst(dst, markWord::monitor_value); + br(Assembler::NE, fast); + + // Fetch displaced header + ldr(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header))); // Fast-path: shift and decode Klass*. + bind(fast); lsr(dst, dst, markWord::klass_shift); - b(done); - - bind(slow); - RegSet saved_regs = RegSet::of(lr); - // We need r0 as argument and return register for the call. Preserve it, if necessary. - if (dst != r0) { - saved_regs += RegSet::of(r0); - } - push(saved_regs, sp); - mov(r0, src); - assert(StubRoutines::load_nklass() != NULL, "Must have stub"); - far_call(RuntimeAddress(StubRoutines::load_nklass())); - if (dst != r0) { - mov(dst, r0); - } - pop(saved_regs, sp); - bind(done); } void MacroAssembler::load_klass(Register dst, Register src) { diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 7820c7435bc..4591c5cd68c 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -6817,29 +6817,6 @@ class StubGenerator: public StubCodeGenerator { } #endif // LINUX - // Pass object argument in r0 (which has to be preserved outside this stub) - // Pass back result in r0 - // Clobbers rscratch1 - address generate_load_nklass() { - __ align(CodeEntryAlignment); - StubCodeMark mark(this, "StubRoutines", "load_nklass"); - - address start = __ pc(); - - __ set_last_Java_frame(sp, rfp, lr, rscratch1); - __ enter(); - __ push(RegSet::of(rscratch1, rscratch2), sp); - __ push_call_clobbered_registers_except(r0); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, oopDesc::load_nklass_runtime), 1); - __ pop_call_clobbered_registers_except(r0); - __ pop(RegSet::of(rscratch1, rscratch2), sp); - __ leave(); - __ reset_last_Java_frame(true); - __ ret(lr); - - return start; - } - address generate_cont_thaw(Continuation::thaw_kind kind) { bool return_barrier = Continuation::is_thaw_return_barrier(kind); bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind); @@ -8007,8 +7984,6 @@ class StubGenerator: public StubCodeGenerator { if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) { StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true); } - - StubRoutines::_load_nklass = generate_load_nklass(); } void generate_phase1() { diff --git a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp index d264ced5af4..940b84bde61 100644 --- a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp +++ b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp @@ -30,6 +30,7 @@ #include "c1/c1_Runtime1.hpp" #include "classfile/javaClasses.hpp" #include "nativeInst_x86.hpp" +#include "runtime/objectMonitor.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/align.hpp" #include "utilities/macros.hpp" @@ -303,18 +304,8 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) { void LoadKlassStub::emit_code(LIR_Assembler* ce) { __ bind(_entry); #ifdef _LP64 - Register res = _result->as_register(); - ce->store_parameter(_obj->as_register(), 0); - if (res != rax) { - // This preserves rax and allows it to be used as return-register, - // without messing with the stack. - __ xchgptr(rax, res); - } - __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::load_klass_id))); - if (res != rax) { - // Swap back rax, and move result to correct register. - __ xchgptr(rax, res); - } + Register d = _result->as_register(); + __ movq(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header))); __ jmp(_continuation); #else __ should_not_reach_here(); diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index f06fe92ad73..5b9d27fb8b1 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -3541,16 +3541,12 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { assert_different_registers(tmp, result); // Check if we can take the (common) fast path, if obj is unlocked. - __ movq(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); - __ xorq(tmp, markWord::unlocked_value); - __ testb(tmp, markWord::lock_mask_in_place); + __ movq(result, Address(obj, oopDesc::mark_offset_in_bytes())); + __ testb(result, markWord::monitor_value); __ jcc(Assembler::notZero, *op->stub()->entry()); - + __ bind(*op->stub()->continuation()); // Fast-path: shift and decode Klass*. - __ movq(result, tmp); __ shrq(result, markWord::klass_shift); - - __ bind(*op->stub()->continuation()); __ decode_klass_not_null(result, tmp); #else __ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes())); diff --git a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp index e0b2b756146..fcc20640d22 100644 --- a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp +++ b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp @@ -1040,17 +1040,6 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { } break; -#ifdef _LP64 - case load_klass_id: - { - StubFrame f(sasm, "load_klass", dont_gc_arguments); - sasm->save_live_registers_no_oop_map(true); - f.load_argument(0, c_rarg0); // obj - __ call_VM_leaf(CAST_FROM_FN_PTR(address, oopDesc::load_nklass_runtime), c_rarg0); - sasm->restore_live_registers_except_rax(true); - } - break; -#endif case counter_overflow_id: { Register bci = rax, method = rbx; diff --git a/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp b/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp index 8f69c463179..7209e9968a6 100644 --- a/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp +++ b/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "opto/c2_MacroAssembler.hpp" #include "opto/c2_CodeStubs.hpp" +#include "runtime/objectMonitor.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" @@ -83,4 +84,17 @@ void C2CheckLockStackStub::emit(C2_MacroAssembler& masm) { __ jmp(continuation(), false /* maybe_short */); } +#ifdef _LP64 +int C2LoadNKlassStub::max_size() const { + return 10; +} + +void C2LoadNKlassStub::emit(C2_MacroAssembler& masm) { + __ bind(entry()); + Register d = dst(); + __ movq(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header))); + __ jmp(continuation()); +} +#endif + #undef __ diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 2f9dcc42b56..a6e36302b43 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -5096,35 +5096,18 @@ void MacroAssembler::load_method_holder(Register holder, Register method) { #ifdef _LP64 void MacroAssembler::load_nklass(Register dst, Register src) { - assert_different_registers(src, dst); assert(UseCompressedClassPointers, "expect compressed class pointers"); - Label slow, done; + Label fast; movq(dst, Address(src, oopDesc::mark_offset_in_bytes())); - // NOTE: While it would seem nice to use xorb instead (for which we don't have an encoding in our assembler), - // the encoding for xorq uses the signed version (0x81/6) of xor, which encodes as compact as xorb would, - // and does't make a difference performance-wise. - xorq(dst, markWord::unlocked_value); - testb(dst, markWord::lock_mask_in_place); - jccb(Assembler::notZero, slow); + testb(dst, markWord::monitor_value); + jccb(Assembler::zero, fast); - shrq(dst, markWord::klass_shift); - jmp(done); - bind(slow); + // Fetch displaced header + movq(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header))); - if (dst != rax) { - push(rax); - } - if (src != rax) { - mov(rax, src); - } - call(RuntimeAddress(StubRoutines::load_nklass())); - if (dst != rax) { - mov(dst, rax); - pop(rax); - } - - bind(done); + bind(fast); + shrq(dst, markWord::klass_shift); } #endif @@ -5133,17 +5116,10 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp, bool n assert_different_registers(dst, tmp); #ifdef _LP64 assert(UseCompressedClassPointers, "expect compressed class pointers"); - Register d = dst; - if (src == dst) { - d = tmp; - } if (null_check_src) { null_check(src, oopDesc::mark_offset_in_bytes()); } - load_nklass(d, src); - if (src == dst) { - mov(dst, d); - } + load_nklass(dst, src); decode_klass_not_null(dst, tmp); #else if (null_check_src) { diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index f66efaf4da5..a2b2662e7a3 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -3402,46 +3402,6 @@ void StubGenerator::generate_libm_stubs() { } - // Call stub to call runtime oopDesc::load_nklass_runtime(). - // rax: call argument (object) - // rax: return object's narrowKlass - // Preserves all caller-saved registers, except rax -#ifdef _LP64 -address StubGenerator::generate_load_nklass() { - __ align(CodeEntryAlignment); - StubCodeMark(this, "StubRoutines", "load_nklass"); - address start = __ pc(); - __ enter(); // save rbp - - __ andptr(rsp, -(StackAlignmentInBytes)); // Align stack - __ push_FPU_state(); - - __ push(rdi); - __ push(rsi); - __ push(rdx); - __ push(rcx); - __ push(r8); - __ push(r9); - __ push(r10); - __ push(r11); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, oopDesc::load_nklass_runtime), rax); - __ pop(r11); - __ pop(r10); - __ pop(r9); - __ pop(r8); - __ pop(rcx); - __ pop(rdx); - __ pop(rsi); - __ pop(rdi); - - __ pop_FPU_state(); - - __ leave(); - __ ret(0); - return start; -} -#endif // _LP64 - address StubGenerator::generate_cont_thaw(const char* label, Continuation::thaw_kind kind) { if (!Continuations::enabled()) return nullptr; @@ -3817,10 +3777,6 @@ void StubGenerator::generate_initial() { } void StubGenerator::generate_phase1() { -#ifdef _LP64 - StubRoutines::_load_nklass = generate_load_nklass(); -#endif - // Continuation stubs: StubRoutines::_cont_thaw = generate_cont_thaw(); StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index a1ba67ef9a9..d3fc4c29cae 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -5322,7 +5322,15 @@ instruct loadNKlass(rRegN dst, indOffset8 mem, rFlagsReg cr) ins_encode %{ assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset 4, but got: %d", $mem$$disp); assert($mem$$index == 4, "expect no index register: %d", $mem$$index); - __ load_nklass($dst$$Register, $mem$$base$$Register); + Register dst = $dst$$Register; + Register obj = $mem$$base$$Register; + C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst); + Compile::current()->output()->add_stub(stub); + __ movq(dst, Address(obj, oopDesc::mark_offset_in_bytes())); + __ testb(dst, markWord::monitor_value); + __ jcc(Assembler::notZero, stub->entry()); + __ bind(stub->continuation()); + __ shrq(dst, markWord::klass_shift); %} ins_pipe(pipe_slow); // XXX %} diff --git a/src/hotspot/share/c1/c1_CodeStubs.hpp b/src/hotspot/share/c1/c1_CodeStubs.hpp index 82f3bbde0d2..28c1ab21162 100644 --- a/src/hotspot/share/c1/c1_CodeStubs.hpp +++ b/src/hotspot/share/c1/c1_CodeStubs.hpp @@ -556,16 +556,15 @@ class ArrayCopyStub: public CodeStub { class LoadKlassStub: public CodeStub { private: - LIR_Opr _obj; LIR_Opr _result; public: - LoadKlassStub(LIR_Opr obj, LIR_Opr result) : - CodeStub(), _obj(obj), _result(result) {}; + LoadKlassStub(LIR_Opr result) : + CodeStub(), _result(result) {}; virtual void emit_code(LIR_Assembler* e); virtual void visit(LIR_OpVisitState* visitor) { - visitor->do_input(_obj); + visitor->do_temp(_result); visitor->do_output(_result); } #ifndef PRODUCT diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index effd4754dcf..8f8f0110b4a 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -1241,7 +1241,7 @@ void LIRGenerator::do_isInstance(Intrinsic* x) { } void LIRGenerator::load_klass(LIR_Opr obj, LIR_Opr klass, CodeEmitInfo* null_check_info) { - CodeStub* slow_path = new LoadKlassStub(obj, klass); + CodeStub* slow_path = new LoadKlassStub(klass); __ load_klass(obj, klass, null_check_info, slow_path); } diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index 7ec33017cdb..da8d1bb7f96 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -244,7 +244,6 @@ void Runtime1::generate_blob_for(BufferBlob* buffer_blob, StubID id) { // Make sure that stubs that need oopmaps have them switch (id) { // These stubs don't need to have an oopmap - case load_klass_id: case dtrace_object_alloc_id: case slow_subtype_check_id: case fpu2long_stub_id: diff --git a/src/hotspot/share/c1/c1_Runtime1.hpp b/src/hotspot/share/c1/c1_Runtime1.hpp index d2632b6e0f9..3dcb27476a6 100644 --- a/src/hotspot/share/c1/c1_Runtime1.hpp +++ b/src/hotspot/share/c1/c1_Runtime1.hpp @@ -63,7 +63,6 @@ class StubAssembler; stub(monitorenter_nofpu) /* optimized version that does not preserve fpu registers */ \ stub(monitorexit) \ stub(monitorexit_nofpu) /* optimized version that does not preserve fpu registers */ \ - stub(load_klass) \ stub(deoptimize) \ stub(access_field_patching) \ stub(load_klass_patching) \ diff --git a/src/hotspot/share/oops/oop.cpp b/src/hotspot/share/oops/oop.cpp index 7146e66f4c7..67846c88bdd 100644 --- a/src/hotspot/share/oops/oop.cpp +++ b/src/hotspot/share/oops/oop.cpp @@ -168,21 +168,6 @@ void* oopDesc::load_oop_raw(oop obj, int offset) { } } -#ifdef _LP64 -JRT_LEAF(narrowKlass, oopDesc::load_nklass_runtime(oopDesc* o)) - assert(o != NULL, "null-check"); - oop obj = oop(o); - assert(oopDesc::is_oop(obj), "need a valid oop here: " PTR_FORMAT, p2i(o)); - markWord header = obj->mark(); - if (!header.is_neutral()) { - header = ObjectSynchronizer::stable_mark(obj); - } - assert(header.is_neutral() || header.is_fast_locked(), "expect neutral or fast-locked header here"); - narrowKlass nklass = header.narrow_klass(); - return nklass; -JRT_END -#endif - oop oopDesc::obj_field_acquire(int offset) const { return HeapAccess::oop_load_at(as_oop(), offset); } void oopDesc::obj_field_put_raw(int offset, oop value) { RawAccess<>::oop_store_at(as_oop(), offset, value); } diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 22beb83bfde..6091b17a046 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -79,6 +79,8 @@ class oopDesc { inline markWord cas_set_mark(markWord new_mark, markWord old_mark); inline markWord cas_set_mark(markWord new_mark, markWord old_mark, atomic_memory_order order); + inline markWord resolve_mark() const; + // Used only to re-initialize the mark word (e.g., of promoted // objects during a GC) -- requires a valid klass pointer inline void init_mark(); @@ -321,11 +323,6 @@ class oopDesc { static void* load_klass_raw(oop obj); static void* load_oop_raw(oop obj, int offset); - // Runtime entry -#ifdef _LP64 - static narrowKlass load_nklass_runtime(oopDesc* o); -#endif - DEBUG_ONLY(bool size_might_change();) }; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index c050361c0fc..36a3251acc1 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -38,6 +38,7 @@ #include "oops/oopsHierarchy.hpp" #include "runtime/atomic.hpp" #include "runtime/globals.hpp" +#include "runtime/objectMonitor.inline.hpp" #include "runtime/safepoint.hpp" #include "runtime/synchronizer.hpp" #include "utilities/align.hpp" @@ -84,9 +85,18 @@ markWord oopDesc::cas_set_mark(markWord new_mark, markWord old_mark, atomic_memo return Atomic::cmpxchg(&_mark, old_mark, new_mark, order); } +markWord oopDesc::resolve_mark() const { + markWord hdr = mark(); + if (hdr.has_monitor()) { + ObjectMonitor* monitor = hdr.monitor(); + return monitor->header(); + } + return hdr; +} + void oopDesc::init_mark() { #ifdef _LP64 - markWord header = ObjectSynchronizer::stable_mark(cast_to_oop(this)); + markWord header = resolve_mark(); assert(UseCompressedClassPointers, "expect compressed klass pointers"); header = markWord((header.value() & markWord::klass_mask_in_place) | markWord::prototype().value()); #else @@ -98,10 +108,7 @@ void oopDesc::init_mark() { Klass* oopDesc::klass() const { #ifdef _LP64 assert(UseCompressedClassPointers, "only with compressed class pointers"); - markWord header = mark(); - if (!header.is_neutral()) { - header = ObjectSynchronizer::stable_mark(cast_to_oop(this)); - } + markWord header = resolve_mark(); return header.klass(); #else return _klass; @@ -111,10 +118,7 @@ Klass* oopDesc::klass() const { Klass* oopDesc::klass_or_null() const { #ifdef _LP64 assert(UseCompressedClassPointers, "only with compressed class pointers"); - markWord header = mark(); - if (!header.is_neutral()) { - header = ObjectSynchronizer::stable_mark(cast_to_oop(this)); - } + markWord header = resolve_mark(); return header.klass_or_null(); #else return _klass; @@ -125,8 +129,8 @@ Klass* oopDesc::klass_or_null_acquire() const { #ifdef _LP64 assert(UseCompressedClassPointers, "only with compressed class pointers"); markWord header = mark_acquire(); - if (!header.is_neutral()) { - header = ObjectSynchronizer::stable_mark(cast_to_oop(this)); + if (header.has_monitor()) { + header = header.monitor()->header(); } return header.klass_or_null(); #else diff --git a/src/hotspot/share/opto/c2_CodeStubs.hpp b/src/hotspot/share/opto/c2_CodeStubs.hpp index 628392d3775..9450d3ff6e9 100644 --- a/src/hotspot/share/opto/c2_CodeStubs.hpp +++ b/src/hotspot/share/opto/c2_CodeStubs.hpp @@ -94,4 +94,16 @@ class C2CheckLockStackStub : public C2CodeStub { void emit(C2_MacroAssembler& masm); }; +#ifdef _LP64 +class C2LoadNKlassStub : public C2CodeStub { +private: + Register _dst; +public: + C2LoadNKlassStub(Register dst) : C2CodeStub(), _dst(dst) {} + Register dst() { return _dst; } + int max_size() const; + void emit(C2_MacroAssembler& masm); +}; +#endif + #endif // SHARE_OPTO_C2_CODESTUBS_HPP diff --git a/src/hotspot/share/runtime/objectMonitor.hpp b/src/hotspot/share/runtime/objectMonitor.hpp index 91db59a0589..cbeb9f26afa 100644 --- a/src/hotspot/share/runtime/objectMonitor.hpp +++ b/src/hotspot/share/runtime/objectMonitor.hpp @@ -212,6 +212,7 @@ class ObjectMonitor : public CHeapObj { // TODO-FIXME: the "offset" routines should return a type of off_t instead of int ... // ByteSize would also be an appropriate type. + static int header_offset_in_bytes() { return offset_of(ObjectMonitor, _header); } static int owner_offset_in_bytes() { return offset_of(ObjectMonitor, _owner); } static int recursions_offset_in_bytes() { return offset_of(ObjectMonitor, _recursions); } static int cxq_offset_in_bytes() { return offset_of(ObjectMonitor, _cxq); } diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 925793ae1f0..872cf358879 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -171,8 +171,6 @@ address StubRoutines::_dtan = NULL; address StubRoutines::_vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{NULL}, {NULL}}; address StubRoutines::_vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{NULL}, {NULL}}; -address StubRoutines::_load_nklass = NULL; - address StubRoutines::_cont_thaw = NULL; address StubRoutines::_cont_returnBarrier = NULL; address StubRoutines::_cont_returnBarrierExc = NULL; diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index 4995a65e701..80e77beb8aa 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -260,9 +260,7 @@ class StubRoutines: AllStatic { static address _vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP]; static address _vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP]; - static address _load_nklass; - -public: + public: // Initialization/Testing static void initialize1(); // must happen before universe::genesis static void initialize2(); // must happen after universe::genesis @@ -426,8 +424,6 @@ class StubRoutines: AllStatic { static address dlibm_tan_cot_huge() { return _dlibm_tan_cot_huge; } static address dtan() { return _dtan; } - static address load_nklass() { return _load_nklass; } - static address cont_thaw() { return _cont_thaw; } static address cont_returnBarrier() { return _cont_returnBarrier; } static address cont_returnBarrierExc(){return _cont_returnBarrierExc; } diff --git a/src/hotspot/share/runtime/synchronizer.cpp b/src/hotspot/share/runtime/synchronizer.cpp index 7d5d8850d1c..0e350e6b252 100644 --- a/src/hotspot/share/runtime/synchronizer.cpp +++ b/src/hotspot/share/runtime/synchronizer.cpp @@ -795,7 +795,7 @@ struct SharedGlobals { static SharedGlobals GVars; -markWord ObjectSynchronizer::read_stable_mark(oop obj) { +static markWord read_stable_mark(oop obj) { markWord mark = obj->mark_acquire(); if (!mark.is_being_inflated() || UseFastLocking) { return mark; // normal fast-path return @@ -855,73 +855,6 @@ markWord ObjectSynchronizer::read_stable_mark(oop obj) { } } -// Safely load a mark word from an object, even with racing stack-locking or monitor inflation. -// The protocol is a partial inflation-protocol: it installs INFLATING into the object's mark -// word in order to prevent an stack-locks or inflations from interferring (or detect such -// interference and retry), but then, instead of creating and installing a monitor, simply -// read and return the real mark word. -markWord ObjectSynchronizer::stable_mark(oop object) { - for (;;) { - const markWord mark = read_stable_mark(object); - assert(!mark.is_being_inflated(), "read_stable_mark must prevent inflating mark"); - - // The mark can be in one of the following states: - // * Inflated - just return mark from inflated monitor - // * Stack-locked - coerce it to inflating, and then return displaced mark - // * Neutral - return mark - // * Marked - return mark - - // CASE: inflated - if (mark.has_monitor()) { - ObjectMonitor* inf = mark.monitor(); - markWord dmw = inf->header(); - assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); - return dmw; - } - - // CASE: stack-locked - // Could be stack-locked either by this thread or by some other thread. - if (mark.has_locker()) { - BasicLock* lock = mark.locker(); - if (Thread::current()->is_lock_owned((address)lock)) { - // If locked by this thread, it is safe to access the displaced header. - markWord dmw = lock->displaced_header(); - assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); - return dmw; - } - - // Otherwise, attempt to temporarily install INFLATING into the mark-word, - // to prevent inflation or unlocking by competing thread. - markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark); - if (cmp != mark) { - continue; // Interference -- just retry - } - - // fetch the displaced mark from the owner's stack. - // The owner can't die or unwind past the lock while our INFLATING - // object is in the mark. Furthermore the owner can't complete - // an unlock on the object, either. - markWord dmw = mark.displaced_mark_helper(); - // Catch if the object's header is not neutral (not locked and - // not marked is what we care about here). - assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); - - // Must preserve store ordering. The monitor state must - // be stable at the time of publishing the monitor address. - assert(object->mark() == markWord::INFLATING(), "invariant"); - // Release semantics so that above set_object() is seen first. - object->release_set_mark(mark); - - return dmw; - } - - // CASE: neutral or marked (for GC) - // Catch if the object's header is not neutral or marked (it must not be locked). - assert(mark.is_neutral() || mark.is_marked() || mark.is_fast_locked(), "invariant: header=" INTPTR_FORMAT, mark.value()); - return mark; - } -} - // hashCode() generation : // // Possibilities: diff --git a/src/hotspot/share/runtime/synchronizer.hpp b/src/hotspot/share/runtime/synchronizer.hpp index 832b4fab93a..9592d1a43f7 100644 --- a/src/hotspot/share/runtime/synchronizer.hpp +++ b/src/hotspot/share/runtime/synchronizer.hpp @@ -170,11 +170,6 @@ class ObjectSynchronizer : AllStatic { // NOTE: It may cause monitor inflation static intptr_t FastHashCode(Thread* current, oop obj); - // Read mark-word and spin-wait as long as INFLATING is observed. - static markWord read_stable_mark(oop obj); - - static markWord stable_mark(oop obj); - // java.lang.Thread support static bool current_thread_holds_lock(JavaThread* current, Handle h_obj);