diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 39c5c3af1bc..2dea48dc614 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -708,7 +708,7 @@ void MacroAssembler::emit_static_call_stub() { isb(); mov_metadata(rmethod, (Metadata*)NULL); - // Jump to the entry point of the i2c stub. + // Jump to the entry point of the c2i stub. movptr(rscratch1, 0); br(rscratch1); } diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index 845f10df71a..3f8ae75b228 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -1015,22 +1015,65 @@ static void gen_continuation_enter(MacroAssembler* masm, int& exception_offset, OopMapSet*oop_maps, int& frame_complete, - int& stack_slots) { + int& stack_slots, + int& interpreted_entry_offset, + int& compiled_entry_offset) { //verify_oop_args(masm, method, sig_bt, regs); - Address resolve(SharedRuntime::get_resolve_static_call_stub(), - relocInfo::static_call_type); + Address resolve(SharedRuntime::get_resolve_static_call_stub(), relocInfo::static_call_type); - stack_slots = 2; // will be overwritten address start = __ pc(); Label call_thaw, exit; - __ enter(); + // i2i entry used at interp_only_mode only + interpreted_entry_offset = __ pc() - start; + { - OopMap* map = continuation_enter_setup(masm, stack_slots); +#ifdef ASSERT + Label is_interp_only; + __ ldrw(rscratch1, Address(rthread, JavaThread::interp_only_mode_offset())); + __ cbnzw(rscratch1, is_interp_only); + __ stop("enterSpecial interpreter entry called when not in interp_only_mode"); + __ bind(is_interp_only); +#endif - // Frame is now completed as far as size and linkage. - frame_complete =__ pc() - start; + // Read interpreter arguments into registers (this is an ad-hoc i2c adapter) + __ ldr(c_rarg1, Address(esp, Interpreter::stackElementSize*2)); + __ ldr(c_rarg2, Address(esp, Interpreter::stackElementSize*1)); + __ ldr(c_rarg3, Address(esp, Interpreter::stackElementSize*0)); + __ push_cont_fastpath(rthread); + + __ enter(); + stack_slots = 2; // will be adjusted in setup + OopMap* map = continuation_enter_setup(masm, stack_slots); + // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe, + // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway. + + fill_continuation_entry(masm); + + __ cmp(c_rarg2, (u1)0); + __ br(Assembler::NE, call_thaw); + + address mark = __ pc(); + __ trampoline_call1(resolve, NULL, false); + + oop_maps->add_gc_map(__ pc() - start, map); + __ post_call_nop(); + + __ b(exit); + + CodeBuffer* cbuf = masm->code_section()->outer(); + CompiledStaticCall::emit_to_interp_stub(*cbuf, mark); + } + + // compiled entry + __ align(CodeEntryAlignment); + compiled_entry_offset = __ pc() - start; + + __ enter(); + stack_slots = 2; // will be adjusted in setup + OopMap* map = continuation_enter_setup(masm, stack_slots); + frame_complete = __ pc() - start; fill_continuation_entry(masm); @@ -1038,7 +1081,6 @@ static void gen_continuation_enter(MacroAssembler* masm, __ br(Assembler::NE, call_thaw); address mark = __ pc(); - __ trampoline_call1(resolve, NULL, false); oop_maps->add_gc_map(__ pc() - start, map); @@ -1081,7 +1123,7 @@ static void gen_continuation_enter(MacroAssembler* masm, } CodeBuffer* cbuf = masm->code_section()->outer(); - address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, mark); + CompiledStaticCall::emit_to_interp_stub(*cbuf, mark); } static void gen_special_dispatch(MacroAssembler* masm, @@ -1171,11 +1213,12 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, if (method->is_continuation_enter_intrinsic()) { vmIntrinsics::ID iid = method->intrinsic_id(); intptr_t start = (intptr_t)__ pc(); - int vep_offset = ((intptr_t)__ pc()) - start; + int vep_offset = 0; int exception_offset = 0; int frame_complete = 0; int stack_slots = 0; OopMapSet* oop_maps = new OopMapSet(); + int interpreted_entry_offset = -1; gen_continuation_enter(masm, method, in_sig_bt, @@ -1183,7 +1226,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, exception_offset, oop_maps, frame_complete, - stack_slots); + stack_slots, + interpreted_entry_offset, + vep_offset); __ flush(); nmethod* nm = nmethod::new_native_nmethod(method, compile_id, @@ -1195,7 +1240,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, in_ByteSize(-1), oop_maps, exception_offset); - ContinuationEntry::set_enter_code(nm); + ContinuationEntry::set_enter_code(nm, interpreted_entry_offset); return nm; } diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index acb4f850ec1..25eeb46ae50 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -1276,7 +1276,9 @@ static void gen_continuation_enter(MacroAssembler* masm, int& exception_offset, OopMapSet* oop_maps, int& frame_complete, - int& stack_slots) { + int& stack_slots, + int& interpreted_entry_offset, + int& compiled_entry_offset) { // enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread) int pos_cont_obj = 0; @@ -1298,8 +1300,68 @@ static void gen_continuation_enter(MacroAssembler* masm, // Utility methods kill rax, make sure there are no collisions assert_different_registers(rax, reg_cont_obj, reg_is_cont, reg_is_virtual); + AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(), + relocInfo::static_call_type); + address start = __ pc(); + Label L_thaw, L_exit; + + // i2i entry used at interp_only_mode only + interpreted_entry_offset = __ pc() - start; + { +#ifdef ASSERT + Label is_interp_only; + __ cmpb(Address(r15_thread, JavaThread::interp_only_mode_offset()), 0); + __ jcc(Assembler::notEqual, is_interp_only); + __ stop("enterSpecial interpreter entry called when not in interp_only_mode"); + __ bind(is_interp_only); +#endif + + __ pop(rax); // return address + // Read interpreter arguments into registers (this is an ad-hoc i2c adapter) + __ movptr(c_rarg1, Address(rsp, Interpreter::stackElementSize*2)); + __ movl(c_rarg2, Address(rsp, Interpreter::stackElementSize*1)); + __ movl(c_rarg3, Address(rsp, Interpreter::stackElementSize*0)); + __ andptr(rsp, -16); // Ensure compiled code always sees stack at proper alignment + __ push(rax); // return address + __ push_cont_fastpath(); + + __ enter(); + + stack_slots = 2; // will be adjusted in setup + OopMap* map = continuation_enter_setup(masm, stack_slots); + // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe, + // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway. + + __ verify_oop(reg_cont_obj); + + fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual); + + // If continuation, call to thaw. Otherwise, resolve the call and exit. + __ testptr(reg_is_cont, reg_is_cont); + __ jcc(Assembler::notZero, L_thaw); + + // --- Resolve path + + // Make sure the call is patchable + __ align(BytesPerWord, __ offset() + NativeCall::displacement_offset); + // Emit stub for static call + CodeBuffer* cbuf = masm->code_section()->outer(); + address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, __ pc()); + if (stub == nullptr) { + fatal("CodeCache is full at gen_continuation_enter"); + } + __ call(resolve); + oop_maps->add_gc_map(__ pc() - start, map); + __ post_call_nop(); + + __ jmp(L_exit); + } + + // compiled entry + __ align(CodeEntryAlignment); + compiled_entry_offset = __ pc() - start; __ enter(); stack_slots = 2; // will be adjusted in setup @@ -1312,8 +1374,6 @@ static void gen_continuation_enter(MacroAssembler* masm, fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual); - Label L_thaw, L_exit; - // If continuation, call to thaw. Otherwise, resolve the call and exit. __ testptr(reg_is_cont, reg_is_cont); __ jccb(Assembler::notZero, L_thaw); @@ -1330,9 +1390,6 @@ static void gen_continuation_enter(MacroAssembler* masm, fatal("CodeCache is full at gen_continuation_enter"); } - // Call the resolve stub - AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(), - relocInfo::static_call_type); __ call(resolve); oop_maps->add_gc_map(__ pc() - start, map); @@ -1471,17 +1528,20 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, if (method->is_continuation_enter_intrinsic()) { vmIntrinsics::ID iid = method->intrinsic_id(); intptr_t start = (intptr_t)__ pc(); - int vep_offset = ((intptr_t)__ pc()) - start; + int vep_offset = 0; int exception_offset = 0; int frame_complete = 0; int stack_slots = 0; OopMapSet* oop_maps = new OopMapSet(); + int interpreted_entry_offset = -1; gen_continuation_enter(masm, in_regs, exception_offset, oop_maps, frame_complete, - stack_slots); + stack_slots, + interpreted_entry_offset, + vep_offset); __ flush(); nmethod* nm = nmethod::new_native_nmethod(method, compile_id, @@ -1493,7 +1553,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, in_ByteSize(-1), oop_maps, exception_offset); - ContinuationEntry::set_enter_code(nm); + ContinuationEntry::set_enter_code(nm, interpreted_entry_offset); return nm; } diff --git a/src/hotspot/share/code/compiledIC.cpp b/src/hotspot/share/code/compiledIC.cpp index e26d27f10e8..f0329ba2142 100644 --- a/src/hotspot/share/code/compiledIC.cpp +++ b/src/hotspot/share/code/compiledIC.cpp @@ -39,6 +39,7 @@ #include "oops/method.inline.hpp" #include "oops/oop.inline.hpp" #include "oops/symbol.hpp" +#include "runtime/continuationEntry.hpp" #include "runtime/handles.inline.hpp" #include "runtime/icache.hpp" #include "runtime/safepoint.hpp" @@ -653,6 +654,13 @@ void CompiledStaticCall::compute_entry(const methodHandle& m, bool caller_is_nme } } +void CompiledStaticCall::compute_entry_for_continuation_entry(const methodHandle& m, StaticCallInfo& info) { + if (ContinuationEntry::is_interpreted_call(instruction_address())) { + info._to_interpreter = true; + info._entry = m()->get_c2i_entry(); + } +} + address CompiledDirectStaticCall::find_stub_for(address instruction) { // Find reloc. information containing this call-site RelocIterator iter((nmethod*)NULL, instruction); diff --git a/src/hotspot/share/code/compiledIC.hpp b/src/hotspot/share/code/compiledIC.hpp index 6cdaf20c22d..18bd3765507 100644 --- a/src/hotspot/share/code/compiledIC.hpp +++ b/src/hotspot/share/code/compiledIC.hpp @@ -345,6 +345,7 @@ class CompiledStaticCall : public ResourceObj { // Compute entry point given a method static void compute_entry(const methodHandle& m, bool caller_is_nmethod, StaticCallInfo& info); + void compute_entry_for_continuation_entry(const methodHandle& m, StaticCallInfo& info); public: // Clean static call (will force resolving on next use) diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index 716b8a74bf3..5bf7b14e297 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -61,6 +61,7 @@ #include "prims/methodHandles.hpp" #include "runtime/arguments.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuationEntry.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/init.hpp" @@ -1328,7 +1329,7 @@ void Method::set_code(const methodHandle& mh, CompiledMethod *code) { assert(mh->_from_interpreted_entry == NULL, "initialized incorrectly"); // see link_method // This is the entry used when we're in interpreter-only mode; see InterpreterMacroAssembler::jump_from_interpreted - mh->_i2i_entry = mh->get_i2c_entry(); + mh->_i2i_entry = ContinuationEntry::interpreted_entry(); // This must come last, as it is what's tested in LinkResolver::resolve_static_call Atomic::release_store(&mh->_from_interpreted_entry , mh->get_i2c_entry()); } else if (!mh->is_method_handle_intrinsic()) { diff --git a/src/hotspot/share/runtime/continuationEntry.cpp b/src/hotspot/share/runtime/continuationEntry.cpp index f0882d3bd65..37b91b5385a 100644 --- a/src/hotspot/share/runtime/continuationEntry.cpp +++ b/src/hotspot/share/runtime/continuationEntry.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "code/compiledIC.hpp" #include "code/nmethod.hpp" #include "runtime/continuation.hpp" #include "runtime/continuationEntry.inline.hpp" @@ -34,10 +35,32 @@ int ContinuationEntry::_return_pc_offset = 0; address ContinuationEntry::_return_pc = nullptr; +CompiledMethod* ContinuationEntry::_enter_special = nullptr; +int ContinuationEntry::_interpreted_entry_offset = 0; -void ContinuationEntry::set_enter_code(CompiledMethod* cm) { +void ContinuationEntry::set_enter_code(CompiledMethod* cm, int interpreted_entry_offset) { assert(_return_pc_offset != 0, ""); _return_pc = cm->code_begin() + _return_pc_offset; + + _enter_special = cm; + _interpreted_entry_offset = interpreted_entry_offset; + assert(_enter_special->code_contains(compiled_entry()), "entry not in enterSpecial"); + assert(_enter_special->code_contains(interpreted_entry()), "entry not in enterSpecial"); + assert(interpreted_entry() < compiled_entry(), "unexpected code layout"); +} + +address ContinuationEntry::compiled_entry() { + return _enter_special->verified_entry_point(); +} + +address ContinuationEntry::interpreted_entry() { + return _enter_special->code_begin() + _interpreted_entry_offset; +} + +bool ContinuationEntry::is_interpreted_call(address call_address) { + assert(_enter_special->code_contains(call_address), "call not in enterSpecial"); + assert(call_address >= interpreted_entry(), "unexpected location"); + return call_address < compiled_entry(); } ContinuationEntry* ContinuationEntry::from_frame(const frame& f) { diff --git a/src/hotspot/share/runtime/continuationEntry.hpp b/src/hotspot/share/runtime/continuationEntry.hpp index 11c01d2f6e8..b6e56977f4c 100644 --- a/src/hotspot/share/runtime/continuationEntry.hpp +++ b/src/hotspot/share/runtime/continuationEntry.hpp @@ -52,10 +52,13 @@ class ContinuationEntry { public: static int _return_pc_offset; // friend gen_continuation_enter - static void set_enter_code(CompiledMethod* nm); // friend SharedRuntime::generate_native_wrapper + static void set_enter_code(CompiledMethod* cm, int interpreted_entry_offset); + static bool is_interpreted_call(address call_address); private: static address _return_pc; + static CompiledMethod* _enter_special; + static int _interpreted_entry_offset; private: ContinuationEntry* _parent; @@ -89,6 +92,11 @@ class ContinuationEntry { intptr_t* entry_sp() const { return (intptr_t*)this; } intptr_t* entry_fp() const; + static address compiled_entry(); + static address interpreted_entry(); + + static CompiledMethod* enter_special() { return _enter_special; } + int argsize() const { return _argsize; } void set_argsize(int value) { _argsize = value; } diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index d5582e2d9c6..1b5c5883401 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -1343,6 +1343,9 @@ bool SharedRuntime::resolve_sub_helper_internal(methodHandle callee_method, cons return true; // skip patching for JVMCI } CompiledStaticCall* ssc = caller_nm->compiledStaticCall_before(caller_frame.pc()); + if (is_nmethod && caller_nm->method()->is_continuation_enter_intrinsic()) { + ssc->compute_entry_for_continuation_entry(callee_method, static_call_info); + } if (ssc->is_clean()) ssc->set(static_call_info); } } @@ -1557,10 +1560,32 @@ JRT_END // resolve a static call and patch code JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_static_call_C(JavaThread* current )) methodHandle callee_method; + bool enter_special = false; JRT_BLOCK callee_method = SharedRuntime::resolve_helper(false, false, CHECK_NULL); current->set_vm_result_2(callee_method()); + + if (current->is_interp_only_mode()) { + RegisterMap reg_map(current, false); + frame stub_frame = current->last_frame(); + assert(stub_frame.is_runtime_frame(), "must be a runtimeStub"); + frame caller = stub_frame.sender(®_map); + enter_special = caller.cb() != NULL && caller.cb()->is_compiled() + && caller.cb()->as_compiled_method()->method()->is_continuation_enter_intrinsic(); + } JRT_BLOCK_END + + if (current->is_interp_only_mode() && enter_special) { + // enterSpecial is compiled and calls this method to resolve the call to Continuation::enter + // but in interp_only_mode we need to go to the interpreted entry + // The c2i won't patch in this mode -- see fixup_callers_callsite + // + // This should probably be done in all cases, not just enterSpecial (see JDK-8218403), + // but that's part of a larger fix, and the situation is worse for enterSpecial, as it has no + // interpreted version. + return callee_method->get_c2i_entry(); + } + // return compiled code entry point after potential safepoints assert(callee_method->verified_code_entry() != NULL, " Jump to zero!"); return callee_method->verified_code_entry(); @@ -1991,6 +2016,9 @@ JRT_LEAF(void, SharedRuntime::fixup_callers_callsite(Method* method, address cal // Get the return PC for the passed caller PC. address return_pc = caller_pc + frame::pc_return_offset; + assert(!JavaThread::current()->is_interp_only_mode() || !nm->method()->is_continuation_enter_intrinsic() + || ContinuationEntry::is_interpreted_call(return_pc), "interp_only_mode but not in enterSpecial interpreted entry"); + // There is a benign race here. We could be attempting to patch to a compiled // entry point at the same time the callee is being deoptimized. If that is // the case then entry_point may in fact point to a c2i and we'd patch the @@ -2027,6 +2055,13 @@ JRT_LEAF(void, SharedRuntime::fixup_callers_callsite(Method* method, address cal typ != relocInfo::static_stub_type) { return; } + if (nm->method()->is_continuation_enter_intrinsic()) { + assert(ContinuationEntry::is_interpreted_call(call->instruction_address()) == JavaThread::current()->is_interp_only_mode(), + "mode: %d", JavaThread::current()->is_interp_only_mode()); + if (ContinuationEntry::is_interpreted_call(call->instruction_address())) { + return; + } + } address destination = call->destination(); if (should_fixup_call_destination(destination, entry_point, caller_pc, moop, cb)) { call->set_destination_mt_safe(entry_point); @@ -3056,7 +3091,7 @@ void AdapterHandlerLibrary::create_native_wrapper(const methodHandle& method) { CodeBuffer buffer(buf); if (method->is_continuation_enter_intrinsic()) { - buffer.initialize_stubs_size(64); + buffer.initialize_stubs_size(128); } struct { double data[20]; } locs_buf; diff --git a/test/hotspot/jtreg/ProblemList-Xcomp.txt b/test/hotspot/jtreg/ProblemList-Xcomp.txt index 38b43dd2425..d8a9681ee1b 100644 --- a/test/hotspot/jtreg/ProblemList-Xcomp.txt +++ b/test/hotspot/jtreg/ProblemList-Xcomp.txt @@ -36,5 +36,3 @@ serviceability/sa/TestJhsdbJstackMixed.java 8248675 linux-aarch64 vmTestbase/nsk/jvmti/scenarios/sampling/SP07/sp07t002/TestDescription.java 8245680 windows-x64 vmTestbase/vm/mlvm/mixed/stress/regression/b6969574/INDIFY_Test.java 8265295 linux-x64,windows-x64 - -serviceability/jvmti/vthread/ContStackDepthTest/ContStackDepthTest.java 8288949 generic-all