8338383: Implement JEP 491: Synchronize Virtual Threads without Pinning

src/hotspot/cpu/aarch64/aarch64.ad

@@ -1648,8 +1648,8 @@ int MachCallRuntimeNode::ret_addr_offset() {
   // for real runtime callouts it will be six instructions
   // see aarch64_enc_java_to_runtime
   //   adr(rscratch2, retaddr)
+  //   str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
   //   lea(rscratch1, RuntimeAddress(addr)
-  //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
   //   blr(rscratch1)
   CodeBlob *cb = CodeCache::find_blob(_entry_point);
   if (cb) {
@@ -3692,14 +3692,13 @@ encode %{
       __ post_call_nop();
     } else {
       Label retaddr;
+      // Make the anchor frame walkable
       __ adr(rscratch2, retaddr);
+      __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
       __ lea(rscratch1, RuntimeAddress(entry));
-      // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
-      __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
       __ blr(rscratch1);
       __ bind(retaddr);
       __ post_call_nop();
-      __ add(sp, sp, 2 * wordSize);
     }
     if (Compile::current()->max_vector_size() > 0) {
       __ reinitialize_ptrue();

src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp

@@ -292,28 +292,6 @@ inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {
   patch_callee_link(f, fp);
 }
 
-inline intptr_t* ThawBase::possibly_adjust_frame(frame& top) {
-  intptr_t* sp = top.sp();
-  CodeBlob* cb = top.cb();
-
-  if (cb->frame_size() == 2) {
-    // C2 runtime stub case. For aarch64 the real size of the c2 runtime stub is 2 words bigger
-    // than what we think, i.e. size is 4. This is because the _last_Java_sp is not set to the
-    // sp right before making the call to the VM, but rather it is artificially set 2 words above
-    // this real sp so that we can store the return address at last_Java_sp[-1], and keep this
-    // property where we can retrieve the last_Java_pc from the last_Java_sp. But that means that
-    // once we return to the runtime stub, the code will adjust sp according to this real size.
-    // So we must adjust the frame size back here and we copy lr/rfp again.
-    sp -= 2;
-    sp[-2] = sp[0];
-    sp[-1] = sp[1];
-
-    log_develop_trace(continuations, preempt)("adjusted sp for c2 runtime stub, initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT
-                                              " fp: " INTPTR_FORMAT, p2i(sp + 2), p2i(sp), sp[-2]);
-  }
-  return sp;
-}
-
 inline intptr_t* ThawBase::push_cleanup_continuation() {
   frame enterSpecial = new_entry_frame();
   intptr_t* sp = enterSpecial.sp();

src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp

@@ -91,11 +91,6 @@ inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {
   Unimplemented();
 }
 
-inline intptr_t* ThawBase::possibly_adjust_frame(frame& top) {
-  Unimplemented();
-  return nullptr;
-}
-
 inline intptr_t* ThawBase::push_cleanup_continuation() {
   Unimplemented();
   return nullptr;

src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp

@@ -550,11 +550,6 @@ inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, c
   // Keep top_frame_sp relativized.
 }
 
-inline intptr_t* ThawBase::possibly_adjust_frame(frame& top) {
-  // Nothing to do
-  return top.sp();
-}
-
 inline intptr_t* ThawBase::push_cleanup_continuation() {
   frame enterSpecial = new_entry_frame();
   frame::common_abi* enterSpecial_abi = (frame::common_abi*)enterSpecial.sp();

src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp

@@ -298,28 +298,6 @@ inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {
   patch_callee_link(f, fp);
 }
 
-inline intptr_t* ThawBase::possibly_adjust_frame(frame& top) {
-  intptr_t* sp = top.sp();
-  CodeBlob* cb = top.cb();
-
-  if (cb->frame_size() == 2) {
-    // C2 runtime stub case. For riscv64 the real size of the c2 runtime stub is 2 words bigger
-    // than what we think, i.e. size is 4. This is because the _last_Java_sp is not set to the
-    // sp right before making the call to the VM, but rather it is artificially set 2 words above
-    // this real sp so that we can store the return address at last_Java_sp[-1], and keep this
-    // property where we can retrieve the last_Java_pc from the last_Java_sp. But that means that
-    // once we return to the runtime stub, the code will adjust sp according to this real size.
-    // So we must adjust the frame size back here and we copy ra/fp again.
-    sp -= 2;
-    sp[-2] = sp[0];
-    sp[-1] = sp[1];
-
-    log_develop_trace(continuations, preempt)("adjusted sp for c2 runtime stub, initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT
-                                              " fp: " INTPTR_FORMAT, p2i(sp + 2), p2i(sp), sp[-2]);
-  }
-  return sp;
-}
-
 inline intptr_t* ThawBase::push_cleanup_continuation() {
   frame enterSpecial = new_entry_frame();
   intptr_t* sp = enterSpecial.sp();

src/hotspot/cpu/riscv/riscv.ad

@@ -1259,21 +1259,20 @@ int MachCallRuntimeNode::ret_addr_offset() {
   //   jal(addr)
   // or with far branches
   //   jal(trampoline_stub)
-  // for real runtime callouts it will be 9 instructions
+  // for real runtime callouts it will be 8 instructions
   // see riscv_enc_java_to_runtime
-  //   la(t0, retaddr)                ->  auipc + addi
-  //   addi(sp, sp, -2 * wordSize)    ->  addi
-  //   sd(t0, Address(sp, wordSize))  ->  sd
-  //   movptr(t1, addr, offset, t0)   ->  lui + lui + slli + add
-  //   jalr(t1, offset)               ->  jalr
+  //   la(t0, retaddr)                                             ->  auipc + addi
+  //   sd(t0, Address(xthread, JavaThread::last_Java_pc_offset())) ->  sd
+  //   movptr(t1, addr, offset, t0)                                ->  lui + lui + slli + add
+  //   jalr(t1, offset)                                            ->  jalr
   CodeBlob *cb = CodeCache::find_blob(_entry_point);
   if (cb != nullptr) {
     if (UseTrampolines) {
       return 1 * NativeInstruction::instruction_size;
     }
     return 3 * NativeInstruction::instruction_size;
   } else {
-    return 9 * NativeInstruction::instruction_size;
+    return 8 * NativeInstruction::instruction_size;
   }
 }
 
@@ -2504,17 +2503,15 @@ encode %{
       __ post_call_nop();
     } else {
       Label retaddr;
+      // Make the anchor frame walkable
       __ la(t0, retaddr);
-      // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
-      __ addi(sp, sp, -2 * wordSize);
-      __ sd(t0, Address(sp, wordSize));
+      __ sd(t0, Address(xthread, JavaThread::last_Java_pc_offset()));
       int32_t offset = 0;
       // No relocation needed
       __ movptr(t1, entry, offset, t0); // lui + lui + slli + add
       __ jalr(t1, offset);
       __ bind(retaddr);
       __ post_call_nop();
-      __ addi(sp, sp, 2 * wordSize);
     }
   %}
 

src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp

@@ -91,11 +91,6 @@ inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {
   Unimplemented();
 }
 
-inline intptr_t* ThawBase::possibly_adjust_frame(frame& top) {
-  Unimplemented();
-  return nullptr;
-}
-
 inline intptr_t* ThawBase::push_cleanup_continuation() {
   Unimplemented();
   return nullptr;

src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp

@@ -282,11 +282,6 @@ inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {
   patch_callee_link(f, fp);
 }
 
-inline intptr_t* ThawBase::possibly_adjust_frame(frame& top) {
-  // Nothing to do
-  return top.sp();
-}
-
 inline intptr_t* ThawBase::push_cleanup_continuation() {
   frame enterSpecial = new_entry_frame();
   intptr_t* sp = enterSpecial.sp();

src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp

@@ -91,11 +91,6 @@ inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {
   Unimplemented();
 }
 
-inline intptr_t* ThawBase::possibly_adjust_frame(frame& top) {
-  Unimplemented();
-  return nullptr;
-}
-
 inline intptr_t* ThawBase::push_cleanup_continuation() {
   Unimplemented();
   return nullptr;

src/hotspot/share/runtime/continuationFreezeThaw.cpp

@@ -309,11 +309,7 @@ inline void clear_anchor(JavaThread* thread) {
   thread->frame_anchor()->clear();
 }
 
-static void set_anchor(JavaThread* thread, intptr_t* sp, address pc = nullptr) {
-  if (pc == nullptr) {
-    pc = ContinuationHelper::return_address_at(
-           sp - frame::sender_sp_ret_address_offset());
-  }
+static void set_anchor(JavaThread* thread, intptr_t* sp, address pc) {
   assert(pc != nullptr, "");
 
   JavaFrameAnchor* anchor = thread->frame_anchor();
@@ -325,6 +321,12 @@ static void set_anchor(JavaThread* thread, intptr_t* sp, address pc = nullptr) {
   assert(thread->last_frame().cb() != nullptr, "");
 }
 
+static void set_anchor(JavaThread* thread, intptr_t* sp) {
+  address pc = ContinuationHelper::return_address_at(
+           sp - frame::sender_sp_ret_address_offset());
+  set_anchor(thread, sp, pc);
+}
+
 static void set_anchor_to_entry(JavaThread* thread, ContinuationEntry* entry) {
   JavaFrameAnchor* anchor = thread->frame_anchor();
   anchor->set_last_Java_sp(entry->entry_sp());
@@ -1839,7 +1841,6 @@ static int thaw_size(stackChunkOop chunk) {
   int size = chunk->max_thawing_size();
   size += frame::metadata_words; // For the top pc+fp in push_return_frame or top = stack_sp - frame::metadata_words in thaw_fast
   size += 2*frame::align_wiggle; // in case of alignments at the top and bottom
-  size += frame::metadata_words; // for preemption case (see possibly_adjust_frame)
   return size;
 }
 
@@ -1925,7 +1926,6 @@ class ThawBase : public StackObj {
   void patch_return(intptr_t* sp, bool is_last);
 
   intptr_t* handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case);
-  inline intptr_t* possibly_adjust_frame(frame& top);
   inline intptr_t* push_cleanup_continuation();
   void throw_interrupted_exception(JavaThread* current, frame& top);
 
@@ -2481,8 +2481,6 @@ intptr_t* ThawBase::handle_preempted_continuation(intptr_t* sp, Continuation::pr
     // we need to adjust the current thread saved in the stub frame before restoring registers.
     JavaThread** thread_addr = frame::saved_thread_address(top);
     if (thread_addr != nullptr) *thread_addr = _thread;
-    // Some platforms require the size of the runtime frame to be adjusted.
-    sp = possibly_adjust_frame(top);
   }
   return sp;
 }
@@ -2963,29 +2961,15 @@ static void log_frames(JavaThread* thread) {
 static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempted) {
   intptr_t* sp0 = sp;
   address pc0 = *(address*)(sp - frame::sender_sp_ret_address_offset());
-  bool use_cont_entry = false;
-
-  // Some preemption cases need to use and adjusted version of sp.
-  if (preempted) {
-    if (sp0 == cont.entrySP()) {
-      // Still preempted (monitor not acquired) so no frames were thawed.
-      assert(cont.tail()->preempted(), "");
-      use_cont_entry = true;
-    }
-#if defined (AARCH64) || defined (RISCV64)
-    else {
-      CodeBlob* cb = CodeCache::find_blob(pc0);
-      if (cb->frame_size() == 2) {
-        assert(cb->is_runtime_stub(), "");
-        // Returning to c2 runtime stub requires extra adjustment on aarch64
-        // and riscv64 (see possibly_adjust_frame()).
-        sp0 += frame::metadata_words;
-      }
-    }
-#endif
+
+  if (preempted && sp0 == cont.entrySP()) {
+    // Still preempted (monitor not acquired) so no frames were thawed.
+    assert(cont.tail()->preempted(), "");
+    set_anchor(thread, cont.entrySP(), cont.entryPC());
+  } else {
+    set_anchor(thread, sp0);
   }
 
-  set_anchor(thread, use_cont_entry ? cont.entrySP() : sp0, use_cont_entry ? cont.entryPC() : nullptr);
   log_frames(thread);
   if (LoomVerifyAfterThaw) {
     assert(do_verify_after_thaw(thread, cont.tail(), tty), "");

src/hotspot/share/runtime/objectMonitor.cpp

@@ -132,23 +132,34 @@ static void post_virtual_thread_pinned_event(JavaThread* current, const char* re
 // Theory of operations -- Monitors lists, thread residency, etc:
 //
 // * A thread acquires ownership of a monitor by successfully
-//   CAS()ing the _owner field from null to non-null.
+//   CAS()ing the _owner field from NO_OWNER/DEFLATER_MARKER to
+//   its tid (return value from owner_from()).
 //
 // * Invariant: A thread appears on at most one monitor list --
 //   cxq, EntryList or WaitSet -- at any one time.
 //
 // * Contending threads "push" themselves onto the cxq with CAS
 //   and then spin/park.
+//   If the thread is a virtual thread it will first attempt to
+//   unmount itself. The virtual thread will first try to freeze
+//   all frames in the heap. If the operation fails it will just
+//   follow the regular path for platform threads. If the operation
+//   succeeds, it will push itself onto the cxq with CAS and then
+//   return back to Java to continue the unmount logic.
 //
 // * After a contending thread eventually acquires the lock it must
 //   dequeue itself from either the EntryList or the cxq.
 //
 // * The exiting thread identifies and unparks an "heir presumptive"
-//   tentative successor thread on the EntryList.  Critically, the
-//   exiting thread doesn't unlink the successor thread from the EntryList.
-//   After having been unparked, the wakee will recontend for ownership of
-//   the monitor.   The successor (wakee) will either acquire the lock or
-//   re-park itself.
+//   tentative successor thread on the EntryList. In case the successor
+//   is an unmounted virtual thread, the exiting thread will first try
+//   to add it to the list of vthreads waiting to be unblocked, and on
+//   success it will unpark the special unblocker thread instead, which
+//   will be in charge of submitting the vthread back to the scheduler
+//   queue. Critically, the exiting thread doesn't unlink the successor
+//   thread from the EntryList. After having been unparked/re-scheduled,
+//   the wakee will recontend for ownership of the monitor. The successor
+//   (wakee) will either acquire the lock or re-park/unmount itself.
 //
 //   Succession is provided for by a policy of competitive handoff.
 //   The exiting thread does _not_ grant or pass ownership to the
@@ -210,20 +221,18 @@ static void post_virtual_thread_pinned_event(JavaThread* current, const char* re
 //
 // * The monitor synchronization subsystem avoids the use of native
 //   synchronization primitives except for the narrow platform-specific
-//   park-unpark abstraction.  See the comments in os_solaris.cpp regarding
+//   park-unpark abstraction.  See the comments in os_posix.cpp regarding
 //   the semantics of park-unpark.  Put another way, this monitor implementation
-//   depends only on atomic operations and park-unpark.  The monitor subsystem
-//   manages all RUNNING->BLOCKED and BLOCKED->READY transitions while the
-//   underlying OS manages the READY<->RUN transitions.
+//   depends only on atomic operations and park-unpark.
 //
 // * Waiting threads reside on the WaitSet list -- wait() puts
 //   the caller onto the WaitSet.
 //
 // * notify() or notifyAll() simply transfers threads from the WaitSet to
 //   either the EntryList or cxq.  Subsequent exit() operations will
-//   unpark the notifyee.  Unparking a notifee in notify() is inefficient -
-//   it's likely the notifyee would simply impale itself on the lock held
-//   by the notifier.
+//   unpark/re-schedule the notifyee. Unparking/re-scheduling a notifyee in
+//   notify() is inefficient - it's likely the notifyee would simply impale
+//   itself on the lock held by the notifier.
 
 // Check that object() and set_object() are called from the right context:
 static void check_object_context() {

src/hotspot/share/services/heapDumper.cpp

@@ -1875,9 +1875,12 @@ void ThreadDumper::dump_stack_refs(AbstractDumpWriter * writer) {
         // native frame
         blk.set_frame_number(depth);
         if (is_top_frame) {
-          // JNI locals for the top frame.
-          assert(_java_thread != nullptr, "impossible for unmounted vthread");
-          _java_thread->active_handles()->oops_do(&blk);
+          // JNI locals for the top frame if mounted
+          assert(_java_thread != nullptr || jvf->method()->is_synchronized()
+                 || jvf->method()->is_object_wait0(), "impossible for unmounted vthread");
+          if (_java_thread != nullptr) {
+            _java_thread->active_handles()->oops_do(&blk);
+          }
         } else {
           if (last_entry_frame != nullptr) {
             // JNI locals for the entry frame