8284161: Implementation of Virtual Threads (Preview)

Co-authored-by: Ron Pressler <rpressler@openjdk.org>
Co-authored-by: Alan Bateman <alanb@openjdk.org>
Co-authored-by: Erik Österlund <eosterlund@openjdk.org>
Co-authored-by: Andrew Haley <aph@openjdk.org>
Co-authored-by: Rickard Bäckman <rbackman@openjdk.org>
Co-authored-by: Markus Grönlund <mgronlun@openjdk.org>
Co-authored-by: Leonid Mesnik <lmesnik@openjdk.org>
Co-authored-by: Serguei Spitsyn <sspitsyn@openjdk.org>
Co-authored-by: Chris Plummer <cjplummer@openjdk.org>
Co-authored-by: Coleen Phillimore <coleenp@openjdk.org>
Co-authored-by: Robbin Ehn <rehn@openjdk.org>
Co-authored-by: Stefan Karlsson <stefank@openjdk.org>
Co-authored-by: Thomas Schatzl <tschatzl@openjdk.org>
Co-authored-by: Sergey Kuksenko <skuksenko@openjdk.org>
Reviewed-by: lancea, eosterlund, rehn, sspitsyn, stefank, tschatzl, dfuchs, lmesnik, dcubed, kevinw, amenkov, dlong, mchung, psandoz, bpb, coleenp, smarks, egahlin, mseledtsov, coffeys, darcy

make/data/hotspot-symbols/symbols-unix

@@ -46,7 +46,9 @@ JVM_ConstantPoolGetSize
 JVM_ConstantPoolGetStringAt
 JVM_ConstantPoolGetTagAt
 JVM_ConstantPoolGetUTF8At
+JVM_CurrentCarrierThread
 JVM_CurrentThread
+JVM_SetCurrentThread
 JVM_CurrentTimeMillis
 JVM_DefineClass
 JVM_DefineClassWithSource
@@ -120,6 +122,7 @@ JVM_GetMethodTypeAnnotations
 JVM_GetNanoTimeAdjustment
 JVM_GetNestHost
 JVM_GetNestMembers
+JVM_GetNextThreadIdOffset
 JVM_GetPermittedSubclasses
 JVM_GetPrimitiveArrayElement
 JVM_GetProperties
@@ -135,6 +138,7 @@ JVM_GetVmArguments
 JVM_Halt
 JVM_HasReferencePendingList
 JVM_HoldsLock
+JVM_GetStackTrace
 JVM_IHashCode
 JVM_InitClassName
 JVM_InitStackTraceElement
@@ -150,6 +154,7 @@ JVM_IsDumpingClassList
 JVM_IsFinalizationEnabled
 JVM_IsHiddenClass
 JVM_IsInterface
+JVM_IsPreviewEnabled
 JVM_IsPrimitiveClass
 JVM_IsRecord
 JVM_IsSameClassPackage
@@ -182,15 +187,19 @@ JVM_RawMonitorEnter
 JVM_RawMonitorExit
 JVM_ReferenceClear
 JVM_ReferenceRefersTo
+JVM_RegisterContinuationMethods
 JVM_RegisterLambdaProxyClassForArchiving
 JVM_RegisterSignal
 JVM_ReleaseUTF
 JVM_ReportFinalizationComplete
 JVM_ResumeThread
+JVM_ExtentLocalCache
+JVM_SetExtentLocalCache
 JVM_SetArrayElement
 JVM_SetClassSigners
 JVM_SetNativeThreadName
 JVM_SetPrimitiveArrayElement
+JVM_SetStackWalkContinuation
 JVM_SetThreadPriority
 JVM_Sleep
 JVM_StartThread
@@ -210,3 +219,10 @@ JVM_AddReadsModule
 JVM_DefineArchivedModules
 JVM_DefineModule
 JVM_SetBootLoaderUnnamedModule
+
+# Virtual thread notifications for JVMTI
+JVM_VirtualThreadMountBegin
+JVM_VirtualThreadMountEnd
+JVM_VirtualThreadUnmountBegin
+JVM_VirtualThreadUnmountEnd
+#

make/test/BuildMicrobenchmark.gmk

@@ -95,8 +95,12 @@ $(eval $(call SetupJavaCompilation, BUILD_JDK_MICROBENCHMARK, \
     SRC := $(MICROBENCHMARK_SRC), \
     BIN := $(MICROBENCHMARK_CLASSES), \
     JAVAC_FLAGS := --add-exports java.base/sun.security.util=ALL-UNNAMED \
-        --add-exports java.base/sun.invoke.util=ALL-UNNAMED, \
-    JAVA_FLAGS := --add-modules jdk.unsupported --limit-modules java.management, \
+        --add-exports java.base/sun.invoke.util=ALL-UNNAMED \
+        --add-exports java.base/jdk.internal.vm=ALL-UNNAMED \
+        --enable-preview, \
+    JAVA_FLAGS := --add-modules jdk.unsupported --limit-modules java.management \
+        --add-exports java.base/jdk.internal.vm=ALL-UNNAMED \
+        --enable-preview, \
 ))
 
 $(BUILD_JDK_MICROBENCHMARK): $(JMH_COMPILE_JARS)

make/test/JtregNativeHotspot.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -148,8 +148,10 @@ BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libNoFramePointer := $(NO_FRAMEPOINTER_CFLA
 # Optimization -O3 needed, HIGH == -O3
 BUILD_HOTSPOT_JTREG_LIBRARIES_OPTIMIZATION_libNoFramePointer := HIGH
 
-BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS := -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS
-BUILD_HOTSPOT_JTREG_EXECUTABLES_CFLAGS := -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS
+JVMTI_COMMON_INCLUDES=-I$(TOPDIR)/test/lib/jdk/test/lib/jvmti
+
+BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS := -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS $(JVMTI_COMMON_INCLUDES)
+BUILD_HOTSPOT_JTREG_EXECUTABLES_CFLAGS := -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS $(JVMTI_COMMON_INCLUDES)
 
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libProcessUtils := $(VM_SHARE_INCLUDES)
 
@@ -346,6 +348,7 @@ BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libvminit001 := $(NSK_JVMTI_AGENT_INCLUDES)
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libsuspendthrd001 := $(NSK_JVMTI_AGENT_INCLUDES)
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libsuspendthrd002 := $(NSK_JVMTI_AGENT_INCLUDES)
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libsuspendthrd003 := $(NSK_JVMTI_AGENT_INCLUDES)
+BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libsuspendvthr001 := $(NSK_JVMTI_AGENT_INCLUDES)
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libdeclcls002 := $(NSK_JVMTI_AGENT_INCLUDES)
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libdeclcls003 := $(NSK_JVMTI_AGENT_INCLUDES)
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libdeclcls001 := $(NSK_JVMTI_AGENT_INCLUDES)
@@ -1012,6 +1015,7 @@ else
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libsuspendthrd001 += -lpthread
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libsuspendthrd002 += -lpthread
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libsuspendthrd003 += -lpthread
+    BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libsuspendvthr001 += -lpthread
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libdeclcls002 += -lpthread
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libdeclcls003 += -lpthread
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libdeclcls001 += -lpthread

src/hotspot/cpu/aarch64/aarch64.ad

@@ -3829,6 +3829,8 @@ encode %{
       }
     }
 
+    __ post_call_nop();
+
     // Only non uncommon_trap calls need to reinitialize ptrue.
     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
       __ reinitialize_ptrue();
@@ -3842,7 +3844,9 @@ encode %{
     if (call == NULL) {
       ciEnv::current()->record_failure("CodeCache is full");
       return;
-    } else if (Compile::current()->max_vector_size() > 0) {
+    }
+    __ post_call_nop();
+    if (Compile::current()->max_vector_size() > 0) {
       __ reinitialize_ptrue();
     }
   %}
@@ -3870,6 +3874,7 @@ encode %{
         ciEnv::current()->record_failure("CodeCache is full");
         return;
       }
+      __ post_call_nop();
     } else {
       Label retaddr;
       __ adr(rscratch2, retaddr);
@@ -3878,6 +3883,7 @@ encode %{
       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
       __ blr(rscratch1);
       __ bind(retaddr);
+      __ post_call_nop();
       __ add(sp, sp, 2 * wordSize);
     }
     if (Compile::current()->max_vector_size() > 0) {

src/hotspot/cpu/aarch64/abstractInterpreter_aarch64.cpp

@@ -123,9 +123,9 @@ void AbstractInterpreter::layout_activation(Method* method,
   // It is also guaranteed to be walkable even though it is in a
   // skeletal state
 
-  int max_locals = method->max_locals() * Interpreter::stackElementWords;
-  int extra_locals = (method->max_locals() - method->size_of_parameters()) *
-    Interpreter::stackElementWords;
+  const int max_locals = method->max_locals() * Interpreter::stackElementWords;
+  const int params = method->size_of_parameters() * Interpreter::stackElementWords;
+  const int extra_locals = max_locals - params;
 
 #ifdef ASSERT
   assert(caller->sp() == interpreter_frame->sender_sp(), "Frame not properly walkable");
@@ -144,12 +144,9 @@ void AbstractInterpreter::layout_activation(Method* method,
   // align the incoming parameters with the caller's temporary
   // expression stack. For other types of caller frame it doesn't
   // matter.
-  intptr_t* locals;
-  if (caller->is_interpreted_frame()) {
-    locals = caller->interpreter_frame_last_sp() + caller_actual_parameters - 1;
-  } else {
-    locals = interpreter_frame->sender_sp() + max_locals - 1;
-  }
+  intptr_t* const locals = caller->is_interpreted_frame()
+    ? caller->interpreter_frame_last_sp() + caller_actual_parameters - 1
+    : interpreter_frame->sender_sp() + max_locals - 1;
 
 #ifdef ASSERT
   if (caller->is_interpreted_frame()) {
@@ -171,14 +168,10 @@ void AbstractInterpreter::layout_activation(Method* method,
   // All frames but the initial (oldest) interpreter frame we fill in have
   // a value for sender_sp that allows walking the stack but isn't
   // truly correct. Correct the value here.
-  if (extra_locals != 0 &&
-      interpreter_frame->sender_sp() ==
-      interpreter_frame->interpreter_frame_sender_sp()) {
-    interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() +
-                                                       extra_locals);
+  if (extra_locals != 0 && interpreter_frame->sender_sp() == interpreter_frame->interpreter_frame_sender_sp()) {
+    interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() + extra_locals);
   }
-  *interpreter_frame->interpreter_frame_cache_addr() =
-    method->constants()->cache();
-  *interpreter_frame->interpreter_frame_mirror_addr() =
-    method->method_holder()->java_mirror();
+
+  *interpreter_frame->interpreter_frame_cache_addr()  = method->constants()->cache();
+  *interpreter_frame->interpreter_frame_mirror_addr() = method->method_holder()->java_mirror();
 }

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -438,6 +438,7 @@ int LIR_Assembler::emit_unwind_handler() {
       __ unlock_object(r5, r4, r0, *stub->entry());
     }
     __ bind(*stub->continuation());
+    __ dec_held_monitor_count(rthread);
   }
 
   if (compilation()->env()->dtrace_method_probes()) {
@@ -2037,6 +2038,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
     return;
   }
   add_call_info(code_offset(), op->info());
+  __ post_call_nop();
 }
 
 
@@ -2047,6 +2049,7 @@ void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
     return;
   }
   add_call_info(code_offset(), op->info());
+  __ post_call_nop();
 }
 
 void LIR_Assembler::emit_static_call_stub() {
@@ -2571,7 +2574,18 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   } else {
     Unimplemented();
   }
+  if (op->code() == lir_lock) {
+    // If deoptimization happens in Runtime1::monitorenter, inc_held_monitor_count after backing from slowpath
+    // will be skipped. Solution is:
+    // 1. Increase only in fastpath
+    // 2. Runtime1::monitorenter increase count after locking
+    __ inc_held_monitor_count(rthread);
+  }
   __ bind(*op->stub()->continuation());
+  if (op->code() == lir_unlock) {
+    // unlock in slowpath is JRT_Leaf stub, no deoptimization can happen
+    __ dec_held_monitor_count(rthread);
+  }
 }
 
 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
@@ -2899,6 +2913,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg
   if (info != NULL) {
     add_call_info_here(info);
   }
+  __ post_call_nop();
 }
 
 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {

src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -338,6 +338,17 @@ void LIRGenerator::do_MonitorExit(MonitorExit* x) {
   monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no());
 }
 
+void LIRGenerator::do_continuation_doYield(Intrinsic* x) {
+  BasicTypeList signature(0);
+  CallingConvention* cc = frame_map()->java_calling_convention(&signature, true);
+
+  const LIR_Opr result_reg = result_register_for(x->type());
+  address entry = StubRoutines::cont_doYield();
+  LIR_Opr result = rlock_result(x);
+  CodeEmitInfo* info = state_for(x, x->state());
+  __ call_runtime(entry, LIR_OprFact::illegalOpr, result_reg, cc->args(), info);
+  __ move(result_reg, result);
+}
 
 void LIRGenerator::do_NegateOp(NegateOp* x) {
 

src/hotspot/cpu/aarch64/continuationEntry_aarch64.inline.hpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_AARCH64_CONTINUATIONENTRY_AARCH64_INLINE_HPP
+#define CPU_AARCH64_CONTINUATIONENTRY_AARCH64_INLINE_HPP
+
+#include "runtime/continuationEntry.hpp"
+
+#include "code/codeCache.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/registerMap.hpp"
+
+inline frame ContinuationEntry::to_frame() const {
+  static CodeBlob* cb = CodeCache::find_blob_fast(entry_pc());
+  assert(cb != nullptr, "");
+  assert(cb->as_compiled_method()->method()->is_continuation_enter_intrinsic(), "");
+  return frame(entry_sp(), entry_sp(), entry_fp(), entry_pc(), cb);
+}
+
+inline intptr_t* ContinuationEntry::entry_fp() const {
+  return (intptr_t*)((address)this + size());
+}
+
+inline void ContinuationEntry::update_register_map(RegisterMap* map) const {
+  intptr_t** fp = (intptr_t**)(bottom_sender_sp() - frame::sender_sp_offset);
+  frame::update_map_with_saved_link(map, fp);
+}
+
+
+#endif //  CPU_AARCH64_CONTINUATIONENTRY_AARCH64_INLINE_HPP