8368159: Significant performance overhead when started with jdwp agent and unattached debugger

Reviewed-by: lmesnik, cjplummer

Author: Serguei Spitsyn

src/hotspot/share/prims/jvmtiExport.cpp

@@ -1690,6 +1690,23 @@ void JvmtiExport::post_vthread_unmount(jobject vthread) {
   }
 }
 
+bool JvmtiExport::has_frame_pops(JavaThread* thread) {
+  if (!can_post_frame_pop()) {
+    return false;
+  }
+  JvmtiThreadState *state = get_jvmti_thread_state(thread);
+  if (state == nullptr) {
+    return false;
+  }
+  JvmtiEnvThreadStateIterator it(state);
+  for (JvmtiEnvThreadState* ets = it.first(); ets != nullptr; ets = it.next(ets)) {
+    if (ets->has_frame_pops()) {
+      return true;
+    }
+  }
+  return false;
+}
+
 void JvmtiExport::continuation_yield_cleanup(JavaThread* thread, jint continuation_frame_count) {
   if (JvmtiEnv::get_phase() < JVMTI_PHASE_PRIMORDIAL) {
     return;

src/hotspot/share/prims/jvmtiExport.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -373,6 +373,7 @@ class JvmtiExport : public AllStatic {
     JVMTI_ONLY(return _should_post_class_file_load_hook);
     NOT_JVMTI(return false;)
   }
+  static bool has_frame_pops(JavaThread* thread) NOT_JVMTI_RETURN_(false);
   static bool is_early_phase() NOT_JVMTI_RETURN_(false);
   static bool has_early_class_hook_env() NOT_JVMTI_RETURN_(false);
   static bool has_early_vmstart_env() NOT_JVMTI_RETURN_(false);

src/hotspot/share/runtime/continuationFreezeThaw.cpp

@@ -1619,15 +1619,14 @@ static int num_java_frames(ContinuationWrapper& cont) {
 }
 
 static void invalidate_jvmti_stack(JavaThread* thread) {
-  if (thread->is_interp_only_mode()) {
-    JvmtiThreadState *state = thread->jvmti_thread_state();
-    if (state != nullptr)
-      state->invalidate_cur_stack_depth();
+  JvmtiThreadState *state = thread->jvmti_thread_state();
+  if (state != nullptr) {
+    state->invalidate_cur_stack_depth();
   }
 }
 
 static void jvmti_yield_cleanup(JavaThread* thread, ContinuationWrapper& cont) {
-  if (JvmtiExport::can_post_frame_pop()) {
+  if (JvmtiExport::has_frame_pops(thread)) {
     int num_frames = num_java_frames(cont);
 
     ContinuationWrapper::SafepointOp so(Thread::current(), cont);