feat(vt): expose carrier frames and fix truncated traces for virtual threads on JDK 21+

ddprof-lib/src/main/cpp/arguments.cpp

@@ -292,9 +292,11 @@ Error Arguments::parse(const char *args) {
         } else if (strcmp(value, "vm") == 0) {
           _cstack = CSTACK_VM;
         } else if (strcmp(value, "vmx") == 0) {
-          // cstack=vmx is a shorthand for cstack=vm,features=mixed
+          // cstack=vmx is a shorthand for cstack=vm,features=mixed; carrier-frame
+          // unwinding is enabled automatically since vmx already traverses entry frames
           _cstack = CSTACK_VM;
           _features.mixed = 1;
+          _features.carrier_frames = 1;
         } else {
           _cstack = CSTACK_NO;
         }

ddprof-lib/src/main/cpp/arguments.h

@@ -122,7 +122,8 @@ struct StackWalkFeatures {
     unsigned short vtable_target : 1;  // show receiver classes of vtable/itable stubs
     unsigned short comp_task     : 1;  // display current compilation task for JIT threads
     unsigned short pc_addr       : 1;  // record exact PC address for each sample
-    unsigned short _padding      : 3;  // pad structure to 16 bits
+    unsigned short carrier_frames: 1;  // walk through VT continuation boundary to carrier frames (enabled automatically with cstack=vmx)
+    unsigned short _padding      : 2;  // pad structure to 16 bits
 };
 
 struct Multiplier {

ddprof-lib/src/main/cpp/counters.h

@@ -91,7 +91,11 @@
   X(WALKVM_STUB_FRAMESIZE_FALLBACK, "walkvm_stub_framesize_fallback")        \
   X(WALKVM_FP_CHAIN_ATTEMPT, "walkvm_fp_chain_attempt")                      \
   X(WALKVM_FP_CHAIN_REACHED_CODEHEAP, "walkvm_fp_chain_reached_codeheap")    \
-  X(WALKVM_ANCHOR_NOT_IN_JAVA, "walkvm_anchor_not_in_java")                    \
+  X(WALKVM_ANCHOR_NOT_IN_JAVA,  "walkvm_anchor_not_in_java")                   \
+  X(WALKVM_CONT_BARRIER_HIT,    "walkvm_cont_barrier_hit")                     \
+  X(WALKVM_ENTER_SPECIAL_HIT,   "walkvm_enter_special_hit")                    \
+  X(WALKVM_CONT_SPECULATIVE_HIT,"walkvm_cont_speculative_hit")                 \
+  X(WALKVM_CONT_ENTRY_NULL,     "walkvm_cont_entry_null")                      \
   X(NATIVE_LIBS_DROPPED, "native_libs_dropped")                                \
   X(SIGACTION_PATCHED_LIBS, "sigaction_patched_libs")                          \
   X(SIGACTION_INTERCEPTED, "sigaction_intercepted")

ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+#include <cstdlib>
 #include <setjmp.h>
 #include "asyncSampleMutex.h"
 #include "hotspot/hotspotSupport.h"
@@ -115,6 +116,19 @@ static void fillFrameTypes(ASGCT_CallFrame *frames, int num_frames, VMNMethod *n
 
 static ucontext_t empty_ucontext{};
 
+#ifdef NDEBUG
+static const bool CONT_UNWIND_DISABLED = false;
+#else
+// DEBUG-only: when set, both continuation-unwind detection branches
+// (cont_entry_return_pc for fully-thawed VTs, cont_returnBarrier for VTs
+// with frozen frames) are skipped, reproducing pre-fix behaviour.
+// Used by negative integration tests to verify that carrier frames are not
+// visible and walk-error sentinels do appear without the fix.
+// NOTE: the env var is evaluated once at library load time; it must be set
+// in the environment before the profiler agent is attached.
+static const bool CONT_UNWIND_DISABLED = (std::getenv("DDPROF_DISABLE_CONT_UNWIND") != nullptr);
+#endif
+
 __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth,
                         StackWalkFeatures features, EventType event_type, int lock_index, bool* truncated) {
     if (ucontext == NULL) {
@@ -183,6 +197,9 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
 
     const void* prev_native_pc = NULL;
 
+    // Last ContinuationEntry crossed; advanced via parent() for nested continuations.
+    VMContinuationEntry* cont_entry = nullptr;
+
     // Saved anchor data — preserved across anchor consumption so inline
     // recovery can redirect even after the anchor pointer has been set to NULL.
     // Recovery is one-shot: once attempted, we do not retry to avoid
@@ -199,6 +216,80 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
         anchor = vm_thread->anchor();
     }
 
+    static const char* CONT_ROOT_FRAME = "JVM Continuation";
+
+    // Advances through a continuation boundary to the carrier frame.
+    // Without carrier_frames (default, cstack=vm): always stops with a "JVM Continuation"
+    // synthetic root frame — VT frames are complete, carrier internals are noise.
+    // With carrier_frames (cstack=vmx): attempts to walk through; failures emit BCI_ERROR
+    // so the sample is truthfully marked truncated.
+    // Walks cont_entry->parent() on repeated calls to handle nested continuations
+    // (_parent not triggered by standard single-level VTs today, but required
+    // once any runtime layers continuations on top of VTs).
+    //
+    // all_frames_thawed: true when the bottom VT frame's return PC is
+    //         cont_entry_return_pc (all VT frames are thawed — CPU-bound VT),
+    //         false when it is cont_returnBarrier (frozen frames remain in the
+    //         StackChunk — VT parked and just remounted).
+    //         Needed to derive entry_fp on JDK 21-26 where ContinuationEntry
+    //         type size is absent from vmStructs and contEntry() returns nullptr.
+    //
+    // Returns true to continue the walk, false to break.
+    auto walkThroughContinuation = [&](bool all_frames_thawed) -> bool {
+        if (depth >= actual_max_depth) return false;
+        if (!features.carrier_frames) {
+            fillFrame(frames[depth++], BCI_NATIVE_FRAME, CONT_ROOT_FRAME);
+            return false;
+        }
+
+        uintptr_t entry_fp;
+
+        if (VMContinuationEntry::type_size() > 0) {
+            // ContinuationEntry is known via vmStructs (JDK 27+, added by
+            // JDK-8378985).  Walk the linked list of entries for nested-
+            // continuation support and derive the enterSpecial frame FP from
+            // the struct layout (entry + type_size).
+            cont_entry = (cont_entry != nullptr) ? cont_entry->parent() : vm_thread->contEntry();
+            if (cont_entry == nullptr) {
+                Counters::increment(WALKVM_CONT_ENTRY_NULL);
+                fillFrame(frames[depth++], BCI_ERROR, "break_cont_entry_null");
+                return false;
+            }
+            entry_fp = cont_entry->entryFP();
+        } else {
+            // ContinuationEntry absent from vmStructs (JDK 21-26).
+            // Derive the enterSpecial frame FP from the current fp:
+            //   all frames thawed (pc == cont_entry_return_pc): fp IS the
+            //     enterSpecial frame FP.
+            //   frozen frames remain (pc == cont_returnBarrier): the saved
+            //     caller FP at *fp leads to the enterSpecial frame on the
+            //     carrier stack.
+            // Nested continuation tracking is unavailable without type_size().
+            entry_fp = all_frames_thawed ? fp : (uintptr_t)SafeAccess::load((void**)fp);
+        }
+
+        if (!StackWalkValidation::isValidFP(entry_fp)) {
+            fillFrame(frames[depth++], BCI_ERROR, "break_cont_entry_fp");
+            return false;
+        }
+        // entry_fp has been range-checked by isValidFP above; any remaining
+        // SIGSEGV from a stale/concurrently-freed pointer is caught by the
+        // setjmp crash protection in walkVM (checkFault -> longjmp).
+        uintptr_t carrier_fp = *(uintptr_t*)entry_fp;
+        const void* carrier_pc = ((const void**)entry_fp)[FRAME_PC_SLOT];
+        uintptr_t carrier_sp = entry_fp + (FRAME_PC_SLOT + 1) * sizeof(void*);
+        if (!StackWalkValidation::isValidFP(carrier_fp) ||
+            StackWalkValidation::inDeadZone(carrier_pc) ||
+            !StackWalkValidation::isValidSP(carrier_sp, sp, bottom)) {
+            fillFrame(frames[depth++], BCI_ERROR, "break_cont_carrier_sp");
+            return false;
+        }
+        sp = carrier_sp;
+        fp = carrier_fp;
+        pc = carrier_pc;
+        return true;
+    };
+
     unwind_loop:
 
     // Walk until the bottom of the stack or until the first Java frame
@@ -229,8 +320,42 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                 break;
             }
             prev_native_pc = NULL; // we are in JVM code, no previous 'native' PC
+            // Both continuation boundary PCs are JVM stubs whose findNMethod()
+            // returns NULL; detect them by exact-PC match before the nmethod
+            // dispatch below.
+            // cont_returnBarrier: bottom thawed frame returns here when frozen
+            //   frames remain in the StackChunk (blocking/remounted VT).
+            // cont_entry_return_pc: bottom thawed frame returns here when the
+            //   continuation is fully thawed (CPU-bound VT, never yielded).
+            if (!CONT_UNWIND_DISABLED && VMStructs::isContReturnBarrier(pc)) {
+                Counters::increment(WALKVM_CONT_BARRIER_HIT);
+                if (walkThroughContinuation(false)) continue;
+                break;
+            }
+            if (!CONT_UNWIND_DISABLED && VMStructs::isContEntryReturnPc(pc)) {
+                Counters::increment(WALKVM_ENTER_SPECIAL_HIT);
+                if (walkThroughContinuation(true)) continue;
+                break;
+            }
             VMNMethod* nm = CodeHeap::findNMethod(pc);
             if (nm == NULL) {
+                // On JDK 21+ builds, the continuation entry PC may be absent
+                // from vmStructs OR resolved but pointing to the wrong address
+                // (some distributions expose the symbol at the wrong address, so
+                // the exact-PC check above never fires).  Attempt a fully-thawed
+                // continuation walk whenever we see an unknown nmethod after
+                // collecting Java frames.  walkThroughContinuation validates the
+                // fp chain and emits BCI_ERROR cleanly on mismatch, so false
+                // positives are safe.
+                if (!CONT_UNWIND_DISABLED
+                        && features.carrier_frames
+                        && VM::hotspot_version() >= 21
+                        && depth > 0
+                        && vm_thread != NULL && vm_thread->isCarryingVirtualThread()) {
+                    Counters::increment(WALKVM_CONT_SPECULATIVE_HIT);
+                    if (walkThroughContinuation(true)) continue;
+                    break;
+                }
                 if (anchor == NULL) {
                     // Add an error frame only if we cannot recover
                     fillFrame(frames[depth++], BCI_ERROR, "unknown_nmethod");
@@ -241,7 +366,14 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
             // Always prefer JavaFrameAnchor when it is available,
             // since it provides reliable SP and FP.
             // Do not treat the topmost stub as Java frame.
-            if (anchor != NULL && (depth > 0 || !nm->isStub())) {
+            // Exception: when VT carrier-frame unwinding is active, skip the anchor
+            // redirect — it can bypass the continuation boundary by jumping directly
+            // into carrier frames, causing walkThroughContinuation to never fire.
+            // The continuation mechanism finds carrier frames on its own.
+            bool anchor_eligible = anchor != NULL && (depth > 0 || !nm->isStub());
+            bool cont_unwind_active = features.carrier_frames && !CONT_UNWIND_DISABLED
+                && vm_thread != NULL && vm_thread->isCarryingVirtualThread();
+            if (anchor_eligible && !cont_unwind_active) {
                 Counters::increment(WALKVM_ANCHOR_CONSUMED);
                 // Preserve anchor data before consumption — getFrame() is read-only
                 // but we set anchor=NULL below, losing the pointer for later recovery.
@@ -255,6 +387,10 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                     continue;  // NMethod has changed as a result of correction
                 }
                 anchor = NULL;
+            } else if (anchor_eligible && cont_unwind_active) {
+                // Clear the anchor without redirecting so it doesn't corrupt fp
+                // for the continuation boundary walk.
+                anchor = NULL;
             }
 
             if (nm->isInterpreter()) {
@@ -304,6 +440,15 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                 fillFrame(frames[depth++], BCI_ERROR, "break_interpreted");
                 break;
             } else if (nm->isNMethod()) {
+                // enterSpecial is a generated native nmethod that acts as the
+                // continuation entry stub on JDK 27+. It has no JavaCallWrapper, so
+                // isEntryFrame() will not fire for it. Detect it by identity
+                // and navigate to the carrier thread via ContinuationEntry.
+                if (!CONT_UNWIND_DISABLED && nm == VMStructs::enterSpecialNMethod()) {
+                    Counters::increment(WALKVM_ENTER_SPECIAL_HIT);
+                    if (walkThroughContinuation(true)) continue;
+                    break;
+                }
                 // Check if deoptimization is in progress before walking compiled frames
                 if (vm_thread != NULL && vm_thread->inDeopt()) {
                     fillFrame(frames[depth++], BCI_ERROR, "break_deopt_compiled");
@@ -970,6 +1115,17 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) {
                 }
             }
         }
+        // ASGCT stops at the continuation boundary for virtual threads (JDK 21+).
+        // Append a synthetic root frame so the UI does not show "Missing Frames".
+        if (java_frames > 0 && VM::hotspot_version() >= 21 && java_frames < max_depth) {
+            VMThread* carrier = VMThread::current();
+            if (carrier != nullptr && carrier->isCarryingVirtualThread()) {
+                frames[java_frames].bci = BCI_NATIVE_FRAME;
+                frames[java_frames].method_id = (jmethodID) "JVM Continuation";
+                LP64_ONLY(frames[java_frames].padding = 0;)
+                java_frames++;
+            }
+        }
     }
   }
   return java_frames; 

ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp

@@ -35,6 +35,9 @@ int VMStructs::_narrow_klass_shift = -1;
 int VMStructs::_interpreter_frame_bcp_offset = 0;
 unsigned char VMStructs::_unsigned5_base = 0;
 const void* VMStructs::_call_stub_return = nullptr;
+const void* VMStructs::_cont_return_barrier   = nullptr;
+const void* VMStructs::_cont_entry_return_pc  = nullptr;
+VMNMethod*  VMStructs::_enter_special_nm       = nullptr;
 const void* VMStructs::_interpreter_start = nullptr;
 VMNMethod* VMStructs::_interpreter_nm = nullptr;
 const void* VMStructs::_interpreted_frame_valid_start = nullptr;
@@ -44,6 +47,7 @@ const void* VMStructs::_interpreted_frame_valid_end = nullptr;
 // Initialize type size to 0
 #define INIT_TYPE_SIZE(name, names) uint64_t VMStructs::TYPE_SIZE_NAME(name) = 0;
 DECLARE_TYPES_DO(INIT_TYPE_SIZE)
+DECLARE_V27_TYPES_DO(INIT_TYPE_SIZE)
 #undef INIT_TYPE_SIZE
 
 #define offset_value -1
@@ -62,6 +66,7 @@ DECLARE_TYPES_DO(INIT_TYPE_SIZE)
     field_type VMStructs::var = field_type##_value;
 
 DECLARE_TYPE_FIELD_DO(DO_NOTHING, INIT_FIELD, INIT_FIELD_WITH_VERSION, DO_NOTHING)
+DECLARE_V27_TYPE_FIELD_DO(DO_NOTHING, INIT_FIELD, INIT_FIELD_WITH_VERSION, DO_NOTHING)
 
 #undef INIT_FIELD
 #undef INIT_FIELD_WITH_VERSION
@@ -175,6 +180,7 @@ void VMStructs::init_offsets_and_addresses() {
 
 #define END_TYPE() continue; }
         DECLARE_TYPE_FIELD_DO(MATCH_TYPE_NAMES, READ_FIELD_VALUE, READ_FIELD_VALUE_WITH_VERSION, END_TYPE)
+        DECLARE_V27_TYPE_FIELD_DO(MATCH_TYPE_NAMES, READ_FIELD_VALUE, READ_FIELD_VALUE_WITH_VERSION, END_TYPE)
 #undef MATCH_TYPE_NAMES
 #undef READ_FIELD_VALUE
 #undef READ_FIELD_VALUE_WITH_VERSION
@@ -205,6 +211,7 @@ void VMStructs::init_type_sizes() {
         }
 
         DECLARE_TYPES_DO(READ_TYPE_SIZE)
+        DECLARE_V27_TYPES_DO(READ_TYPE_SIZE)
 
 #undef READ_TYPE_SIZE   
 
@@ -271,12 +278,21 @@ void VMStructs::verify_offsets() {
     }
 
 // Verify type sizes
+// Note: DECLARE_V27_TYPES_DO (VMContinuationEntry) is intentionally excluded here.
+// ContinuationEntry is not exported in gHotSpotVMTypes before JDK 27 (added via JDK-8378985);
+// asserting type_size() > 0 would SIGABRT on any JDK 21-26 build.
 #define VERIFY_TYPE_SIZE(name, names) assert(TYPE_SIZE_NAME(name) > 0);
     DECLARE_TYPES_DO(VERIFY_TYPE_SIZE);
 #undef VERIFY_TYPE_SIZE
 
 
 // Verify offsets and addresses
+// Note: DECLARE_V27_TYPE_FIELD_DO is intentionally excluded here.
+// Continuation-related fields (_cont_entry_offset, _cont_return_barrier_addr,
+// _cont_entry_return_pc_addr, _cont_entry_parent_offset) are absent from
+// gHotSpotVMStructs in all JDK 21-26 builds: ContinuationEntry was not
+// exported in the vmStructs table until JDK 27 (JDK-8378985). walkVM degrades
+// gracefully when they are missing.
 #define offset_value -1
 #define address_value nullptr
 
@@ -391,6 +407,25 @@ void VMStructs::resolveOffsets() {
     if (_call_stub_return_addr != NULL) {
         _call_stub_return = *(const void**)_call_stub_return_addr;
     }
+    if (_cont_return_barrier_addr != NULL) {
+        _cont_return_barrier = *(const void**)_cont_return_barrier_addr;
+    }
+    if (_cont_entry_return_pc_addr != NULL) {
+        _cont_entry_return_pc = *(const void**)_cont_entry_return_pc_addr;
+    }
+    // Fallback for JDK 21-26: StubRoutines::_cont_returnBarrier and
+    // ContinuationEntry::_return_pc are absent from gHotSpotVMStructs before
+    // JDK 27 (added via JDK-8378985).  Resolve them via C++ symbol lookup.
+    // Symbol names use Itanium C++ ABI mangling (GCC/Clang), which matches
+    // the HotSpot build toolchain on all supported platforms.
+    if (_cont_return_barrier == nullptr && VM::hotspot_version() >= 21) {
+        const void** sym = (const void**)_libjvm->findSymbol("_ZN12StubRoutines19_cont_returnBarrierE");
+        if (sym != nullptr) _cont_return_barrier = *sym;
+    }
+    if (_cont_entry_return_pc == nullptr && VM::hotspot_version() >= 21) {
+        const void** sym = (const void**)_libjvm->findSymbol("_ZN17ContinuationEntry10_return_pcE");
+        if (sym != nullptr) _cont_entry_return_pc = *sym;
+    }
 
     // Since JDK 23, _metadata_offset is relative to _data_offset. See metadata()
     if (_nmethod_immutable_offset < 0) {
@@ -440,6 +475,14 @@ void VMStructs::resolveOffsets() {
     if (_interpreter_nm == NULL && _interpreter_start != NULL) {
         _interpreter_nm = CodeHeap::findNMethod(_interpreter_start);
     }
+    if (_enter_special_nm == NULL && _cont_entry_return_pc != NULL) {
+        // On JDK 27+, enterSpecial is a proper nmethod; findNMethod succeeds.
+        // On JDK 21-26, it is a RuntimeBlob; findNMethod returns NULL and
+        // _enter_special_nm stays NULL.  The cont_entry_return_pc boundary is
+        // then detected via isContEntryReturnPc() in the walk loop rather than
+        // nmethod identity.
+        _enter_special_nm = CodeHeap::findNMethod(_cont_entry_return_pc);
+    }
 }
 
 void VMStructs::initJvmFunctions() {