8298377: JfrVframeStream causes deadlocks in ZGC

Backport-of: 453dbd1

src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp

@@ -41,6 +41,7 @@
 #include "runtime/javaThread.inline.hpp"
 #include "runtime/os.hpp"
 #include "runtime/semaphore.hpp"
+#include "runtime/stackWatermark.hpp"
 #include "runtime/suspendedThreadTask.hpp"
 #include "runtime/threadCrashProtection.hpp"
 #include "runtime/threadSMR.hpp"
@@ -256,6 +257,11 @@ void JfrNativeSamplerCallback::call() {
 }
 
 bool JfrThreadSampleClosure::sample_thread_in_java(JavaThread* thread, JfrStackFrame* frames, u4 max_frames) {
+  // Process the oops in the thread head before calling into code that wants to
+  // stack walk over Loom continuations. The stack walking code will otherwise
+  // skip frames in stack chunks on the Java heap.
+  StackWatermarkSet::start_processing(thread, StackWatermarkKind::gc);
+
   OSThreadSampler sampler(thread, *this, frames, max_frames);
   sampler.take_sample();
   /* We don't want to allocate any memory using malloc/etc while the thread
@@ -274,6 +280,11 @@ bool JfrThreadSampleClosure::sample_thread_in_java(JavaThread* thread, JfrStackF
 }
 
 bool JfrThreadSampleClosure::sample_thread_in_native(JavaThread* thread, JfrStackFrame* frames, u4 max_frames) {
+  // Process the oops in the thread head before calling into code that wants to
+  // stack walk over Loom continuations. The stack walking code will otherwise
+  // skip frames in stack chunks on the Java heap.
+  StackWatermarkSet::start_processing(thread, StackWatermarkKind::gc);
+
   JfrNativeSamplerCallback cb(*this, thread, frames, max_frames);
   if (JfrOptionSet::sample_protection()) {
     ThreadCrashProtection crash_protection;

src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.cpp

@@ -148,11 +148,23 @@ class JfrVframeStream : public vframeStreamCommon {
   void next_vframe();
 };
 
+static RegisterMap::WalkContinuation walk_continuation(JavaThread* jt) {
+  // NOTE: WalkContinuation::skip, because of interactions with ZGC relocation
+  //       and load barriers. This code is run while generating stack traces for
+  //       the ZPage allocation event, even when ZGC is relocating  objects.
+  //       When ZGC is relocating, it is forbidden to run code that performs
+  //       load barriers. With WalkContinuation::include, we visit heap stack
+  //       chunks and could be using load barriers.
+  return (UseZGC && !StackWatermarkSet::processing_started(jt))
+      ? RegisterMap::WalkContinuation::skip
+      : RegisterMap::WalkContinuation::include;
+}
+
 JfrVframeStream::JfrVframeStream(JavaThread* jt, const frame& fr, bool stop_at_java_call_stub, bool async_mode) :
   vframeStreamCommon(RegisterMap(jt,
                                  RegisterMap::UpdateMap::skip,
                                  RegisterMap::ProcessFrames::skip,
-                                 RegisterMap::WalkContinuation::include)),
+                                 walk_continuation(jt))),
     _cont_entry(JfrThreadLocal::is_vthread(jt) ? jt->last_continuation() : nullptr),
     _async_mode(async_mode), _vthread(JfrThreadLocal::is_vthread(jt)) {
   assert(!_vthread || _cont_entry != nullptr, "invariant");