8326106: Write and clear stack trace table outside of safepoint

Backport-of: a776104e210db212c4e32894844d3c0cbaac53c3

src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp

@@ -214,9 +214,6 @@ class StackTraceBlobInstaller {
   StackTraceBlobInstaller() : _cache(JfrOptionSet::old_object_queue_size()) {
     prepare_for_resolution();
   }
-  ~StackTraceBlobInstaller() {
-    JfrStackTraceRepository::clear_leak_profiler();
-  }
   void sample_do(ObjectSample* sample) {
     if (stack_trace_precondition(sample)) {
       install(sample);
@@ -270,11 +267,14 @@ void ObjectSampleCheckpoint::on_rotation(const ObjectSampler* sampler) {
   assert(LeakProfiler::is_running(), "invariant");
   JavaThread* const thread = JavaThread::current();
   DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(thread);)
-  // can safepoint here
-  ThreadInVMfromNative transition(thread);
-  MutexLocker lock(ClassLoaderDataGraph_lock);
-  // the lock is needed to ensure the unload lists do not grow in the middle of inspection.
-  install_stack_traces(sampler);
+  {
+    // can safepoint here
+    ThreadInVMfromNative transition(thread);
+    MutexLocker lock(ClassLoaderDataGraph_lock);
+    // the lock is needed to ensure the unload lists do not grow in the middle of inspection.
+    install_stack_traces(sampler);
+  }
+  JfrStackTraceRepository::clear_leak_profiler();
 }
 
 static bool is_klass_unloaded(traceid klass_id) {

src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp

@@ -569,9 +569,7 @@ void JfrRecorderService::pre_safepoint_write() {
     ObjectSampleCheckpoint::on_rotation(ObjectSampler::acquire());
   }
   write_storage(_storage, _chunkwriter);
-  if (_stack_trace_repository.is_modified()) {
-    write_stacktrace(_stack_trace_repository, _chunkwriter, false);
-  }
+  write_stacktrace(_stack_trace_repository, _chunkwriter, true);
 }
 
 void JfrRecorderService::invoke_safepoint_write() {

src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp

@@ -98,11 +98,10 @@ bool JfrStackTraceRepository::is_modified() const {
 }
 
 size_t JfrStackTraceRepository::write(JfrChunkWriter& sw, bool clear) {
+  MutexLocker lock(JfrStacktrace_lock, Mutex::_no_safepoint_check_flag);
   if (_entries == 0) {
     return 0;
   }
-  MutexLocker lock(JfrStacktrace_lock, Mutex::_no_safepoint_check_flag);
-  assert(_entries > 0, "invariant");
   int count = 0;
   for (u4 i = 0; i < TABLE_SIZE; ++i) {
     JfrStackTrace* stacktrace = _table[i];