Add threads dump for troubleshooting purposes

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
nextflow-io · May 13, 2024 · 8992ebd · 8992ebd
1 parent 6e91285
commit 8992ebd
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 18 deletions.
diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy
@@ -41,6 +41,9 @@ import nextflow.util.LoggerHelper
 import nextflow.util.ProxyConfig
 import nextflow.util.SpuriousDeps
 import org.eclipse.jgit.api.errors.GitAPIException
+
+import static nextflow.util.SysHelper.dumpThreads
+
 /**
  * Main application entry point. It parses the command line and
  * launch the pipeline execution.
@@ -553,24 +556,6 @@ class Launcher {
 
     }
 
-    /**
-     * Dump th stack trace of current running threads
-     * @return
-     */
-    private String dumpThreads() {
-
-        def buffer = new StringBuffer()
-        Map<Thread, StackTraceElement[]> m = Thread.getAllStackTraces();
-        for(Map.Entry<Thread,  StackTraceElement[]> e : m.entrySet()) {
-            buffer.append('\n').append(e.getKey().toString()).append('\n')
-            for (StackTraceElement s : e.getValue()) {
-                buffer.append("  " + s).append('\n')
-            }
-        }
-
-        return buffer.toString()
-    }
-
     /**
      * set up environment and system properties. It checks the following
      * environment variables:

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskPollingMonitor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskPollingMonitor.groovy
@@ -34,6 +34,8 @@ import nextflow.executor.GridTaskHandler
 import nextflow.util.Duration
 import nextflow.util.Threads
 import nextflow.util.Throttle
+import static nextflow.util.SysHelper.dumpThreads
+
 /**
  * Monitors the queued tasks waiting for their termination
  *
@@ -449,10 +451,15 @@ class TaskPollingMonitor implements TaskMonitor {
             // dump this line every two minutes
             Throttle.after(dumpInterval) {
                 dumpRunningQueue()
+                dumpCurrentThreads()
             }
         }
     }
 
+    protected dumpCurrentThreads() {
+        log.trace "Current runnign threads:\n${dumpThreads()}"
+    }
+
     protected void dumpRunningQueue() {
 
         try {

diff --git a/modules/nf-commons/src/main/nextflow/util/SysHelper.groovy b/modules/nf-commons/src/main/nextflow/util/SysHelper.groovy
@@ -178,4 +178,21 @@ class SysHelper {
         return os.getArch()
     }
 
+    /**
+     * Dump th stack trace of current running threads
+     * @return
+     */
+    static String dumpThreads() {
+
+        def buffer = new StringBuffer()
+        Map<Thread, StackTraceElement[]> m = Thread.getAllStackTraces();
+        for(Map.Entry<Thread,  StackTraceElement[]> e : m.entrySet()) {
+            buffer.append('\n').append(e.getKey().toString()).append('\n')
+            for (StackTraceElement s : e.getValue()) {
+                buffer.append("  " + s).append('\n')
+            }
+        }
+
+        return buffer.toString()
+    }
 }