[WFCORE-5968] log thread dump upon management timeout

controller/src/main/java/org/jboss/as/controller/OperationContextImpl.java

@@ -501,6 +501,8 @@ void awaitServiceContainerStability() throws InterruptedException, TimeoutExcept
                 // Deliberate log and throw; we want to log this but the caller method passes a slightly different
                 // message to the user as part of the operation response
                 MGMT_OP_LOGGER.timeoutExecutingOperation(timeout / 1000, containerMonitorStep.operationId.name, containerMonitorStep.address);
+                // Produce and log thread dump for diagnostics
+                MGMT_OP_LOGGER.threadDump(ThreadDumpUtil.threadDump(""));
                 throw te;
             } finally {
                 executionStatus = originalExecutionStatus;
@@ -867,6 +869,8 @@ private void ensureWriteLockForRuntime() {
                     // different messages for both so just throwing a RuntimeException to get the automatic handling
                     // in AbstractOperationContext.executeStep is not what I wanted
                     ControllerLogger.MGMT_OP_LOGGER.timeoutAwaitingInitialStability(timeout / 1000, activeStep.operationId.name, activeStep.operationId.address);
+                    // Produce and log thread dump for diagnostics
+                    MGMT_OP_LOGGER.threadDump(ThreadDumpUtil.threadDump(""));
                     setRollbackOnly();
                     throw new OperationFailedRuntimeException(ControllerLogger.ROOT_LOGGER.timeoutAwaitingInitialStability());
                 } finally {
@@ -1233,6 +1237,8 @@ void releaseStepLocks(AbstractOperationContext.Step step) {
                     // it's almost certain we never stabilized during execution or we are rolling back and destabilized there.
                     // Either one means there is already a failure message associated with this op.
                     MGMT_OP_LOGGER.timeoutCompletingOperation(timeout / 1000, activeStep.operationId.name, activeStep.operationId.address);
+                    // Produce and log thread dump for diagnostics
+                    MGMT_OP_LOGGER.threadDump(ThreadDumpUtil.threadDump(""));
                 }
             }
 

controller/src/main/java/org/jboss/as/controller/ThreadDumpUtil.java

@@ -0,0 +1,142 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * Copyright 2011, Red Hat, Inc., and individual contributors
+ * as indicated by the @author tags. See the copyright.txt file in the
+ * distribution for a full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.as.controller;
+
+import org.jboss.as.controller.logging.ControllerLogger;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.management.LockInfo;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+
+public final class ThreadDumpUtil {
+
+   public static String threadDump(final String msg) {
+
+      try (
+         StringWriter str = new StringWriter();
+         PrintWriter out = new PrintWriter(str)
+      ) {
+
+         ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+
+         out.println("*******************************************************************************");
+         out.println("Complete Thread dump " + msg);
+
+         for (ThreadInfo threadInfo : threadMXBean.dumpAllThreads(true, true)) {
+            out.println(threadInfoToString(threadInfo));
+         }
+
+         long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
+
+         if (deadlockedThreads != null && deadlockedThreads.length > 0) {
+            out.println("Deadlock detected!");
+            out.println();
+
+            for (ThreadInfo threadInfo : threadMXBean.getThreadInfo(deadlockedThreads, true, true)) {
+               out.println(threadInfoToString(threadInfo));
+            }
+         }
+
+         out.println("===============================================================================");
+         out.println("End Thread dump " + msg);
+         out.println("*******************************************************************************");
+
+         return str.toString();
+
+      } catch (IOException e) {
+         ControllerLogger.ROOT_LOGGER.error("Exception thrown during generation of thread dump.", e);
+      }
+
+      return "Generating of thread dump failed " + msg;
+
+   }
+
+   private static String threadInfoToString(ThreadInfo threadInfo) {
+      StringBuilder sb = new StringBuilder("\"" + threadInfo.getThreadName() + "\"" +
+            " Id=" + threadInfo.getThreadId() + " " +
+            threadInfo.getThreadState());
+      if (threadInfo.getLockName() != null) {
+         sb.append(" on " + threadInfo.getLockName());
+      }
+      if (threadInfo.getLockOwnerName() != null) {
+         sb.append(" owned by \"" + threadInfo.getLockOwnerName() +
+               "\" Id=" + threadInfo.getLockOwnerId());
+      }
+      if (threadInfo.isSuspended()) {
+         sb.append(" (suspended)");
+      }
+      if (threadInfo.isInNative()) {
+         sb.append(" (in native)");
+      }
+      sb.append('\n');
+      int i = 0;
+      for (; i < threadInfo.getStackTrace().length; i++) {
+         StackTraceElement ste = threadInfo.getStackTrace()[i];
+         sb.append("\tat " + ste.toString());
+         sb.append('\n');
+         if (i == 0 && threadInfo.getLockInfo() != null) {
+            Thread.State ts = threadInfo.getThreadState();
+            switch (ts) {
+               case BLOCKED:
+                  sb.append("\t-  blocked on " + threadInfo.getLockInfo());
+                  sb.append('\n');
+                  break;
+               case WAITING:
+                  sb.append("\t-  waiting on " + threadInfo.getLockInfo());
+                  sb.append('\n');
+                  break;
+               case TIMED_WAITING:
+                  sb.append("\t-  waiting on " + threadInfo.getLockInfo());
+                  sb.append('\n');
+                  break;
+               default:
+            }
+         }
+
+         for (MonitorInfo mi : threadInfo.getLockedMonitors()) {
+            if (mi.getLockedStackDepth() == i) {
+               sb.append("\t-  locked " + mi);
+               sb.append('\n');
+            }
+         }
+      }
+
+      LockInfo[] locks = threadInfo.getLockedSynchronizers();
+      if (locks.length > 0) {
+         sb.append("\n\tNumber of locked synchronizers = " + locks.length);
+         sb.append('\n');
+         for (LockInfo li : locks) {
+            sb.append("\t- " + li);
+            sb.append('\n');
+         }
+      }
+      sb.append('\n');
+      return sb.toString();
+   }
+
+}

controller/src/main/java/org/jboss/as/controller/logging/ControllerLogger.java

@@ -3707,6 +3707,10 @@ OperationFailedRuntimeException capabilityAlreadyRegisteredInContext(String capa
     @Message(id = 495, value = "\"fetch-from-master\" is a deprecated value for \"domain-controller.remote.admin-only-policy\", \"fetch-from-domain-controller\" will be used instead.")
     void adminOnlyPolicyDeprecatedValue();
 
+    @LogMessage(level = WARN)
+    @Message(id = 496, value = "Thread dump: {0}", format = Message.Format.MESSAGE_FORMAT)
+    void threadDump(String manager);
+
     @Message(id = NONE, value = "While constructing a mapping; %s; expected a mapping for merging, but found %s")
     String errorConstructingYAMLMapping(Mark mark, NodeId node);