Sched:Automatically find deadlocks when assert

arch/Kconfig

@@ -902,6 +902,12 @@ config ARCH_USBDUMP
 	---help---
 		Enable to do USB trace after assertions
 
+config ARCH_DEADLOCKDUMP
+	bool "Dump dead lock thread"
+	default "n"
+	---help---
+		This option will dump the dead lock thread when assert happen..
+
 config ENDIAN_BIG
 	bool "Big Endian Architecture"
 	default n

include/nuttx/sched.h

@@ -1555,6 +1555,23 @@ pid_t nxsched_getpid(void);
 
 pid_t nxsched_getppid(void);
 
+/****************************************************************************
+ * Name: nxsched_collect_deadlock
+ *
+ * Description:
+ *   Check if there is a deadlock and get the thread pid of the deadlock.
+ *
+ * Input parameters:
+ *   pid   - The array to store the thread pid of the deadlock.
+ *   count - The size of the pid array.
+ *
+ * Returned Value:
+ *   The number of thread deadlocks.
+ *
+ ****************************************************************************/
+
+size_t nxsched_collect_deadlock(FAR pid_t *pid, size_t count);
+
 #undef EXTERN
 #if defined(__cplusplus)
 }

libs/libc/misc/lib_mutex.c

@@ -89,7 +89,11 @@ int nxmutex_init(FAR mutex_t *mutex)
     }
 
   mutex->holder = NXMUTEX_NO_HOLDER;
+#ifdef CONFIG_PRIORITY_INHERITANCE
   _SEM_SETPROTOCOL(&mutex->sem, SEM_TYPE_MUTEX | SEM_PRIO_INHERIT);
+#else
+  _SEM_SETPROTOCOL(&mutex->sem, SEM_TYPE_MUTEX);
+#endif
   return ret;
 }
 

libs/libc/semaphore/sem_setprotocol.c

@@ -77,20 +77,21 @@ int nxsem_set_protocol(FAR sem_t *sem, int protocol)
 {
   DEBUGASSERT(sem != NULL);
 
-  switch (protocol)
+  switch (protocol & SEM_PRIO_MASK)
     {
       case SEM_PRIO_NONE:
-        return OK;
+        break;
 
       case SEM_PRIO_INHERIT:
       case SEM_PRIO_PROTECT:
         return -ENOTSUP;
 
       default:
-        break;
+        return -EINVAL;
     }
 
-  return -EINVAL;
+  sem->flags = protocol;
+  return OK;
 }
 
 /****************************************************************************

sched/misc/Make.defs

@@ -20,6 +20,10 @@
 
 CSRCS += assert.c panic_notifier.c reboot_notifier.c
 
+ifeq ($(CONFIG_ARCH_DEADLOCKDUMP),y)
+CSRCS += deadlock.c
+endif
+
 # Include init build support
 
 DEPPATH += --dep-path misc

sched/misc/assert.c

@@ -50,6 +50,8 @@
  * Pre-processor Definitions
  ****************************************************************************/
 
+#define DEADLOCK_MAX 8
+
 #ifndef CONFIG_BOARD_RESET_ON_ASSERT
 #  define CONFIG_BOARD_RESET_ON_ASSERT 0
 #endif
@@ -511,6 +513,31 @@ static void dump_core(pid_t pid)
 }
 #endif
 
+/****************************************************************************
+ * Name: dump_deadlock
+ ****************************************************************************/
+
+#ifdef CONFIG_ARCH_DEADLOCKDUMP
+static void dump_deadlock(void)
+{
+  pid_t deadlock[DEADLOCK_MAX];
+  size_t i = nxsched_collect_deadlock(deadlock, DEADLOCK_MAX);
+
+  if (i > 0)
+    {
+      _alert("Deadlock detected\n");
+      while (i-- > 0)
+        {
+#ifdef CONFIG_SCHED_BACKTRACE
+          sched_dumpstack(deadlock[i]);
+#else
+          _alert("deadlock pid: %d\n", deadlock[i])
+#endif
+        }
+    }
+}
+#endif
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
@@ -594,6 +621,12 @@ void _assert(FAR const char *filename, int linenum,
     {
       show_tasks();
 
+#ifdef CONFIG_ARCH_DEADLOCKDUMP
+      /* Deadlock Dump */
+
+      dump_deadlock();
+#endif
+
 #ifdef CONFIG_ARCH_USBDUMP
       /* Dump USB trace data */
 

sched/misc/deadlock.c

@@ -0,0 +1,115 @@
+/****************************************************************************
+ * sched/misc/deadlock.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/mutex.h>
+#include <nuttx/sched.h>
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+struct deadlock_info_s
+{
+  FAR pid_t *pid;
+  size_t count;
+  size_t found;
+};
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name:find_circular_lock
+ ****************************************************************************/
+
+static void collect_deadlock(FAR struct tcb_s *tcb, FAR void *arg)
+{
+  FAR struct deadlock_info_s *info = arg;
+  size_t index;
+
+  for (index = 0; index < info->found; index++)
+    {
+      if (info->pid[index] == tcb->pid)
+        {
+          return;
+        }
+    }
+
+  for (index = info->found; index < info->count; index++)
+    {
+      FAR sem_t *sem = tcb->waitobj;
+      pid_t next;
+      size_t i;
+
+      if (tcb->task_state != TSTATE_WAIT_SEM || sem == NULL ||
+          !(sem->flags & SEM_TYPE_MUTEX))
+        {
+          return;
+        }
+
+      next = ((FAR mutex_t *)sem)->holder;
+      for (i = info->found; i < index; i++)
+        {
+          if (info->pid[i] == next)
+            {
+              info->found = index;
+              return;
+            }
+        }
+
+      info->pid[index] = tcb->pid;
+      tcb = nxsched_get_tcb(next);
+    }
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: nxsched_collect_deadlock
+ *
+ * Description:
+ *   Check if there is a deadlock and get the thread pid of the deadlock.
+ *
+ * Input parameters:
+ *   pid   - The array to store the thread pid of the deadlock.
+ *   count - The size of the pid array.
+ *
+ * Returned Value:
+ *   The number of thread deadlocks.
+ *
+ ****************************************************************************/
+
+size_t nxsched_collect_deadlock(FAR pid_t *pid, size_t count)
+{
+  struct deadlock_info_s info;
+
+  info.pid = pid;
+  info.count = count;
+  info.found = 0;
+  nxsched_foreach(collect_deadlock, &info);
+  return info.found;
+}