From b5fbc746115308475100c53c17420234da9a2a10 Mon Sep 17 00:00:00 2001
From: Jack Thomasson <jkt@moonlitsw.com>
Date: Mon, 3 Sep 2012 09:21:54 -0600
Subject: [PATCH] Remove potential deadlock on MythSystem IO threads.

This performs finer grained locking against the file descriptor type map
in the IO handling mechanisms for MythSystem. If the read thread wakes
up and has data to process at the exact moment the external thread is
handing off a newly started child to the manager for handling, the two
may hit fdLock and MythSystemIOHandler::m_pLock from opposite
directions, deadlocking both threads.

Fixes #11066

Signed-off-by: Raymond Wagner <rwagner@mythtv.org>
---
 mythtv/libs/libmythbase/system-unix.cpp    | 6 ++++--
 mythtv/libs/libmythbase/system-windows.cpp | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/mythtv/libs/libmythbase/system-unix.cpp b/mythtv/libs/libmythbase/system-unix.cpp
index eb713d71895..873a6d034bd 100644
--- a/mythtv/libs/libmythbase/system-unix.cpp
+++ b/mythtv/libs/libmythbase/system-unix.cpp
@@ -444,7 +444,6 @@ void MythSystemManager::append(MythSystemUnix *ms)
     m_pMap.insert(ms->m_pid, ms);
     m_mapLock.unlock();
 
-    fdLock.lock();
     if( ms->GetSetting("UseStdin") )
         writeThread->insert(ms->m_stdpipe[0], ms->GetBuffer(0));
 
@@ -453,7 +452,9 @@ void MythSystemManager::append(MythSystemUnix *ms)
         FDType_t *fdType = new FDType_t;
         fdType->ms = ms;
         fdType->type = 1;
+        fdLock.lock();
         fdMap.insert( ms->m_stdpipe[1], fdType );
+        fdLock.unlock();
         readThread->insert(ms->m_stdpipe[1], ms->GetBuffer(1));
     }
 
@@ -462,10 +463,11 @@ void MythSystemManager::append(MythSystemUnix *ms)
         FDType_t *fdType = new FDType_t;
         fdType->ms = ms;
         fdType->type = 2;
+        fdLock.lock();
         fdMap.insert( ms->m_stdpipe[2], fdType );
+        fdLock.unlock();
         readThread->insert(ms->m_stdpipe[2], ms->GetBuffer(2));
     }
-    fdLock.unlock();
 }
 
 void MythSystemManager::jumpAbort(void)
diff --git a/mythtv/libs/libmythbase/system-windows.cpp b/mythtv/libs/libmythbase/system-windows.cpp
index 64adb5b9998..011654379c1 100644
--- a/mythtv/libs/libmythbase/system-windows.cpp
+++ b/mythtv/libs/libmythbase/system-windows.cpp
@@ -402,7 +402,6 @@ void MythSystemManager::append(MythSystemWindows *ms)
     ChildListRebuild();
     m_mapLock.unlock();
 
-    fdLock.lock();
     if( ms->GetSetting("UseStdin") )
         writeThread->insert(ms->m_stdpipe[0], ms->GetBuffer(0));
 
@@ -411,7 +410,9 @@ void MythSystemManager::append(MythSystemWindows *ms)
         FDType_t *fdType = new FDType_t;
         fdType->ms = ms;
         fdType->type = 1;
+        fdLock.lock();
         fdMap.insert( ms->m_stdpipe[1], fdType );
+        fdLock.unlock();
         readThread->insert(ms->m_stdpipe[1], ms->GetBuffer(1));
     }
 
@@ -420,10 +421,11 @@ void MythSystemManager::append(MythSystemWindows *ms)
         FDType_t *fdType = new FDType_t;
         fdType->ms = ms;
         fdType->type = 2;
+        fdLock.lock();
         fdMap.insert( ms->m_stdpipe[2], fdType );
+        fdLock.unlock();
         readThread->insert(ms->m_stdpipe[2], ms->GetBuffer(2));
     }
-    fdLock.unlock();
 }
 
 void MythSystemManager::jumpAbort(void)