Protect the reschedule queue with a seperate lock to fix a backend de…

…adlock.

 Fixes a deadlock in mythbackend caused when a reschedule request occurs
 immedately prior to a recording starting.

 The recorder thread locks the scheduler when starting a recording. The pending
 reschedule request handled in the event loop waits for the scheduler lock to be
 released blocking the event loop. The backend is now stopped from handling any
 further events or protocol requests, meanwhile the recorder is waiting on
 events to be processed before it will release the lock resulting in an
 irrecoverable deadlock (or hang) and the failure of the recording.

 This deadlock was more likely to be triggered for users of EIT because it will
 ask for a reschedule every 5 minutes in addition to any reschedules triggered
 through other means.

 Refs #10771

mythtv/programs/mythbackend/scheduler.cpp

@@ -57,6 +57,7 @@ Scheduler::Scheduler(bool runthread, QMap<int, EncoderLink *> *tvList,
     recordTable(tmptable),
     priorityTable("powerpriority"),
     schedLock(),
+    m_queueLock(),
     reclist_changed(false),
     specsched(master_sched),
     schedMoveHigher(false),
@@ -593,7 +594,9 @@ void Scheduler::UpdateRecStatus(RecordingInfo *pginfo)
                 p->AddHistory(false);
                 if (resched)
                 {
+                    m_queueLock.lock();
                     reschedQueue.enqueue(0);
+                    m_queueLock.unlock();
                     reschedWait.wakeOne();
                 }
                 else
@@ -643,7 +646,9 @@ void Scheduler::UpdateRecStatus(uint cardid, uint chanid,
                 p->AddHistory(false);
                 if (resched)
                 {
+                    m_queueLock.lock();
                     reschedQueue.enqueue(0);
+                    m_queueLock.unlock();
                     reschedWait.wakeOne();
                 }
                 else
@@ -1615,7 +1620,7 @@ void Scheduler::GetAllScheduled(QStringList &strList)
 
 void Scheduler::Reschedule(int recordid)
 {
-    QMutexLocker locker(&schedLock);
+    QMutexLocker locker(&m_queueLock);
 
     if (recordid == -1)
         reschedQueue.clear();
@@ -1661,7 +1666,9 @@ void Scheduler::AddRecording(const RecordingInfo &pi)
     new_pi->GetRecordingRule();
 
     // Trigger reschedule..
+    m_queueLock.lock();
     reschedQueue.enqueue(pi.GetRecordingRuleID());
+    m_queueLock.unlock();
     reschedWait.wakeOne();
 }
 
@@ -1765,11 +1772,13 @@ void Scheduler::run(void)
     // wait for slaves to connect
     sleep(3);
 
-    QMutexLocker lockit(&schedLock);
-
-    reschedQueue.clear();
+    ClearRequestQueue();
+    m_queueLock.lock();
     reschedQueue.enqueue(-1);
+    m_queueLock.unlock();
 
+    QMutexLocker lockit(&schedLock);
+    
     int       prerollseconds  = 0;
     int       wakeThreshold   = 300;
     int       idleTimeoutSecs = 0;
@@ -1807,7 +1816,7 @@ void Scheduler::run(void)
         }
         else
         {
-            if (reschedQueue.empty())
+            if (!HaveQueuedRequests())
             {
                 int sched_sleep = (secs_to_next - schedRunTime - 1) * 1000;
                 sched_sleep = min(sched_sleep, maxSleep);
@@ -1822,7 +1831,7 @@ void Scheduler::run(void)
             }
 
             QTime t; t.start();
-            if (!reschedQueue.empty() && HandleReschedule())
+            if (HaveQueuedRequests() && HandleReschedule())
             {
                 statuschanged = true;
                 startIter = reclist.begin();
@@ -2003,17 +2012,19 @@ bool Scheduler::HandleReschedule(void)
     QString msg;
     bool deleteFuture = false;
 
-    while (!reschedQueue.empty())
+    while (HaveQueuedRequests())
     {
+        m_queueLock.lock();
         int recordid = reschedQueue.dequeue();
+        m_queueLock.unlock();
 
         LOG(VB_GENERAL, LOG_INFO, QString("Reschedule requested for id %1.")
                 .arg(recordid));
 
         if (recordid != 0)
         {
             if (recordid == -1)
-                reschedQueue.clear();
+                ClearRequestQueue();
 
             deleteFuture = true;
             schedLock.unlock();
@@ -2056,7 +2067,9 @@ bool Scheduler::HandleReschedule(void)
     else
     {
         LOG(VB_GENERAL, LOG_INFO, "Reschedule interrupted, will retry");
+        m_queueLock.lock();
         reschedQueue.enqueue(0);
+        m_queueLock.unlock();
         return false;
     }
 
@@ -2216,7 +2229,11 @@ void Scheduler::HandleWakeSlave(RecordingInfo &ri, int prerollseconds)
                 .arg(nexttv->GetHostName()).arg(ri.GetTitle()));
 
         if (!WakeUpSlave(nexttv->GetHostName()))
+        {
+            m_queueLock.lock();
             reschedQueue.enqueue(0);
+            m_queueLock.unlock();
+        }
     }
     else if ((nexttv->IsWaking()) &&
              ((secsleft - prerollseconds) < 210) &&
@@ -2229,7 +2246,11 @@ void Scheduler::HandleWakeSlave(RecordingInfo &ri, int prerollseconds)
                 .arg(nexttv->GetHostName()));
 
         if (!WakeUpSlave(nexttv->GetHostName(), false))
+        {
+            m_queueLock.lock();
             reschedQueue.enqueue(0);
+            m_queueLock.unlock();
+        }
     }
     else if ((nexttv->IsWaking()) &&
              ((secsleft - prerollseconds) < 150) &&
@@ -2249,7 +2270,9 @@ void Scheduler::HandleWakeSlave(RecordingInfo &ri, int prerollseconds)
                 (*it)->SetSleepStatus(sStatus_Undefined);
         }
 
+        m_queueLock.lock();
         reschedQueue.enqueue(0);
+        m_queueLock.unlock();
     }
 }
 
@@ -2287,7 +2310,9 @@ bool Scheduler::HandleRecording(
             livetvTime = (livetvTime < nextrectime) ?
                 nextrectime : livetvTime;
 
+            m_queueLock.lock();
             reschedQueue.enqueue(0);
+            m_queueLock.unlock();
         }
     }
 
@@ -2370,7 +2395,9 @@ bool Scheduler::HandleRecording(
                     enc->SetSleepStatus(sStatus_Undefined);
             }
 
+            m_queueLock.lock();
             reschedQueue.enqueue(0);
+            m_queueLock.unlock();
         }
 
         return false;

mythtv/programs/mythbackend/scheduler.h

@@ -165,10 +165,16 @@ class Scheduler : public MThread, public MythScheduler
         bool &blockShutdown, QDateTime &idleSince, int prerollseconds,
         int idleTimeoutSecs, int idleWaitForRecordingTime,
         bool &statuschanged);
+    
+    bool HaveQueuedRequests(void)
+    {  QMutexLocker locker(&m_queueLock); return !reschedQueue.empty(); };
+    void ClearRequestQueue(void)
+    {  QMutexLocker locker(&m_queueLock); reschedQueue.clear(); };
 
 
     MythDeque<int> reschedQueue;
     mutable QMutex schedLock;
+    mutable QMutex m_queueLock;
     QMutex recordmatchLock;
     QWaitCondition reschedWait;
     RecList reclist;