Adds recording quality tracking to DTV recorders.

This ads a recording gap list to RecorderBase to track gaps in the recording.

DTVRecorder updates this with any gaps in video dts values over one second in
duration or if it can't extract the dts then gap in pts values then a gap
is two seconds or greater in duration. In addition we count transport stream
continuity errors and if these are greater than 1% or 0.1% we apply a 40%
and 20% derating on the quality metric respectively.

The NuppelVideoRecorder could update the recording gap list, but would need to
use some other means to detect and measure the length of gaps. For now we just
assume there are no gaps in these recordings.

Once the recording completes we look at the gaps and score them, giving higher
priority to gaps near the start and end of the recording, on the assumption
that these are more information rich, i.e. "Who shot JR?"

If the score doesn't meet a quality threshold the recording is flagged as a
damaged recording and MythTV will try to re-record it when it re-airs.

Note: There are a few thresholds that may need additional tuning, only one
of which is DB tunable, "MinimumRecordingQuality". I'd appreciate
"mythbackend -v record" logs if you see any good recordings marked as bad
or bad recordings marked as good. So I can suggest new values for this
threshold or try other tweaks to the grading algorithm.

mythtv/libs/libmythtv/NuppelVideoRecorder.cpp

@@ -2833,6 +2833,8 @@ void NuppelVideoRecorder::ResetForNewFile(void)
 
     seektable->clear();
 
+    ClearStatistics();
+
     positionMapLock.lock();
     positionMap.clear();
     positionMapDelta.clear();

mythtv/libs/libmythtv/asirecorder.cpp

@@ -79,8 +79,6 @@ void ASIRecorder::run(void)
         return;        
     }
 
-    _continuity_error_count = 0;
-
     {
         QMutexLocker locker(&pauseLock);
         request_recording = true;
@@ -163,10 +161,7 @@ bool ASIRecorder::Open(void)
         return true;
     }
 
-    memset(_stream_id,  0, sizeof(_stream_id));
-    memset(_pid_status, 0, sizeof(_pid_status));
-    memset(_continuity_counter, 0xff, sizeof(_continuity_counter));
-    _continuity_error_count = 0;
+    ResetForNewFile();
 
     m_stream_handler = ASIStreamHandler::Get(m_channel->GetDevice());
 

mythtv/libs/libmythtv/cetonrecorder.cpp

@@ -27,9 +27,7 @@ bool CetonRecorder::Open(void)
         return true;
     }
 
-    memset(_stream_id,  0, sizeof(_stream_id));
-    memset(_pid_status, 0, sizeof(_pid_status));
-    memset(_continuity_counter, 0xff, sizeof(_continuity_counter));
+    ResetForNewFile();
 
     _stream_handler = CetonStreamHandler::Get(_channel->GetDevice());
 
@@ -60,8 +58,6 @@ void CetonRecorder::run(void)
         return;
     }
 
-    _continuity_error_count = 0;
-
     {
         QMutexLocker locker(&pauseLock);
         request_recording = true;

mythtv/libs/libmythtv/dtvrecorder.cpp

@@ -73,15 +73,14 @@ DTVRecorder::DTVRecorder(TVRec *rec) :
     _input_pmt(NULL),
     _has_no_av(false),
     // statistics
+    _use_pts(false),
     _packet_count(0),
     _continuity_error_count(0),
     _frames_seen_count(0),          _frames_written_count(0)
 {
     SetPositionMapType(MARK_GOP_BYFRAME);
     _payload_buffer.reserve(TSPacket::kSize * (50 + 1));
-    memset(_stream_id, 0, sizeof(_stream_id));
-    memset(_pid_status, 0, sizeof(_pid_status));
-    memset(_continuity_counter, 0, sizeof(_continuity_counter));
+    ResetForNewFile();
 }
 
 DTVRecorder::~DTVRecorder()
@@ -187,15 +186,30 @@ void DTVRecorder::ResetForNewFile(void)
     memset(_pid_status, 0, sizeof(_pid_status));
     memset(_continuity_counter, 0xff, sizeof(_continuity_counter));
 
-    _packet_count               = 0;
-    _continuity_error_count     = 0;
-    _frames_seen_count          = 0;
-    _frames_written_count       = 0;
     _pes_synced                 = false;
     //_seen_sps
     positionMap.clear();
     positionMapDelta.clear();
     _payload_buffer.clear();
+
+    locker.unlock();
+    ClearStatistics();
+}
+
+void DTVRecorder::ClearStatistics(void)
+{
+    RecorderBase::ClearStatistics();
+
+    memset(_ts_count, 0, sizeof(_ts_count));
+    for (int i = 0; i < 256; i++)
+        _ts_last[i] = -1LL;
+    for (int i = 0; i < 256; i++)
+        _ts_first[i] = -1LL;
+    //_ts_first_dt -- doesn't need to be cleared only used if _ts_first>=0
+    _packet_count.fetchAndStoreRelaxed(0);
+    _continuity_error_count.fetchAndStoreRelaxed(0);
+    _frames_seen_count          = 0;
+    _frames_written_count       = 0;
 }
 
 // documented in recorderbase.h
@@ -248,6 +262,19 @@ void DTVRecorder::BufferedWrite(const TSPacket &tspacket)
     if (_wait_for_keyframe_option && _first_keyframe<0)
         return;
 
+    if (!timeOfFirstData.isValid() && curRecording)
+    {
+        QMutexLocker locker(&statisticsLock);
+        timeOfFirstData = mythCurrentDateTime();
+    }
+
+    uint64_t now = mythCurrentDateTime().toTime_t();
+    if (!timeOfLatestData.isValid() || (now - timeOfLatestData.toTime_t() >= 5))
+    {
+        QMutexLocker locker(&statisticsLock);
+        timeOfLatestData = mythCurrentDateTime();
+    }
+
     // Do we have to buffer the packet for exact keyframe detection?
     if (_buffer_packets)
     {
@@ -270,6 +297,44 @@ void DTVRecorder::BufferedWrite(const TSPacket &tspacket)
         ringBuffer->Write(tspacket.data(), TSPacket::kSize);
 }
 
+enum { kExtractPTS, kExtractDTS };
+static int64_t extract_timestamp(
+    const uint8_t *bufptr, int bytes_left, int pts_or_dts)
+{
+    if (bytes_left < 4)
+        return -1LL;
+
+    bool has_pts = bufptr[3] & 0x80;
+    int offset = 5;
+    if (((kExtractPTS == pts_or_dts) && !has_pts) || (offset + 5 > bytes_left))
+        return -1LL;
+
+    bool has_dts = bufptr[3] & 0x40;
+    if (kExtractDTS == pts_or_dts)
+    {
+        if (!has_dts)
+            return -1LL;
+        offset += has_pts ? 5 : 0;
+        if (offset + 5 > bytes_left)
+            return -1LL;
+    }
+
+    return ((uint64_t(bufptr[offset+0] & 0x0e) << 29) |
+            (uint64_t(bufptr[offset+1]       ) << 22) |
+            (uint64_t(bufptr[offset+2] & 0xfe) << 14) |
+            (uint64_t(bufptr[offset+3]       ) <<  7) |
+            (uint64_t(bufptr[offset+4] & 0xfe) >>  1));
+}
+
+static QDateTime ts_to_qdatetime(
+    uint64_t pts, uint64_t pts_first, QDateTime &pts_first_dt)
+{
+    if (pts < pts_first)
+        pts += 0x1FFFFFFFF;
+    QDateTime dt = pts_first_dt;
+    return dt.addMSecs((pts - pts_first)/90);
+}
+
 static const uint frameRateMap[16] = {
     0, 23796, 24000, 25000, 29970, 30000, 50000, 59940, 60000, 
     0, 0, 0, 0, 0, 0, 0 
@@ -412,6 +477,15 @@ bool DTVRecorder::FindMPEG2Keyframes(const TSPacket* tspacket)
                     }
                 }
             }
+            if ((stream_id >= PESStreamID::MPEGVideoStreamBegin) &&
+                (stream_id <= PESStreamID::MPEGVideoStreamEnd))
+            {
+                int64_t pts = extract_timestamp(
+                    bufptr, bytes_left, kExtractPTS);
+                int64_t dts = extract_timestamp(
+                    bufptr, bytes_left, kExtractPTS);
+                HandleTimestamps(stream_id, pts, dts);
+            }
         }
     }
 
@@ -463,6 +537,72 @@ bool DTVRecorder::FindMPEG2Keyframes(const TSPacket* tspacket)
     return hasKeyFrame || (_payload_buffer.size() >= (188*50));
 }
 
+void DTVRecorder::HandleTimestamps(int stream_id, int64_t pts, int64_t dts)
+{
+    if (pts < 0)
+    {
+        _ts_last[stream_id] = -1;
+        return;
+    }
+
+    if ((dts < 0) && !_use_pts)
+    {
+        _ts_last[stream_id] = -1;
+        _use_pts = true;
+        LOG(VB_RECORD, LOG_DEBUG,
+            "Switching from dts tracking to pts tracking." +
+            QString("TS count is %1").arg(_ts_count[stream_id]));
+    }
+
+    int64_t ts = dts;
+    int64_t gap_threshold = 90000; // 1 second
+    if (_use_pts)
+    {
+        ts = dts;
+        gap_threshold = 2*90000; // two seconds, compensate for GOP ordering
+    }
+
+    if (_ts_last[stream_id] >= 0)
+    {
+        int64_t diff = ts - _ts_last[stream_id];
+        if ((diff < 0) && (diff < (10 * -90000)))
+            diff += 0x1ffffffffLL;
+        if (diff < 0)
+            diff = -diff;
+        if (diff > gap_threshold)
+        {
+            QMutexLocker locker(&statisticsLock);
+            recordingGaps.push_back(
+                RecordingGap(
+                    ts_to_qdatetime(
+                        _ts_last[stream_id], _ts_first[stream_id],
+                        _ts_first_dt[stream_id]),
+                    ts_to_qdatetime(
+                        ts, _ts_first[stream_id], _ts_first_dt[stream_id])));
+            LOG(VB_RECORD, LOG_DEBUG, LOC + QString("Inserted gap %1 dur %2")
+                .arg(recordingGaps.back().toString()).arg(diff/90000.0));
+        }
+    }
+
+    _ts_last[stream_id] = ts;
+
+    if (_ts_count[stream_id] < 30)
+    {
+        if (!_ts_count[stream_id])
+        {
+            _ts_first[stream_id] = ts;
+            _ts_first_dt[stream_id] = mythCurrentDateTime();
+        }
+        else if (ts < _ts_first[stream_id])
+        {
+            _ts_first[stream_id] = ts;
+            _ts_first_dt[stream_id] = mythCurrentDateTime();
+        }
+    }
+
+    _ts_count[stream_id]++;
+}
+
 bool DTVRecorder::FindAudioKeyframes(const TSPacket*)
 {
     bool hasKeyFrame = false;
@@ -1070,14 +1210,14 @@ bool DTVRecorder::ProcessTSPacket(const TSPacket &tspacket)
     const uint pid = tspacket.PID();
 
     if (pid != 0x1fff)
-        _packet_count++;
+        _packet_count.fetchAndAddAcquire(1);
 
     // Check continuity counter
     uint old_cnt = _continuity_counter[pid];
     if ((pid != 0x1fff) && !CheckCC(pid, tspacket.ContinuityCounter()))
     {
-        _continuity_error_count++;
-        double erate = _continuity_error_count * 100.0 / _packet_count;
+        int v = _continuity_error_count.fetchAndAddRelaxed(1) + 1;
+        double erate = v * 100.0 / _packet_count.fetchAndAddRelaxed(0);
         LOG(VB_RECORD, LOG_WARNING, LOC +
             QString("PID 0x%1 discontinuity detected ((%2+1)%16!=%3) %4\%")
                 .arg(pid,0,16).arg(old_cnt,2)
@@ -1142,14 +1282,14 @@ bool DTVRecorder::ProcessAVTSPacket(const TSPacket &tspacket)
     const uint pid = tspacket.PID();
 
     if (pid != 0x1fff)
-        _packet_count++;
+        _packet_count.fetchAndAddAcquire(1);
 
     // Check continuity counter
     uint old_cnt = _continuity_counter[pid];
     if ((pid != 0x1fff) && !CheckCC(pid, tspacket.ContinuityCounter()))
     {
-        _continuity_error_count++;
-        double erate = _continuity_error_count * 100.0 / _packet_count;
+        int v = _continuity_error_count.fetchAndAddRelaxed(1) + 1;
+        double erate = v * 100.0 / _packet_count.fetchAndAddRelaxed(0);
         LOG(VB_RECORD, LOG_WARNING, LOC +
             QString("A/V PID 0x%1 discontinuity detected ((%2+1)%16!=%3) %4\%")
                 .arg(pid,0,16).arg(old_cnt).arg(tspacket.ContinuityCounter())
@@ -1178,4 +1318,13 @@ bool DTVRecorder::ProcessAVTSPacket(const TSPacket &tspacket)
     return true;
 }
 
+RecordingQuality *DTVRecorder::GetRecordingQuality(void) const
+{
+    RecordingQuality *recq = RecorderBase::GetRecordingQuality();
+    recq->AddTSStatistics(
+        _continuity_error_count.fetchAndAddRelaxed(0),
+        _packet_count.fetchAndAddRelaxed(0));
+    return recq;
+}
+
 /* vim: set expandtab tabstop=4 shiftwidth=4: */

mythtv/libs/libmythtv/dtvrecorder.h

@@ -12,6 +12,7 @@
 #include <vector>
 using namespace std;
 
+#include <QAtomicInt>
 #include <QString>
 
 #include "streamlisteners.h"
@@ -54,7 +55,9 @@ class DTVRecorder :
     void SetStreamData(MPEGStreamData* sd);
     MPEGStreamData *GetStreamData(void) const { return _stream_data; }
 
-    virtual void Reset();
+    virtual void Reset(void);
+    virtual void ClearStatistics(void);
+    virtual RecordingQuality *GetRecordingQuality(void) const;
 
     // MPEG Stream Listener
     void HandlePAT(const ProgramAssociationTable*);
@@ -91,6 +94,7 @@ class DTVRecorder :
     void ResetForNewFile(void);
 
     void HandleKeyframe(uint64_t frameNum, int64_t extra = 0);
+    void HandleTimestamps(int stream_id, int64_t pts, int64_t dts);
 
     void BufferedWrite(const TSPacket &tspacket);
 
@@ -165,8 +169,13 @@ class DTVRecorder :
     vector<TSPacket> _scratch;
 
     // Statistics
-    mutable unsigned long long _packet_count;
-    mutable unsigned long long _continuity_error_count;
+    bool          _use_pts; // vs use dts
+    uint64_t      _ts_count[256];
+    int64_t       _ts_last[256];
+    int64_t       _ts_first[256];
+    QDateTime     _ts_first_dt[256];
+    mutable QAtomicInt _packet_count;
+    mutable QAtomicInt _continuity_error_count;
     unsigned long long _frames_seen_count;
     unsigned long long _frames_written_count;
 

mythtv/libs/libmythtv/dvbrecorder.cpp

@@ -49,9 +49,7 @@ bool DVBRecorder::Open(void)
     if (videodevice.isEmpty())
         return false;
 
-    memset(_stream_id,  0, sizeof(_stream_id));
-    memset(_pid_status, 0, sizeof(_pid_status));
-    memset(_continuity_counter, 0xff, sizeof(_continuity_counter));
+    ResetForNewFile();
 
     _stream_handler = DVBStreamHandler::Get(videodevice);
 
@@ -83,8 +81,6 @@ void DVBRecorder::run(void)
         return;
     }
 
-    _continuity_error_count = 0;
-
     {
         QMutexLocker locker(&pauseLock);
         request_recording = true;

mythtv/libs/libmythtv/firewirerecorder.cpp

@@ -28,8 +28,10 @@ FirewireRecorder::~FirewireRecorder()
 bool FirewireRecorder::Open(void)
 {
     if (!isopen)
+    {
         isopen = channel->GetFirewireDevice()->OpenPort();
-
+        ResetForNewFile();
+    }
     return isopen;
 }
 
@@ -63,8 +65,6 @@ void FirewireRecorder::run(void)
         return;
     }
 
-    _continuity_error_count = 0;
-
     {
         QMutexLocker locker(&pauseLock);
         request_recording = true;