Multidisc issues dev

tools/ld-diffdod/diffdod.cpp

@@ -30,7 +30,8 @@ Diffdod::Diffdod(QObject *parent) : QObject(parent)
 }
 
 bool Diffdod::process(QVector<QString> inputFilenames, bool reverse,
-                      qint32 dodThreshold, bool noLumaClip)
+                      qint32 dodThreshold, bool noLumaClip,
+                      qint32 startVbi, qint32 lengthVbi)
 {
     // Show input filenames
     qInfo() << "Processing" << inputFilenames.size() << "input TBC files:";
@@ -52,8 +53,19 @@ bool Diffdod::process(QVector<QString> inputFilenames, bool reverse,
     qInfo() << "Sources have VBI frame number range of" << tbcSources.getMinimumVbiFrameNumber() << "to" << tbcSources.getMaximumVbiFrameNumber();
 
     // Check start and length
-    qint32 vbiStartFrame = tbcSources.getMinimumVbiFrameNumber();
-    qint32 length = tbcSources.getMaximumVbiFrameNumber() - tbcSources.getMinimumVbiFrameNumber() + 1;
+    qint32 vbiStartFrame = startVbi;
+    if (vbiStartFrame < tbcSources.getMinimumVbiFrameNumber())
+        vbiStartFrame = tbcSources.getMinimumVbiFrameNumber();
+
+    qint32 length = lengthVbi;
+    if (length > (tbcSources.getMaximumVbiFrameNumber() - vbiStartFrame + 1))
+        length = tbcSources.getMaximumVbiFrameNumber() - vbiStartFrame + 1;
+    if (length == -1) length = tbcSources.getMaximumVbiFrameNumber() - tbcSources.getMinimumVbiFrameNumber() + 1;
+
+    // Verify frame source availablity
+    qInfo() << "";
+    qInfo() << "Verifying VBI frame multi-source availablity:";
+    tbcSources.verifySources(vbiStartFrame, length);
 
     qInfo() << "Processing" << length << "frames starting from VBI frame" << vbiStartFrame;
     if (!tbcSources.saveSources(vbiStartFrame, length, dodThreshold, noLumaClip)) {

tools/ld-diffdod/diffdod.h

@@ -38,7 +38,7 @@ class Diffdod : public QObject
     explicit Diffdod(QObject *parent = nullptr);
 
     bool process(QVector<QString> inputFilenames, bool reverse,
-                 qint32 dodThreshold, bool noLumaClip);
+                 qint32 dodThreshold, bool noLumaClip, qint32 startVbi, qint32 lengthVbi);
 
 private:
     TbcSources tbcSources;

tools/ld-diffdod/main.cpp

@@ -68,13 +68,24 @@ int main(int argc, char *argv[])
                                        QCoreApplication::translate("main", "Do not perform luma clip dropout detection"));
     parser.addOption(setNoLumaOption);
 
-
-    // Option to select DOD threshold (dod-threshold) (-x)
+    // Option to select DOD threshold (-x / --dod-threshold)
     QCommandLineOption dodThresholdOption(QStringList() << "x" << "dod-threshold",
-                                        QCoreApplication::translate("main", "Specify the DOD threshold (100-65435 default: 700"),
+                                        QCoreApplication::translate("main", "Specify the DOD threshold (100-65435 default: 1200"),
                                         QCoreApplication::translate("main", "number"));
     parser.addOption(dodThresholdOption);
 
+    // Option to select the start VBI frame (-s / --start)
+    QCommandLineOption startVbiOption(QStringList() << "s" << "start",
+                                        QCoreApplication::translate("main", "Specify the start VBI frame"),
+                                        QCoreApplication::translate("main", "number"));
+    parser.addOption(startVbiOption);
+
+    // Option to select the maximum number of VBI frames to process (-l / --length)
+    QCommandLineOption lengthVbiOption(QStringList() << "l" << "length",
+                                        QCoreApplication::translate("main", "Specify the maximum number of VBI frames to process"),
+                                        QCoreApplication::translate("main", "number"));
+    parser.addOption(lengthVbiOption);
+
     // Positional argument to specify input TBC files
     parser.addPositionalArgument("input", QCoreApplication::translate("main", "Specify input TBC files (minimum of 3)"));
 
@@ -108,8 +119,7 @@ int main(int argc, char *argv[])
         return -1;
     }
 
-    qint32 dodThreshold = 700;
-
+    qint32 dodThreshold = 1200;
     if (parser.isSet(dodThresholdOption)) {
         dodThreshold = parser.value(dodThresholdOption).toInt();
 
@@ -120,9 +130,31 @@ int main(int argc, char *argv[])
         }
     }
 
+    qint32 vbiFrameStart = 0;
+    if (parser.isSet(startVbiOption)) {
+        vbiFrameStart = parser.value(startVbiOption).toInt();
+
+        if (vbiFrameStart < 1 || vbiFrameStart > 160000) {
+            // Quit with error
+            qCritical("Start VBI frame must be between 1 and 160000");
+            return -1;
+        }
+    }
+
+    qint32 vbiFrameLength = -1;
+    if (parser.isSet(lengthVbiOption)) {
+        vbiFrameLength = parser.value(lengthVbiOption).toInt();
+
+        if (vbiFrameLength < 1 || vbiFrameLength > 160000) {
+            // Quit with error
+            qCritical("VBI frame length must be between 1 and 160000");
+            return -1;
+        }
+    }
+
     // Process the TBC file
     Diffdod diffdod;
-    if (!diffdod.process(inputFilenames, reverse, dodThreshold, noLumaClip)) {
+    if (!diffdod.process(inputFilenames, reverse, dodThreshold, noLumaClip, vbiFrameStart, vbiFrameLength)) {
         return 1;
     }
 

tools/ld-diffdod/tbcsources.cpp

@@ -196,6 +196,32 @@ qint32 TbcSources::getMaximumVbiFrameNumber()
     return maximumFrameNumber;
 }
 
+// Verify that at least 3 sources are available for every VBI frame
+void TbcSources::verifySources(qint32 vbiStartFrame, qint32 length)
+{
+    qint32 uncorrectableFrameCount = 0;
+
+    // Process the sources frame by frame
+    for (qint32 vbiFrame = vbiStartFrame; vbiFrame < vbiStartFrame + length; vbiFrame++) {
+        // Check how many source frames are available for the current frame
+        QVector<qint32> availableSourcesForFrame = getAvailableSourcesForFrame(vbiFrame);
+
+        // DiffDOD requires at least three source frames.  If 3 sources are not available leave any existing DO records in place
+        // and output a warning to the user
+        if (availableSourcesForFrame.size() < 3) {
+            // Differential DOD requires at least 3 valid source frames
+            qInfo().nospace() << "Frame #" << vbiFrame << " has only " << availableSourcesForFrame.size() << " source frames available - cannot correct";
+            uncorrectableFrameCount++;
+        }
+    }
+
+    if (uncorrectableFrameCount != 0) {
+        qInfo() << "Warning:" << uncorrectableFrameCount << "frame(s) cannot be corrected!";
+    } else {
+        qInfo() << "All frames have at least 3 sources available";
+    }
+}
+
 // Private methods ----------------------------------------------------------------------------------------------------
 
 // Perform differential dropout detection to determine (for each source) which frame pixels are valid
@@ -427,8 +453,8 @@ void TbcSources::performFrameDiffDod(qint32 targetVbiFrame, qint32 dodThreshold,
     // Write the first and second field line metadata back to the source
     for (qint32 sourceNo = 0; sourceNo < availableSourcesForFrame.size(); sourceNo++) {
         // Get the required field numbers
-        qint32 firstFieldNumber = sourceVideos[availableSourcesForFrame[sourceNo]]->ldDecodeMetaData.getFirstFieldNumber(convertVbiFrameNumberToSequential(targetVbiFrame, sourceNo));
-        qint32 secondFieldNumber = sourceVideos[availableSourcesForFrame[sourceNo]]->ldDecodeMetaData.getSecondFieldNumber(convertVbiFrameNumberToSequential(targetVbiFrame, sourceNo));
+        qint32 firstFieldNumber = sourceVideos[availableSourcesForFrame[sourceNo]]->ldDecodeMetaData.getFirstFieldNumber(convertVbiFrameNumberToSequential(targetVbiFrame, availableSourcesForFrame[sourceNo]));
+        qint32 secondFieldNumber = sourceVideos[availableSourcesForFrame[sourceNo]]->ldDecodeMetaData.getSecondFieldNumber(convertVbiFrameNumberToSequential(targetVbiFrame, availableSourcesForFrame[sourceNo]));
 
         // Calculate the total number of dropouts detected for the frame
         qint32 totalFirstDropouts = frameDropouts[sourceNo].firstFieldDropOuts.startx.size();

tools/ld-diffdod/tbcsources.h

@@ -49,6 +49,7 @@ class TbcSources : public QObject
     qint32 getNumberOfAvailableSources();
     qint32 getMinimumVbiFrameNumber();
     qint32 getMaximumVbiFrameNumber();
+    void verifySources(qint32 vbiStartFrame, qint32 length);
 
 private:
     // Source definition

tools/ld-discmap/vbimapper.cpp

@@ -40,6 +40,7 @@ bool VbiMapper::create(LdDecodeMetaData &ldDecodeMetaData)
     if (!discCheck(ldDecodeMetaData)) return false;
     if (!createInitialMap(ldDecodeMetaData)) return false;
     correctFrameNumbering();
+    removeCorruptFrames();
     removeDuplicateFrames();
     detectMissingFrames();
 
@@ -297,9 +298,10 @@ void VbiMapper::correctFrameNumbering()
     qInfo() << "";
     qInfo() << "Performing frame number correction...";
 
-    qint32 correctedFrameNumber;
+    qint32 correctedFrameNumber = -1;
     qint32 frameNumberErrorCount = 0;
     qint32 frameMissingFrameNumberCount = 0;
+    qint32 frameNumberCorruptCount = 0;
     qint32 searchDistance = 5;
 
     // Set the maximum frame number limit
@@ -319,6 +321,9 @@ void VbiMapper::correctFrameNumbering()
 
             // Set the frame number to a sane value ready for correction
             frames[frameElement].vbiFrameNumber = frames[frameElement - 1].vbiFrameNumber + 1;
+            frames[frameElement].isCorruptVbi = true; // Flag that the VBI should be rewritten
+            qDebug() << "Seq. frame" << frameElement << "has a VBI frame number of -1 - Setting to" <<
+                       frames[frameElement].vbiFrameNumber << "before correction";
 
             isMissing = true;
         }
@@ -329,6 +334,11 @@ void VbiMapper::correctFrameNumbering()
         // Try up to a distance of 'searchDistance' frames to find the sequence
         for (qint32 gap = 1; gap < searchDistance; gap++) {
             if (frames[frameElement].vbiFrameNumber != (frames[frameElement - 1].vbiFrameNumber + 1)) {
+                // Is the previous frame invalid?
+                if (frames[frameElement - 1].vbiFrameNumber == -1) {
+                    qInfo() << "Previous frame number is invalid - cannot correct, skipping";
+                    break;
+                }
 
                 // Did the player stall and repeat the last frame?
                 if (frames[frameElement - 1].vbiFrameNumber == frames[frameElement].vbiFrameNumber) {
@@ -358,12 +368,21 @@ void VbiMapper::correctFrameNumbering()
 
                         // Update the frame number
                         frames[frameElement].vbiFrameNumber = correctedFrameNumber;
-                        frames[frameElement].isCorruptVbi = true;
+                        frames[frameElement].isCorruptVbi = true; // Flag that the VBI should be rewritten
 
                         frameNumberErrorCount++;
                         break; // done
                     } else {
-                        qDebug() << "VbiMapper::correctFrameNumbering(): No match found with gap" << gap;
+                        if (gap == searchDistance - 1) {
+                            qDebug() << "VbiMapper::correctFrameNumbering(): Search distance reached with no match found - previous" <<
+                                        frames[frameElement - 1].vbiFrameNumber << "current" << frames[frameElement + gap].vbiFrameNumber <<
+                                        "target" << frames[frameElement].vbiFrameNumber;
+
+                            // Set the VBI as invalid
+                            frames[frameElement].isMarkedForDeletion = true; // Flag that the frame should be removed
+                            frameNumberCorruptCount++;
+                            frameNumberErrorCount++;
+                        }
                     }
                 }
             } else {
@@ -375,20 +394,43 @@ void VbiMapper::correctFrameNumbering()
                     qInfo() << "    VBI frame number corrected to" << frames[frameElement].vbiFrameNumber;
                     frameNumberErrorCount++;
                     isMissing = false;
+                    frames[frameElement].isCorruptVbi = true; // Flag that the VBI should be rewritten
                 }
             }
         }
     }
 
-    // All frame numbers are now checked and corrected except the first frame; so we do that here
-    // since the second frame should have been corrected already
+    // All frame numbers are now checked and corrected except the first frame and last frame; so we do that here
+    // since the second frame, and second from last frame should have been corrected already
     if (frames[0].vbiFrameNumber != frames[1].vbiFrameNumber - 1) {
-        qInfo() << "The first frame does not have a valid frame number; correcting based on second frame...";
+        qInfo().nospace() << "The first frame does not have a valid frame number (" << frames[0].vbiFrameNumber <<
+                             ") correcting to " << frames[1].vbiFrameNumber - 1 << " based on second frame VBI";
         frames[0].vbiFrameNumber = frames[1].vbiFrameNumber - 1;
+        frames[0].isCorruptVbi = true;
+        frameNumberErrorCount++;
+    }
+
+    if (frames[frames.size() - 1].vbiFrameNumber != frames[frames.size() - 2].vbiFrameNumber + 1) {
+        qInfo().nospace() << "The last frame does not have a valid frame number (" << frames[frames.size() - 1].vbiFrameNumber <<
+                             ") correcting to " << frames[frames.size() - 2].vbiFrameNumber + 1 << " based on second from last frame VBI";
+        frames[frames.size() - 1].vbiFrameNumber = frames[frames.size() - 2].vbiFrameNumber + 1;
+        frames[frames.size() - 1].isCorruptVbi = true; // Flag that the VBI should be rewritten
+        frameNumberErrorCount++;
     }
 
     qInfo() << "Found and corrected" << frameNumberErrorCount << "bad/missing VBI frame numbers (of which" <<
-                frameMissingFrameNumberCount << "had no frame number set in the VBI)";
+                frameMissingFrameNumberCount << "had no frame number set in the VBI and" << frameNumberCorruptCount << "were unrecoverable)";
+}
+
+void VbiMapper::removeCorruptFrames()
+{
+    qInfo() << "";
+    qInfo() << "Removing frames with unrecoverable corrupt VBI...";
+
+    // Remove all frames marked for deletion from the map
+    qint32 previousSize = frames.size();
+    frames.erase(std::remove_if(frames.begin(), frames.end(), [](const Frame& f) {return f.isMarkedForDeletion == true;}), frames.end());
+    qInfo() << "Removed" << previousSize - frames.size() << "corrupt VBI frames from the map -" << frames.size() << "sequential frames remaining.";
 }
 
 void VbiMapper::removeDuplicateFrames()

tools/ld-discmap/vbimapper.h

@@ -84,6 +84,7 @@ public slots:
     bool discCheck(LdDecodeMetaData &ldDecodeMetaData);
     bool createInitialMap(LdDecodeMetaData &ldDecodeMetaData);
     void correctFrameNumbering();
+    void removeCorruptFrames();
     void removeDuplicateFrames();
     void detectMissingFrames();
     bool isNtscAmendment2ClvFrameNumber(qint32 frameNumber);

tools/ld-dropout-correct/correctorpool.cpp

@@ -134,7 +134,7 @@ bool CorrectorPool::getInputFrame(qint32& frameNumber,
                                   QVector<qint32>& secondFieldNumber, QVector<QByteArray>& secondFieldVideoData, QVector<LdDecodeMetaData::Field>& secondFieldMetadata,
                                   QVector<LdDecodeMetaData::VideoParameters>& videoParameters,
                                   bool& _reverse, bool& _intraField, bool& _overCorrect,
-                                  QVector<qint32>& minVbiForSource, QVector<qint32>& maxVbiForSource, QVector<qint32>& availableSourcesForFrame, QVector<qreal>& sourceFrameQuality)
+                                  QVector<qint32>& availableSourcesForFrame, QVector<qreal>& sourceFrameQuality)
 {
     QMutexLocker locker(&inputMutex);
 
@@ -167,6 +167,10 @@ bool CorrectorPool::getInputFrame(qint32& frameNumber,
     if (numberOfSources > 1) currentVbiFrame = convertSequentialFrameNumberToVbi(frameNumber, 0);
     for (qint32 sourceNo = 0; sourceNo < numberOfSources; sourceNo++) {
         // Determine the fields for the input frame
+        firstFieldNumber[sourceNo] = -1;
+        secondFieldNumber[sourceNo] = -1;
+        sourceFrameQuality[sourceNo] = -1;
+
         if (sourceNo == 0) {
             // No need to perform VBI frame number mapping on the first source
             firstFieldNumber[sourceNo] = ldDecodeMetaData[sourceNo]->getFirstFieldNumber(frameNumber);
@@ -196,9 +200,6 @@ bool CorrectorPool::getInputFrame(qint32& frameNumber,
                         " and fields " << firstFieldNumber[sourceNo] << "/" << secondFieldNumber[sourceNo] <<
                         " (quality is " << sourceFrameQuality[sourceNo] << ")";
         } else {
-            firstFieldNumber[sourceNo] = -1;
-            secondFieldNumber[sourceNo] = -1;
-            sourceFrameQuality[sourceNo] = -1;
             qDebug().nospace() << "CorrectorPool::getInputFrame(): Source #" << sourceNo << " does not contain a usable frame";
         }
 
@@ -232,9 +233,6 @@ bool CorrectorPool::getInputFrame(qint32& frameNumber,
     _intraField = intraField;
     _overCorrect = overCorrect;
 
-    minVbiForSource = sourceMinimumVbiFrame;
-    maxVbiForSource = sourceMaximumVbiFrame;
-
     return true;
 }
 
@@ -328,8 +326,6 @@ bool CorrectorPool::setMinAndMaxVbiFrames()
     sourceMinimumVbiFrame.resize(numberOfSources);
 
     for (qint32 sourceNumber = 0; sourceNumber < numberOfSources; sourceNumber++) {
-        sourceDiscTypeCav[sourceNumber] = false;
-
         // Determine the disc type and max/min VBI frame numbers
         VbiDecoder vbiDecoder;
         qint32 cavCount = 0;
@@ -339,6 +335,10 @@ bool CorrectorPool::setMinAndMaxVbiFrames()
         qint32 clvMin = 1000000;
         qint32 clvMax = 0;
 
+        sourceMinimumVbiFrame[sourceNumber] = 0;
+        sourceMaximumVbiFrame[sourceNumber] = 0;
+        sourceDiscTypeCav[sourceNumber] = false;
+
         // Using sequential frame numbering starting from 1
         for (qint32 seqFrame = 1; seqFrame <= ldDecodeMetaData[sourceNumber]->getNumberOfFrames(); seqFrame++) {
             // Get the VBI data and then decode

tools/ld-dropout-correct/correctorpool.h

@@ -51,8 +51,7 @@ class CorrectorPool : public QObject
                        QVector<qint32> &firstFieldNumber, QVector<QByteArray> &firstFieldVideoData, QVector<LdDecodeMetaData::Field> &firstFieldMetadata,
                        QVector<qint32> &secondFieldNumber, QVector<QByteArray> &secondFieldVideoData, QVector<LdDecodeMetaData::Field> &secondFieldMetadata,
                        QVector<LdDecodeMetaData::VideoParameters> &videoParameters,
-                       bool& _reverse, bool& _intraField, bool& _overCorrect, QVector<qint32> &minVbiForSource,
-                       QVector<qint32> &maxVbiForSource, QVector<qint32> &availableSourcesForFrame, QVector<qreal> &sourceFrameQuality);
+                       bool& _reverse, bool& _intraField, bool& _overCorrect, QVector<qint32> &availableSourcesForFrame, QVector<qreal> &sourceFrameQuality);
 
     bool setOutputFrame(qint32 frameNumber,
                         QByteArray firstTargetFieldData, QByteArray secondTargetFieldData,

tools/ld-dropout-correct/dropoutcorrect.cpp

@@ -43,8 +43,6 @@ void DropOutCorrect::run()
     QVector<LdDecodeMetaData::Field> firstFieldMetadata;
     QVector<LdDecodeMetaData::Field> secondFieldMetadata;
     bool reverse, intraField, overCorrect;
-    QVector<qint32> minVbiForSource;
-    QVector<qint32> maxVbiForSource;
     QVector<qint32> availableSourcesForFrame;
     QVector<qreal> sourceFrameQuality;
 
@@ -56,8 +54,7 @@ void DropOutCorrect::run()
         if (!correctorPool.getInputFrame(frameNumber, firstFieldSeqNo, firstSourceField, firstFieldMetadata,
                                        secondFieldSeqNo, secondSourceField, secondFieldMetadata,
                                        videoParameters, reverse, intraField, overCorrect,
-                                       minVbiForSource, maxVbiForSource, availableSourcesForFrame,
-                                       sourceFrameQuality)) {
+                                       availableSourcesForFrame, sourceFrameQuality)) {
             // No more input fields -- exit
             break;
         }