diff --git a/tools/ld-diffdod/diffdod.cpp b/tools/ld-diffdod/diffdod.cpp index f6cc3432c..13c9b9a3b 100644 --- a/tools/ld-diffdod/diffdod.cpp +++ b/tools/ld-diffdod/diffdod.cpp @@ -30,7 +30,8 @@ Diffdod::Diffdod(QObject *parent) : QObject(parent) } bool Diffdod::process(QVector inputFilenames, bool reverse, - qint32 dodThreshold, bool noLumaClip) + qint32 dodThreshold, bool noLumaClip, + qint32 startVbi, qint32 lengthVbi) { // Show input filenames qInfo() << "Processing" << inputFilenames.size() << "input TBC files:"; @@ -52,8 +53,19 @@ bool Diffdod::process(QVector inputFilenames, bool reverse, qInfo() << "Sources have VBI frame number range of" << tbcSources.getMinimumVbiFrameNumber() << "to" << tbcSources.getMaximumVbiFrameNumber(); // Check start and length - qint32 vbiStartFrame = tbcSources.getMinimumVbiFrameNumber(); - qint32 length = tbcSources.getMaximumVbiFrameNumber() - tbcSources.getMinimumVbiFrameNumber() + 1; + qint32 vbiStartFrame = startVbi; + if (vbiStartFrame < tbcSources.getMinimumVbiFrameNumber()) + vbiStartFrame = tbcSources.getMinimumVbiFrameNumber(); + + qint32 length = lengthVbi; + if (length > (tbcSources.getMaximumVbiFrameNumber() - vbiStartFrame + 1)) + length = tbcSources.getMaximumVbiFrameNumber() - vbiStartFrame + 1; + if (length == -1) length = tbcSources.getMaximumVbiFrameNumber() - tbcSources.getMinimumVbiFrameNumber() + 1; + + // Verify frame source availablity + qInfo() << ""; + qInfo() << "Verifying VBI frame multi-source availablity:"; + tbcSources.verifySources(vbiStartFrame, length); qInfo() << "Processing" << length << "frames starting from VBI frame" << vbiStartFrame; if (!tbcSources.saveSources(vbiStartFrame, length, dodThreshold, noLumaClip)) { diff --git a/tools/ld-diffdod/diffdod.h b/tools/ld-diffdod/diffdod.h index cc7ed4efb..da35beb31 100644 --- a/tools/ld-diffdod/diffdod.h +++ b/tools/ld-diffdod/diffdod.h @@ -38,7 +38,7 @@ class Diffdod : public QObject explicit Diffdod(QObject *parent = nullptr); bool process(QVector inputFilenames, bool reverse, - qint32 dodThreshold, bool noLumaClip); + qint32 dodThreshold, bool noLumaClip, qint32 startVbi, qint32 lengthVbi); private: TbcSources tbcSources; diff --git a/tools/ld-diffdod/main.cpp b/tools/ld-diffdod/main.cpp index 27e59658f..b2021f6ca 100644 --- a/tools/ld-diffdod/main.cpp +++ b/tools/ld-diffdod/main.cpp @@ -68,13 +68,24 @@ int main(int argc, char *argv[]) QCoreApplication::translate("main", "Do not perform luma clip dropout detection")); parser.addOption(setNoLumaOption); - - // Option to select DOD threshold (dod-threshold) (-x) + // Option to select DOD threshold (-x / --dod-threshold) QCommandLineOption dodThresholdOption(QStringList() << "x" << "dod-threshold", - QCoreApplication::translate("main", "Specify the DOD threshold (100-65435 default: 700"), + QCoreApplication::translate("main", "Specify the DOD threshold (100-65435 default: 1200"), QCoreApplication::translate("main", "number")); parser.addOption(dodThresholdOption); + // Option to select the start VBI frame (-s / --start) + QCommandLineOption startVbiOption(QStringList() << "s" << "start", + QCoreApplication::translate("main", "Specify the start VBI frame"), + QCoreApplication::translate("main", "number")); + parser.addOption(startVbiOption); + + // Option to select the maximum number of VBI frames to process (-l / --length) + QCommandLineOption lengthVbiOption(QStringList() << "l" << "length", + QCoreApplication::translate("main", "Specify the maximum number of VBI frames to process"), + QCoreApplication::translate("main", "number")); + parser.addOption(lengthVbiOption); + // Positional argument to specify input TBC files parser.addPositionalArgument("input", QCoreApplication::translate("main", "Specify input TBC files (minimum of 3)")); @@ -108,8 +119,7 @@ int main(int argc, char *argv[]) return -1; } - qint32 dodThreshold = 700; - + qint32 dodThreshold = 1200; if (parser.isSet(dodThresholdOption)) { dodThreshold = parser.value(dodThresholdOption).toInt(); @@ -120,9 +130,31 @@ int main(int argc, char *argv[]) } } + qint32 vbiFrameStart = 0; + if (parser.isSet(startVbiOption)) { + vbiFrameStart = parser.value(startVbiOption).toInt(); + + if (vbiFrameStart < 1 || vbiFrameStart > 160000) { + // Quit with error + qCritical("Start VBI frame must be between 1 and 160000"); + return -1; + } + } + + qint32 vbiFrameLength = -1; + if (parser.isSet(lengthVbiOption)) { + vbiFrameLength = parser.value(lengthVbiOption).toInt(); + + if (vbiFrameLength < 1 || vbiFrameLength > 160000) { + // Quit with error + qCritical("VBI frame length must be between 1 and 160000"); + return -1; + } + } + // Process the TBC file Diffdod diffdod; - if (!diffdod.process(inputFilenames, reverse, dodThreshold, noLumaClip)) { + if (!diffdod.process(inputFilenames, reverse, dodThreshold, noLumaClip, vbiFrameStart, vbiFrameLength)) { return 1; } diff --git a/tools/ld-diffdod/tbcsources.cpp b/tools/ld-diffdod/tbcsources.cpp index 42a15888c..38a09c507 100644 --- a/tools/ld-diffdod/tbcsources.cpp +++ b/tools/ld-diffdod/tbcsources.cpp @@ -196,6 +196,32 @@ qint32 TbcSources::getMaximumVbiFrameNumber() return maximumFrameNumber; } +// Verify that at least 3 sources are available for every VBI frame +void TbcSources::verifySources(qint32 vbiStartFrame, qint32 length) +{ + qint32 uncorrectableFrameCount = 0; + + // Process the sources frame by frame + for (qint32 vbiFrame = vbiStartFrame; vbiFrame < vbiStartFrame + length; vbiFrame++) { + // Check how many source frames are available for the current frame + QVector availableSourcesForFrame = getAvailableSourcesForFrame(vbiFrame); + + // DiffDOD requires at least three source frames. If 3 sources are not available leave any existing DO records in place + // and output a warning to the user + if (availableSourcesForFrame.size() < 3) { + // Differential DOD requires at least 3 valid source frames + qInfo().nospace() << "Frame #" << vbiFrame << " has only " << availableSourcesForFrame.size() << " source frames available - cannot correct"; + uncorrectableFrameCount++; + } + } + + if (uncorrectableFrameCount != 0) { + qInfo() << "Warning:" << uncorrectableFrameCount << "frame(s) cannot be corrected!"; + } else { + qInfo() << "All frames have at least 3 sources available"; + } +} + // Private methods ---------------------------------------------------------------------------------------------------- // Perform differential dropout detection to determine (for each source) which frame pixels are valid @@ -427,8 +453,8 @@ void TbcSources::performFrameDiffDod(qint32 targetVbiFrame, qint32 dodThreshold, // Write the first and second field line metadata back to the source for (qint32 sourceNo = 0; sourceNo < availableSourcesForFrame.size(); sourceNo++) { // Get the required field numbers - qint32 firstFieldNumber = sourceVideos[availableSourcesForFrame[sourceNo]]->ldDecodeMetaData.getFirstFieldNumber(convertVbiFrameNumberToSequential(targetVbiFrame, sourceNo)); - qint32 secondFieldNumber = sourceVideos[availableSourcesForFrame[sourceNo]]->ldDecodeMetaData.getSecondFieldNumber(convertVbiFrameNumberToSequential(targetVbiFrame, sourceNo)); + qint32 firstFieldNumber = sourceVideos[availableSourcesForFrame[sourceNo]]->ldDecodeMetaData.getFirstFieldNumber(convertVbiFrameNumberToSequential(targetVbiFrame, availableSourcesForFrame[sourceNo])); + qint32 secondFieldNumber = sourceVideos[availableSourcesForFrame[sourceNo]]->ldDecodeMetaData.getSecondFieldNumber(convertVbiFrameNumberToSequential(targetVbiFrame, availableSourcesForFrame[sourceNo])); // Calculate the total number of dropouts detected for the frame qint32 totalFirstDropouts = frameDropouts[sourceNo].firstFieldDropOuts.startx.size(); diff --git a/tools/ld-diffdod/tbcsources.h b/tools/ld-diffdod/tbcsources.h index 024e7e498..38dc3d034 100644 --- a/tools/ld-diffdod/tbcsources.h +++ b/tools/ld-diffdod/tbcsources.h @@ -49,6 +49,7 @@ class TbcSources : public QObject qint32 getNumberOfAvailableSources(); qint32 getMinimumVbiFrameNumber(); qint32 getMaximumVbiFrameNumber(); + void verifySources(qint32 vbiStartFrame, qint32 length); private: // Source definition diff --git a/tools/ld-discmap/vbimapper.cpp b/tools/ld-discmap/vbimapper.cpp index c407d4d44..3be2a1d6e 100644 --- a/tools/ld-discmap/vbimapper.cpp +++ b/tools/ld-discmap/vbimapper.cpp @@ -40,6 +40,7 @@ bool VbiMapper::create(LdDecodeMetaData &ldDecodeMetaData) if (!discCheck(ldDecodeMetaData)) return false; if (!createInitialMap(ldDecodeMetaData)) return false; correctFrameNumbering(); + removeCorruptFrames(); removeDuplicateFrames(); detectMissingFrames(); @@ -297,9 +298,10 @@ void VbiMapper::correctFrameNumbering() qInfo() << ""; qInfo() << "Performing frame number correction..."; - qint32 correctedFrameNumber; + qint32 correctedFrameNumber = -1; qint32 frameNumberErrorCount = 0; qint32 frameMissingFrameNumberCount = 0; + qint32 frameNumberCorruptCount = 0; qint32 searchDistance = 5; // Set the maximum frame number limit @@ -319,6 +321,9 @@ void VbiMapper::correctFrameNumbering() // Set the frame number to a sane value ready for correction frames[frameElement].vbiFrameNumber = frames[frameElement - 1].vbiFrameNumber + 1; + frames[frameElement].isCorruptVbi = true; // Flag that the VBI should be rewritten + qDebug() << "Seq. frame" << frameElement << "has a VBI frame number of -1 - Setting to" << + frames[frameElement].vbiFrameNumber << "before correction"; isMissing = true; } @@ -329,6 +334,11 @@ void VbiMapper::correctFrameNumbering() // Try up to a distance of 'searchDistance' frames to find the sequence for (qint32 gap = 1; gap < searchDistance; gap++) { if (frames[frameElement].vbiFrameNumber != (frames[frameElement - 1].vbiFrameNumber + 1)) { + // Is the previous frame invalid? + if (frames[frameElement - 1].vbiFrameNumber == -1) { + qInfo() << "Previous frame number is invalid - cannot correct, skipping"; + break; + } // Did the player stall and repeat the last frame? if (frames[frameElement - 1].vbiFrameNumber == frames[frameElement].vbiFrameNumber) { @@ -358,12 +368,21 @@ void VbiMapper::correctFrameNumbering() // Update the frame number frames[frameElement].vbiFrameNumber = correctedFrameNumber; - frames[frameElement].isCorruptVbi = true; + frames[frameElement].isCorruptVbi = true; // Flag that the VBI should be rewritten frameNumberErrorCount++; break; // done } else { - qDebug() << "VbiMapper::correctFrameNumbering(): No match found with gap" << gap; + if (gap == searchDistance - 1) { + qDebug() << "VbiMapper::correctFrameNumbering(): Search distance reached with no match found - previous" << + frames[frameElement - 1].vbiFrameNumber << "current" << frames[frameElement + gap].vbiFrameNumber << + "target" << frames[frameElement].vbiFrameNumber; + + // Set the VBI as invalid + frames[frameElement].isMarkedForDeletion = true; // Flag that the frame should be removed + frameNumberCorruptCount++; + frameNumberErrorCount++; + } } } } else { @@ -375,20 +394,43 @@ void VbiMapper::correctFrameNumbering() qInfo() << " VBI frame number corrected to" << frames[frameElement].vbiFrameNumber; frameNumberErrorCount++; isMissing = false; + frames[frameElement].isCorruptVbi = true; // Flag that the VBI should be rewritten } } } } - // All frame numbers are now checked and corrected except the first frame; so we do that here - // since the second frame should have been corrected already + // All frame numbers are now checked and corrected except the first frame and last frame; so we do that here + // since the second frame, and second from last frame should have been corrected already if (frames[0].vbiFrameNumber != frames[1].vbiFrameNumber - 1) { - qInfo() << "The first frame does not have a valid frame number; correcting based on second frame..."; + qInfo().nospace() << "The first frame does not have a valid frame number (" << frames[0].vbiFrameNumber << + ") correcting to " << frames[1].vbiFrameNumber - 1 << " based on second frame VBI"; frames[0].vbiFrameNumber = frames[1].vbiFrameNumber - 1; + frames[0].isCorruptVbi = true; + frameNumberErrorCount++; + } + + if (frames[frames.size() - 1].vbiFrameNumber != frames[frames.size() - 2].vbiFrameNumber + 1) { + qInfo().nospace() << "The last frame does not have a valid frame number (" << frames[frames.size() - 1].vbiFrameNumber << + ") correcting to " << frames[frames.size() - 2].vbiFrameNumber + 1 << " based on second from last frame VBI"; + frames[frames.size() - 1].vbiFrameNumber = frames[frames.size() - 2].vbiFrameNumber + 1; + frames[frames.size() - 1].isCorruptVbi = true; // Flag that the VBI should be rewritten + frameNumberErrorCount++; } qInfo() << "Found and corrected" << frameNumberErrorCount << "bad/missing VBI frame numbers (of which" << - frameMissingFrameNumberCount << "had no frame number set in the VBI)"; + frameMissingFrameNumberCount << "had no frame number set in the VBI and" << frameNumberCorruptCount << "were unrecoverable)"; +} + +void VbiMapper::removeCorruptFrames() +{ + qInfo() << ""; + qInfo() << "Removing frames with unrecoverable corrupt VBI..."; + + // Remove all frames marked for deletion from the map + qint32 previousSize = frames.size(); + frames.erase(std::remove_if(frames.begin(), frames.end(), [](const Frame& f) {return f.isMarkedForDeletion == true;}), frames.end()); + qInfo() << "Removed" << previousSize - frames.size() << "corrupt VBI frames from the map -" << frames.size() << "sequential frames remaining."; } void VbiMapper::removeDuplicateFrames() diff --git a/tools/ld-discmap/vbimapper.h b/tools/ld-discmap/vbimapper.h index a43f1806f..5c715807f 100644 --- a/tools/ld-discmap/vbimapper.h +++ b/tools/ld-discmap/vbimapper.h @@ -84,6 +84,7 @@ public slots: bool discCheck(LdDecodeMetaData &ldDecodeMetaData); bool createInitialMap(LdDecodeMetaData &ldDecodeMetaData); void correctFrameNumbering(); + void removeCorruptFrames(); void removeDuplicateFrames(); void detectMissingFrames(); bool isNtscAmendment2ClvFrameNumber(qint32 frameNumber); diff --git a/tools/ld-dropout-correct/correctorpool.cpp b/tools/ld-dropout-correct/correctorpool.cpp index 2f3041064..06bdddf89 100644 --- a/tools/ld-dropout-correct/correctorpool.cpp +++ b/tools/ld-dropout-correct/correctorpool.cpp @@ -134,7 +134,7 @@ bool CorrectorPool::getInputFrame(qint32& frameNumber, QVector& secondFieldNumber, QVector& secondFieldVideoData, QVector& secondFieldMetadata, QVector& videoParameters, bool& _reverse, bool& _intraField, bool& _overCorrect, - QVector& minVbiForSource, QVector& maxVbiForSource, QVector& availableSourcesForFrame, QVector& sourceFrameQuality) + QVector& availableSourcesForFrame, QVector& sourceFrameQuality) { QMutexLocker locker(&inputMutex); @@ -167,6 +167,10 @@ bool CorrectorPool::getInputFrame(qint32& frameNumber, if (numberOfSources > 1) currentVbiFrame = convertSequentialFrameNumberToVbi(frameNumber, 0); for (qint32 sourceNo = 0; sourceNo < numberOfSources; sourceNo++) { // Determine the fields for the input frame + firstFieldNumber[sourceNo] = -1; + secondFieldNumber[sourceNo] = -1; + sourceFrameQuality[sourceNo] = -1; + if (sourceNo == 0) { // No need to perform VBI frame number mapping on the first source firstFieldNumber[sourceNo] = ldDecodeMetaData[sourceNo]->getFirstFieldNumber(frameNumber); @@ -196,9 +200,6 @@ bool CorrectorPool::getInputFrame(qint32& frameNumber, " and fields " << firstFieldNumber[sourceNo] << "/" << secondFieldNumber[sourceNo] << " (quality is " << sourceFrameQuality[sourceNo] << ")"; } else { - firstFieldNumber[sourceNo] = -1; - secondFieldNumber[sourceNo] = -1; - sourceFrameQuality[sourceNo] = -1; qDebug().nospace() << "CorrectorPool::getInputFrame(): Source #" << sourceNo << " does not contain a usable frame"; } @@ -232,9 +233,6 @@ bool CorrectorPool::getInputFrame(qint32& frameNumber, _intraField = intraField; _overCorrect = overCorrect; - minVbiForSource = sourceMinimumVbiFrame; - maxVbiForSource = sourceMaximumVbiFrame; - return true; } @@ -328,8 +326,6 @@ bool CorrectorPool::setMinAndMaxVbiFrames() sourceMinimumVbiFrame.resize(numberOfSources); for (qint32 sourceNumber = 0; sourceNumber < numberOfSources; sourceNumber++) { - sourceDiscTypeCav[sourceNumber] = false; - // Determine the disc type and max/min VBI frame numbers VbiDecoder vbiDecoder; qint32 cavCount = 0; @@ -339,6 +335,10 @@ bool CorrectorPool::setMinAndMaxVbiFrames() qint32 clvMin = 1000000; qint32 clvMax = 0; + sourceMinimumVbiFrame[sourceNumber] = 0; + sourceMaximumVbiFrame[sourceNumber] = 0; + sourceDiscTypeCav[sourceNumber] = false; + // Using sequential frame numbering starting from 1 for (qint32 seqFrame = 1; seqFrame <= ldDecodeMetaData[sourceNumber]->getNumberOfFrames(); seqFrame++) { // Get the VBI data and then decode diff --git a/tools/ld-dropout-correct/correctorpool.h b/tools/ld-dropout-correct/correctorpool.h index 85ffbe5bf..5fe84f342 100644 --- a/tools/ld-dropout-correct/correctorpool.h +++ b/tools/ld-dropout-correct/correctorpool.h @@ -51,8 +51,7 @@ class CorrectorPool : public QObject QVector &firstFieldNumber, QVector &firstFieldVideoData, QVector &firstFieldMetadata, QVector &secondFieldNumber, QVector &secondFieldVideoData, QVector &secondFieldMetadata, QVector &videoParameters, - bool& _reverse, bool& _intraField, bool& _overCorrect, QVector &minVbiForSource, - QVector &maxVbiForSource, QVector &availableSourcesForFrame, QVector &sourceFrameQuality); + bool& _reverse, bool& _intraField, bool& _overCorrect, QVector &availableSourcesForFrame, QVector &sourceFrameQuality); bool setOutputFrame(qint32 frameNumber, QByteArray firstTargetFieldData, QByteArray secondTargetFieldData, diff --git a/tools/ld-dropout-correct/dropoutcorrect.cpp b/tools/ld-dropout-correct/dropoutcorrect.cpp index cd9307bf0..17b22d59c 100644 --- a/tools/ld-dropout-correct/dropoutcorrect.cpp +++ b/tools/ld-dropout-correct/dropoutcorrect.cpp @@ -43,8 +43,6 @@ void DropOutCorrect::run() QVector firstFieldMetadata; QVector secondFieldMetadata; bool reverse, intraField, overCorrect; - QVector minVbiForSource; - QVector maxVbiForSource; QVector availableSourcesForFrame; QVector sourceFrameQuality; @@ -56,8 +54,7 @@ void DropOutCorrect::run() if (!correctorPool.getInputFrame(frameNumber, firstFieldSeqNo, firstSourceField, firstFieldMetadata, secondFieldSeqNo, secondSourceField, secondFieldMetadata, videoParameters, reverse, intraField, overCorrect, - minVbiForSource, maxVbiForSource, availableSourcesForFrame, - sourceFrameQuality)) { + availableSourcesForFrame, sourceFrameQuality)) { // No more input fields -- exit break; } diff --git a/tools/ld-dropout-correct/main.cpp b/tools/ld-dropout-correct/main.cpp index a0a5461a7..2ec3b779e 100644 --- a/tools/ld-dropout-correct/main.cpp +++ b/tools/ld-dropout-correct/main.cpp @@ -300,9 +300,9 @@ int main(int argc, char *argv[]) } // Verify TBC and JSON input fields match - if (sourceVideos[i]->getNumberOfAvailableFields() != ldDecodeMetaData[0]->getNumberOfFields()) { + if (sourceVideos[i]->getNumberOfAvailableFields() != ldDecodeMetaData[i]->getNumberOfFields()) { qInfo() << "Warning: TBC file contains" << sourceVideos[i]->getNumberOfAvailableFields() << - "fields but the JSON indicates" << ldDecodeMetaData[0]->getNumberOfFields() << + "fields but the JSON indicates" << ldDecodeMetaData[i]->getNumberOfFields() << "fields - some fields will be ignored"; qInfo() << "Update your copy of ld-decode and try again, this shouldn't happen unless the JSON metadata has been corrupted"; }