Fix glitches that sometimes happen near the scene changes. (#3707)

* Add missing iLastWrittenPOC setting in unbuffered reordering.

In CWelsDecoder::ReorderPicturesInDisplay(), iLastWrittenPOC was
not set in unbuffered-reordering case. Due to this problem, it
sometimes reordered the frames incorrectly. This patch fixes the
issue.

* Prevent frame buffer from overwrite at GOP change.

When bNewSeqBegin got true, iRefCount was forcibly reset even if the
buffer was still used for reordering. Due to this problem, the buffer
in use was sometimes overwritten with newly decoded frame. This commit
is for fixing that problem.

* Fix test failure due to the issue of the previous commit.

* Fix glitch occurs when multiple scene changes happen in a short time.

Previously, when more than 2 frame sequences exist in the reordering
buffer, the reordering does not work correctly. This is because the
management based on bLastGOP flag does not work in this case.
Instead, this patch introduces frame sequence number (iSeqNum) to
distingish frames belonging to each sequence and drops bLastGOP flag.

codec/decoder/core/inc/decoder_context.h

@@ -285,14 +285,15 @@ typedef struct tagPictInfo {
   int32_t                 iPOC;
   int32_t                 iPicBuffIdx;
   uint32_t                uiDecodingTimeStamp;
-  bool                    bLastGOP;
+  int32_t                 iSeqNum;
 } SPictInfo, *PPictInfo;
 
 typedef struct tagPictReoderingStatus {
   int32_t iPictInfoIndex;
+  int32_t iMinSeqNum;
   int32_t iMinPOC;
   int32_t iNumOfPicts;
-  int32_t iLastGOPRemainPicts;
+  int32_t iLastWrittenSeqNum;
   int32_t iLastWrittenPOC;
   int32_t iLargestBufferedPicIndex;
   bool    bHasBSlice;
@@ -431,6 +432,7 @@ typedef struct TagWelsDecoderContext {
 #endif
   bool       bNewSeqBegin;
   bool       bNextNewSeqBegin;
+  int32_t    iSeqNum;
 
 //for Parse only
   bool bFramePending;

codec/decoder/core/inc/picture.h

@@ -70,6 +70,7 @@ struct SPicture {
   bool            bUsedAsRef;                                                     //for ref pic management
   bool            bIsLongRef;     // long term reference frame flag       //for ref pic management
   int8_t          iRefCount;
+  void            (*pSetUnRef)(WelsDec::SPicture*);
 
   bool            bIsComplete;    // indicate whether current picture is complete, not from EC
   /*******************************for future use****************************/

codec/decoder/core/src/decoder.cpp

@@ -152,6 +152,7 @@ static int32_t IncreasePicBuff (PWelsDecoderContext pCtx, PPicBuff* ppPicBuf, co
     pPicNewBuf->ppPic[i]->bUsedAsRef = false;
     pPicNewBuf->ppPic[i]->bIsLongRef = false;
     pPicNewBuf->ppPic[i]->iRefCount = 0;
+    pPicNewBuf->ppPic[i]->pSetUnRef = NULL;
     pPicNewBuf->ppPic[i]->bIsComplete = false;
   }
 // remove old PicBuf
@@ -240,6 +241,7 @@ static int32_t DecreasePicBuff (PWelsDecoderContext pCtx, PPicBuff* ppPicBuf, co
     pPicNewBuf->ppPic[i]->bUsedAsRef = false;
     pPicNewBuf->ppPic[i]->bIsLongRef = false;
     pPicNewBuf->ppPic[i]->iRefCount = 0;
+    pPicNewBuf->ppPic[i]->pSetUnRef = NULL;
     pPicNewBuf->ppPic[i]->bIsComplete = false;
   }
   // remove old PicBuf
@@ -296,11 +298,9 @@ void ResetReorderingPictureBuffers (PPictReoderingStatus pPictReoderingStatus, P
     pPictReoderingStatus->iPictInfoIndex = 0;
     pPictReoderingStatus->iMinPOC = IMinInt32;
     pPictReoderingStatus->iNumOfPicts = 0;
-    pPictReoderingStatus->iLastGOPRemainPicts = 0;
     pPictReoderingStatus->iLastWrittenPOC = IMinInt32;
     pPictReoderingStatus->iLargestBufferedPicIndex = 0;
     for (int32_t i = 0; i < pictInfoListCount; ++i) {
-      pPictInfo[i].bLastGOP = false;
       pPictInfo[i].iPOC = IMinInt32;
     }
     pPictInfo->sBufferInfo.iBufferStatus = 0;
@@ -621,6 +621,7 @@ int32_t WelsOpenDecoder (PWelsDecoderContext pCtx, SLogContext* pLogCtx) {
   pCtx->bPrintFrameErrorTraceFlag = true;
   pCtx->iIgnoredErrorInfoPacketCount = 0;
   pCtx->bFrameFinish = true;
+  pCtx->iSeqNum = 0;
   return iRet;
 }
 

codec/decoder/core/src/decoder_core.cpp

@@ -2261,6 +2261,8 @@ int32_t WelsDecodeInitAccessUnitStart (PWelsDecoderContext pCtx, SBufferInfo* pD
   pCtx->bAuReadyFlag = false;
   pCtx->pLastDecPicInfo->bLastHasMmco5 = false;
   bool bTmpNewSeqBegin = CheckNewSeqBeginAndUpdateActiveLayerSps (pCtx);
+  if (bTmpNewSeqBegin)
+    pCtx->iSeqNum++;
   pCtx->bNewSeqBegin = pCtx->bNewSeqBegin || bTmpNewSeqBegin;
   iErr = WelsDecodeAccessUnitStart (pCtx);
   GetVclNalTemporalId (pCtx);

codec/decoder/core/src/manage_dec_ref.cpp

@@ -67,7 +67,9 @@ int32_t GetLTRFrameIndex (PRefPic pRefPic, int32_t iAncLTRFrameNum);
 static int32_t RemainOneBufferInDpbForEC (PWelsDecoderContext pCtx, PRefPic pRefPic);
 
 static void SetUnRef (PPicture pRef) {
-  if (NULL != pRef) {
+  if (pRef == NULL) return;
+
+  if (pRef->iRefCount <= 0) {
     pRef->bUsedAsRef = false;
     pRef->bIsLongRef = false;
     pRef->iFrameNum = -1;
@@ -81,19 +83,19 @@ static void SetUnRef (PPicture pRef) {
     pRef->iSpsId = -1;
     pRef->bIsComplete = false;
     pRef->iRefCount = 0;
+    pRef->pSetUnRef = NULL;
 
     if (pRef->eSliceType == I_SLICE) {
       return;
     }
     int32_t lists = pRef->eSliceType == P_SLICE ? 1 : 2;
     for (int32_t i = 0; i < MAX_DPB_COUNT; ++i) {
       for (int32_t list = 0; list < lists; ++list) {
-        if (pRef->pRefPic[list][i] != NULL) {
-          pRef->pRefPic[list][i]->iRefCount = 0;
-          pRef->pRefPic[list][i] = NULL;
-        }
+        pRef->pRefPic[list][i] = NULL;
       }
     }
+  } else {
+    pRef->pSetUnRef = SetUnRef;
   }
 }
 

codec/decoder/core/src/pic_queue.cpp

@@ -107,6 +107,7 @@ PPicture AllocPicture (PWelsDecoderContext pCtx, const int32_t kiPicWidth, const
   pPic->iHeightInPixel = kiPicHeight;
   pPic->iFrameNum      = -1;
   pPic->iRefCount = 0;
+  pPic->pSetUnRef = NULL;
 
   uint32_t uiMbWidth = (kiPicWidth + 15) >> 4;
   uint32_t uiMbHeight = (kiPicHeight + 15) >> 4;

codec/decoder/plus/src/welsDecoderExt.cpp

@@ -993,46 +993,17 @@ void CWelsDecoder::BufferingReadyPicture (PWelsDecoderContext pCtx, unsigned cha
     if (pCtx->pSliceHeader->eSliceType == B_SLICE) {
       m_sReoderingStatus.bHasBSlice = true;
     }
-    if (m_sReoderingStatus.iNumOfPicts && pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb
-        && pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb->bNewSeqBegin) {
-      m_sReoderingStatus.iLastGOPRemainPicts = m_sReoderingStatus.iNumOfPicts;
-
-      for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
-        if (m_sPictInfoList[i].iPOC > IMinInt32) {
-          m_sPictInfoList[i].bLastGOP = true;
-        }
-      }
-    } else {
-      if (m_sReoderingStatus.iNumOfPicts > 0) {
-        //This can happen when decoder moves to next GOP without being able to decoder first picture PicOrderCntLsb = 0
-        bool hasGOPChanged = false;
-        for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
-          if (m_sPictInfoList[i].iPOC == pCtx->pSliceHeader->iPicOrderCntLsb) {
-            hasGOPChanged = true;
-            break;
-          }
-        }
-        if (hasGOPChanged) {
-          m_sReoderingStatus.iLastGOPRemainPicts = m_sReoderingStatus.iNumOfPicts;
-          for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
-            if (m_sPictInfoList[i].iPOC > IMinInt32) {
-              m_sPictInfoList[i].bLastGOP = true;
-            }
-          }
-        }
-      }
-    }
   }
   for (int32_t i = 0; i < 16; ++i) {
     if (m_sPictInfoList[i].iPOC == IMinInt32) {
       memcpy (&m_sPictInfoList[i].sBufferInfo, pDstInfo, sizeof (SBufferInfo));
       m_sPictInfoList[i].iPOC = pCtx->pSliceHeader->iPicOrderCntLsb;
+      m_sPictInfoList[i].iSeqNum = pCtx->iSeqNum;
       m_sPictInfoList[i].uiDecodingTimeStamp = pCtx->uiDecodingTimeStamp;
       if (pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb != NULL) {
         m_sPictInfoList[i].iPicBuffIdx = pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb->iPicBuffIdx;
         if (GetThreadCount (pCtx) <= 1) ++pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb->iRefCount;
       }
-      m_sPictInfoList[i].bLastGOP = false;
       m_iLastBufferedIdx = i;
       pDstInfo->iBufferStatus = 0;
       ++m_sReoderingStatus.iNumOfPicts;
@@ -1050,65 +1021,24 @@ void CWelsDecoder::ReleaseBufferedReadyPictureReorder (PWelsDecoderContext pCtx,
   if (pCtx == NULL && m_iThreadCount <= 1) {
     pCtx = m_pDecThrCtx[0].pCtx;
   }
-  if (m_sReoderingStatus.iLastGOPRemainPicts > 0) {
-    m_sReoderingStatus.iMinPOC = IMinInt32;
-    int32_t firstValidIdx = -1;
-    for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
-      if (m_sReoderingStatus.iMinPOC == IMinInt32 && m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].bLastGOP) {
-        m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
-        m_sReoderingStatus.iPictInfoIndex = i;
-        firstValidIdx = i;
-        break;
-      }
-    }
-    for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
-      if (i == firstValidIdx) continue;
-      if (m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].iPOC < m_sReoderingStatus.iMinPOC
-          && m_sPictInfoList[i].bLastGOP) {
-        m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
-        m_sReoderingStatus.iPictInfoIndex = i;
-      }
-    }
-    m_sReoderingStatus.iLastWrittenPOC = m_sReoderingStatus.iMinPOC;
-#if defined (_DEBUG)
-#ifdef _MOTION_VECTOR_DUMP_
-    fprintf (stderr, "Output POC: #%d uiDecodingTimeStamp=%d\n", m_sReoderingStatus.iLastWrittenPOC,
-             m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].uiDecodingTimeStamp);
-#endif
-#endif
-    memcpy (pDstInfo, &m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].sBufferInfo, sizeof (SBufferInfo));
-    ppDst[0] = pDstInfo->pDst[0];
-    ppDst[1] = pDstInfo->pDst[1];
-    ppDst[2] = pDstInfo->pDst[2];
-    m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC = IMinInt32;
-    if (pPicBuff != NULL) {
-      PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
-      --pPic->iRefCount;
-    }
-    m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
-    m_sReoderingStatus.iMinPOC = IMinInt32;
-    --m_sReoderingStatus.iNumOfPicts;
-    --m_sReoderingStatus.iLastGOPRemainPicts;
-    if (m_sReoderingStatus.iLastGOPRemainPicts == 0) {
-      m_sReoderingStatus.iLastWrittenPOC = IMinInt32;
-    }
-    return;
-  }
   if (m_sReoderingStatus.iNumOfPicts > 0) {
     m_sReoderingStatus.iMinPOC = IMinInt32;
     int32_t firstValidIdx = -1;
     for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
       if (m_sReoderingStatus.iMinPOC == IMinInt32 && m_sPictInfoList[i].iPOC > IMinInt32) {
         m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
+        m_sReoderingStatus.iMinSeqNum = m_sPictInfoList[i].iSeqNum;
         m_sReoderingStatus.iPictInfoIndex = i;
         firstValidIdx = i;
         break;
       }
     }
     for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
       if (i == firstValidIdx) continue;
-      if (m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].iPOC < m_sReoderingStatus.iMinPOC) {
+      if (m_sPictInfoList[i].iPOC > IMinInt32
+          && ((m_sPictInfoList[i].iSeqNum == m_sReoderingStatus.iMinSeqNum) ? (m_sPictInfoList[i].iPOC < m_sReoderingStatus.iMinPOC) : (m_sPictInfoList[i].iSeqNum - m_sReoderingStatus.iMinSeqNum < 0))) {
         m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
+        m_sReoderingStatus.iMinSeqNum = m_sPictInfoList[i].iSeqNum;
         m_sReoderingStatus.iPictInfoIndex = i;
       }
     }
@@ -1117,12 +1047,15 @@ void CWelsDecoder::ReleaseBufferedReadyPictureReorder (PWelsDecoderContext pCtx,
     bool isReady = true;
     if (!isFlush) {
       int32_t iLastPOC = pCtx != NULL ? pCtx->pSliceHeader->iPicOrderCntLsb : m_sPictInfoList[m_iLastBufferedIdx].iPOC;
+      int32_t iLastSeqNum = pCtx != NULL ? pCtx->iSeqNum : m_sPictInfoList[m_iLastBufferedIdx].iSeqNum;
       isReady = (m_sReoderingStatus.iLastWrittenPOC > IMinInt32
         && m_sReoderingStatus.iMinPOC - m_sReoderingStatus.iLastWrittenPOC <= 1)
-        || m_sReoderingStatus.iMinPOC < iLastPOC;
+        || m_sReoderingStatus.iMinPOC < iLastPOC
+        || m_sReoderingStatus.iMinSeqNum - iLastSeqNum < 0;
     }
     if (isReady) {
       m_sReoderingStatus.iLastWrittenPOC = m_sReoderingStatus.iMinPOC;
+      m_sReoderingStatus.iLastWrittenSeqNum = m_sReoderingStatus.iMinSeqNum;
 #if defined (_DEBUG)
 #ifdef _MOTION_VECTOR_DUMP_
       fprintf (stderr, "Output POC: #%d uiDecodingTimeStamp=%d\n", m_sReoderingStatus.iLastWrittenPOC,
@@ -1140,9 +1073,10 @@ void CWelsDecoder::ReleaseBufferedReadyPictureReorder (PWelsDecoderContext pCtx,
         {
             PPicture pPic = pPicBuff->ppPic[iPicBuffIdx];
             --pPic->iRefCount;
+            if (pPic->iRefCount <= 0 && pPic->pSetUnRef)
+              pPic->pSetUnRef(pPic);
         }
       }
-      m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
       m_sReoderingStatus.iMinPOC = IMinInt32;
       --m_sReoderingStatus.iNumOfPicts;
     }
@@ -1178,6 +1112,7 @@ void CWelsDecoder::ReleaseBufferedReadyPictureNoReorder(PWelsDecoderContext pCtx
 #endif
 #endif
     m_sReoderingStatus.iLastWrittenPOC = m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC;
+    m_sReoderingStatus.iLastWrittenSeqNum = m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iSeqNum;
     memcpy(pDstInfo, &m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].sBufferInfo, sizeof(SBufferInfo));
     ppDst[0] = pDstInfo->pDst[0];
     ppDst[1] = pDstInfo->pDst[1];
@@ -1187,10 +1122,8 @@ void CWelsDecoder::ReleaseBufferedReadyPictureNoReorder(PWelsDecoderContext pCtx
       PPicBuff pPicBuff = pCtx ? pCtx->pPicBuff : m_pPicBuff;
       PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
       --pPic->iRefCount;
-    }
-    if (m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP) {
-      --m_sReoderingStatus.iLastGOPRemainPicts;
-      m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
+      if (pPic->iRefCount <= 0 && pPic->pSetUnRef)
+        pPic->pSetUnRef(pPic);
     }
     --m_sReoderingStatus.iNumOfPicts;
   }
@@ -1204,8 +1137,12 @@ DECODING_STATE CWelsDecoder::ReorderPicturesInDisplay(PWelsDecoderContext pDecCo
     m_bIsBaseline = pDecContext->pSps->uiProfileIdc == 66 || pDecContext->pSps->uiProfileIdc == 83;
     if (!m_bIsBaseline) {
       if (pDstInfo->iBufferStatus == 1) {
-        if (m_sReoderingStatus.iLastGOPRemainPicts == 0 && pDecContext->pSliceHeader->eSliceType == B_SLICE && 
-            pDecContext->pSliceHeader->iPicOrderCntLsb <= m_sReoderingStatus.iLastWrittenPOC + 2) {
+        if (pDecContext->pSliceHeader->eSliceType == B_SLICE &&
+            ((pDecContext->iSeqNum == m_sReoderingStatus.iLastWrittenSeqNum) ?
+              (pDecContext->pSliceHeader->iPicOrderCntLsb <= m_sReoderingStatus.iLastWrittenPOC + 2) :
+              (pDecContext->iSeqNum - m_sReoderingStatus.iLastWrittenSeqNum == 1 && pDecContext->pSliceHeader->iPicOrderCntLsb == 0))) {
+          m_sReoderingStatus.iLastWrittenPOC = pDecContext->pSliceHeader->iPicOrderCntLsb;
+          m_sReoderingStatus.iLastWrittenSeqNum = pDecContext->iSeqNum;
           //issue #3478, use b-slice type to determine correct picture order as the first priority as POC order is not as reliable as based on b-slice 
           ppDst[0] = pDstInfo->pDst[0];
           ppDst[1] = pDstInfo->pDst[1];