Skip to content
This repository
Browse code

xvba: fix heap corruption on mode switches

  • Loading branch information...
commit 47f44ac5cfacc786b81a68f5df2a9952052b0289 1 parent dd91c94
Rainer Hochecker authored
100 xbmc/cores/dvdplayer/DVDCodecs/Video/XVBA.cpp
@@ -483,10 +483,20 @@ void CDecoder::Close()
483 483 m_context->Release();
484 484 m_context = 0;
485 485
  486 + while (!m_videoSurfaces.empty())
  487 + {
  488 + xvba_render_state *render = m_videoSurfaces.back();
  489 + if(render->buffers_alllocated > 0)
  490 + m_dllAvUtil.av_free(render->buffers);
  491 + m_videoSurfaces.pop_back();
  492 + free(render);
  493 + }
  494 +
486 495 if (m_flipBuffer)
487 496 delete [] m_flipBuffer;
488 497
489 498 g_Windowing.Unregister(this);
  499 + m_dllAvUtil.Unload();
490 500 }
491 501
492 502 void CDecoder::ResetState()
@@ -646,16 +656,16 @@ void CDecoder::DestroySession()
646 656 }
647 657 m_xvbaBufferPool.iq_matrix_buffer = 0;
648 658
649   - while (!m_videoSurfaces.empty())
  659 + for (unsigned int i = 0; i < m_videoSurfaces.size(); ++i)
650 660 {
651   - xvba_render_state *render = m_videoSurfaces.back();
652   - if(render->buffers_alllocated > 0)
653   - m_dllAvUtil.av_free(render->buffers);
654   -
655   - m_videoSurfaces.pop_back();
656   - if (m_xvbaSession)
  661 + xvba_render_state *render = m_videoSurfaces[i];
  662 + if (m_xvbaSession && render->surface)
  663 + {
657 664 g_XVBA_vtable.DestroySurface(render->surface);
658   - free(render);
  665 + render->surface = 0;
  666 + render->picture_descriptor = 0;
  667 + render->iq_matrix = 0;
  668 + }
659 669 }
660 670
661 671 if (m_xvbaSession)
@@ -663,6 +673,33 @@ void CDecoder::DestroySession()
663 673 m_xvbaSession = 0;
664 674 }
665 675
  676 +bool CDecoder::IsSurfaceValid(xvba_render_state *render)
  677 +{
  678 + // find render state in queue
  679 + bool found(false);
  680 + unsigned int i;
  681 + for(i = 0; i < m_videoSurfaces.size(); ++i)
  682 + {
  683 + if(m_videoSurfaces[i] == render)
  684 + {
  685 + found = true;
  686 + break;
  687 + }
  688 + }
  689 + if (!found)
  690 + {
  691 + CLog::Log(LOGERROR,"%s - video surface not found", __FUNCTION__);
  692 + return false;
  693 + }
  694 + if (m_videoSurfaces[i]->surface == 0)
  695 + {
  696 + m_videoSurfaces[i]->state = 0;
  697 + return false;
  698 + }
  699 +
  700 + return true;
  701 +}
  702 +
666 703 bool CDecoder::EnsureDataControlBuffers(unsigned int num)
667 704 {
668 705 if (m_xvbaBufferPool.data_control_buffers.size() >= num)
@@ -714,16 +751,7 @@ void CDecoder::FFReleaseBuffer(AVCodecContext *avctx, AVFrame *pic)
714 751 pic->data[i]= NULL;
715 752
716 753 // find render state in queue
717   - bool found(false);
718   - for(unsigned int i = 0; i < xvba->m_videoSurfaces.size(); ++i)
719   - {
720   - if(xvba->m_videoSurfaces[i] == render)
721   - {
722   - found = true;
723   - break;
724   - }
725   - }
726   - if (!found)
  754 + if (!xvba->IsSurfaceValid(render))
727 755 {
728 756 CLog::Log(LOGDEBUG, "XVBA::FFReleaseBuffer - ignoring invalid buffer");
729 757 return;
@@ -763,16 +791,7 @@ void CDecoder::FFDrawSlice(struct AVCodecContext *avctx,
763 791 }
764 792
765 793 // ffmpeg vc-1 decoder does not flush, make sure the data buffer is still valid
766   - bool found(false);
767   - for(unsigned int i = 0; i < xvba->m_videoSurfaces.size(); ++i)
768   - {
769   - if(xvba->m_videoSurfaces[i] == render)
770   - {
771   - found = true;
772   - break;
773   - }
774   - }
775   - if (!found)
  794 + if (!xvba->IsSurfaceValid(render))
776 795 {
777 796 CLog::Log(LOGWARNING, "XVBA::FFDrawSlice - ignoring invalid buffer");
778 797 return;
@@ -964,7 +983,7 @@ int CDecoder::FFGetBuffer(AVCodecContext *avctx, AVFrame *pic)
964 983 }
965 984 }
966 985
967   - // create a new surface
  986 + // create a new render state
968 987 if (render == NULL)
969 988 {
970 989 render = (xvba_render_state*)calloc(sizeof(xvba_render_state), 1);
@@ -973,6 +992,15 @@ int CDecoder::FFGetBuffer(AVCodecContext *avctx, AVFrame *pic)
973 992 CLog::Log(LOGERROR, "XVBA::FFGetBuffer - calloc failed");
974 993 return -1;
975 994 }
  995 + render->surface = 0;
  996 + render->buffers_alllocated = 0;
  997 + CSingleLock lock(xvba->m_videoSurfaceSec);
  998 + xvba->m_videoSurfaces.push_back(render);
  999 + }
  1000 +
  1001 + // create a new surface
  1002 + if (render->surface == 0)
  1003 + {
976 1004 XVBA_Create_Surface_Input surfaceInput;
977 1005 XVBA_Create_Surface_Output surfaceOutput;
978 1006 surfaceInput.size = sizeof(surfaceInput);
@@ -988,12 +1016,9 @@ int CDecoder::FFGetBuffer(AVCodecContext *avctx, AVFrame *pic)
988 1016 return -1;
989 1017 }
990 1018 }
991   - CSingleLock lock(xvba->m_videoSurfaceSec);
992 1019 render->surface = surfaceOutput.surface;
993   - render->buffers_alllocated = 0;
994 1020 render->picture_descriptor = (XVBAPictureDescriptor *)xvba->m_xvbaBufferPool.picture_descriptor_buffer->bufferXVBA;
995 1021 render->iq_matrix = (XVBAQuantMatrixAvc *)xvba->m_xvbaBufferPool.iq_matrix_buffer->bufferXVBA;
996   - xvba->m_videoSurfaces.push_back(render);
997 1022 CLog::Log(LOGDEBUG, "XVBA::FFGetBuffer - created video surface");
998 1023 }
999 1024
@@ -1041,16 +1066,7 @@ int CDecoder::Decode(AVCodecContext* avctx, AVFrame* frame)
1041 1066 return VC_ERROR;
1042 1067
1043 1068 // ffmpeg vc-1 decoder does not flush, make sure the data buffer is still valid
1044   - bool found(false);
1045   - for(unsigned int i = 0; i < m_videoSurfaces.size(); ++i)
1046   - {
1047   - if(m_videoSurfaces[i] == render)
1048   - {
1049   - found = true;
1050   - break;
1051   - }
1052   - }
1053   - if (!found)
  1069 + if (!IsSurfaceValid(render))
1054 1070 {
1055 1071 CLog::Log(LOGWARNING, "XVBA::Decode - ignoring invalid buffer");
1056 1072 return VC_BUFFER;
1  xbmc/cores/dvdplayer/DVDCodecs/Video/XVBA.h
@@ -102,6 +102,7 @@ class CDecoder : public CDVDVideoCodecFFmpeg::IHardwareDecoder,
102 102 bool DiscardPresentPicture();
103 103 void ResetState();
104 104 void SetError(const char* function, const char* msg, int line);
  105 + bool IsSurfaceValid(xvba_render_state *render);
105 106
106 107 // callbacks for ffmpeg
107 108 static void FFReleaseBuffer(AVCodecContext *avctx, AVFrame *pic);

0 comments on commit 47f44ac

Please sign in to comment.
Something went wrong with that request. Please try again.