dxva: only release hw decoder if opening of new one fails

xbmc/cores/dvdplayer/DVDCodecs/Video/DXVA.cpp

@@ -507,20 +507,30 @@ bool CDXVAContext::CreateSurfaces(int width, int height, D3DFORMAT format, unsig
 bool CDXVAContext::CreateDecoder(GUID &inGuid, DXVA2_VideoDesc *format, const DXVA2_ConfigPictureDecode *config, LPDIRECT3DSURFACE9 *surfaces, unsigned int count, IDirectXVideoDecoder **decoder)
 {
   CSingleLock lock(m_section);
-  std::vector<CDecoder*>::iterator it;
-  for (it = m_decoders.begin(); it != m_decoders.end(); ++it)
-  {
-    (*it)->CloseDXVADecoder();
-  }
 
-  HRESULT res = m_service->CreateVideoDecoder(inGuid, format, config, surfaces, count, decoder);
-  if (FAILED(res))
+  int retry = 0;
+  while (retry < 2)
   {
-    CLog::Log(LOGNOTICE, "%s - failed creating decoder", __FUNCTION__);
-    return false;
+    HRESULT res = m_service->CreateVideoDecoder(inGuid, format, config, surfaces, count, decoder);
+    if (!FAILED(res))
+    {
+      return true;
+    }
+
+    if (retry == 0)
+    {
+      CLog::Log(LOGERROR, "%s - hw may not support multiple decoders, releasing existing ones", __FUNCTION__);
+      std::vector<CDecoder*>::iterator it;
+      for (it = m_decoders.begin(); it != m_decoders.end(); ++it)
+      {
+        (*it)->CloseDXVADecoder();
+      }
+    }
+    retry++;
   }
 
-  return true;
+  CLog::Log(LOGERROR, "%s - failed creating decoder", __FUNCTION__);
+  return false;
 }
 
 bool CDXVAContext::IsValidDecoder(CDecoder *decoder)
@@ -908,6 +918,12 @@ int CDecoder::Check(AVCodecContext* avctx)
 {
   CSingleLock lock(m_section);
 
+  // we may not have a hw decoder on systems (AMD HD2xxx, HD3xxx) which are only capable
+  // of opening a single decoder and DVDPlayer opened a new stream without having flushed
+  // current one.
+  if (!m_decoder)
+    return VC_BUFFER;
+
   if(m_state == DXVA_RESET)
     Close();