Reject H.264 Level 6.0+ for hardware decoding

Nevcairiel · Aug 18, 2020 · 8ea3959 · 8ea3959
1 parent 32cc429
commit 8ea3959
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 7 deletions.
diff --git a/decoder/LAVVideo/decoders/cuvid.cpp b/decoder/LAVVideo/decoders/cuvid.cpp
@@ -1297,12 +1297,12 @@ STDMETHODIMP CDecCuvid::CheckH264Sequence(const BYTE *buffer, int buflen)
         m_bARPresent = h264parser.sps.ar_present;
         DbgLog((LOG_TRACE, 10, L"-> SPS found"));
         if (h264parser.sps.profile > 100 || h264parser.sps.chroma != 1 || h264parser.sps.luma_bitdepth != 8 ||
-            h264parser.sps.chroma_bitdepth != 8)
+            h264parser.sps.chroma_bitdepth != 8 || h264parser.sps.level >= 60)
         {
             DbgLog((LOG_TRACE, 10,
-                    L"  -> SPS indicates video incompatible with CUVID, aborting (profile: %d, chroma: %d, bitdepth: "
+                    L"  -> SPS indicates video incompatible with CUVID, aborting (profile: %d, level: %d, chroma: %d, bitdepth: "
                     L"%d/%d)",
-                    h264parser.sps.profile, h264parser.sps.chroma, h264parser.sps.luma_bitdepth,
+                    h264parser.sps.profile, h264parser.sps.level, h264parser.sps.chroma, h264parser.sps.luma_bitdepth,
                     h264parser.sps.chroma_bitdepth));
             return E_FAIL;
         }

diff --git a/decoder/LAVVideo/decoders/d3d11va.cpp b/decoder/LAVVideo/decoders/d3d11va.cpp
@@ -506,7 +506,7 @@ STDMETHODIMP CDecD3D11::InitDecoder(AVCodecID codec, const CMediaType *pmt)
     if (FAILED(hr))
         return hr;
 
-    if (check_dxva_codec_profile(m_pAVCtx->codec_id, m_pAVCtx->pix_fmt, m_pAVCtx->profile, AV_PIX_FMT_D3D11))
+    if (check_dxva_codec_profile(m_pAVCtx->codec_id, m_pAVCtx->pix_fmt, m_pAVCtx->profile, m_pAVCtx->level, AV_PIX_FMT_D3D11))
     {
         DbgLog((LOG_TRACE, 10, L"-> Incompatible profile detected, falling back to software decoding"));
         return E_FAIL;

diff --git a/decoder/LAVVideo/decoders/dxva2/dxva_common.cpp b/decoder/LAVVideo/decoders/dxva2/dxva_common.cpp
@@ -157,7 +157,7 @@ int check_dxva_mode_compatibility(const dxva_mode_t *mode, int codec, int profil
   return 1;
 }
 
-int check_dxva_codec_profile(AVCodecID codec, AVPixelFormat pix_fmt, int profile, int hwpixfmt)
+int check_dxva_codec_profile(AVCodecID codec, AVPixelFormat pix_fmt, int profile, int level, int hwpixfmt)
 {
   // check mpeg2 pixfmt
   if (codec == AV_CODEC_ID_MPEG2VIDEO && pix_fmt != AV_PIX_FMT_YUV420P && pix_fmt != AV_PIX_FMT_YUVJ420P && pix_fmt != hwpixfmt && pix_fmt != AV_PIX_FMT_NONE)
@@ -171,6 +171,10 @@ int check_dxva_codec_profile(AVCodecID codec, AVPixelFormat pix_fmt, int profile
   if (codec == AV_CODEC_ID_H264 && profile != FF_PROFILE_UNKNOWN && !H264_CHECK_PROFILE(profile))
     return 1;
 
+  // check h264 level
+  if (codec == AV_CODEC_ID_H264 && level >= 60)
+      return 1;
+
   // check wmv/vc1 profile
   if ((codec == AV_CODEC_ID_WMV3 || codec == AV_CODEC_ID_VC1) && profile == FF_PROFILE_VC1_COMPLEX)
     return 1;

diff --git a/decoder/LAVVideo/decoders/dxva2/dxva_common.h b/decoder/LAVVideo/decoders/dxva2/dxva_common.h
@@ -37,7 +37,7 @@ extern const dxva_mode_t dxva_modes[];
 const dxva_mode_t *get_dxva_mode_from_guid(const GUID *guid);
 int check_dxva_mode_compatibility(const dxva_mode_t *mode, int codec, int profile);
 
-int check_dxva_codec_profile(AVCodecID codec, AVPixelFormat pix_fmt, int profile, int hwpixfmt);
+int check_dxva_codec_profile(AVCodecID codec, AVPixelFormat pix_fmt, int profile, int level, int hwpixfmt);
 
 #define H264_CHECK_PROFILE(profile) (((profile) & ~FF_PROFILE_H264_CONSTRAINED) <= FF_PROFILE_H264_HIGH)
 #define HEVC_CHECK_PROFILE(profile) ((profile) <= FF_PROFILE_HEVC_MAIN_10)

diff --git a/decoder/LAVVideo/decoders/dxva2dec.cpp b/decoder/LAVVideo/decoders/dxva2dec.cpp
@@ -1067,7 +1067,7 @@ STDMETHODIMP CDecDXVA2::InitDecoder(AVCodecID codec, const CMediaType *pmt)
             output = (D3DFORMAT)FOURCC_NV12;
     }
 
-    if (check_dxva_codec_profile(m_pAVCtx->codec_id, m_pAVCtx->pix_fmt, m_pAVCtx->profile, AV_PIX_FMT_DXVA2_VLD))
+    if (check_dxva_codec_profile(m_pAVCtx->codec_id, m_pAVCtx->pix_fmt, m_pAVCtx->profile, m_pAVCtx->level, AV_PIX_FMT_DXVA2_VLD))
     {
         DbgLog((LOG_TRACE, 10, L"-> Incompatible profile detected, falling back to software decoding"));
         return E_FAIL;