Fixing buffer issues with RGBA

DefenceX · May 13, 2024 · a361d6c · a361d6c
1 parent 043065c
commit a361d6c
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/src/qt6/QtRtpOpenH264Depayloader.cc b/src/qt6/QtRtpOpenH264Depayloader.cc
@@ -27,8 +27,10 @@ void QtRtpOpenH264Depayloader::setStreamInfo(const mediax::rtp::StreamInformatio
   m_depayloader.RegisterCallback(
       [this](const mediax::rtp::RtpDepayloader &depay [[maybe_unused]], mediax::rtp::RtpFrameData frame) {
         Frame frame_data;
-        frame_data.video.resize(frame.resolution.height * frame.resolution.width * 3);
-        memcpy(frame_data.video.data(), frame.cpu_buffer, frame.resolution.height * frame.resolution.width * 3);
+        int bits_per_pixel = mediax::BitsPerPixel(frame.encoding);
+        frame_data.video.resize(frame.resolution.height * frame.resolution.width * bits_per_pixel);
+        memcpy(frame_data.video.data(), frame.cpu_buffer,
+               frame.resolution.height * frame.resolution.width * bits_per_pixel);
 
         frame_data.height = m_depayloader.GetHeight();
         frame_data.width = m_depayloader.GetWidth();

diff --git a/src/qt6/QtRtpVaapiH264Depayloader.cc b/src/qt6/QtRtpVaapiH264Depayloader.cc
@@ -27,8 +27,11 @@ void QtRtpVaapiH264Depayloader::setStreamInfo(const mediax::rtp::StreamInformati
   m_depayloader.RegisterCallback(
       [this](const mediax::rtp::RtpDepayloader &depay [[maybe_unused]], mediax::rtp::RtpFrameData frame) {
         Frame frame_data;
-        frame_data.video.resize(frame.resolution.height * frame.resolution.width * 3);
-        memcpy(frame_data.video.data(), frame.cpu_buffer, frame.resolution.height * frame.resolution.width * 3);
+
+        int bits_per_pixel = mediax::BitsPerPixel(frame.encoding);
+        frame_data.video.resize(frame.resolution.height * frame.resolution.width * bits_per_pixel);
+        memcpy(frame_data.video.data(), frame.cpu_buffer,
+               frame.resolution.height * frame.resolution.width * bits_per_pixel);
 
         frame_data.height = m_depayloader.GetHeight();
         frame_data.width = m_depayloader.GetWidth();