Caffe2 uses FFMPEG functions that are deprecated in FFMPEG 4.0 and gone in 5.0

[gspr]

Describe the bug

caffe2/video/video_decoder.cc use several parts of the FFMPEG API that have been deprecated for a while, and that have been removed in the recent FFMPEG 5.0 release. This naturally makes the Caffe2 part of PyTorch unbuildable with FFMPEG 5.0.

See also this Debian bug.

Partial fix

Many of the changes are very mechanical to implement. As an example, this patch against an older PyTorch (but representative of the changes needed in git HEAD too)

---
 caffe2/video/video_decoder.cc | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/caffe2/video/video_decoder.cc b/caffe2/video/video_decoder.cc
index d40bc05..a496380 100644
--- a/caffe2/video/video_decoder.cc
+++ b/caffe2/video/video_decoder.cc
@@ -12,8 +12,10 @@ VideoDecoder::VideoDecoder() {
   static std::mutex gMutex;
   std::unique_lock<std::mutex> lock(gMutex);
   if (!gInitialized) {
+#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
     av_register_all();
     avcodec_register_all();
+#endif
     avformat_network_init();
     gInitialized = true;
   }
@@ -185,12 +187,12 @@ void VideoDecoder::decodeLoop(
     if (params.streamIndex_ == -1) {
       for (int i = 0; i < inputContext->nb_streams; i++) {
         auto stream = inputContext->streams[i];
-        if (stream->codec->codec_type == AVMEDIA_TYPE_VIDEO &&
+        if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
             videoStreamIndex_ == -1) {
           videoStreamIndex_ = i;
           videoStream_ = stream;
         } else if (
-            stream->codec->codec_type == AVMEDIA_TYPE_AUDIO &&
+            stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO &&
             audioStreamIndex_ == -1) {
           audioStreamIndex_ = i;
         }
@@ -207,7 +209,11 @@ void VideoDecoder::decodeLoop(
 
     // Initialize codec
     AVDictionary* opts = nullptr;
-    videoCodecContext_ = videoStream_->codec;
+    ret = avcodec_parameters_to_context(videoCodecContext_, videoStream_->codecpar);
+    if (ret < 0) {
+      LOG(ERROR) << "Cannot get codec context from parameters";
+      return;
+    }
     try {
       ret = avcodec_open2(
           videoCodecContext_,
@@ -452,12 +458,12 @@ void VideoDecoder::decodeLoop(
           ret = av_read_frame(inputContext, &packet);
           if (ret == AVERROR_EOF) {
             eof = 1;
-            av_free_packet(&packet);
+            av_packet_unref(&packet);
             packet.data = nullptr;
             packet.size = 0;
             // stay in the while loop to flush frames
           } else if (ret == AVERROR(EAGAIN)) {
-            av_free_packet(&packet);
+            av_packet_unref(&packet);
             continue;
           } else if (ret < 0) {
             LOG(ERROR) << "Error reading packet : " << ffmpegErrorStr(ret);
@@ -483,7 +489,7 @@ void VideoDecoder::decodeLoop(
           }
 
           if (si != videoStreamIndex_) {
-            av_free_packet(&packet);
+            av_packet_unref(&packet);
             continue;
           }
         }
@@ -497,7 +503,7 @@ void VideoDecoder::decodeLoop(
         try {
           // Nothing to do without a picture
           if (!gotPicture) {
-            av_free_packet(&packet);
+            av_packet_unref(&packet);
             continue;
           }
           frameIndex++;
@@ -534,7 +540,7 @@ void VideoDecoder::decodeLoop(
             if (!keyFrame) {
               // if fps == SpecialFps::SAMPLE_NO_FRAME (0), don't sample at all
               if (currFps == SpecialFps::SAMPLE_NO_FRAME) {
-                av_free_packet(&packet);
+                av_packet_unref(&packet);
                 continue;
               }
 
@@ -560,7 +566,7 @@ void VideoDecoder::decodeLoop(
                    timestamp >= lastFrameTimestamp + (1 / currFps));
 
               if (!fpsReached) {
-                av_free_packet(&packet);
+                av_packet_unref(&packet);
                 continue;
               }
             }
@@ -571,7 +577,7 @@ void VideoDecoder::decodeLoop(
             if (params.maximumOutputFrames_ != -1 &&
                 outputFrameIndex >= params.maximumOutputFrames_) {
               // enough frames
-              av_free_packet(&packet);
+              av_packet_unref(&packet);
               break;
             }
 
@@ -628,9 +634,9 @@ void VideoDecoder::decodeLoop(
           av_frame_unref(audioStreamFrame_);
         }
 
-        av_free_packet(&packet);
+        av_packet_unref(&packet);
       } catch (const std::exception&) {
-        av_free_packet(&packet);
+        av_packet_unref(&packet);
       }
     } // of while loop
     callback.videoDecodingEnded(timestamp);

seems to take care of a lot of them. However, replacing avcodec_decode_video2 and friends with the newer avcodec_receive_frame probably takes some care and thought.

Versions

This behavior is present in the current master HEAD at the time of writing, although the partial patch above is for an older version.

[gspr]

This is probably a duplicate of #72250. My apologies.

[rien333]

Coming here from #72250 (a duplicate issue). Would just like to point out that this build problem is most likely going to hit rolling-release distros soon, ffmpeg 5.0 is soon to be released on Arch Linux, for example.

Most of the compilation errors have to do with libavcodec in particulair. To give an incomplete sample:

/tmp/makepkg/python-pytorch/src/pytorch-1.10.2/caffe2/video/video_decoder.cc:15:5: error: ‘av_register_all’ was not declared in this scope
   15 |     av_register_all();
/tmp/makepkg/python-pytorch/src/pytorch-1.10.2/caffe2/video/video_decoder.cc: In memberfunction ‘void caffe2::VideoDecoder::getAudioSample(AVPacket&, AVCodecContext*, AVFrame*, SwrContext*, caffe2::Callback&, const caffe2::Params&)’:
/tmp/makepkg/python-pytorch/src/pytorch-1.10.2/caffe2/video/video_decoder.cc:30:17: error: ‘avcodec_decode_audio4’ was not declared in this scope; did you mean ‘avcodec_decode_subtitle2’?
   30 |   auto result = avcodec_decode_audio4(
      |                 ^~~~~~~~~~~~~~~~~~~~~
      |                 avcodec_decode_subtitle2
/tmp/makepkg/python-pytorch/src/pytorch-1.10.2/caffe2/video/video_decoder.cc:193:21: error: ‘struct AVStream’ has no member named ‘codec’
  193 |             stream->codec->codec_type == AVMEDIA_TYPE_AUDIO &&

[gspr]

The following is a patch (again against an older PyTorch) that makes building with FFMPEG 5.0 succeed. THE PATCH IS FUNCTIONALLY UNTESTED. It is based on my limited understanding of how to replace deprecated and removed FFMPEG functionality. Maybe someone can suggest how to test it, or maybe it will at least serve as a useful starting point.

From: Gard Spreemann <gspr@nonempty.org>
Date: Fri, 11 Feb 2022 20:59:35 +0100
Subject: FFMPEG 5.0 support

---
 caffe2/video/video_decoder.cc | 77 ++++++++++++++++++++++++++++++-------------
 caffe2/video/video_decoder.h  |  1 +
 2 files changed, 55 insertions(+), 23 deletions(-)

diff --git a/caffe2/video/video_decoder.cc b/caffe2/video/video_decoder.cc
index d40bc05..9ab7271 100644
--- a/caffe2/video/video_decoder.cc
+++ b/caffe2/video/video_decoder.cc
@@ -12,8 +12,10 @@ VideoDecoder::VideoDecoder() {
   static std::mutex gMutex;
   std::unique_lock<std::mutex> lock(gMutex);
   if (!gInitialized) {
+#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
     av_register_all();
     avcodec_register_all();
+#endif
     avformat_network_init();
     gInitialized = true;
   }
@@ -27,8 +29,20 @@ void VideoDecoder::getAudioSample(
     Callback& callback,
     const Params& params) {
   int frame_finished = 0;
-  auto result = avcodec_decode_audio4(
-      audioCodecContext_, audioStreamFrame_, &frame_finished, &packet);
+
+  int result = avcodec_send_packet(audioCodecContext_, &packet);
+  if (result < 0) {
+    LOG(ERROR) << "Failed to decode audio packet: " << ffmpegErrorStr(result);
+    return;
+  }
+  result = avcodec_receive_frame(audioCodecContext_, audioStreamFrame_);
+  if (result == 0) {
+    frame_finished = 1;
+  }
+  else if (result < 0 && result != AVERROR(EAGAIN) && result != AVERROR_EOF) {
+    LOG(ERROR) << "Failed to decode audio packet: " << ffmpegErrorStr(result);
+    return;
+  }
 
   if (frame_finished) {
     // from
@@ -185,12 +199,12 @@ void VideoDecoder::decodeLoop(
     if (params.streamIndex_ == -1) {
       for (int i = 0; i < inputContext->nb_streams; i++) {
         auto stream = inputContext->streams[i];
-        if (stream->codec->codec_type == AVMEDIA_TYPE_VIDEO &&
+        if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
             videoStreamIndex_ == -1) {
           videoStreamIndex_ = i;
           videoStream_ = stream;
         } else if (
-            stream->codec->codec_type == AVMEDIA_TYPE_AUDIO &&
+            stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO &&
             audioStreamIndex_ == -1) {
           audioStreamIndex_ = i;
         }
@@ -207,7 +221,11 @@ void VideoDecoder::decodeLoop(
 
     // Initialize codec
     AVDictionary* opts = nullptr;
-    videoCodecContext_ = videoStream_->codec;
+    ret = avcodec_parameters_to_context(videoCodecContext_, videoStream_->codecpar);
+    if (ret < 0) {
+      LOG(ERROR) << "Cannot get codec context from parameters";
+      return;
+    }
     try {
       ret = avcodec_open2(
           videoCodecContext_,
@@ -226,7 +244,11 @@ void VideoDecoder::decodeLoop(
 
     if (params.getAudio_ && audioStreamIndex_ >= 0) {
       // see e.g. ridge/decoder/StreamDecoder.cpp
-      audioCodecContext_ = inputContext->streams[audioStreamIndex_]->codec;
+      ret = avcodec_parameters_to_context(audioCodecContext_, inputContext->streams[audioStreamIndex_]->codecpar);
+      if (ret < 0) {
+        LOG(ERROR) << "Failed to get codec parameters: " << ffmpegErrorStr(ret);
+        return;
+      }
       ret = avcodec_open2(
           audioCodecContext_,
           avcodec_find_decoder(audioCodecContext_->codec_id),
@@ -452,12 +474,12 @@ void VideoDecoder::decodeLoop(
           ret = av_read_frame(inputContext, &packet);
           if (ret == AVERROR_EOF) {
             eof = 1;
-            av_free_packet(&packet);
+            av_packet_unref(&packet);
             packet.data = nullptr;
             packet.size = 0;
             // stay in the while loop to flush frames
           } else if (ret == AVERROR(EAGAIN)) {
-            av_free_packet(&packet);
+            av_packet_unref(&packet);
             continue;
           } else if (ret < 0) {
             LOG(ERROR) << "Error reading packet : " << ffmpegErrorStr(ret);
@@ -483,13 +505,17 @@ void VideoDecoder::decodeLoop(
           }
 
           if (si != videoStreamIndex_) {
-            av_free_packet(&packet);
+            av_packet_unref(&packet);
             continue;
           }
         }
 
-        ret = avcodec_decode_video2(
-            videoCodecContext_, videoStreamFrame_, &gotPicture, &packet);
+        ret = avcodec_send_packet(videoCodecContext_, &packet);
+        if (ret < 0) {
+          LOG(ERROR) << "Error decoding video frame : " << ffmpegErrorStr(ret);
+          return;
+        }
+        ret = avcodec_receive_frame(videoCodecContext_, videoStreamFrame_);
         if (ret < 0) {
           LOG(ERROR) << "Error decoding video frame : " << ffmpegErrorStr(ret);
           return;
@@ -497,13 +523,12 @@ void VideoDecoder::decodeLoop(
         try {
           // Nothing to do without a picture
           if (!gotPicture) {
-            av_free_packet(&packet);
+            av_packet_unref(&packet);
             continue;
           }
           frameIndex++;
 
-          long int frame_ts =
-              av_frame_get_best_effort_timestamp(videoStreamFrame_);
+          long int frame_ts = videoStreamFrame_->best_effort_timestamp;
           timestamp = frame_ts * av_q2d(videoStream_->time_base);
           if ((frame_ts >= start_ts && !mustDecodeAll) || mustDecodeAll) {
             /* process current frame if:
@@ -534,7 +559,7 @@ void VideoDecoder::decodeLoop(
             if (!keyFrame) {
               // if fps == SpecialFps::SAMPLE_NO_FRAME (0), don't sample at all
               if (currFps == SpecialFps::SAMPLE_NO_FRAME) {
-                av_free_packet(&packet);
+                av_packet_unref(&packet);
                 continue;
               }
 
@@ -560,7 +585,7 @@ void VideoDecoder::decodeLoop(
                    timestamp >= lastFrameTimestamp + (1 / currFps));
 
               if (!fpsReached) {
-                av_free_packet(&packet);
+                av_packet_unref(&packet);
                 continue;
               }
             }
@@ -571,7 +596,7 @@ void VideoDecoder::decodeLoop(
             if (params.maximumOutputFrames_ != -1 &&
                 outputFrameIndex >= params.maximumOutputFrames_) {
               // enough frames
-              av_free_packet(&packet);
+              av_packet_unref(&packet);
               break;
             }
 
@@ -583,16 +608,22 @@ void VideoDecoder::decodeLoop(
 
             try {
               // Determine required buffer size and allocate buffer
-              int numBytes = avpicture_get_size(pixFormat, outWidth, outHeight);
+              int numBytes = av_image_get_buffer_size(pixFormat, outWidth, outHeight, 1);
               DecodedFrame::AvDataPtr buffer(
                   (uint8_t*)av_malloc(numBytes * sizeof(uint8_t)));
 
-              int size = avpicture_fill(
-                  (AVPicture*)rgbFrame,
+              int size = av_image_fill_arrays(
+                  rgbFrame->data,
+                  rgbFrame->linesize,
                   buffer.get(),
                   pixFormat,
                   outWidth,
-                  outHeight);
+                  outHeight,
+                  1);
+              if (size < 0) {
+                LOG(ERROR) << "Failed to fill frame";
+                return;
+              }
 
               sws_scale(
                   scaleContext_,
@@ -628,9 +659,9 @@ void VideoDecoder::decodeLoop(
           av_frame_unref(audioStreamFrame_);
         }
 
-        av_free_packet(&packet);
+        av_packet_unref(&packet);
       } catch (const std::exception&) {
-        av_free_packet(&packet);
+        av_packet_unref(&packet);
       }
     } // of while loop
     callback.videoDecodingEnded(timestamp);
diff --git a/caffe2/video/video_decoder.h b/caffe2/video/video_decoder.h
index a091142..8c061fd 100644
--- a/caffe2/video/video_decoder.h
+++ b/caffe2/video/video_decoder.h
@@ -11,6 +11,7 @@ extern "C" {
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
 #include <libavformat/avio.h>
+#include <libavutil/imgutils.h>
 #include <libavutil/log.h>
 #include <libavutil/motion_vector.h>
 #include <libswresample/swresample.h>

[sl1pkn07]

Hi

any bump of this?

for note, the patch above apply clearly in pytorch 2.1.1. now in build process

greetings