Skip to content

Commit

Permalink
videoio: Add raw encoded video stream encapsulation to cv::VideoWrite…
Browse files Browse the repository at this point in the history 
…r with CAP_FFMEG
  • Loading branch information
@cudawarped
cudawarped committed Oct 19, 2023
1 parent c96f48e commit b0f4231
Show file tree
Hide file tree
Showing 4 changed files with 351 additions and 81 deletions.
5 changes: 4 additions & 1 deletion modules/videoio/include/opencv2/videoio.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,10 +225,13 @@ enum VideoWriterProperties {
VIDEOWRITER_PROP_NSTRIPES = 3, //!< Number of stripes for parallel encoding. -1 for auto detection.
VIDEOWRITER_PROP_IS_COLOR = 4, //!< If it is not zero, the encoder will expect and encode color frames, otherwise it
//!< will work with grayscale frames.
VIDEOWRITER_PROP_DEPTH = 5, //!< Defaults to CV_8U.
VIDEOWRITER_PROP_DEPTH = 5, //!< Defaults to \ref CV_8U.
VIDEOWRITER_PROP_HW_ACCELERATION = 6, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in VideoWriter constructor / .open() method. Default value is backend-specific.
VIDEOWRITER_PROP_HW_DEVICE = 7, //!< (**open-only**) Hardware device index (select GPU if multiple available). Device enumeration is acceleration type specific.
VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL= 8, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between cv::UMat and HW accelerated encoder.
VIDEOWRITER_PROP_RAW_VIDEO = 9, //!< (**open-only**) Set to non-zero to enable encapsulation of an encoded raw video stream. Each raw encoded video frame should be passed to VideoWriter::write() as single row or column of a \ref CV_8UC1 Mat. \note If the key frame interval is not 1 then it must be manually specified by the user. This can either be performed during initialization passing \ref VIDEOWRITER_PROP_KEY_INTERVAL as one of the extra encoder params to \ref VideoWriter::VideoWriter(const String &, int, double, const Size &, const std::vector< int > &params) or afterwards by setting the \ref VIDEOWRITER_PROP_KEY_FLAG with \ref VideoWriter::set() before writing each frame. FFMpeg backend only.
VIDEOWRITER_PROP_KEY_INTERVAL = 10, //!< (**open-only**) Set the key frame interval using raw video encapsulation (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). Defaults to 1 when not set. FFMpeg backend only.
VIDEOWRITER_PROP_KEY_FLAG = 11, //!< Set to non-zero to signal that the following frames are key frames or zero if not, when encapsulating raw video (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). FFMpeg backend only.
#ifndef CV_DOXYGEN
CV__VIDEOWRITER_PROP_LATEST
#endif
Expand Down
6 changes: 5 additions & 1 deletion modules/videoio/src/cap_ffmpeg.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,11 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL :
return ffmpegWriter->getProperty(propId);
}

virtual bool setProperty(int, double) CV_OVERRIDE { return false; }
virtual bool setProperty(int propId, double value) CV_OVERRIDE {
if (!ffmpegWriter)
return 0;
return ffmpegWriter->setProperty(propId, value);
}
virtual bool isOpened() const CV_OVERRIDE { return ffmpegWriter != 0; }

protected:
Expand Down
224 changes: 145 additions & 79 deletions modules/videoio/src/cap_ffmpeg_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2069,6 +2069,7 @@ struct CvVideoWriter_FFMPEG
bool writeFrame( const unsigned char* data, int step, int width, int height, int cn, int origin );
bool writeHWFrame(cv::InputArray input);
double getProperty(int propId) const;
bool setProperty(int, double);

void init();

Expand All @@ -2092,6 +2093,9 @@ struct CvVideoWriter_FFMPEG
VideoAccelerationType va_type;
int hw_device;
int use_opencl;
bool encode_video;
int idr_period;
bool key_frame;
};

static const char * icvFFMPEGErrStr(int err)
Expand Down Expand Up @@ -2157,6 +2161,9 @@ void CvVideoWriter_FFMPEG::init()
hw_device = -1;
use_opencl = 0;
ok = false;
encode_video = true;
idr_period = 0;
key_frame = false;
}

/**
Expand Down Expand Up @@ -2202,7 +2209,7 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc,
AVStream *st,
const AVCodec* codec,
int w, int h, int bitrate,
double fps, AVPixelFormat pixel_format, int fourcc)
double fps, AVPixelFormat pixel_format, int fourcc, AVCodecID codec_id)
{
#ifdef CV_FFMPEG_CODECPAR
AVCodecContext *c = avcodec_alloc_context3(codec);
Expand All @@ -2213,9 +2220,7 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc,

int frame_rate, frame_rate_base;

c->codec_id = codec->id;
c->codec_type = AVMEDIA_TYPE_VIDEO;
c->codec_tag = fourcc;
c->codec_id = codec ? codec->id : codec_id;

#ifndef CV_FFMPEG_CODECPAR
// Set per-codec defaults
Expand All @@ -2225,6 +2230,9 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc,
c->codec_id = c_id;
#endif

c->codec_type = AVMEDIA_TYPE_VIDEO;
c->codec_tag = fourcc;

/* put sample parameters */
int64_t lbit_rate = (int64_t)bitrate;
lbit_rate += (bitrate / 2);
Expand Down Expand Up @@ -2323,6 +2331,29 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc,

static const int OPENCV_NO_FRAMES_WRITTEN_CODE = 1000;

static int icv_av_encapsulate_video_FFMPEG(AVFormatContext* oc, AVStream* video_st, AVCodecContext* c,
uint8_t* data, int sz, const int frame_idx, const bool key_frame)
{
#if LIBAVFORMAT_BUILD < CALC_FFMPEG_VERSION(57, 0, 0)
AVPacket pkt_;
av_init_packet(&pkt_);
AVPacket* pkt = &pkt_;
#else
AVPacket* pkt = av_packet_alloc();
#endif
if(key_frame)
pkt->flags |= PKT_FLAG_KEY;
pkt->pts = frame_idx;
pkt->size = sz;
pkt->data = data;
av_packet_rescale_ts(pkt, c->time_base, video_st->time_base);
int ret = av_write_frame(oc, pkt);
#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(57, 0, 0)
av_packet_free(&pkt);
#endif
return ret;
}

static int icv_av_write_frame_FFMPEG( AVFormatContext * oc, AVStream * video_st, AVCodecContext * c,
uint8_t *, uint32_t,
AVFrame * picture, int frame_idx)
Expand Down Expand Up @@ -2404,6 +2435,14 @@ static int icv_av_write_frame_FFMPEG( AVFormatContext * oc, AVStream * video_st,
/// write a frame with FFMPEG
bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int width, int height, int cn, int origin )
{
if (!encode_video) {
CV_Assert(cn == 1 && ((width > 0 && height == 1) || (width == 1 && height > 0 && step == 1)));
const bool set_key_frame = key_frame ? key_frame : idr_period ? frame_idx % idr_period == 0 : 1;
bool ret = icv_av_encapsulate_video_FFMPEG(oc, video_st, context, (uint8_t*)data, width, frame_idx, set_key_frame);
frame_idx++;
return ret;
}

// check parameters
if (input_pix_fmt == AV_PIX_FMT_BGR24) {
if (cn != 3) {
Expand Down Expand Up @@ -2592,6 +2631,21 @@ double CvVideoWriter_FFMPEG::getProperty(int propId) const
return 0;
}

bool CvVideoWriter_FFMPEG::setProperty(int property_id, double value)
{
if (!video_st) return false;

switch (property_id)
{
case VIDEOWRITER_PROP_KEY_FLAG:
key_frame = static_cast<bool>(value);
break;
default:
return false;
}
return true;
}

/// close video output stream and free associated memory
void CvVideoWriter_FFMPEG::close()
{
Expand All @@ -2601,17 +2655,19 @@ void CvVideoWriter_FFMPEG::close()
// TODO -- do we need to account for latency here?

/* write the trailer, if any */
if (picture && ok && oc)
if ((!encode_video || picture) && ok && oc)
{
#if LIBAVFORMAT_BUILD < CALC_FFMPEG_VERSION(57, 0, 0)
if (!(oc->oformat->flags & AVFMT_RAWPICTURE))
#endif
{
for(;;)
{
int ret = icv_av_write_frame_FFMPEG( oc, video_st, context, outbuf, outbuf_size, NULL, frame_idx);
if( ret == OPENCV_NO_FRAMES_WRITTEN_CODE || ret < 0 )
break;
if (encode_video) {
for (;;)
{
int ret = icv_av_write_frame_FFMPEG(oc, video_st, context, outbuf, outbuf_size, NULL, frame_idx);
if (ret == OPENCV_NO_FRAMES_WRITTEN_CODE || ret < 0)
break;
}
}
}
av_write_trailer(oc);
Expand Down Expand Up @@ -2720,6 +2776,8 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,

close();

encode_video = !params.get(VIDEOWRITER_PROP_RAW_VIDEO, false);
idr_period = params.get(VIDEOWRITER_PROP_KEY_INTERVAL, 0);
const bool is_color = params.get(VIDEOWRITER_PROP_IS_COLOR, true);
const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U);
const bool is_supported = depth == CV_8U || (depth == CV_16U && !is_color);
Expand Down Expand Up @@ -2770,13 +2828,15 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
if(fps <= 0)
return false;

// we allow frames of odd width or height, but in this case we truncate
// the rightmost column/the bottom row. Probably, this should be handled more elegantly,
// but some internal functions inside FFMPEG swscale require even width/height.
width &= -2;
height &= -2;
if( width <= 0 || height <= 0 )
return false;
if (encode_video) {
// we allow frames of odd width or height, but in this case we truncate
// the rightmost column/the bottom row. Probably, this should be handled more elegantly,
// but some internal functions inside FFMPEG swscale require even width/height.
width &= -2;
height &= -2;
if (width <= 0 || height <= 0)
return false;
}

/* auto detect the output format from the name and fourcc code. */

Expand Down Expand Up @@ -3027,49 +3087,54 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
HWAccelIterator accel_iter(va_type, true/*isEncoder*/, dict);
while (accel_iter.good())
{
AVPixelFormat hw_format = AV_PIX_FMT_NONE;
AVHWDeviceType hw_type = AV_HWDEVICE_TYPE_NONE;
#else
do {
#endif
if (encode_video) {
#if USE_AV_HW_CODECS
accel_iter.parse_next();
AVHWDeviceType hw_type = accel_iter.hw_type();
codec = NULL;
AVPixelFormat hw_format = AV_PIX_FMT_NONE;
if (hw_device_ctx)
av_buffer_unref(&hw_device_ctx);
if (hw_type != AV_HWDEVICE_TYPE_NONE)
{
codec = hw_find_codec(codec_id, hw_type, av_codec_is_encoder, accel_iter.disabled_codecs().c_str(), &hw_format);
if (!codec)
continue;
accel_iter.parse_next();
hw_type = accel_iter.hw_type();
codec = NULL;
hw_format = AV_PIX_FMT_NONE;
if (hw_device_ctx)
av_buffer_unref(&hw_device_ctx);
if (hw_type != AV_HWDEVICE_TYPE_NONE)
{
codec = hw_find_codec(codec_id, hw_type, av_codec_is_encoder, accel_iter.disabled_codecs().c_str(), &hw_format);
if (!codec)
continue;

hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname(), use_opencl != 0);
if (!hw_device_ctx)
continue;
}
else if (hw_type == AV_HWDEVICE_TYPE_NONE)
hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname(), use_opencl != 0);
if (!hw_device_ctx)
continue;
}
else if (hw_type == AV_HWDEVICE_TYPE_NONE)
#endif
{
codec = avcodec_find_encoder(codec_id);
if (!codec) {
CV_LOG_ERROR(NULL, "Could not find encoder for codec_id=" << (int)codec_id << ", error: "
{
codec = avcodec_find_encoder(codec_id);
if (!codec) {
CV_LOG_ERROR(NULL, "Could not find encoder for codec_id=" << (int)codec_id << ", error: "
<< icvFFMPEGErrStr(AVERROR_ENCODER_NOT_FOUND));
}
}
if (!codec)
continue;
}
if (!codec)
continue;

#if USE_AV_HW_CODECS
AVPixelFormat format = (hw_format != AV_PIX_FMT_NONE) ? hw_format : codec_pix_fmt;
AVPixelFormat format = (hw_format != AV_PIX_FMT_NONE) ? hw_format : codec_pix_fmt;
#else
AVPixelFormat format = codec_pix_fmt;
AVPixelFormat format = codec_pix_fmt;
#endif

#ifdef CV_FFMPEG_CODECPAR
avcodec_free_context(&context);
#endif
context = icv_configure_video_stream_FFMPEG(oc, video_st, codec,
width, height, (int) (bitrate + 0.5),
fps, format, fourcc);
fps, format, fourcc, codec_id);
if (!context)
{
continue;
Expand All @@ -3082,17 +3147,18 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
av_dump_format(oc, 0, filename, 1);
#endif
#endif

if (encode_video) {
#if USE_AV_HW_CODECS
if (hw_device_ctx) {
context->hw_device_ctx = av_buffer_ref(hw_device_ctx);
if (hw_format != AV_PIX_FMT_NONE) {
context->hw_frames_ctx = hw_create_frames(NULL, hw_device_ctx, width, height, hw_format);
if (!context->hw_frames_ctx)
continue;
if (hw_device_ctx) {
context->hw_device_ctx = av_buffer_ref(hw_device_ctx);
if (hw_format != AV_PIX_FMT_NONE) {
context->hw_frames_ctx = hw_create_frames(NULL, hw_device_ctx, width, height, hw_format);
if (!context->hw_frames_ctx)
continue;
}
}
}
#endif
}

int64_t lbit_rate = (int64_t) context->bit_rate;
lbit_rate += (int64_t)(bitrate / 2);
Expand All @@ -3101,7 +3167,7 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
context->bit_rate = (int) lbit_rate;

/* open the codec */
err = avcodec_open2(context, codec, NULL);
err = !encode_video ? 0 : avcodec_open2(context, codec, NULL);
if (err >= 0) {
#if USE_AV_HW_CODECS
va_type = hw_type_to_va_type(hw_type);
Expand Down Expand Up @@ -3137,43 +3203,43 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
avcodec_parameters_from_context(video_st->codecpar, context);
#endif

outbuf = NULL;


if (encode_video) {
outbuf = NULL;
#if LIBAVFORMAT_BUILD < CALC_FFMPEG_VERSION(57, 0, 0)
if (!(oc->oformat->flags & AVFMT_RAWPICTURE))
if (!(oc->oformat->flags & AVFMT_RAWPICTURE))
#endif
{
/* allocate output buffer */
/* assume we will never get codec output with more than 4 bytes per pixel... */
outbuf_size = width*height*4;
outbuf = (uint8_t *) av_malloc(outbuf_size);
}
{
/* allocate output buffer */
/* assume we will never get codec output with more than 4 bytes per pixel... */
outbuf_size = width * height * 4;
outbuf = (uint8_t*)av_malloc(outbuf_size);
}

bool need_color_convert;
AVPixelFormat sw_pix_fmt = context->pix_fmt;
bool need_color_convert;
AVPixelFormat sw_pix_fmt = context->pix_fmt;
#if USE_AV_HW_CODECS
if (context->hw_frames_ctx)
sw_pix_fmt = ((AVHWFramesContext*)context->hw_frames_ctx->data)->sw_format;
if (context->hw_frames_ctx)
sw_pix_fmt = ((AVHWFramesContext*)context->hw_frames_ctx->data)->sw_format;
#endif

need_color_convert = (sw_pix_fmt != input_pix_fmt);

/* allocate the encoded raw picture */
picture = icv_alloc_picture_FFMPEG(sw_pix_fmt, context->width, context->height, need_color_convert);
if (!picture) {
return false;
}
need_color_convert = (sw_pix_fmt != input_pix_fmt);

/* if the output format is not our input format, then a temporary
picture of the input format is needed too. It is then converted
to the required output format */
input_picture = NULL;
if ( need_color_convert ) {
input_picture = icv_alloc_picture_FFMPEG(input_pix_fmt, context->width, context->height, false);
if (!input_picture) {
/* allocate the encoded raw picture */
picture = icv_alloc_picture_FFMPEG(sw_pix_fmt, context->width, context->height, need_color_convert);
if (!picture) {
return false;
}

/* if the output format is not our input format, then a temporary
picture of the input format is needed too. It is then converted
to the required output format */
input_picture = NULL;
if (need_color_convert) {
input_picture = icv_alloc_picture_FFMPEG(input_pix_fmt, context->width, context->height, false);
if (!input_picture) {
return false;
}
}
}

/* open the output file, if needed */
Expand Down

0 comments on commit b0f4231

Please sign in to comment.