Encoding with PyAV results in missing frames

[tbenst]

Overview

I have a monochrome numpy array. Encoding with PyAV results in missing frames.

Traceback:
No exceptions

Investigation

Tried encoding same numpy array with Moviepy using same codec. No missing frames.

Reproduction

file: (1.4GB): https://nix-science.s3-us-west-2.amazonaws.com/f2_e1_FishVR.mat

hdf5_to_video --codec libx265 f2_e1_FishVR.mat "/gROI" results in 22 missing frames.

hdf5_to_video --codec libx264 f2_e1_FishVR.mat "/gROI" results in 31 missing frames.

hdf5_to_video --codec libx264 --moviepy f2_e1_FishVR.mat "/gROI" results in 0 missing frames.

hdf5_to_video:

import h5py, click, os, numpy as np
from tqdm import tqdm


CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

@click.command(context_settings=CONTEXT_SETTINGS)
@click.argument("hdf5_file", type=click.Path(exists=True))
@click.argument("dataset", type=str)
@click.argument("output_path", type=click.Path(exists=False), required=False)
@click.option("-r", "--frame-rate", default=60, type=float, help="Hz")
@click.option('-f', '--pix-format', default='gray8', help="pixel format")
# @click.option('-b', '--bitrate', type=int, default=8000000)
@click.option('-c', '--codec', default='libx265')
@click.option('-q', '--quality', default='22', type=str, help="CRF quality")
@click.option("--av", "lib", flag_value="av", default=True)
@click.option("--moviepy", "lib", flag_value="moviepy")
def hdf5_to_video(hdf5_file: str, dataset: str, quality: str, codec, pix_format,
                          output_path: str, frame_rate: float, lib: float):
    """Convert HDF5_FILE["DATASET"] to [output_path], an .mp4 file.
    
    Dataset must be 3D tensor. Example: 
    ~/code/babelfish/scripts/hdf5_to_video f2_e1_FishVR.mat  "/gROI"
    """
    if hdf5_file is None or dataset is None:
        # print help and exit
        ctx = click.get_current_context()
        click.echo(ctx.get_help())
        exit(0)
    
    if output_path is None:
        name, _ = os.path.splitext(hdf5_file)
        name, _ = os.path.splitext(name)
        output_path = name + ".mp4"
    # scripts/hdf5_to_video /data/dlab/zfish_2p/20191101_6f/f2_e1_FishVR.mat "/gROI" x264 benchmark at 205.88 it/s
    # x265 at 110.8 it/s
    with h5py.File(hdf5_file, 'r', swmr=True) as h5:
        imaging = h5[dataset]
        print(f"found {imaging.shape[0]} frames")
        assert len(imaging.shape)==4
        assert imaging.shape[1]==1 # empty dim
        
        if lib=="av":
            import av
            output = av.open(output_path, 'w')
            stream = output.add_stream(codec, frame_rate)
            # stream.bit_rate = bitrate
            stream.pix_fmt = pix_format
            stream.options = {'crf': quality}

            for array in tqdm(imaging):
                frame = av.VideoFrame.from_ndarray(array[0], format='gray8')
                packet = stream.encode(frame)
                output.mux(packet)
            
            output.close()
            
        elif lib=="moviepy":
            import moviepy.editor as mpy
            global c
            c = 0
            print("imaging", imaging.shape)
            def make_frame(t):
                global c
                frame = np.repeat(np.array(imaging[c,0])[:,:,None],3,axis=2)
                c += 1
                return frame 
            fps = 60
            duration = len(imaging)/fps
            clip = mpy.VideoClip(make_frame, duration = duration)

            clip.write_videofile(output_path, fps=fps, codec=codec,
                ffmpeg_params=['-crf', quality])
    
    print("created video at " + output_path)

if __name__ == '__main__':
    hdf5_to_video()

count_vid_frames (e.g. ./count_vid_frames f2_e1_FishVR_x265.mp4):

import av, click, moviepy.editor as mpy

CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

@click.command(context_settings=CONTEXT_SETTINGS)
@click.argument("mp4_file", type=click.Path(exists=True))
@click.option("--av", "lib", flag_value="av", default=True)
@click.option("--moviepy", "lib", flag_value="moviepy")
def main(mp4_file: str, lib: str):
    """Count frames of .mp4"""
    if mp4_file is None:
        # print help and exit
        ctx = click.get_current_context()
        click.echo(ctx.get_help())
        exit(0)
    
    frame_count = 0

    if lib=="av":
        container = av.open(mp4_file)
        for _ in container.decode(video=0):
            frame_count += 1
    elif lib=="moviepy":    
        myclip = mpy.VideoFileClip(mp4_file)
        for _ in myclip.iter_frames():
            frame_count += 1
        
    print(f"number of frames: {frame_count}")

if __name__ == '__main__':
    main()

Versions

• OS: Linux
• PyAV runtime: 6.2.0

PyAV v6.2.0
git origin: git@github.com:mikeboers/PyAV
git commit: unknown-commit
library configuration: --disable-static --prefix=/nix/store/af3vqy0bqz38s91ay0m19ynh6nh2x2f2-ffmpeg-4.2.1 --arch=x86_64 --target_os=linux --enable-gpl --enable-version3 --enable-shared --enable-pic --enable-runtime-cpudetect --enable-hardcoded-tables --enable-pthreads --disable-w32threads --disable-os2threads --enable-network --enable-pixelutils --enable-ffmpeg --disable-ffplay --enable-ffprobe --enable-avcodec --enable-avdevice --enable-avfilter --enable-avformat --enable-avresample --enable-avutil --enable-postproc --enable-swresample --enable-swscale --disable-doc --enable-bzlib --enable-gnutls --enable-fontconfig --enable-libfreetype --enable-libmp3lame --enable-iconv --enable-libtheora --enable-libssh --enable-vaapi --enable-libdrm --enable-vdpau --enable-libvorbis --enable-libvpx --enable-lzma --disable-opengl --disable-libmfx --disable-libaom --enable-libpulse --enable-sdl2 --enable-libsoxr --enable-libx264 --enable-libxvid --enable-zlib --enable-libopus --enable-libspeex --enable-libx265 --enable-libdav1d --disable-debug --enable-optimizations --disable-extra-warnings --disable-stripping
library license: GPL version 3 or later
libavcodec     58. 54.100
libavdevice    58.  8.100
libavfilter     7. 57.100
libavformat    58. 29.100
libavutil      56. 31.100
libswresample   3.  5.100
libswscale      5.  5.100

• PyAV build:

N/A, installed pypi 6.2.0

• FFmpeg:

ffmpeg version 4.2.1 Copyright (c) 2000-2019 the FFmpeg developers
built with gcc 8.3.0 (GCC)
configuration: --disable-static --prefix=/nix/store/3dkxqw4r625xqli01xms8k5v2z8k0jri-ffmpeg-full-4.2.1 --target_os=linux --arch=x86_64 --enable-gpl --enable-version3 --enable-nonfree --enable-shared --enable-pic --disable-small --enable-runtime-cpudetect --disable-lto --enable-gray --enable-swscale-alpha --enable-hardcoded-tables --enable-safe-bitstream-reader --enable-pthreads --disable-w32threads --disable-os2threads --enable-network --enable-pixelutils --enable-ffmpeg --enable-ffplay --enable-ffprobe --enable-avcodec --enable-avdevice --enable-avfilter --enable-avformat --enable-avresample --enable-avutil --enable-postproc --enable-swresample --enable-swscale --enable-doc --disable-htmlpages --enable-manpages --disable-podpages --disable-txtpages --enable-bzlib --enable-libcelt --enable-libdav1d --disable-libfdk-aac --disable-libflite --enable-fontconfig --enable-libfreetype --enable-frei0r --enable-libfribidi --enable-libgme --enable-gnutls --enable-libgsm --enable-ladspa --enable-libmp3lame --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libdc1394 --enable-iconv --enable-libjack --enable-libmfx --enable-libmodplug --enable-libmysofa --enable-libopus --enable-libssh --enable-libtheora --enable-libv4l2 --enable-vaapi --enable-vdpau --enable-libvorbis --enable-libvmaf --enable-libvpx --enable-libwebp --enable-xlib --enable-libxcb --enable-libxcb-shm --enable-libxcb-xfixes --enable-libxcb-shape --enable-lzma --enable-nvenc --enable-openal --enable-libopencore-amrnb --disable-opengl --enable-libopenjpeg --disable-openssl --enable-libpulse --enable-librtmp --enable-sdl2 --enable-libsoxr --enable-libspeex --enable-libvidstab --enable-libvo-amrwbenc --enable-libwavpack --enable-libx264 --enable-libx265 --enable-libxavs --enable-libxvid --enable-libzmq --enable-zlib --disable-debug --enable-optimizations --disable-extra-warnings --disable-stripping
libavutil      56. 31.100 / 56. 31.100
libavcodec     58. 54.100 / 58. 54.100
libavformat    58. 29.100 / 58. 29.100
libavdevice    58.  8.100 / 58.  8.100
libavfilter     7. 57.100 /  7. 57.100
libavresample   4.  0.  0 /  4.  0.  0
libswscale      5.  5.100 /  5.  5.100
libswresample   3.  5.100 /  3.  5.100
libpostproc    55.  5.100 / 55.  5.100

Research

I have done the following:

• Checked the PyAV documentation
• Searched on Google
• Searched on Stack Overflow
• Looked through old GitHub issues
• Asked on PyAV Gitter
• ... and waited 72 hours for a response.

[tbenst]

Figured out the issue by searching on gitter: https://gitter.im/mikeboers/PyAV?at=5dba3f509c39821509b4d5c9. Also see #272

Adding output.mux(stream.encode()) right before output.close() fixed the issue. This is shown at http://docs.mikeboers.com/pyav/develop/cookbook/numpy.html?highlight=flush