diff --git a/benchmarks/decoders/benchmark_readme_chart.png b/benchmarks/decoders/benchmark_readme_chart.png index 5dec9348a..28ceabe4a 100644 Binary files a/benchmarks/decoders/benchmark_readme_chart.png and b/benchmarks/decoders/benchmark_readme_chart.png differ diff --git a/benchmarks/decoders/benchmark_readme_data.json b/benchmarks/decoders/benchmark_readme_data.json index 28c79303d..9639d107e 100644 --- a/benchmarks/decoders/benchmark_readme_data.json +++ b/benchmarks/decoders/benchmark_readme_data.json @@ -1,290 +1,218 @@ [ { - "decoder": "TorchCodec", + "decoder": "torchcodec", "description": "uniform 10 seek()+next()", - "fps_median": 2.874245162330032, - "fps_p25": 2.890669218048133, - "fps_p75": 2.862767020928218, + "fps_median": 2.888099110308182, + "fps_p25": 2.9416150999827617, + "fps_p75": 2.8451474402295633, "frame_count": 10, - "iqr": 0.03371739387512207, - "median": 3.4791743345558643, - "type": "uniform:seek()+next()", + "iqr": 0.1152633186429739, + "median": 3.4624850526452065, + "type": "uniform seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "TorchCodec", + "decoder": "torchcodec", "description": "random 10 seek()+next()", - "fps_median": 3.658458368739899, - "fps_p25": 3.679569429876782, - "fps_p75": 3.606357184854245, + "fps_median": 3.679673367614735, + "fps_p25": 3.7010260633892753, + "fps_p75": 3.6730411925903006, "frame_count": 10, - "iqr": 0.05517190555110574, - "median": 2.733391771093011, - "type": "random:seek()+next()", + "iqr": 0.020586160942912102, + "median": 2.7176325181499124, + "type": "random seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "TorchCodec", + "decoder": "torchcodec", "description": "100 next()", - "fps_median": 248.32328677696285, - "fps_p25": 250.6501813668334, - "fps_p75": 244.31468271091225, + "fps_median": 332.62889220665653, + "fps_p25": 335.30421912694004, + "fps_p75": 329.14228955425125, "frame_count": 100, - "iqr": 0.010345779359340668, - "median": 0.402700855396688, - "type": "next()", + "iqr": 0.005583339370787144, + "median": 0.300635339692235, + "type": "100 next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "TorchVision[video_reader]", + "decoder": "torchvision[video_reader]", "description": "uniform 10 seek()+next()", - "fps_median": 0.4195638935454161, - "fps_p25": 0.42014650010734295, - "fps_p75": 0.4189829005177118, + "fps_median": 0.4183496816958986, + "fps_p25": 0.42028123679840773, + "fps_p75": 0.41643579969655253, "frame_count": 10, - "iqr": 0.06610076874494553, - "median": 23.834272095002234, - "type": "uniform:seek()+next()", + "iqr": 0.21971396077424288, + "median": 23.90344832930714, + "type": "uniform seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "TorchVision[video_reader]", + "decoder": "torchvision[video_reader]", "description": "random 10 seek()+next()", - "fps_median": 0.32245125406966435, - "fps_p25": 0.32245125406966435, - "fps_p75": 0.32245125406966435, + "fps_median": 0.3201664495067429, + "fps_p25": 0.3201664495067429, + "fps_p75": 0.3201664495067429, "frame_count": 10, "iqr": 0.0, - "median": 31.01243947353214, - "type": "random:seek()+next()", + "median": 31.23375361599028, + "type": "random seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "TorchVision[video_reader]", + "decoder": "torchvision[video_reader]", "description": "100 next()", - "fps_median": 176.82997455955987, - "fps_p25": 177.54379221046926, - "fps_p75": 175.14625035317215, + "fps_median": 183.6059253297136, + "fps_p25": 184.88581057175523, + "fps_p75": 180.4772551158132, "frame_count": 100, - "iqr": 0.00771009735763073, - "median": 0.5655149826779962, - "type": "next()", + "iqr": 0.01321205124258995, + "median": 0.5446447320282459, + "type": "100 next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "TorchAudio", + "decoder": "torchaudio", "description": "uniform 10 seek()+next()", - "fps_median": 0.5316661236830815, - "fps_p25": 0.5318804166321828, - "fps_p75": 0.5314520033403498, + "fps_median": 0.5240865109999612, + "fps_p25": 0.5245880071682041, + "fps_p75": 0.5235859727575772, "frame_count": 10, - "iqr": 0.015156010165810585, - "median": 18.80879663862288, - "type": "uniform:seek()+next()", + "iqr": 0.03648180142045021, + "median": 19.080819273367524, + "type": "uniform seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "TorchAudio", + "decoder": "torchaudio", "description": "random 10 seek()+next()", - "fps_median": 0.417209378798961, - "fps_p25": 0.41758998612516984, - "fps_p75": 0.4168294646408316, + "fps_median": 0.41470785541122906, + "fps_p25": 0.4150239639037595, + "fps_p75": 0.41439222808863957, "frame_count": 10, - "iqr": 0.04369210824370384, - "median": 23.968780444934964, - "type": "random:seek()+next()", + "iqr": 0.03673251997679472, + "median": 24.113360452465713, + "type": "random seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "TorchAudio", + "decoder": "torchaudio", "description": "100 next()", - "fps_median": 179.4697720392447, - "fps_p25": 181.05508626841646, - "fps_p75": 173.49148405860208, + "fps_median": 189.09233561323046, + "fps_p25": 190.02373443273453, + "fps_p75": 187.75804451204894, "frame_count": 100, - "iqr": 0.024079074384644628, - "median": 0.5571968965232372, - "type": "next()", + "iqr": 0.0063502974808216095, + "median": 0.5288421642035246, + "type": "100 next()", "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" }, { - "decoder": "Decord", + "decoder": "torchcodec", "description": "uniform 10 seek()+next()", - "fps_median": 2.9254250604823127, - "fps_p25": 2.928776037979067, - "fps_p75": 2.9179279307467434, + "fps_median": 33.23248091744692, + "fps_p25": 33.74429257495837, + "fps_p75": 32.677051134717146, "frame_count": 10, - "iqr": 0.012693846598267555, - "median": 3.4183066710829735, - "type": "uniform:seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "Decord", - "description": "random 10 seek()+next()", - "fps_median": 2.3913952159683447, - "fps_p25": 2.409423905905687, - "fps_p75": 2.379609551240287, - "frame_count": 10, - "iqr": 0.05200037732720375, - "median": 4.181659281253815, - "type": "random:seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "Decord", - "description": "100 next()", - "fps_median": 301.64058598625485, - "fps_p25": 304.4257754819803, - "fps_p75": 297.8336145342091, - "frame_count": 100, - "iqr": 0.0072706404607743025, - "median": 0.33152037439867854, - "type": "next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1280x720_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "TorchCodec", - "description": "uniform 10 seek()+next()", - "fps_median": 32.797487611763884, - "fps_p25": 33.16049598127707, - "fps_p75": 32.34604228526151, - "frame_count": 10, - "iqr": 0.007593189366161823, - "median": 0.30490140337496996, - "type": "uniform:seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "TorchCodec", - "description": "random 10 seek()+next()", - "fps_median": 31.86583108909739, - "fps_p25": 32.422775044534646, - "fps_p75": 31.51582155415764, - "frame_count": 10, - "iqr": 0.008875773288309574, - "median": 0.313815759960562, - "type": "random:seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "TorchCodec", - "description": "100 next()", - "fps_median": 478.59019817346393, - "fps_p25": 483.5137156807738, - "fps_p75": 469.99482336428133, - "frame_count": 100, - "iqr": 0.00594893516972661, - "median": 0.20894702896475792, - "type": "next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "TorchVision[video_reader]", - "description": "uniform 10 seek()+next()", - "fps_median": 5.863026312201653, - "fps_p25": 5.883880399939407, - "fps_p75": 5.821213575132222, - "frame_count": 10, - "iqr": 0.01829617563635111, - "median": 1.705603807233274, - "type": "uniform:seek()+next()", + "iqr": 0.009678754955530167, + "median": 0.3009104263037443, + "type": "uniform seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { - "decoder": "TorchVision[video_reader]", + "decoder": "torchcodec", "description": "random 10 seek()+next()", - "fps_median": 3.5804448256145283, - "fps_p25": 3.610017214846564, - "fps_p75": 3.573725565752598, + "fps_median": 32.0340867355421, + "fps_p25": 32.33995696335928, + "fps_p75": 31.48660833047371, "frame_count": 10, - "iqr": 0.028130420949310064, - "median": 2.7929490571841598, - "type": "random:seek()+next()", + "iqr": 0.008380331099033356, + "median": 0.3121674759313464, + "type": "random seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { - "decoder": "TorchVision[video_reader]", + "decoder": "torchcodec", "description": "100 next()", - "fps_median": 220.60506211225706, - "fps_p25": 221.92824221952606, - "fps_p75": 219.83775682122163, + "fps_median": 373.9297931168172, + "fps_p25": 377.8974427908539, + "fps_p75": 366.9518403987283, "frame_count": 100, - "iqr": 0.004284817026928067, - "median": 0.45329875499010086, - "type": "next()", + "iqr": 0.007893264293670654, + "median": 0.267429880797863, + "type": "100 next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { - "decoder": "TorchAudio", + "decoder": "torchvision[video_reader]", "description": "uniform 10 seek()+next()", - "fps_median": 10.562701139154996, - "fps_p25": 10.594120999307123, - "fps_p75": 10.475383401305544, + "fps_median": 5.826583628637648, + "fps_p25": 5.869409916500439, + "fps_p75": 5.80448564059718, "frame_count": 10, - "iqr": 0.010699251666665077, - "median": 0.9467275338247418, - "type": "uniform:seek()+next()", + "iqr": 0.01905675418674946, + "median": 1.7162715988233685, + "type": "uniform seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { - "decoder": "TorchAudio", + "decoder": "torchvision[video_reader]", "description": "random 10 seek()+next()", - "fps_median": 7.143372898069971, - "fps_p25": 7.190431420792876, - "fps_p75": 6.984323268168379, + "fps_median": 3.5913843964539565, + "fps_p25": 3.641605801630432, + "fps_p75": 3.557561124756978, "frame_count": 10, - "iqr": 0.041040806798264384, - "median": 1.3998989192768931, - "type": "random:seek()+next()", + "iqr": 0.06487313844263554, + "median": 2.7844415679574013, + "type": "random seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { - "decoder": "TorchAudio", + "decoder": "torchvision[video_reader]", "description": "100 next()", - "fps_median": 234.31507730203276, - "fps_p25": 235.5241203289182, - "fps_p75": 233.2609776710573, + "fps_median": 220.82061460318502, + "fps_p25": 222.48418761498934, + "fps_p75": 218.84938919419412, "frame_count": 100, - "iqr": 0.004119404591619968, - "median": 0.42677578050643206, - "type": "next()", + "iqr": 0.007465103641152382, + "median": 0.452856270596385, + "type": "100 next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { - "decoder": "Decord", + "decoder": "torchaudio", "description": "uniform 10 seek()+next()", - "fps_median": 28.951774908715404, - "fps_p25": 29.60339324217526, - "fps_p75": 28.130013392739134, + "fps_median": 10.54381002725746, + "fps_p25": 10.612585301638902, + "fps_p75": 10.516430888261935, "frame_count": 10, - "iqr": 0.017693073954433203, - "median": 0.3454019669443369, - "type": "uniform:seek()+next()", + "iqr": 0.008615483529865742, + "median": 0.9484237646684051, + "type": "uniform seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { - "decoder": "Decord", + "decoder": "torchaudio", "description": "random 10 seek()+next()", - "fps_median": 23.55224702573544, - "fps_p25": 23.829998507803513, - "fps_p75": 23.327513783118945, + "fps_median": 7.206591092692971, + "fps_p25": 7.231744857114745, + "fps_p75": 7.073061841995869, "frame_count": 10, - "iqr": 0.00903920829296112, - "median": 0.424587938003242, - "type": "random:seek()+next()", + "iqr": 0.0310227214358747, + "median": 1.3876186218112707, + "type": "random seek()+next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { - "decoder": "Decord", + "decoder": "torchaudio", "description": "100 next()", - "fps_median": 526.5873947695661, - "fps_p25": 534.5632246100912, - "fps_p75": 514.0865116923063, + "fps_median": 204.94896843488522, + "fps_p25": 206.320878222405, + "fps_p75": 203.29397516227598, "frame_count": 100, - "iqr": 0.007451178273186088, - "median": 0.18990200106054544, - "type": "next()", + "iqr": 0.007216569967567921, + "median": 0.4879263397306204, + "type": "100 next()", "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" }, { diff --git a/benchmarks/decoders/generate_readme_data.py b/benchmarks/decoders/generate_readme_data.py index 277cd3752..91be59dc1 100644 --- a/benchmarks/decoders/generate_readme_data.py +++ b/benchmarks/decoders/generate_readme_data.py @@ -11,7 +11,6 @@ from pathlib import Path from benchmark_decoders_library import ( - DecordAccurateBatch, generate_videos, retrieve_videos, run_benchmarks, @@ -56,10 +55,9 @@ def main() -> None: retrieve_videos(urls_and_dest_paths) decoder_dict = {} - decoder_dict["TorchCodec"] = TorchCodecPublic() - decoder_dict["TorchVision[video_reader]"] = TorchVision("video_reader") - decoder_dict["TorchAudio"] = TorchAudioDecoder() - decoder_dict["Decord"] = DecordAccurateBatch() + decoder_dict["torchcodec"] = TorchCodecPublic() + decoder_dict["torchvision[video_reader]"] = TorchVision("video_reader") + decoder_dict["torchaudio"] = TorchAudioDecoder() # These are the number of uniform seeks we do in the seek+decode benchmark. num_samples = 10