diff --git a/benchmarks/decoders/benchmark_decoders_library.py b/benchmarks/decoders/benchmark_decoders_library.py index 0991c146d..4646d80ad 100644 --- a/benchmarks/decoders/benchmark_decoders_library.py +++ b/benchmarks/decoders/benchmark_decoders_library.py @@ -444,11 +444,11 @@ def retrieve_videos(urls_and_dest_paths): urllib.request.urlretrieve(url, path) -def plot_data(df_data, plot_path): +def plot_data(json_data, plot_path): plt.rcParams["font.size"] = 18 # Creating the DataFrame - df = pd.DataFrame(df_data) + df = pd.DataFrame(json_data["experiments"]) # Sorting by video, type, and frame_count df_sorted = df.sort_values(by=["video", "type", "frame_count"]) @@ -518,6 +518,14 @@ def plot_data(df_data, plot_path): for col in range(video_type_combinations[unique_videos[row]], max_combinations): fig.delaxes(axes[row, col]) + plt.gcf().text( + 0.005, + 0.87, + "\n".join([f"{k}: {v}" for k, v in json_data["system_metadata"].items()]), + fontsize=11, + bbox=dict(facecolor="white"), + ) + # Adjust layout to avoid overlap plt.tight_layout() diff --git a/benchmarks/decoders/benchmark_readme_chart.png b/benchmarks/decoders/benchmark_readme_chart.png index 6e67180c1..2ead24b5e 100644 Binary files a/benchmarks/decoders/benchmark_readme_chart.png and b/benchmarks/decoders/benchmark_readme_chart.png differ diff --git a/benchmarks/decoders/benchmark_readme_data.json b/benchmarks/decoders/benchmark_readme_data.json index dd5411674..3bfa1f182 100644 --- a/benchmarks/decoders/benchmark_readme_data.json +++ b/benchmarks/decoders/benchmark_readme_data.json @@ -1,394 +1,397 @@ -[ - { - "decoder": "torchcodec", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 2.1909987258958803, - "fps_p25": 2.1909987258958803, - "fps_p75": 2.1909987258958803, - "frame_count": 500, - "iqr": 0.0, - "median": 228.2064311997965, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "uniform 10 seek()+next()", - "fps_median": 0.22373366699708405, - "fps_p25": 0.22373366699708405, - "fps_p75": 0.22373366699708405, - "frame_count": 10, - "iqr": 0.0, - "median": 44.69600008893758, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "random 10 seek()+next()", - "fps_median": 0.2870396511009182, - "fps_p25": 0.2870396511009182, - "fps_p75": 0.2870396511009182, - "frame_count": 10, - "iqr": 0.0, - "median": 34.83839240204543, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "100 next()", - "fps_median": 79.89466672412033, - "fps_p25": 99.18266043890266, - "fps_p75": 79.07153270302595, - "frame_count": 100, - "iqr": 0.2564368974417448, - "median": 1.2516480023041368, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 7.324784294018666, - "fps_p25": 7.324784294018666, - "fps_p75": 7.324784294018666, - "frame_count": 500, - "iqr": 0.0, - "median": 68.26139582134783, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "uniform 10 seek()+next()", - "fps_median": 1.232367350732705, - "fps_p25": 1.233843742370965, - "fps_p75": 1.2314023481824525, - "frame_count": 10, - "iqr": 0.016068589873611927, - "median": 8.114463592413813, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "random 10 seek()+next()", - "fps_median": 1.5437990877014915, - "fps_p25": 1.5438508087203877, - "fps_p75": 1.5435480997081767, - "frame_count": 10, - "iqr": 0.0012702811509370804, - "median": 6.477526823058724, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "100 next()", - "fps_median": 179.5141829898575, - "fps_p25": 182.56822073549603, - "fps_p75": 124.4290827477999, - "frame_count": 100, - "iqr": 0.25593018205836415, - "median": 0.5570590486750007, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 1.951097912335046, - "fps_p25": 1.951097912335046, - "fps_p75": 1.951097912335046, - "frame_count": 500, - "iqr": 0.0, - "median": 256.265970476903, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "uniform 10 seek()+next()", - "fps_median": 0.23515671574954122, - "fps_p25": 0.23515671574954122, - "fps_p75": 0.23515671574954122, - "frame_count": 10, - "iqr": 0.0, - "median": 42.524832719005644, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "random 10 seek()+next()", - "fps_median": 0.17733950193054088, - "fps_p25": 0.17733950193054088, - "fps_p75": 0.17733950193054088, - "frame_count": 10, - "iqr": 0.0, - "median": 56.38901593349874, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "100 next()", - "fps_median": 91.93830916027736, - "fps_p25": 93.48146430884644, - "fps_p75": 89.72122041281625, - "frame_count": 100, - "iqr": 0.04483274556696415, - "median": 1.087685872334987, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 0.23115768079016835, - "fps_p25": 0.23115768079016835, - "fps_p75": 0.23115768079016835, - "frame_count": 500, - "iqr": 0.0, - "median": 2163.0256813913584, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "uniform 10 seek()+next()", - "fps_median": 0.26119132784303123, - "fps_p25": 0.26119132784303123, - "fps_p75": 0.26119132784303123, - "frame_count": 10, - "iqr": 0.0, - "median": 38.2861103489995, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "random 10 seek()+next()", - "fps_median": 0.19600716639142407, - "fps_p25": 0.19600716639142407, - "fps_p75": 0.19600716639142407, - "frame_count": 10, - "iqr": 0.0, - "median": 51.01854276098311, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "100 next()", - "fps_median": 100.42696877778134, - "fps_p25": 101.82432674120878, - "fps_p75": 98.63909948929518, - "frame_count": 100, - "iqr": 0.0317131788469851, - "median": 0.995748464949429, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 78.89304868408422, - "fps_p25": 79.36504926194816, - "fps_p75": 78.80019881505761, - "frame_count": 500, - "iqr": 0.04515926633030176, - "median": 6.3376939836889505, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "uniform 10 seek()+next()", - "fps_median": 9.854385050842327, - "fps_p25": 10.094368141227985, - "fps_p75": 9.656371003428172, - "frame_count": 10, - "iqr": 0.04493432166054845, - "median": 1.014776665251702, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "random 10 seek()+next()", - "fps_median": 7.965955631552031, - "fps_p25": 8.052124079150976, - "fps_p75": 7.759671780042043, - "frame_count": 10, - "iqr": 0.04680596780963242, - "median": 1.2553421664051712, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "100 next()", - "fps_median": 250.74790033552227, - "fps_p25": 255.02885647654145, - "fps_p75": 248.55035150151298, - "frame_count": 100, - "iqr": 0.010220475262030959, - "median": 0.3988069286569953, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 14.558551600810196, - "fps_p25": 14.558551600810196, - "fps_p75": 14.558551600810196, - "frame_count": 500, - "iqr": 0.0, - "median": 34.34407581947744, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "uniform 10 seek()+next()", - "fps_median": 3.3993119043827935, - "fps_p25": 3.4033747004472383, - "fps_p75": 3.3871671818375297, - "frame_count": 10, - "iqr": 0.014059508219361305, - "median": 2.9417718295007944, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "random 10 seek()+next()", - "fps_median": 4.208422451807454, - "fps_p25": 4.228530445598264, - "fps_p75": 4.202620082236087, - "frame_count": 10, - "iqr": 0.014580214396119118, - "median": 2.376187304034829, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "100 next()", - "fps_median": 108.81798607716902, - "fps_p25": 109.2651801306662, - "fps_p75": 107.98428953245536, - "frame_count": 100, - "iqr": 0.010855996049940586, - "median": 0.9189657298848033, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 41.39450037405011, - "fps_p25": 41.84854039364475, - "fps_p75": 41.139148433617976, - "frame_count": 500, - "iqr": 0.20602534664794803, - "median": 12.078899261541665, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "uniform 10 seek()+next()", - "fps_median": 4.743936824381266, - "fps_p25": 4.806402420821976, - "fps_p75": 4.66718304034089, - "frame_count": 10, - "iqr": 0.06206185230985284, - "median": 2.1079538725316525, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "random 10 seek()+next()", - "fps_median": 2.9809550025126472, - "fps_p25": 3.038024440532653, - "fps_p75": 2.95329673866747, - "frame_count": 10, - "iqr": 0.09443371556699276, - "median": 3.354629637673497, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "100 next()", - "fps_median": 177.20109315629767, - "fps_p25": 182.07930273828418, - "fps_p75": 173.29419285866405, - "frame_count": 100, - "iqr": 0.027842139592394233, - "median": 0.5643306043930352, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 9.23025646940316, - "fps_p25": 9.23025646940316, - "fps_p75": 9.23025646940316, - "frame_count": 500, - "iqr": 0.0, - "median": 54.1696757460013, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "uniform 10 seek()+next()", - "fps_median": 10.100628756440091, - "fps_p25": 10.212753705185772, - "fps_p75": 9.780241639161853, - "frame_count": 10, - "iqr": 0.0433017835021019, - "median": 0.9900373769924045, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "random 10 seek()+next()", - "fps_median": 6.6581950176290565, - "fps_p25": 6.769915708964532, - "fps_p75": 6.587147793106109, - "frame_count": 10, - "iqr": 0.04098446690477431, - "median": 1.5019085463136435, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "100 next()", - "fps_median": 263.36187377747655, - "fps_p25": 272.8191596128445, - "fps_p75": 254.92052325953637, - "frame_count": 100, - "iqr": 0.02573595428839326, - "median": 0.37970568239688873, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "cpu_count": 22, - "is_cuda_available": "True", - "machine": "x86_64", - "processor": "x86_64", - "python_version": "3.12.5", - "system": "Linux" - } -] +{ + "experiments": + [ + { + "decoder": "torchcodec", + "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 2.1909987258958803, + "fps_p25": 2.1909987258958803, + "fps_p75": 2.1909987258958803, + "frame_count": 500, + "iqr": 0.0, + "median": 228.2064311997965, + "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "uniform 10 seek()+next()", + "fps_median": 0.22373366699708405, + "fps_p25": 0.22373366699708405, + "fps_p75": 0.22373366699708405, + "frame_count": 10, + "iqr": 0.0, + "median": 44.69600008893758, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "random 10 seek()+next()", + "fps_median": 0.2870396511009182, + "fps_p25": 0.2870396511009182, + "fps_p75": 0.2870396511009182, + "frame_count": 10, + "iqr": 0.0, + "median": 34.83839240204543, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "100 next()", + "fps_median": 79.89466672412033, + "fps_p25": 99.18266043890266, + "fps_p75": 79.07153270302595, + "frame_count": 100, + "iqr": 0.2564368974417448, + "median": 1.2516480023041368, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 7.324784294018666, + "fps_p25": 7.324784294018666, + "fps_p75": 7.324784294018666, + "frame_count": 500, + "iqr": 0.0, + "median": 68.26139582134783, + "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "uniform 10 seek()+next()", + "fps_median": 1.232367350732705, + "fps_p25": 1.233843742370965, + "fps_p75": 1.2314023481824525, + "frame_count": 10, + "iqr": 0.016068589873611927, + "median": 8.114463592413813, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "random 10 seek()+next()", + "fps_median": 1.5437990877014915, + "fps_p25": 1.5438508087203877, + "fps_p75": 1.5435480997081767, + "frame_count": 10, + "iqr": 0.0012702811509370804, + "median": 6.477526823058724, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "100 next()", + "fps_median": 179.5141829898575, + "fps_p25": 182.56822073549603, + "fps_p75": 124.4290827477999, + "frame_count": 100, + "iqr": 0.25593018205836415, + "median": 0.5570590486750007, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 1.951097912335046, + "fps_p25": 1.951097912335046, + "fps_p75": 1.951097912335046, + "frame_count": 500, + "iqr": 0.0, + "median": 256.265970476903, + "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "uniform 10 seek()+next()", + "fps_median": 0.23515671574954122, + "fps_p25": 0.23515671574954122, + "fps_p75": 0.23515671574954122, + "frame_count": 10, + "iqr": 0.0, + "median": 42.524832719005644, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "random 10 seek()+next()", + "fps_median": 0.17733950193054088, + "fps_p25": 0.17733950193054088, + "fps_p75": 0.17733950193054088, + "frame_count": 10, + "iqr": 0.0, + "median": 56.38901593349874, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "100 next()", + "fps_median": 91.93830916027736, + "fps_p25": 93.48146430884644, + "fps_p75": 89.72122041281625, + "frame_count": 100, + "iqr": 0.04483274556696415, + "median": 1.087685872334987, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 0.23115768079016835, + "fps_p25": 0.23115768079016835, + "fps_p75": 0.23115768079016835, + "frame_count": 500, + "iqr": 0.0, + "median": 2163.0256813913584, + "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "uniform 10 seek()+next()", + "fps_median": 0.26119132784303123, + "fps_p25": 0.26119132784303123, + "fps_p75": 0.26119132784303123, + "frame_count": 10, + "iqr": 0.0, + "median": 38.2861103489995, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "random 10 seek()+next()", + "fps_median": 0.19600716639142407, + "fps_p25": 0.19600716639142407, + "fps_p75": 0.19600716639142407, + "frame_count": 10, + "iqr": 0.0, + "median": 51.01854276098311, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "100 next()", + "fps_median": 100.42696877778134, + "fps_p25": 101.82432674120878, + "fps_p75": 98.63909948929518, + "frame_count": 100, + "iqr": 0.0317131788469851, + "median": 0.995748464949429, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 78.89304868408422, + "fps_p25": 79.36504926194816, + "fps_p75": 78.80019881505761, + "frame_count": 500, + "iqr": 0.04515926633030176, + "median": 6.3376939836889505, + "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "uniform 10 seek()+next()", + "fps_median": 9.854385050842327, + "fps_p25": 10.094368141227985, + "fps_p75": 9.656371003428172, + "frame_count": 10, + "iqr": 0.04493432166054845, + "median": 1.014776665251702, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "random 10 seek()+next()", + "fps_median": 7.965955631552031, + "fps_p25": 8.052124079150976, + "fps_p75": 7.759671780042043, + "frame_count": 10, + "iqr": 0.04680596780963242, + "median": 1.2553421664051712, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "100 next()", + "fps_median": 250.74790033552227, + "fps_p25": 255.02885647654145, + "fps_p75": 248.55035150151298, + "frame_count": 100, + "iqr": 0.010220475262030959, + "median": 0.3988069286569953, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 14.558551600810196, + "fps_p25": 14.558551600810196, + "fps_p75": 14.558551600810196, + "frame_count": 500, + "iqr": 0.0, + "median": 34.34407581947744, + "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "uniform 10 seek()+next()", + "fps_median": 3.3993119043827935, + "fps_p25": 3.4033747004472383, + "fps_p75": 3.3871671818375297, + "frame_count": 10, + "iqr": 0.014059508219361305, + "median": 2.9417718295007944, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "random 10 seek()+next()", + "fps_median": 4.208422451807454, + "fps_p25": 4.228530445598264, + "fps_p75": 4.202620082236087, + "frame_count": 10, + "iqr": 0.014580214396119118, + "median": 2.376187304034829, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "100 next()", + "fps_median": 108.81798607716902, + "fps_p25": 109.2651801306662, + "fps_p75": 107.98428953245536, + "frame_count": 100, + "iqr": 0.010855996049940586, + "median": 0.9189657298848033, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 41.39450037405011, + "fps_p25": 41.84854039364475, + "fps_p75": 41.139148433617976, + "frame_count": 500, + "iqr": 0.20602534664794803, + "median": 12.078899261541665, + "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "uniform 10 seek()+next()", + "fps_median": 4.743936824381266, + "fps_p25": 4.806402420821976, + "fps_p75": 4.66718304034089, + "frame_count": 10, + "iqr": 0.06206185230985284, + "median": 2.1079538725316525, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "random 10 seek()+next()", + "fps_median": 2.9809550025126472, + "fps_p25": 3.038024440532653, + "fps_p75": 2.95329673866747, + "frame_count": 10, + "iqr": 0.09443371556699276, + "median": 3.354629637673497, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "100 next()", + "fps_median": 177.20109315629767, + "fps_p25": 182.07930273828418, + "fps_p75": 173.29419285866405, + "frame_count": 100, + "iqr": 0.027842139592394233, + "median": 0.5643306043930352, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 9.23025646940316, + "fps_p25": 9.23025646940316, + "fps_p75": 9.23025646940316, + "frame_count": 500, + "iqr": 0.0, + "median": 54.1696757460013, + "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "uniform 10 seek()+next()", + "fps_median": 10.100628756440091, + "fps_p25": 10.212753705185772, + "fps_p75": 9.780241639161853, + "frame_count": 10, + "iqr": 0.0433017835021019, + "median": 0.9900373769924045, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "random 10 seek()+next()", + "fps_median": 6.6581950176290565, + "fps_p25": 6.769915708964532, + "fps_p75": 6.587147793106109, + "frame_count": 10, + "iqr": 0.04098446690477431, + "median": 1.5019085463136435, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "100 next()", + "fps_median": 263.36187377747655, + "fps_p25": 272.8191596128445, + "fps_p75": 254.92052325953637, + "frame_count": 100, + "iqr": 0.02573595428839326, + "median": 0.37970568239688873, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + } + ], + "system_metadata": + { + "cpu_count": 22, + "machine": "x86_64", + "python_version": "3.12.5", + "system": "Linux", + "cuda": "NVIDIA PG509-210" + } +} diff --git a/benchmarks/decoders/generate_readme_data.py b/benchmarks/decoders/generate_readme_data.py index 8feb4e3db..891e5dcaa 100644 --- a/benchmarks/decoders/generate_readme_data.py +++ b/benchmarks/decoders/generate_readme_data.py @@ -69,7 +69,7 @@ def main() -> None: num_samples = 10 video_files_paths = list(Path(videos_dir_path).glob("*.mp4")) assert len(video_files_paths) == 2, "Expected exactly 2 videos" - df_data = run_benchmarks( + results = run_benchmarks( decoder_dict, video_files_paths, num_samples, @@ -83,21 +83,24 @@ def main() -> None: resize_device="cuda", ), ) - df_data.append( - { - "system_metadata": { - "cpu_count": os.cpu_count(), - "system": platform.system(), - "machine": platform.machine(), - "python_version": str(platform.python_version()), - "is_cuda_available": str(torch.cuda.is_available()), - } - } - ) + data_for_writing = { + "experiments": results, + "system_metadata": { + "cpu_count": os.cpu_count(), + "system": platform.system(), + "machine": platform.machine(), + "python_version": str(platform.python_version()), + "cuda": ( + torch.cuda.get_device_properties(0).name + if torch.cuda.is_available() + else "not available" + ), + }, + } data_json = Path(__file__).parent / "benchmark_readme_data.json" with open(data_json, "w") as write_file: - json.dump(df_data, write_file, sort_keys=True, indent=4) + json.dump(data_for_writing, write_file, sort_keys=True, indent=4) if __name__ == "__main__":