Skip to content

Use more interesting videos for README experiment #351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Nov 8, 2024
Merged

Use more interesting videos for README experiment #351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d4cdc20
Use more interesting videos for README experiment
scotts Nov 8, 2024
2d6b82f
Merge branch 'main' of github.com:pytorch/torchcodec into readme_video
scotts Nov 8, 2024
d537a90
Apply linting.
scotts Nov 8, 2024
74db54d
Revert mean back to median
scotts Nov 8, 2024
9d5ed07
Refactor out uses of os library
scotts Nov 8, 2024
51b5146
Add encoding and pixel format to README
scotts Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,12 @@ The following was generated by running [our benchmark script](./benchmarks/decod

![benchmark_results](./benchmarks/decoders/benchmark_readme_chart.png)

The top row is a [Mandelbrot](https://ffmpeg.org/ffmpeg-filters.html#mandelbrot) video
generated from FFmpeg that has a resolution of 1280x720 at 60 fps and is 120 seconds long.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add the codec and pixel format too if you can

The bottom row is [promotional video from NASA](https://download.pytorch.org/torchaudio/tutorial-assets/stream-api/NASAs_Most_Scientifically_Complex_Space_Observatory_Requires_Precision-MP4_small.mp4)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The two charts look very similar. I would remove the fractal and just use the real video

Copy link
Contributor Author

@scotts scotts Nov 8, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm going to merge as-is. Let's discuss later what we want long-term. I have a slight preference to show two videos, to give some indication that the NASA video isn't a fluke and to include a higher resolution video. Put another way, I want someone to look at both videos and think "Huh, they're about the same performance."

that has a resolution of 960x540 at 29.7 fps and is 206 seconds long. Both videos were
encoded with libx264 and yuv420p pixel format.

## Planned future work

We are actively working on the following features:
Expand Down
77 changes: 36 additions & 41 deletions benchmarks/decoders/benchmark_decoders_library.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import abc
import json
import os
import subprocess
import urllib.request
from concurrent.futures import ThreadPoolExecutor, wait
from itertools import product
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
Expand Down Expand Up @@ -330,6 +331,7 @@ def generate_video(command):
def generate_videos(
resolutions,
encodings,
patterns,
fpses,
gop_sizes,
durations,
Expand All @@ -341,23 +343,25 @@ def generate_videos(
video_count = 0

futures = []
for resolution, duration, fps, gop_size, encoding, pix_fmt in product(
resolutions, durations, fpses, gop_sizes, encodings, pix_fmts
for resolution, duration, fps, gop_size, encoding, pattern, pix_fmt in product(
resolutions, durations, fpses, gop_sizes, encodings, patterns, pix_fmts
):
outfile = f"{output_dir}/{resolution}_{duration}s_{fps}fps_{gop_size}gop_{encoding}_{pix_fmt}.mp4"
outfile = f"{output_dir}/{pattern}_{resolution}_{duration}s_{fps}fps_{gop_size}gop_{encoding}_{pix_fmt}.mp4"
command = [
ffmpeg_cli,
"-y",
"-f",
"lavfi",
"-i",
f"color=c=blue:s={resolution}:d={duration}",
f"{pattern}=s={resolution}",
"-t",
str(duration),
"-c:v",
encoding,
"-r",
f"{fps}",
str(fps),
"-g",
f"{gop_size}",
str(gop_size),
"-pix_fmt",
pix_fmt,
outfile,
Expand All @@ -372,6 +376,11 @@ def generate_videos(
print(f"Generated {video_count} videos")


def retrieve_videos(urls_and_dest_paths):
for url, path in urls_and_dest_paths:
urllib.request.urlretrieve(url, path)


def plot_data(df_data, plot_path):
# Creating the DataFrame
df = pd.DataFrame(df_data)
Expand Down Expand Up @@ -400,7 +409,7 @@ def plot_data(df_data, plot_path):
nrows=len(unique_videos),
ncols=max_combinations,
figsize=(max_combinations * 6, len(unique_videos) * 4),
sharex=True,
sharex=False,
sharey=True,
)

Expand All @@ -419,16 +428,19 @@ def plot_data(df_data, plot_path):
ax = axes[row, col] # Select the appropriate axis

# Set the title for the subplot
base_video = os.path.basename(video)
base_video = Path(video).name.removesuffix(".mp4")
ax.set_title(
f"video={base_video}\ndecode_pattern={vcount} x {vtype}", fontsize=12
f"video={base_video}\ndecode_pattern={vcount} x {vtype}", fontsize=10
)

# Plot bars with error bars
ax.barh(
group["decoder"],
group["fps"],
xerr=[group["fps"] - group["fps_p75"], group["fps_p25"] - group["fps"]],
group["fps_median"],
xerr=[
group["fps_median"] - group["fps_p75"],
group["fps_p25"] - group["fps_median"],
],
color=[colors(i) for i in range(len(group))],
align="center",
capsize=5,
Expand All @@ -438,28 +450,11 @@ def plot_data(df_data, plot_path):
# Set the labels
ax.set_xlabel("FPS")

# No need for y-axis label past the plot on the far left
if col == 0:
ax.set_ylabel("Decoder")

# Remove any empty subplots for videos with fewer combinations
for row in range(len(unique_videos)):
for col in range(video_type_combinations[unique_videos[row]], max_combinations):
fig.delaxes(axes[row, col])

# If we just call fig.legend, we'll get duplicate labels, as each label appears on
# each subplot. We take advantage of dicts having unique keys to de-dupe.
handles, labels = plt.gca().get_legend_handles_labels()
unique_labels = dict(zip(labels, handles))

# Reverse the order of the handles and labels to match the order of the bars
fig.legend(
handles=reversed(unique_labels.values()),
labels=reversed(unique_labels.keys()),
frameon=True,
loc="right",
)

# Adjust layout to avoid overlap
plt.tight_layout()

Expand All @@ -475,7 +470,7 @@ def get_metadata(video_file_path: str) -> VideoStreamMetadata:

def run_benchmarks(
decoder_dict: dict[str, AbstractDecoder],
video_files_paths: list[str],
video_files_paths: list[Path],
num_samples: int,
num_sequential_frames_from_start: list[int],
min_runtime_seconds: float,
Expand Down Expand Up @@ -515,7 +510,7 @@ def run_benchmarks(
seeked_result = benchmark.Timer(
stmt="decoder.get_frames_from_video(video_file, pts_list)",
globals={
"video_file": video_file_path,
"video_file": str(video_file_path),
"pts_list": pts_list,
"decoder": decoder,
},
Expand All @@ -528,22 +523,22 @@ def run_benchmarks(
)
df_item = {}
df_item["decoder"] = decoder_name
df_item["video"] = video_file_path
df_item["video"] = str(video_file_path)
df_item["description"] = results[-1].description
df_item["frame_count"] = num_samples
df_item["median"] = results[-1].median
df_item["iqr"] = results[-1].iqr
df_item["type"] = f"{kind}:seek()+next()"
df_item["fps"] = 1.0 * num_samples / results[-1].median
df_item["fps_p75"] = 1.0 * num_samples / results[-1]._p75
df_item["fps_p25"] = 1.0 * num_samples / results[-1]._p25
df_item["fps_median"] = num_samples / results[-1].median
df_item["fps_p75"] = num_samples / results[-1]._p75
df_item["fps_p25"] = num_samples / results[-1]._p25
df_data.append(df_item)

for num_consecutive_nexts in num_sequential_frames_from_start:
consecutive_frames_result = benchmark.Timer(
stmt="decoder.get_consecutive_frames_from_video(video_file, consecutive_frames_to_extract)",
globals={
"video_file": video_file_path,
"video_file": str(video_file_path),
"consecutive_frames_to_extract": num_consecutive_nexts,
"decoder": decoder,
},
Expand All @@ -558,15 +553,15 @@ def run_benchmarks(
)
df_item = {}
df_item["decoder"] = decoder_name
df_item["video"] = video_file_path
df_item["video"] = str(video_file_path)
df_item["description"] = results[-1].description
df_item["frame_count"] = num_consecutive_nexts
df_item["median"] = results[-1].median
df_item["iqr"] = results[-1].iqr
df_item["type"] = "next()"
df_item["fps"] = 1.0 * num_consecutive_nexts / results[-1].median
df_item["fps_p75"] = 1.0 * num_consecutive_nexts / results[-1]._p75
df_item["fps_p25"] = 1.0 * num_consecutive_nexts / results[-1]._p25
df_item["fps_median"] = num_consecutive_nexts / results[-1].median
df_item["fps_p75"] = num_consecutive_nexts / results[-1]._p75
df_item["fps_p25"] = num_consecutive_nexts / results[-1]._p25
df_data.append(df_item)

first_video_file_path = video_files_paths[0]
Expand All @@ -576,7 +571,7 @@ def run_benchmarks(
creation_result = benchmark.Timer(
stmt="create_torchcodec_decoder_from_file(video_file)",
globals={
"video_file": first_video_file_path,
"video_file": str(first_video_file_path),
"create_torchcodec_decoder_from_file": create_torchcodec_decoder_from_file,
},
label=f"video={first_video_file_path} {metadata_label}",
Expand Down
Binary file modified benchmarks/decoders/benchmark_readme_chart.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Loading