Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve setting default encode parameters #798

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
Open

Improve setting default encode parameters #798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
142db5a
main: do not overwrite output by default
FlyingWombat Mar 28, 2023
3cfbf07
determine_workers: consider source resolution and tiles
FlyingWombat Jun 4, 2023
460c189
default_args: set keyint 0 (unlimited)
FlyingWombat Jun 4, 2023
6eaf6a1
settings: merge video_params into get_default_arguments
FlyingWombat Jun 4, 2023
be2df3d
do not set frame limit on encoder
FlyingWombat Jul 7, 2023
259b04c
rav1e has no --fullhelp argument
FlyingWombat Jul 7, 2023
a90b5ff
context: print encoder information to log file
FlyingWombat Jan 5, 2024
881b028
default_args: no scene detection
FlyingWombat Jan 6, 2024
d67cb9e
determine_workers: update resource estimation
FlyingWombat Jan 8, 2024
ef8361e
Only estimate tiles when explicitly instructed
FlyingWombat Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 11 additions & 31 deletions av1an-core/src/context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ use std::sync::atomic::{self, AtomicBool, AtomicUsize};
use std::sync::{mpsc, Arc};
use std::{cmp, fs, iter, thread};

use ansi_term::{Color, Style};
use anyhow::{bail, Context};
use av1_grain::TransferFunction;
use crossbeam_utils;
Expand Down Expand Up @@ -254,36 +253,18 @@ impl Av1anContext {
};

if self.args.workers == 0 {
self.args.workers = determine_workers(self.args.encoder) as usize;
self.args.workers = determine_workers(&self.args) as usize;
}
self.args.workers = cmp::min(self.args.workers, chunk_queue.len());

if atty::is(atty::Stream::Stderr) {
eprintln!(
"{}{} {} {}{} {} {}{} {}\n{}: {}",
Color::Green.bold().paint("Q"),
Color::Green.paint("ueue"),
Color::Green.bold().paint(format!("{}", chunk_queue.len())),
Color::Blue.bold().paint("W"),
Color::Blue.paint("orkers"),
Color::Blue.bold().paint(format!("{}", self.args.workers)),
Color::Purple.bold().paint("P"),
Color::Purple.paint("asses"),
Color::Purple.bold().paint(format!("{}", self.args.passes)),
Style::default().bold().paint("Params"),
Style::default()
.dimmed()
.paint(self.args.video_params.join(" "))
);
} else {
eprintln!(
"Queue {} Workers {} Passes {}\nParams: {}",
chunk_queue.len(),
self.args.workers,
self.args.passes,
self.args.video_params.join(" ")
);
}
info!(
"Queue {} Workers {} Passes {} Encoder {}\nParams: {}",
chunk_queue.len(),
self.args.workers,
self.args.passes,
self.args.encoder,
self.args.video_params.join(" ")
);

if self.args.verbosity == Verbosity::Normal {
init_progress_bar(self.frames as u64, initial_frames as u64);
Expand Down Expand Up @@ -430,17 +411,16 @@ impl Av1anContext {
let mut enc_cmd = if chunk.passes == 1 {
chunk
.encoder
.compose_1_1_pass(video_params, chunk.output(), chunk.frames())
.compose_1_1_pass(video_params, chunk.output())
} else if current_pass == 1 {
chunk
.encoder
.compose_1_2_pass(video_params, fpf_file.to_str().unwrap(), chunk.frames())
.compose_1_2_pass(video_params, fpf_file.to_str().unwrap())
} else {
chunk.encoder.compose_2_2_pass(
video_params,
fpf_file.to_str().unwrap(),
chunk.output(),
chunk.frames(),
)
};

Expand Down
63 changes: 31 additions & 32 deletions av1an-core/src/encoder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,6 @@ impl Encoder {
self,
params: Vec<String>,
output: String,
frame_count: usize,
) -> Vec<String> {
match self {
Self::aom => chain!(
Expand All @@ -114,7 +113,7 @@ impl Encoder {
)
.collect(),
Self::rav1e => chain!(
into_array!["rav1e", "-", "-y", "--limit", frame_count.to_string()],
into_array!["rav1e", "-", "-y"],
params,
into_array!["--output", output]
)
Expand All @@ -139,15 +138,13 @@ impl Encoder {
"error",
"--demuxer",
"y4m",
"--frames",
frame_count.to_string()
],
params,
into_array!["-", "-o", output]
)
.collect(),
Self::x265 => chain!(
into_array!["x265", "--y4m", "--frames", frame_count.to_string()],
into_array!["x265", "--y4m"],
params,
into_array!["-", "-o", output]
)
Expand All @@ -156,7 +153,7 @@ impl Encoder {
}

/// Composes 1st pass command for 2 pass encoding
pub fn compose_1_2_pass(self, params: Vec<String>, fpf: &str, frame_count: usize) -> Vec<String> {
pub fn compose_1_2_pass(self, params: Vec<String>, fpf: &str) -> Vec<String> {
match self {
Self::aom => chain!(
into_array!["aomenc", "--passes=2", "--pass=1"],
Expand All @@ -170,8 +167,6 @@ impl Encoder {
"-",
"-y",
"--quiet",
"--limit",
frame_count.to_string()
],
params,
into_array!["--first-pass", format!("{fpf}.stat"), "--output", NULL]
Expand Down Expand Up @@ -207,8 +202,6 @@ impl Encoder {
"1",
"--demuxer",
"y4m",
"--frames",
frame_count.to_string()
],
params,
into_array!["--stats", format!("{fpf}.log"), "-", "-o", NULL]
Expand All @@ -223,8 +216,6 @@ impl Encoder {
"--pass",
"1",
"--y4m",
"--frames",
frame_count.to_string()
],
params,
into_array![
Expand All @@ -247,7 +238,6 @@ impl Encoder {
params: Vec<String>,
fpf: &str,
output: String,
frame_count: usize,
) -> Vec<String> {
match self {
Self::aom => chain!(
Expand All @@ -262,8 +252,6 @@ impl Encoder {
"-",
"-y",
"--quiet",
"--limit",
frame_count.to_string()
],
params,
into_array!["--second-pass", format!("{fpf}.stat"), "--output", output]
Expand Down Expand Up @@ -306,8 +294,6 @@ impl Encoder {
"2",
"--demuxer",
"y4m",
"--frames",
frame_count.to_string()
],
params,
into_array!["--stats", format!("{fpf}.log"), "-", "-o", output]
Expand All @@ -322,8 +308,6 @@ impl Encoder {
"--pass",
"2",
"--y4m",
"--frames",
frame_count.to_string()
],
params,
into_array![
Expand Down Expand Up @@ -364,6 +348,7 @@ impl Encoder {
"--cpu-used=6",
"--end-usage=q",
"--cq-level=30",
"--disable-kf",
];

if cols > 1 || rows > 1 {
Expand All @@ -380,8 +365,12 @@ impl Encoder {
}
}
Encoder::rav1e => {
let defaults: Vec<String> =
into_vec!["--speed", "6", "--quantizer", "100", "--no-scene-detection"];
let defaults: Vec<String> = into_vec![
"--speed", "6",
"--quantizer", "100",
"--keyint", "0",
"--no-scene-detection",
];

if cols > 1 || rows > 1 {
let tiles: Vec<String> = into_vec!["--tiles", format!("{}", cols * rows)];
Expand All @@ -404,6 +393,7 @@ impl Encoder {
"--cq-level=30",
"--row-mt=1",
"--auto-alt-ref=6",
"--disable-kf",
];

if cols > 1 || rows > 1 {
Expand All @@ -420,7 +410,13 @@ impl Encoder {
}
}
Encoder::svt_av1 => {
let defaults = into_vec!["--preset", "4", "--keyint", "240", "--rc", "0", "--crf", "25"];
let defaults = into_vec![
"--preset", "4",
"--keyint", "0",
"--scd", "0",
"--rc", "0",
"--crf", "25",
];
if cols > 1 || rows > 1 {
let columns = ilog2(cols);
let rows = ilog2(rows);
Expand All @@ -436,16 +432,19 @@ impl Encoder {
defaults
}
}
Encoder::x264 => into_vec!["--preset", "slow", "--crf", "25"],
Encoder::x264 => into_vec![
"--preset", "slow",
"--crf", "25",
"--keyint", "infinite",
"--scenecut", "0",
],
Encoder::x265 => into_vec![
"-p",
"slow",
"--crf",
"25",
"-D",
"10",
"--level-idc",
"5.0"
"--preset", "slow",
"--crf", "25",
"-D", "10",
"--level-idc", "5.0",
"--keyint", "-1",
"--scenecut", "0",
],
}
}
Expand All @@ -472,7 +471,7 @@ impl Encoder {
pub const fn help_command(self) -> [&'static str; 2] {
match self {
Self::aom => ["aomenc", "--help"],
Self::rav1e => ["rav1e", "--fullhelp"],
Self::rav1e => ["rav1e", "--help"],
Self::vpx => ["vpxenc", "--help"],
Self::svt_av1 => ["SvtAv1EncApp", "--help"],
Self::x264 => ["x264", "--fullhelp"],
Expand Down
59 changes: 48 additions & 11 deletions av1an-core/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ use std::thread::available_parallelism;
use std::time::Instant;

use ::ffmpeg::color::TransferCharacteristic;
use ::ffmpeg::format::Pixel;
use ::vapoursynth::api::API;
use ::vapoursynth::map::OwnedMap;
use anyhow::{bail, Context};
Expand All @@ -44,6 +45,7 @@ use serde::{Deserialize, Serialize};
use strum::{Display, EnumString, IntoStaticStr};

use crate::encoder::Encoder;
use crate::settings::{EncodeArgs};
use crate::progress_bar::finish_progress_bar;

pub mod broker;
Expand Down Expand Up @@ -364,24 +366,59 @@ pub enum ChunkOrdering {

/// Determine the optimal number of workers for an encoder
#[must_use]
pub fn determine_workers(encoder: Encoder) -> u64 {
pub fn determine_workers(args: &EncodeArgs) -> u64 {
let res = args.input.resolution().unwrap();
let tiles = args.tiles;
let megapixels = (res.0 * res.1) as f64 / 1e6;
// encoder memory and chunk_method memory usage scales with resolution (megapixels),
// approximately linearly. Expressed as GB/Megapixel
let cm_ram = match args.chunk_method {
ChunkMethod::FFMS2 | ChunkMethod::LSMASH | ChunkMethod::BESTSOURCE => 0.3,
ChunkMethod::DGDECNV => 0.3,
ChunkMethod::Hybrid | ChunkMethod::Select | ChunkMethod::Segment => 0.1,
};
let enc_ram = match args.encoder {
Encoder::aom => 0.4,
Encoder::rav1e => 0.7,
Encoder::svt_av1 => 1.2,
Encoder::vpx => 0.3,
Encoder::x264 => 0.7,
Encoder::x265 => 0.6,
};
// This is a rough estimate of how many cpu cores will be fully loaded by an encoder
// worker. With rav1e, CPU usage scales with tiles, but not 1:1.
// Other encoders don't seem to significantly scale CPU usage with tiles.
// CPU threads/worker here is relative to default threading parameters, e.g. aom
// will use 1 thread/worker if --threads=1 is set.
let cpu_threads = match args.encoder {
Encoder::aom => 4,
Encoder::rav1e => ((tiles.0 * tiles.1) as f32 * 0.7).ceil() as u64,
Encoder::svt_av1 => 6,
Encoder::vpx => 3,
Encoder::x264 | Encoder::x265 => 8,
};
// memory usage scales with pixel format, expressed as a multiplier of memory usage.
// Roughly the same behavior was observed accross all encoders.
let pix_mult = match args.output_pix_format.format {
Pixel::YUV444P | Pixel::YUV444P10LE | Pixel::YUV444P12LE => 1.5,
Pixel::YUV422P | Pixel::YUV422P10LE | Pixel::YUV422P12LE => 1.25,
_ => 1.0,
};

let mut system = sysinfo::System::new();
system.refresh_memory();

let cpu = available_parallelism()
.expect("Unrecoverable: Failed to get thread count")
.get() as u64;
// available_memory returns kb, convert to gb
let ram_gb = system.available_memory() / 10_u64.pow(6);
// sysinfo returns Bytes, convert to GB
// use total instead of available, because av1an does not resize worker pool
let ram_gb = system.total_memory() as f64 / 1e9;

std::cmp::max(
match encoder {
Encoder::aom | Encoder::rav1e | Encoder::vpx => std::cmp::min(
(cpu as f64 / 3.0).round() as u64,
(ram_gb as f64 / 1.5).round() as u64,
),
Encoder::svt_av1 | Encoder::x264 | Encoder::x265 => std::cmp::min(cpu, ram_gb) / 8,
},
std::cmp::min(
cpu / cpu_threads,
(ram_gb / (megapixels * (enc_ram + cm_ram) * pix_mult)).round() as u64,
),
1,
)
}
Expand Down
2 changes: 2 additions & 0 deletions av1an-core/src/scenes.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,8 @@ fn get_test_args() -> Av1anContext {
vmaf: false,
verbosity: Verbosity::Normal,
workers: 1,
tiles: (1, 1),
tile_auto: false,
set_thread_affinity: None,
zones: None,
scaler: String::new(),
Expand Down
Loading