Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add option to encode using baseline/constrained baseline. #1932

Merged
merged 2 commits into from
Jan 2, 2024
Merged

Add option to encode using baseline/constrained baseline. #1932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c8b4358
Add option to encode using baseline/constrained baseline.
20kdc Dec 17, 2023
98855a9
Allow selecting h264 profile with a bit more precision, migrate setti…
20kdc Dec 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions alvr/server/cpp/ALVR-common/packet_types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ enum ALVR_CODEC {
ALVR_CODEC_H265 = 1,
};

enum ALVR_H264_PROFILE {
ALVR_H264_PROFILE_HIGH = 0,
ALVR_H264_PROFILE_MAIN = 1,
ALVR_H264_PROFILE_BASELINE = 2,
};

enum ALVR_RATE_CONTROL_METHOD {
ALVR_CBR = 0,
ALVR_VBR = 1,
Expand Down
5 changes: 3 additions & 2 deletions alvr/server/cpp/alvr_server/Settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ void Settings::Load() {
picojson::value v;
std::string err = picojson::parse(v, json);
if (!err.empty()) {
Error("Error on parsing json: %hs\n", err.c_str());
Error("Error on parsing session config (%s): %hs\n", g_sessionPath, err.c_str());
return;
}

Expand Down Expand Up @@ -60,6 +60,7 @@ void Settings::Load() {
m_sharpening = (float)config.get("sharpening").get<double>();

m_codec = (int32_t)config.get("codec").get<int64_t>();
m_h264Profile = (int32_t)config.get("h264_profile").get<int64_t>();
m_rateControlMode = (uint32_t)config.get("rate_control_mode").get<int64_t>();
m_fillerData = config.get("filler_data").get<bool>();
m_entropyCoding = (uint32_t)config.get("entropy_coding").get<int64_t>();
Expand Down Expand Up @@ -109,6 +110,6 @@ void Settings::Load() {
Info("Refresh Rate: %d\n", m_refreshRate);
m_loaded = true;
} catch (std::exception &e) {
Error("Exception on parsing json: %hs\n", e.what());
Error("Exception on parsing session config (%s): %hs\n", g_sessionPath, e.what());
}
}
1 change: 1 addition & 0 deletions alvr/server/cpp/alvr_server/Settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ class Settings {
float m_sharpening;

int m_codec;
int m_h264Profile;
bool m_use10bitEncoder;
bool m_enableVbaq;
bool m_usePreproc;
Expand Down
13 changes: 13 additions & 0 deletions alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,19 @@ alvr::EncodePipelineNvEnc::EncodePipelineNvEnc(Renderer *render,
break;
}

switch (settings.m_h264Profile) {
case ALVR_H264_PROFILE_BASELINE:
av_opt_set(encoder_ctx->priv_data, "profile", "baseline", 0);
break;
case ALVR_H264_PROFILE_MAIN:
av_opt_set(encoder_ctx->priv_data, "profile", "main", 0);
break;
default:
case ALVR_H264_PROFILE_HIGH:
av_opt_set(encoder_ctx->priv_data, "profile", "high", 0);
break;
}

char preset[] = "p0";
// replace 0 with preset number
preset[1] += settings.m_nvencQualityPreset;
Expand Down
14 changes: 13 additions & 1 deletion alvr/server/cpp/platform/linux/EncodePipelineSW.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ alvr::EncodePipelineSW::EncodePipelineSW(Renderer *render, uint32_t width, uint3
const auto& settings = Settings::Instance();

x264_param_default_preset(&param, "ultrafast", "zerolatency");
x264_param_apply_profile(&param, "high");

param.pf_log = x264_log;
param.i_log_level = X264_LOG_INFO;
Expand All @@ -51,6 +50,19 @@ alvr::EncodePipelineSW::EncodePipelineSW(Renderer *render, uint32_t width, uint3
param.i_height = height;
param.rc.i_rc_method = X264_RC_ABR;

switch (settings.m_h264Profile) {
case ALVR_H264_PROFILE_BASELINE:
x264_param_apply_profile(&param, "baseline");
break;
case ALVR_H264_PROFILE_MAIN:
x264_param_apply_profile(&param, "main");
break;
default:
case ALVR_H264_PROFILE_HIGH:
x264_param_apply_profile(&param, "high");
break;
}

auto params = FfiDynamicEncoderParams {};
params.updated = true;
params.bitrate_bps = 30'000'000;
Expand Down
13 changes: 12 additions & 1 deletion alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,18 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(Renderer *render, VkContext &vk_c
switch (codec_id)
{
case ALVR_CODEC_H264:
encoder_ctx->profile = FF_PROFILE_H264_MAIN;
switch (settings.m_h264Profile) {
case ALVR_H264_PROFILE_BASELINE:
encoder_ctx->profile = FF_PROFILE_H264_BASELINE;
break;
case ALVR_H264_PROFILE_MAIN:
encoder_ctx->profile = FF_PROFILE_H264_MAIN;
break;
default:
case ALVR_H264_PROFILE_HIGH:
encoder_ctx->profile = FF_PROFILE_H264_HIGH;
break;
}

switch (settings.m_entropyCoding) {
case ALVR_CABAC:
Expand Down
13 changes: 12 additions & 1 deletion alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,18 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder(
if (codec == ALVR_CODEC_H264)
{
amfEncoder->SetProperty(AMF_VIDEO_ENCODER_USAGE, AMF_VIDEO_ENCODER_USAGE_ULTRA_LOW_LATENCY);
amfEncoder->SetProperty(AMF_VIDEO_ENCODER_PROFILE, AMF_VIDEO_ENCODER_PROFILE_HIGH);
switch (Settings::Instance().m_h264Profile) {
case ALVR_H264_PROFILE_BASELINE:
amfEncoder->SetProperty(AMF_VIDEO_ENCODER_PROFILE, AMF_VIDEO_ENCODER_PROFILE_BASELINE);
break;
case ALVR_H264_PROFILE_MAIN:
amfEncoder->SetProperty(AMF_VIDEO_ENCODER_PROFILE, AMF_VIDEO_ENCODER_PROFILE_MAIN);
break;
default:
case ALVR_H264_PROFILE_HIGH:
amfEncoder->SetProperty(AMF_VIDEO_ENCODER_PROFILE, AMF_VIDEO_ENCODER_PROFILE_HIGH);
break;
}
amfEncoder->SetProperty(AMF_VIDEO_ENCODER_PROFILE_LEVEL, 42);
switch (Settings::Instance().m_rateControlMode) {
case ALVR_CBR:
Expand Down
15 changes: 13 additions & 2 deletions alvr/server/cpp/platform/win32/VideoEncoderSW.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,18 @@ void VideoEncoderSW::Initialize() {
av_dict_set(&opt, "preset", "ultrafast", 0);
av_dict_set(&opt, "tune", "zerolatency", 0);

m_codecContext->profile = FF_PROFILE_H264_HIGH;
switch (settings.m_h264Profile) {
case ALVR_H264_PROFILE_BASELINE:
m_codecContext->profile = FF_PROFILE_H264_BASELINE;
break;
case ALVR_H264_PROFILE_MAIN:
m_codecContext->profile = FF_PROFILE_H264_MAIN;
break;
default:
case ALVR_H264_PROFILE_HIGH:
m_codecContext->profile = FF_PROFILE_H264_HIGH;
break;
}
switch (settings.m_entropyCoding) {
case ALVR_CABAC:
av_dict_set(&opt, "coder", "ac", 0);
Expand Down Expand Up @@ -245,4 +256,4 @@ AVCodecID VideoEncoderSW::ToFFMPEGCodec(ALVR_CODEC codec) {
}
}

#endif // ALVR_GPL
#endif // ALVR_GPL
1 change: 1 addition & 0 deletions alvr/server/src/connection.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ pub fn contruct_openvr_config(session: &SessionConfig) -> OpenvrConfig {
aggressive_keyframe_resend: settings.connection.aggressive_keyframe_resend,
adapter_index: settings.video.adapter_index,
codec: matches!(settings.video.preferred_codec, CodecType::Hevc) as _,
h264_profile: settings.video.encoder_config.h264_profile as u32,
rate_control_mode: settings.video.encoder_config.rate_control_mode as u32,
filler_data: settings.video.encoder_config.filler_data,
entropy_coding: settings.video.encoder_config.entropy_coding as u32,
Expand Down
1 change: 1 addition & 0 deletions alvr/session/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ pub struct OpenvrConfig {
pub aggressive_keyframe_resend: bool,
pub adapter_index: u32,
pub codec: u32,
pub h264_profile: u32,
pub refresh_rate: u32,
pub use_10bit_encoder: bool,
pub enable_vbaq: bool,
Expand Down
22 changes: 22 additions & 0 deletions alvr/session/src/settings.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,13 @@ VBR: Variable BitRate mode. Not commended because it may throw off the adaptive
#[schema(flag = "steamvr-restart")]
pub filler_data: bool,

#[schema(strings(
display_name = "h264: Profile",
help = "Whenever possible, attempts to use this profile. May increase compatibility with varying mobile devices. Only has an effect for h264. Doesn't affect NVENC on Windows."
))]
#[schema(flag = "steamvr-restart")]
pub h264_profile: H264Profile,

#[schema(strings(help = r#"CAVLC algorithm is recommended.
CABAC produces better compression but it's significantly slower and may lead to runaway latency"#))]
#[schema(flag = "steamvr-restart")]
Expand Down Expand Up @@ -420,6 +427,18 @@ pub enum CodecType {
Hevc = 1,
}

#[repr(u8)]
#[derive(SettingsSchema, Serialize, Deserialize, Debug, Copy, Clone)]
#[schema(gui = "button_group")]
pub enum H264Profile {
#[schema(strings(display_name = "High"))]
High = 0,
#[schema(strings(display_name = "Main"))]
Main = 1,
#[schema(strings(display_name = "Baseline"))]
Baseline = 2,
}
Comment on lines +433 to +440
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's more natural to have Baseline as 0th index, then Main then High. Also, you don't need to specify the display_name, as in this case the dashboard is able to figure it out by itself.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

High was set as zeroth index because I was uncertain of the behaviour of certain defaults. In most safe languages, values which are not strictly defined but are initialized end up zero. Rust is not necessarily the same, but there is also C++ code involved with no default value provided when reading from the JSON. I do not know the behaviour of this code on error, so I decided the zero value should be the default value, so the behaviour is as consistent as reasonably possible.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not too opinionated, that's ok then.


#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
#[schema(collapsible)]
pub struct VideoConfig {
Expand Down Expand Up @@ -1165,6 +1184,9 @@ pub fn session_settings_default() -> SettingsDefault {
variant: RateControlModeDefaultVariant::Cbr,
},
filler_data: false,
h264_profile: H264ProfileDefault {
variant: H264ProfileDefaultVariant::High,
Copy link
Member

@zarik5 zarik5 Dec 24, 2023 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe the default level should be Main. Why was High chosen?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

High was chosen for continuity with the existing default settings in some of the encoders.
Reviewing the diff, it is clear that most encoders default to High right now.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@nowrep Any reason for this? Main should have less latency.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure how reliable this is but this says that high main and base have no difference in latency
https://ipvm.com/reports/h264-high-profile-tested
Given these are just cameras idk

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While higher profiles do increase complexity it seems it may not correlate to latency

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you want to fix software decoder compatibility then just use ffmpeg in the client instead of MediaCodec.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While this is correct, in practice, AHardwareBuffers are involved, and it'd likely involve some refactoring, and there are APIs that expose these AHardwareBuffers. I don't have the necessary knowledge.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe the encoding profile is still good to have.
@20kdc have you checked if the decoder can provide its own supported profiles? We could automate this as a future optimization

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As far as I'm aware decoder can't provide supported profiles outside of successful or failed initialization.
Even if it could, decoder bugs make the output potentially untrustworthy.
The encoder might be able to provide supported profiles to some extent, but that's more like an API-level thing, and it requires getting this information about support to the Dashboard, somehow.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As far as I'm aware decoder can't provide supported profiles outside of successful or failed initialization

That's too bad

it requires getting this information about support to the Dashboard

No we don't. we can do the same thing we do for FPS, we just choose the closes one if unsupported by the client.

},
entropy_coding: EntropyCodingDefault {
variant: EntropyCodingDefaultVariant::Cavlc,
},
Expand Down
4 changes: 2 additions & 2 deletions alvr/vulkan_layer/layer/settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ void Settings::Load()
std::string err = picojson::parse(v, json);
if (!err.empty())
{
Error("Error on parsing json: %hs\n", err.c_str());
Error("Error on parsing session config (%s): %hs\n", g_sessionPath, err.c_str());
return;
}

Expand All @@ -56,6 +56,6 @@ void Settings::Load()
}
catch (std::exception &e)
{
Error("Exception on parsing json: %hs\n", e.what());
Error("Exception on parsing session config (%s): %hs\n", g_sessionPath, e.what());
}
}