Merge pull request #96 from alvr-org/master

[pull] master from alvr-org:master

alvr/client_core/src/c_api.rs

@@ -32,7 +32,8 @@ static HUD_MESSAGE: Lazy<Mutex<String>> = Lazy::new(|| Mutex::new("".into()));
 #[repr(u8)]
 pub enum AlvrCodec {
     H264 = 0,
-    H265 = 1,
+    Hevc = 1,
+    AV1 = 2,
 }
 
 #[repr(u8)]
@@ -235,10 +236,10 @@ pub extern "C" fn alvr_poll_event(out_event: *mut AlvrEvent) -> bool {
                 });
 
                 AlvrEvent::CreateDecoder {
-                    codec: if matches!(codec, CodecType::H264) {
-                        AlvrCodec::H264
-                    } else {
-                        AlvrCodec::H265
+                    codec: match codec {
+                        CodecType::H264 => AlvrCodec::H264,
+                        CodecType::Hevc => AlvrCodec::Hevc,
+                        CodecType::AV1 => AlvrCodec::AV1,
                     },
                 }
             }

alvr/client_core/src/platform/android/decoder.rs

@@ -148,6 +148,7 @@ fn mime_for_codec(codec: CodecType) -> &'static str {
     match codec {
         CodecType::H264 => "video/avc",
         CodecType::Hevc => "video/hevc",
+        CodecType::AV1 => "video/av01",
     }
 }
 
@@ -162,6 +163,7 @@ fn decoder_attempt_setup(
         let sw_codec_name = match codec_type {
             CodecType::H264 => "OMX.google.h264.decoder",
             CodecType::Hevc => "OMX.google.hevc.decoder",
+            CodecType::AV1 => bail!("AV1 is not supported for software decoding"),
         };
         MediaCodec::from_codec_name(&sw_codec_name)
             .ok_or(anyhow!("no such codec: {}", &sw_codec_name))?

alvr/server/cpp/ALVR-common/packet_types.h

@@ -6,7 +6,8 @@
 
 enum ALVR_CODEC {
 	ALVR_CODEC_H264 = 0,
-	ALVR_CODEC_H265 = 1,
+	ALVR_CODEC_HEVC = 1,
+	ALVR_CODEC_AV1 = 2,
 };
 
 enum ALVR_H264_PROFILE {

alvr/server/cpp/alvr_server/NalParsing.cpp

@@ -10,10 +10,10 @@ static const char NAL_PREFIX_3B[] = {0x00, 0x00, 0x01};
 static const char NAL_PREFIX_4B[] = {0x00, 0x00, 0x00, 0x01};
 
 static const unsigned char H264_NAL_TYPE_SPS = 7;
-static const unsigned char H265_NAL_TYPE_VPS = 32;
+static const unsigned char HEVC_NAL_TYPE_VPS = 32;
 
 static const unsigned char H264_NAL_TYPE_AUD = 9;
-static const unsigned char H265_NAL_TYPE_AUD = 35;
+static const unsigned char HEVC_NAL_TYPE_AUD = 35;
 
 int8_t getNalPrefixSize(unsigned char *buf) {
     if (memcmp(buf, NAL_PREFIX_3B, sizeof(NAL_PREFIX_3B)) == 0) {
@@ -81,31 +81,36 @@ void processH264Nals(unsigned char *&buf, int &len) {
     }
 }
 
-void processH265Nals(unsigned char *&buf, int &len) {
+void processHevcNals(unsigned char *&buf, int &len) {
     unsigned char prefixSize = getNalPrefixSize(buf);
     unsigned char nalType = (buf[prefixSize] >> 1) & 0x3F;
 
-    if (nalType == H265_NAL_TYPE_AUD && len > prefixSize * 2 + 3) {
+    if (nalType == HEVC_NAL_TYPE_AUD && len > prefixSize * 2 + 3) {
         buf += prefixSize + 3;
         len -= prefixSize + 3;
         prefixSize = getNalPrefixSize(buf);
         nalType = (buf[prefixSize] >> 1) & 0x3F;
     }
-    if (nalType == H265_NAL_TYPE_VPS) {
-        sendHeaders(ALVR_CODEC_H265, buf, len, 3); // 3 headers VPS, SPS and PPS
+    if (nalType == HEVC_NAL_TYPE_VPS) {
+        sendHeaders(ALVR_CODEC_HEVC, buf, len, 3); // 3 headers VPS, SPS and PPS
     }
 }
 
 void ParseFrameNals(
     int codec, unsigned char *buf, int len, unsigned long long targetTimestampNs, bool isIdr) {
+    static bool av1GotFrame = false;
+
     if ((unsigned)len < sizeof(NAL_PREFIX_4B)) {
         return;
     }
 
     if (codec == ALVR_CODEC_H264) {
         processH264Nals(buf, len);
-    } else if (codec == ALVR_CODEC_H265) {
-        processH265Nals(buf, len);
+    } else if (codec == ALVR_CODEC_HEVC) {
+        processHevcNals(buf, len);
+    } else if (codec == ALVR_CODEC_AV1 && !av1GotFrame) {
+        av1GotFrame = true;
+        SetVideoConfigNals(0, 0, codec);
     }
 
     VideoSend(targetTimestampNs, buf, len, isIdr);

alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp

@@ -1,6 +1,7 @@
 #include "EncodePipelineNvEnc.h"
 #include "ALVR-common/packet_types.h"
 #include "alvr_server/Settings.h"
+#include "alvr_server/Logger.h"
 #include "ffmpeg_helper.h"
 #include <chrono>
 
@@ -15,7 +16,10 @@ const char *encoder(ALVR_CODEC codec) {
     switch (codec) {
     case ALVR_CODEC_H264:
         return "h264_nvenc";
-    case ALVR_CODEC_H265:
+    case ALVR_CODEC_HEVC:
+        return "hevc_nvenc";
+    case ALVR_CODEC_AV1:
+        Warn("AV1 is not supported by NvEnc. Using HEVC instead.");
         return "hevc_nvenc";
     }
     throw std::runtime_error("invalid codec " + std::to_string(codec));
@@ -100,7 +104,9 @@ alvr::EncodePipelineNvEnc::EncodePipelineNvEnc(Renderer *render,
             break;
         }
         break;
-    case ALVR_CODEC_H265:
+    case ALVR_CODEC_HEVC:
+        break;
+    case ALVR_CODEC_AV1:
         break;
     }
 

alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp

@@ -2,6 +2,7 @@
 #include "ALVR-common/packet_types.h"
 #include "ffmpeg_helper.h"
 #include "alvr_server/Settings.h"
+#include "alvr_server/Logger.h"
 #include <chrono>
 
 extern "C" {
@@ -22,8 +23,10 @@ const char * encoder(ALVR_CODEC codec)
   {
     case ALVR_CODEC_H264:
       return "h264_vaapi";
-    case ALVR_CODEC_H265:
+    case ALVR_CODEC_HEVC:
       return "hevc_vaapi";
+    case ALVR_CODEC_AV1:
+      return "av1_vaapi";
   }
   throw std::runtime_error("invalid codec " + std::to_string(codec));
 }
@@ -39,7 +42,7 @@ void set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx)
   }
   frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
   frames_ctx->format = AV_PIX_FMT_VAAPI;
-  frames_ctx->sw_format = Settings::Instance().m_codec == ALVR_CODEC_H265 && Settings::Instance().m_use10bitEncoder ? AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
+  frames_ctx->sw_format = Settings::Instance().m_codec == ALVR_CODEC_HEVC && Settings::Instance().m_use10bitEncoder ? AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
   frames_ctx->width = ctx->width;
   frames_ctx->height = ctx->height;
   frames_ctx->initial_pool_size = 3;
@@ -179,9 +182,12 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(Renderer *render, VkContext &vk_c
       }
 
       break;
-    case ALVR_CODEC_H265:
+    case ALVR_CODEC_HEVC:
       encoder_ctx->profile = Settings::Instance().m_use10bitEncoder ? FF_PROFILE_HEVC_MAIN_10 : FF_PROFILE_HEVC_MAIN;
       break;
+    case ALVR_CODEC_AV1:
+      encoder_ctx->profile = FF_PROFILE_AV1_MAIN;
+      break;
   }
 
   switch (settings.m_rateControlMode)
@@ -203,7 +209,7 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(Renderer *render, VkContext &vk_c
   encoder_ctx->sample_aspect_ratio = AVRational{1, 1};
   encoder_ctx->pix_fmt = AV_PIX_FMT_VAAPI;
   encoder_ctx->max_b_frames = 0;
-  encoder_ctx->gop_size = INT16_MAX;
+  encoder_ctx->gop_size = INT_MAX;
 
   auto params = FfiDynamicEncoderParams {};
   params.updated = true;
@@ -236,7 +242,6 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(Renderer *render, VkContext &vk_c
     break;
   }
 
-  av_opt_set_int(encoder_ctx->priv_data, "idr_interval", INT_MAX, 0);
   av_opt_set_int(encoder_ctx->priv_data, "async_depth", 1, 0);
 
   set_hwframe_ctx(encoder_ctx, hw_ctx);
@@ -285,7 +290,7 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(Renderer *render, VkContext &vk_c
   inputs->next = NULL;
 
   std::string filters = "scale_vaapi=format=";
-  if (Settings::Instance().m_codec == ALVR_CODEC_H265 && Settings::Instance().m_use10bitEncoder) {
+  if (Settings::Instance().m_codec == ALVR_CODEC_HEVC && Settings::Instance().m_use10bitEncoder) {
     filters += "p010";
   } else {
     filters += "nv12";

alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp

@@ -135,7 +135,11 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder(
 		}
 		pCodec = AMFVideoEncoderVCE_AVC;
 		break;
-	case ALVR_CODEC_H265:
+	case ALVR_CODEC_HEVC:
+		pCodec = AMFVideoEncoder_HEVC;
+		break;
+	case ALVR_CODEC_AV1:
+		Warn("AV1 encoding is not supported. Using HEVC instead.");
 		pCodec = AMFVideoEncoder_HEVC;
 		break;
 	default:
@@ -146,7 +150,8 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder(
 	// Create encoder component.
 	AMF_THROW_IF(g_AMFFactory.GetFactory()->CreateComponent(m_amfContext, pCodec, &amfEncoder));
 
-	if (codec == ALVR_CODEC_H264)
+	switch (codec) {
+	case ALVR_CODEC_H264:
 	{
 		amfEncoder->SetProperty(AMF_VIDEO_ENCODER_USAGE, AMF_VIDEO_ENCODER_USAGE_ULTRA_LOW_LATENCY);
       	switch (Settings::Instance().m_h264Profile) {
@@ -223,7 +228,7 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder(
 			amfEncoder->SetProperty(AMF_VIDEO_ENCODER_QUERY_TIMEOUT, 1000); // 1s timeout
 		}
 	}
-	else
+	case ALVR_CODEC_HEVC:
 	{
 		amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_USAGE, AMF_VIDEO_ENCODER_HEVC_USAGE_ULTRA_LOW_LATENCY);
 		switch (Settings::Instance().m_rateControlMode) {
@@ -286,6 +291,7 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder(
 			amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_QUERY_TIMEOUT, 1000); // 1s timeout
 		}
 	}
+	}
 
 	Debug("Configured %s.\n", pCodec);
 	AMF_THROW_IF(amfEncoder->Init(inputFormat, width, height));
@@ -477,7 +483,7 @@ void VideoEncoderAMF::ApplyFrameProperties(const amf::AMFSurfacePtr &surface, bo
 			surface->SetProperty(AMF_VIDEO_ENCODER_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_PICTURE_TYPE_IDR);
 		}
 		break;
-	case ALVR_CODEC_H265:
+	case ALVR_CODEC_HEVC:
 		// FIXME: This option works with 22.10.3, but may not work with older drivers
 		surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_AUD, false);
 		if (insertIDR) {
@@ -489,6 +495,8 @@ void VideoEncoderAMF::ApplyFrameProperties(const amf::AMFSurfacePtr &surface, bo
 			surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_HEVC_PICTURE_TYPE_IDR);
 		}
 		break;
+	case ALVR_CODEC_AV1:
+		throw MakeException("AV1 encoding is not supported");
 	default:
 		throw MakeException("Invalid video codec");
 	}

alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp

@@ -125,7 +125,20 @@ void VideoEncoderNVENC::Transmit(ID3D11Texture2D *pTexture, uint64_t presentatio
 void VideoEncoderNVENC::FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializeParams, int refreshRate, int renderWidth, int renderHeight, uint64_t bitrate_bps)
 {
 	auto &encodeConfig = *initializeParams.encodeConfig;
-	GUID encoderGUID = m_codec == ALVR_CODEC_H264 ? NV_ENC_CODEC_H264_GUID : NV_ENC_CODEC_HEVC_GUID;
+
+	GUID encoderGUID;
+	switch (m_codec) {
+		case ALVR_CODEC_H264:
+			encoderGUID = NV_ENC_CODEC_H264_GUID;
+			break;
+		case ALVR_CODEC_HEVC:
+			encoderGUID = NV_ENC_CODEC_HEVC_GUID;
+			break;
+		case ALVR_CODEC_AV1:
+			Warn("AV1 is not supported yet. Using HEVC instead.");
+			encoderGUID = NV_ENC_CODEC_HEVC_GUID;
+			break;
+	}
 
 	GUID qualityPreset;
 	// See recommended NVENC settings for low-latency encoding.
@@ -182,7 +195,9 @@ void VideoEncoderNVENC::FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializePar
 		gopLength = Settings::Instance().m_nvencGopLength;
 	}
 
-	if (m_codec == ALVR_CODEC_H264) {
+	switch (m_codec) {
+	case ALVR_CODEC_H264:
+	{
 		auto &config = encodeConfig.encodeCodecConfig.h264Config;
 		config.repeatSPSPPS = 1;
 		config.enableIntraRefresh = Settings::Instance().m_nvencEnableIntraRefresh;
@@ -209,8 +224,9 @@ void VideoEncoderNVENC::FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializePar
 		if (Settings::Instance().m_fillerData) {
 			config.enableFillerDataInsertion = Settings::Instance().m_rateControlMode == ALVR_CBR;
 		}
-	} 
-	else {
+	}
+	case ALVR_CODEC_HEVC:
+	{
 		auto &config = encodeConfig.encodeCodecConfig.hevcConfig;
 		config.repeatSPSPPS = 1;
 		config.enableIntraRefresh = Settings::Instance().m_nvencEnableIntraRefresh;
@@ -233,6 +249,11 @@ void VideoEncoderNVENC::FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializePar
 			config.enableFillerDataInsertion = Settings::Instance().m_rateControlMode == ALVR_CBR;
 		}
 	}
+	case ALVR_CODEC_AV1:
+	{
+		// todo
+	}
+	}
 
 	// Disable automatic IDR insertion by NVENC. We need to manually insert IDR when packet is dropped
 	// if don't use reference frame invalidation.

alvr/server/cpp/platform/win32/VideoEncoderSW.cpp

@@ -249,7 +249,10 @@ AVCodecID VideoEncoderSW::ToFFMPEGCodec(ALVR_CODEC codec) {
 	switch (codec) {
 		case ALVR_CODEC_H264:
 			return AV_CODEC_ID_H264;
-		case ALVR_CODEC_H265:
+		case ALVR_CODEC_HEVC:
+			return AV_CODEC_ID_HEVC;
+		case ALVR_CODEC_AV1:
+			Warn("AV1 is not supported. Using HEVC instead.");
 			return AV_CODEC_ID_HEVC;
 		default:
 			return AV_CODEC_ID_NONE;

alvr/server/src/connection.rs

@@ -28,7 +28,7 @@ use alvr_packets::{
     ServerControlPacket, StreamConfigPacket, Tracking, VideoPacketHeader, AUDIO, HAPTICS,
     STATISTICS, TRACKING, VIDEO,
 };
-use alvr_session::{CodecType, ControllersEmulationMode, FrameSize, OpenvrConfig, SessionConfig};
+use alvr_session::{ControllersEmulationMode, FrameSize, OpenvrConfig, SessionConfig};
 use alvr_sockets::{
     PeerType, ProtoControlSocket, StreamSender, StreamSocketBuilder, KEEPALIVE_INTERVAL,
     KEEPALIVE_TIMEOUT,
@@ -141,7 +141,7 @@ pub fn contruct_openvr_config(session: &SessionConfig) -> OpenvrConfig {
         enable_vive_tracker_proxy: settings.headset.enable_vive_tracker_proxy,
         aggressive_keyframe_resend: settings.connection.aggressive_keyframe_resend,
         adapter_index: settings.video.adapter_index,
-        codec: matches!(settings.video.preferred_codec, CodecType::Hevc) as _,
+        codec: settings.video.preferred_codec as _,
         h264_profile: settings.video.encoder_config.h264_profile as u32,
         rate_control_mode: settings.video.encoder_config.rate_control_mode as u32,
         filler_data: settings.video.encoder_config.filler_data,

alvr/server/src/lib.rs

@@ -105,10 +105,10 @@ fn to_ffi_quat(quat: Quat) -> FfiQuat {
 
 pub fn create_recording_file(settings: &Settings) {
     let codec = settings.video.preferred_codec;
-    let ext = if matches!(codec, CodecType::H264) {
-        "h264"
-    } else {
-        "h265"
+    let ext = match codec {
+        CodecType::H264 => "h264",
+        CodecType::Hevc => "h265",
+        CodecType::AV1 => "av1",
     };
 
     let path = FILESYSTEM_LAYOUT.log_dir.join(format!(
@@ -325,8 +325,10 @@ pub unsafe extern "C" fn HmdDriverFactory(
     extern "C" fn set_video_config_nals(buffer_ptr: *const u8, len: i32, codec: i32) {
         let codec = if codec == 0 {
             CodecType::H264
-        } else {
+        } else if codec == 1 {
             CodecType::Hevc
+        } else {
+            CodecType::AV1
         };
 
         let mut config_buffer = vec![0; len as usize];

alvr/session/src/lib.rs

@@ -45,7 +45,7 @@ pub struct OpenvrConfig {
     pub enable_vive_tracker_proxy: bool,
     pub aggressive_keyframe_resend: bool,
     pub adapter_index: u32,
-    pub codec: u32,
+    pub codec: u8,
     pub h264_profile: u32,
     pub refresh_rate: u32,
     pub use_10bit_encoder: bool,

alvr/session/src/settings.rs

@@ -425,6 +425,8 @@ pub enum CodecType {
     H264 = 0,
     #[schema(strings(display_name = "HEVC"))]
     Hevc = 1,
+    #[schema(strings(display_name = "AV1 (VAAPI only)"))]
+    AV1 = 2,
 }
 
 #[repr(u8)]