alvr-org · deiteris · Jan 16, 2023 · Jan 19, 2023 · Jul 2, 2023 · Jul 2, 2023
diff --git a/alvr/client_core/src/c_api.rs b/alvr/client_core/src/c_api.rs
@@ -411,6 +411,12 @@ pub extern "C" fn alvr_request_idr() {
     crate::request_idr();
 }
 
+/// Call only with external decoder
+#[no_mangle]
+pub extern "C" fn alvr_request_config_nal() {
+    crate::request_config_nal();
+}
+
 /// Call only with external decoder
 #[no_mangle]
 pub extern "C" fn alvr_report_frame_decoded(target_timestamp_ns: u64) {

diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs
@@ -218,6 +218,13 @@ pub fn request_idr() {
     }
 }
 
+/// Call only with external decoder
+pub fn request_config_nal() {
+    if let Some(sender) = &*CONTROL_CHANNEL_SENDER.lock() {
+        sender.send(ClientControlPacket::RequestConfigNAL).ok();
+    }
+}
+
 /// Call only with external decoder
 pub fn report_frame_decoded(target_timestamp: Duration) {
     if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {

diff --git a/alvr/packets/src/lib.rs b/alvr/packets/src/lib.rs
@@ -89,6 +89,7 @@ pub struct ButtonEntry {
 pub enum ClientControlPacket {
     PlayspaceSync(Option<Vec2>),
     RequestIdr,
+    RequestConfigNAL,
     KeepAlive,
     StreamReady,
     ViewsConfig(ViewsConfig),

diff --git a/alvr/server/cpp/alvr_server/alvr_server.cpp b/alvr/server/cpp/alvr_server/alvr_server.cpp
@@ -234,6 +234,12 @@ void RequestIDR() {
     }
 }
 
+void RequestConfigNAL() {
+    if (g_driver_provider.hmd && g_driver_provider.hmd->m_encoder) {
+        g_driver_provider.hmd->m_encoder->GetConfigNAL();
+    }
+}
+
 void SetTracking(unsigned long long targetTimestampNs,
                  float controllerPoseTimeOffsetS,
                  const FfiDeviceMotion *deviceMotions,

diff --git a/alvr/server/cpp/alvr_server/bindings.h b/alvr/server/cpp/alvr_server/bindings.h
@@ -129,6 +129,7 @@ extern "C" void InitializeStreaming();
 extern "C" void DeinitializeStreaming();
 extern "C" void SendVSync();
 extern "C" void RequestIDR();
+extern "C" void RequestConfigNAL();
 extern "C" void SetTracking(unsigned long long targetTimestampNs,
                             float controllerPoseTimeOffsetS,
                             const FfiDeviceMotion *deviceMotions,

diff --git a/alvr/server/cpp/platform/linux/CEncoder.cpp b/alvr/server/cpp/platform/linux/CEncoder.cpp
@@ -218,7 +218,7 @@ void CEncoder::Run() {
 
       alvr::VkFrameCtx vk_frame_ctx(vk_ctx, output.imageInfo);
       alvr::VkFrame frame(vk_ctx, output.image, output.imageInfo, output.size, output.memory, output.drm);
-      auto encode_pipeline = alvr::EncodePipeline::Create(&render, vk_ctx, frame, vk_frame_ctx, render.GetEncodingWidth(), render.GetEncodingHeight());
+      m_encodePipeline = alvr::EncodePipeline::Create(&render, vk_ctx, frame, vk_frame_ctx, render.GetEncodingWidth(), render.GetEncodingHeight());
 
       fprintf(stderr, "CEncoder starting to read present packets");
       present_packet frame_info;
@@ -241,7 +241,7 @@ void CEncoder::Run() {
 
         render.Render(frame_info.image, frame_info.semaphore_value);
 
-        encode_pipeline->PushFrame(pose->targetTimestampNs, m_scheduler.CheckIDRInsertion());
+        m_encodePipeline->PushFrame(pose->targetTimestampNs, m_scheduler.CheckIDRInsertion());
 
         static_assert(sizeof(frame_info.pose) == sizeof(vr::HmdMatrix34_t&));
 
@@ -252,7 +252,7 @@ void CEncoder::Run() {
         }
 
         auto render_timestamps = render.GetTimestamps();
-        auto encode_timestamp = encode_pipeline->GetTimestamp();
+        auto encode_timestamp = m_encodePipeline->GetTimestamp();
 
         uint64_t present_offset = render_timestamps.now - render_timestamps.renderBegin;
         uint64_t composed_offset = 0;
@@ -297,4 +297,6 @@ void CEncoder::OnPacketLoss() { m_scheduler.OnPacketLoss(); }
 
 void CEncoder::InsertIDR() { m_scheduler.InsertIDR(); }
 
+void CEncoder::GetConfigNAL() { m_encodePipeline->GetConfigNAL(); }
+
 void CEncoder::CaptureFrame() { m_captureFrame = true; }
diff --git a/alvr/server/cpp/platform/linux/CEncoder.h b/alvr/server/cpp/platform/linux/CEncoder.h
@@ -7,6 +7,8 @@
 #include <poll.h>
 #include <sys/types.h>
 
+#include "EncodePipeline.h"
+
 class PoseHistory;
 
 class CEncoder : public CThread {
@@ -19,12 +21,14 @@ class CEncoder : public CThread {
     void Stop();
     void OnPacketLoss();
     void InsertIDR();
+    void GetConfigNAL();
     bool IsConnected() { return m_connected; }
     void CaptureFrame();
 
   private:
     void GetFds(int client, int (*fds)[6]);
     std::shared_ptr<PoseHistory> m_poseHistory;
+    std::unique_ptr<alvr::EncodePipeline> m_encodePipeline;
     std::atomic_bool m_exiting{false};
     IDRScheduler m_scheduler;
     pollfd m_socket;

diff --git a/alvr/server/cpp/platform/linux/EncodePipeline.h b/alvr/server/cpp/platform/linux/EncodePipeline.h
@@ -35,6 +35,7 @@ class EncodePipeline
 
   virtual void PushFrame(uint64_t targetTimestampNs, bool idr) = 0;
   virtual bool GetEncoded(FramePacket &data);
+  virtual void GetConfigNAL() = 0;
   virtual Timestamp GetTimestamp() { return timestamp; }
   virtual int GetCodec();
 

diff --git a/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp b/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp
@@ -436,18 +436,12 @@ void EncodePipelineAMF::ApplyFrameProperties(const amf::AMFSurfacePtr &surface,
     case ALVR_CODEC_H264:
         surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_AUD, false);
         if (insertIDR) {
-            Debug("Inserting IDR frame for H.264.\n");
-            surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_SPS, true);
-            surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_PPS, true);
             surface->SetProperty(AMF_VIDEO_ENCODER_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_PICTURE_TYPE_IDR);
         }
         break;
     case ALVR_CODEC_H265:
         surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_AUD, false);
         if (insertIDR) {
-            Debug("Inserting IDR frame for H.265.\n");
-            // Insert VPS,SPS,PPS
-            surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_HEADER, true);
             surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_HEVC_PICTURE_TYPE_IDR);
         }
         break;
@@ -456,4 +450,15 @@ void EncodePipelineAMF::ApplyFrameProperties(const amf::AMFSurfacePtr &surface,
     }
 }
 
+void EncodePipelineAMF::GetConfigNAL() {
+	amf::AMFVariant var;
+	if (m_codec == ALVR_CODEC_H264) {
+		m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_EXTRADATA, &var);
+	} else {
+		m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_HEVC_EXTRADATA, &var);
+	}
+	amf::AMFBufferPtr buffer(var.pInterface);
+	InitializeDecoder(reinterpret_cast<unsigned char *>(buffer->GetNative()), buffer->GetSize(), GetCodec());
+}
+
 };
diff --git a/alvr/server/cpp/platform/linux/EncodePipelineAMF.h b/alvr/server/cpp/platform/linux/EncodePipelineAMF.h
@@ -70,6 +70,7 @@ class EncodePipelineAMF : public EncodePipeline
     void PushFrame(uint64_t targetTimestampNs, bool idr) override;
     bool GetEncoded(FramePacket &packet) override;
     void SetParams(FfiDynamicEncoderParams params) override;
+    void GetConfigNAL() override;
 
 private:
     amf::AMFComponentPtr MakeConverter(amf::AMF_SURFACE_FORMAT inputFormat, int width, int height, amf::AMF_SURFACE_FORMAT outputFormat);
@@ -93,7 +94,7 @@ class EncodePipelineAMF : public EncodePipeline
     int m_renderHeight;
 
     bool m_hasQueryTimeout = false;
-    
+
     amf::AMFBufferPtr m_frameBuffer;
     uint64_t m_targetTimestampNs;
 };

diff --git a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp
@@ -116,6 +116,8 @@ alvr::EncodePipelineNvEnc::EncodePipelineNvEnc(Renderer *render,
     params.framerate = 60.0;
     SetParams(params);
 
+    encoder_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+
     err = avcodec_open2(encoder_ctx, codec, NULL);
     if (err < 0) {
         throw alvr::AvException("Cannot open video encoder codec:", err);
@@ -144,3 +146,7 @@ void alvr::EncodePipelineNvEnc::PushFrame(uint64_t targetTimestampNs, bool idr)
         throw alvr::AvException("avcodec_send_frame failed:", err);
     }
 }
+
+void alvr::EncodePipelineNvEnc::GetConfigNAL() {
+	InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size, GetCodec());
+}
diff --git a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.h b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.h
@@ -19,6 +19,7 @@ class EncodePipelineNvEnc: public EncodePipeline
   EncodePipelineNvEnc(Renderer *render, VkFrame &input_frame, VkFrameCtx& vk_frame_ctx, uint32_t width, uint32_t height);
 
   void PushFrame(uint64_t targetTimestampNs, bool idr) override;
+  void GetConfigNAL() override;
 
 private:
   Renderer *r = nullptr;

diff --git a/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp b/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp
@@ -97,6 +97,10 @@ void alvr::EncodePipelineSW::PushFrame(uint64_t targetTimestampNs, bool idr)
   }
 }
 
+void alvr::EncodePipelineSW::GetConfigNAL() {
+	InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size, GetCodec());
+}
+
 bool alvr::EncodePipelineSW::GetEncoded(FramePacket &packet)
 {
   if (!nal) {

diff --git a/alvr/server/cpp/platform/linux/EncodePipelineSW.h b/alvr/server/cpp/platform/linux/EncodePipelineSW.h
@@ -16,6 +16,7 @@ class EncodePipelineSW: public EncodePipeline
   EncodePipelineSW(Renderer *render, uint32_t width, uint32_t height);
 
   void PushFrame(uint64_t targetTimestampNs, bool idr) override;
+  void GetConfigNAL() override;
   bool GetEncoded(FramePacket &packet) override;
   void SetParams(FfiDynamicEncoderParams params) override;
   int GetCodec() override;

diff --git a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp
@@ -197,6 +197,8 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(Renderer *render, VkContext &vk_c
   params.framerate = settings.m_refreshRate;
   SetParams(params);
 
+  encoder_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+
   vlVaQualityBits quality = {};
   quality.valid_setting = 1;
   quality.vbaq_mode = Settings::Instance().m_enableVbaq;  //No noticable performance difference and should improve subjective quality by allocating more bits to smooth areas
@@ -331,6 +333,10 @@ void alvr::EncodePipelineVAAPI::PushFrame(uint64_t targetTimestampNs, bool idr)
   av_frame_unref(encoder_frame);
 }
 
+void alvr::EncodePipelineVAAPI::GetConfigNAL() {
+	InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size, GetCodec());
+}
+
 void alvr::EncodePipelineVAAPI::SetParams(FfiDynamicEncoderParams params)
 {
   if (!params.updated) {

diff --git a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.h b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.h
@@ -30,6 +30,7 @@ class EncodePipelineVAAPI: public EncodePipeline
   EncodePipelineVAAPI(Renderer *render, VkContext &vk_ctx, VkFrame &input_frame, uint32_t width, uint32_t height);
 
   void PushFrame(uint64_t targetTimestampNs, bool idr) override;
+  void GetConfigNAL() override;
   void SetParams(FfiDynamicEncoderParams params) override;
 
 private:

diff --git a/alvr/server/cpp/platform/win32/CEncoder.cpp b/alvr/server/cpp/platform/win32/CEncoder.cpp
@@ -137,5 +137,9 @@
 			m_scheduler.InsertIDR();
 		}
 
+		void CEncoder::GetConfigNAL() {
+			m_videoEncoder->GetConfigNAL();
+		}
+
 		void CEncoder::CaptureFrame() {
 		}
diff --git a/alvr/server/cpp/platform/win32/CEncoder.h b/alvr/server/cpp/platform/win32/CEncoder.h
@@ -54,6 +54,8 @@
 
 		void CaptureFrame();
 
+		void GetConfigNAL();
+
 	private:
 		CThreadEvent m_newFrameReady, m_encodeFinished;
 		std::shared_ptr<VideoEncoder> m_videoEncoder;

diff --git a/alvr/server/cpp/platform/win32/VideoEncoder.h b/alvr/server/cpp/platform/win32/VideoEncoder.h
@@ -10,8 +10,6 @@ class VideoEncoder {
     virtual void Initialize() = 0;
     virtual void Shutdown() = 0;
 
-    virtual void Transmit(ID3D11Texture2D *pTexture,
-                          uint64_t presentationTime,
-                          uint64_t targetTimestampNs,
-                          bool insertIDR) = 0;
+    virtual void Transmit(ID3D11Texture2D *pTexture, uint64_t presentationTime, uint64_t targetTimestampNs, bool insertIDR) = 0;
+    virtual void GetConfigNAL() = 0;
 };