Nvidia linux. Add encoder nvenc

.github/workflows/rust.yml

@@ -43,10 +43,10 @@ jobs:
           RUST_BACKTRACE: 1
         run: |
           sudo apt update
-          sudo apt install build-essential pkg-config nasm libva-dev libdrm-dev libvulkan-dev libx264-dev libx265-dev
+          sudo apt install build-essential pkg-config nasm libva-dev libdrm-dev libvulkan-dev libx264-dev libx265-dev cmake libasound2-dev libgtk-3-dev libunwind-dev libffmpeg-nvenc-dev nvidia-cuda-toolkit
+          cp packaging/deb/cuda.pc /usr/share/pkgconfig
           cargo xtask build-ffmpeg-linux
           cd deps/linux/FFmpeg-n4.4 && sudo make install && cd ../../..
-          sudo apt install build-essential pkg-config cmake libasound2-dev libgtk-3-dev libvulkan-dev libunwind-dev
 
       - name: Build crates
         uses: actions-rs/cargo@v1

alvr/server/cpp/platform/linux/CEncoder.cpp

@@ -249,7 +249,10 @@ void CEncoder::Run() {
         }
 
         encoded_data.clear();
-        while (encode_pipeline->GetEncoded(encoded_data)) {}
+        // Encoders can req more then once frame, need to accumulate more data before sending it to the client
+        if (!encode_pipeline->GetEncoded(encoded_data)) {
+          continue;
+        }
 
         m_listener->SendVideo(encoded_data.data(), encoded_data.size(), m_poseSubmitIndex + Settings::Instance().m_trackingFrameOffset);
 

alvr/server/cpp/platform/linux/EncodePipeline.cpp

@@ -4,6 +4,7 @@
 #include "alvr_server/Settings.h"
 #include "EncodePipelineSW.h"
 #include "EncodePipelineVAAPI.h"
+#include "EncodePipelineNvEnc.h"
 #include "ffmpeg_helper.h"
 
 extern "C" {
@@ -75,6 +76,12 @@ std::unique_ptr<alvr::EncodePipeline> alvr::EncodePipeline::Create(std::vector<V
   {
     Info("failed to create VAAPI encoder");
   }
+  try {
+    return std::make_unique<alvr::EncodePipelineNvEnc>(input_frames, vk_frame_ctx);
+  } catch (...)
+  {
+    Info("failed to create NvEnc encoder");
+  }
   return std::make_unique<alvr::EncodePipelineSW>(input_frames, vk_frame_ctx);
 }
 

alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp

@@ -0,0 +1,109 @@
+#include "EncodePipelineNvEnc.h"
+#include "ALVR-common/packet_types.h"
+#include "alvr_server/Settings.h"
+#include "ffmpeg_helper.h"
+#include <chrono>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavutil/opt.h>
+#include <libswscale/swscale.h>
+}
+
+namespace {
+
+const char *encoder(ALVR_CODEC codec) {
+    switch (codec) {
+    case ALVR_CODEC_H264:
+        return "h264_nvenc";
+    case ALVR_CODEC_H265:
+        return "hevc_nvenc";
+    }
+    throw std::runtime_error("invalid codec " + std::to_string(codec));
+}
+
+} // namespace
+alvr::EncodePipelineNvEnc::EncodePipelineNvEnc(std::vector<VkFrame> &input_frames,
+                                               VkFrameCtx &vk_frame_ctx) {
+    auto input_frame_ctx = (AVHWFramesContext *)vk_frame_ctx.ctx->data;
+    assert(input_frame_ctx->sw_format == AV_PIX_FMT_BGRA);
+
+    int err;
+    for (auto &input_frame : input_frames) {
+        vk_frames.push_back(std::move(input_frame.make_av_frame(vk_frame_ctx)));
+    }
+
+    const auto &settings = Settings::Instance();
+
+    auto codec_id = ALVR_CODEC(settings.m_codec);
+    const char *encoder_name = encoder(codec_id);
+    AVCodec *codec = AVCODEC.avcodec_find_encoder_by_name(encoder_name);
+    if (codec == nullptr) {
+        throw std::runtime_error(std::string("Failed to find encoder ") + encoder_name);
+    }
+
+    encoder_ctx = AVCODEC.avcodec_alloc_context3(codec);
+    if (not encoder_ctx) {
+        throw std::runtime_error("failed to allocate NvEnc encoder");
+    }
+
+    switch (codec_id) {
+    case ALVR_CODEC_H264:
+        AVUTIL.av_opt_set(encoder_ctx, "preset", "llhq", 0);
+        AVUTIL.av_opt_set(encoder_ctx, "zerolatency", "1", 0);
+        break;
+    case ALVR_CODEC_H265:
+        AVUTIL.av_opt_set(encoder_ctx, "preset", "llhq", 0);
+        AVUTIL.av_opt_set(encoder_ctx, "zerolatency", "1", 0);
+        break;
+    }
+
+    /**
+     * We will recieve a frame from HW as AV_PIX_FMT_VULKAN which will converted to AV_PIX_FMT_BGRA
+     * as SW format when we get it from HW.
+     * But NVEnc support only BGR0 format and we easy can just to force it
+     * Because:
+     * AV_PIX_FMT_BGRA - 28  ///< packed BGRA 8:8:8:8, 32bpp, BGRABGRA...
+     * AV_PIX_FMT_BGR0 - 123 ///< packed BGR 8:8:8,    32bpp, BGRXBGRX...   X=unused/undefined
+     *
+     * We just to ignore the alpha channel and it's done
+     */
+    encoder_ctx->pix_fmt = AV_PIX_FMT_BGR0;
+    encoder_ctx->width = settings.m_renderWidth;
+    encoder_ctx->height = settings.m_renderHeight;
+    encoder_ctx->time_base = {std::chrono::steady_clock::period::num,
+                              std::chrono::steady_clock::period::den};
+    encoder_ctx->framerate = AVRational{settings.m_refreshRate, 1};
+    encoder_ctx->sample_aspect_ratio = AVRational{1, 1};
+    encoder_ctx->max_b_frames = 0;
+    encoder_ctx->gop_size = 30;
+    encoder_ctx->bit_rate = settings.mEncodeBitrateMBs * 1000 * 1000;
+
+    err = AVCODEC.avcodec_open2(encoder_ctx, codec, NULL);
+    if (err < 0) {
+        throw alvr::AvException("Cannot open video encoder codec:", err);
+    }
+
+    hw_frame = AVUTIL.av_frame_alloc();
+}
+
+alvr::EncodePipelineNvEnc::~EncodePipelineNvEnc() {
+    AVUTIL.av_buffer_unref(&hw_ctx);
+    AVUTIL.av_frame_free(&hw_frame);
+}
+
+void alvr::EncodePipelineNvEnc::PushFrame(uint32_t frame_index, bool idr) {
+    assert(frame_index < vk_frames.size());
+
+    int err = AVUTIL.av_hwframe_transfer_data(hw_frame, vk_frames[frame_index].get(), 0);
+    if (err) {
+        throw alvr::AvException("av_hwframe_transfer_data", err);
+    }
+
+    hw_frame->pict_type = idr ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE;
+    hw_frame->pts = std::chrono::steady_clock::now().time_since_epoch().count();
+
+    if ((err = AVCODEC.avcodec_send_frame(encoder_ctx, hw_frame)) < 0) {
+        throw alvr::AvException("avcodec_send_frame failed:", err);
+    }
+}

alvr/server/cpp/platform/linux/EncodePipelineNvEnc.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <functional>
+#include "EncodePipeline.h"
+
+extern "C" struct AVBufferRef;
+extern "C" struct AVCodecContext;
+extern "C" struct AVFrame;
+
+namespace alvr
+{
+
+class EncodePipelineNvEnc: public EncodePipeline
+{
+public:
+  ~EncodePipelineNvEnc();
+  EncodePipelineNvEnc(std::vector<VkFrame> &input_frames, VkFrameCtx& vk_frame_ctx);
+
+  void PushFrame(uint32_t frame_index, bool idr) override;
+
+private:
+  AVBufferRef *hw_ctx = nullptr;
+  std::vector<std::unique_ptr<AVFrame, std::function<void(AVFrame*)>>> vk_frames;
+  AVFrame * hw_frame = nullptr;
+};
+}

alvr/xtask/Cargo.toml

@@ -12,3 +12,4 @@ alvr_filesystem = { path = "../filesystem" }
 fs_extra = "1"
 pico-args = "0.4"
 walkdir = "2"
+pkg-config = "0.3.9"

alvr/xtask/src/dependencies.rs

@@ -70,8 +70,10 @@ fn build_rust_android_gradle() {
     fs::remove_dir_all(temp_build_dir).ok();
 }
 
-pub fn build_ffmpeg_linux() -> std::path::PathBuf {
-    // dependencies: build-essential pkg-config nasm libva-dev libdrm-dev libvulkan-dev libx264-dev libx265-dev
+pub fn build_ffmpeg_linux(nvenc_flag: bool) -> std::path::PathBuf {
+    /* dependencies: build-essential pkg-config nasm libva-dev libdrm-dev libvulkan-dev
+                     libx264-dev libx265-dev libffmpeg-nvenc-dev nvidia-cuda-toolkit
+    */
 
     let download_path = afs::deps_dir().join("linux");
     let ffmpeg_path = download_path.join("FFmpeg-n4.4");
@@ -94,7 +96,32 @@ pub fn build_ffmpeg_linux() -> std::path::PathBuf {
             "--disable-network",
             "--enable-lto",
             format!(
-                "--disable-everything {} {} {} {}",
+                "--disable-everything {} {} {} {} {}",
+                /*
+                   Describing Nvidia specific options --nvccflags: 
+                   nvcc from CUDA toolkit version 11.0 or higher does not support compiling for 'compute_30' (default in ffmpeg)
+                   52 is the minimum required for the current CUDA 11 version (Quadro M6000 , GeForce 900, GTX-970, GTX-980, GTX Titan X)
+                   https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/
+                   Anyway below 50 arch card don't support nvenc encoding hevc https://developer.nvidia.com/nvidia-video-codec-sdk (Supported devices)
+                   Nvidia docs:
+                   https://docs.nvidia.com/video-technologies/video-codec-sdk/ffmpeg-with-nvidia-gpu/#commonly-faced-issues-and-tips-to-resolve-them
+                */
+                (if nvenc_flag {
+                    let cuda = pkg_config::Config::new().probe("cuda").unwrap();
+                    let include_flags = cuda.include_paths
+                        .iter()
+                        .map(|path| format!("-I{:?}", path))
+                        .reduce(|a, b| { format!("{}{}", a, b) })
+                        .expect("pkg-config cuda entry to have include-paths");
+                    let link_flags = cuda.link_paths
+                        .iter()
+                        .map(|path| format!("-L{:?}", path))
+                        .reduce(|a, b| { format!("{}{}", a, b) })
+                        .expect("pkg-config cuda entry to have link-paths");
+
+                    format!("--enable-encoder=h264_nvenc --enable-encoder=hevc_nvenc --enable-nonfree --enable-cuda-nvcc --enable-libnpp --nvccflags=\"-gencode arch=compute_52,code=sm_52 -O2\" --extra-cflags=\"{}\" --extra-ldflags=\"{}\" --enable-hwaccel=h264_nvenc --enable-hwaccel=hevc_nvenc",
+                            include_flags, link_flags)
+                } else {"".to_string()}),
                 "--enable-encoder=h264_vaapi --enable-encoder=hevc_vaapi",
                 "--enable-encoder=libx264 --enable-encoder=libx264rgb --enable-encoder=libx265",
                 "--enable-hwaccel=h264_vaapi --enable-hwaccel=hevc_vaapi",