Port to GStreamer 1.22

Dockerfile.example

@@ -3,7 +3,7 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 # Supported base images: ubuntu:22.04, ubuntu:20.04, ubuntu:18.04
-ARG UBUNTU_RELEASE=20.04
+ARG UBUNTU_RELEASE=22.04
 ARG GSTREAMER_BASE_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gstreamer
 ARG GSTREAMER_BASE_IMAGE_RELEASE=master
 ARG PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:master
@@ -53,6 +53,11 @@ RUN \
         libvpx-dev \
         zlib1g-dev \
         x264 \
+        va-driver-all \
+        i965-va-driver-shaders \
+        libva2 \
+        vainfo \
+        intel-gpu-tools \
         git && \
     rm -rf /var/lib/apt/lists/*
 

addons/gstreamer/Dockerfile

@@ -2,12 +2,11 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-ARG UBUNTU_RELEASE=20.04
+ARG UBUNTU_RELEASE=22.04
 FROM ubuntu:${UBUNTU_RELEASE}
 
 # Install essentials
-RUN \
-    apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         curl \
         build-essential \
         ca-certificates \
@@ -16,8 +15,7 @@ RUN \
     rm -rf /var/lib/apt/lists/*
 
 # Install build deps
-RUN \
-    apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         autopoint \
         autoconf \
         automake \
@@ -27,6 +25,7 @@ RUN \
         bison \
         flex \
         gtk-doc-tools \
+        nasm \
         libtool-bin \
         libgtk2.0-dev \
         libgl1-mesa-dev \
@@ -37,12 +36,13 @@ RUN \
         libssl-dev \
         libsrtp2-dev \
         libx264-dev \
-        libvpx-dev && \
+        libvpx-dev \
+        libva-dev \
+        libdrm-dev && \
     rm -rf /var/lib/apt/lists/*
 
 # Install meson and ninja
-RUN \
-    apt-get update && apt install -y \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         python3-pip \
         python-gi-dev \
         ninja-build && \
@@ -54,11 +54,11 @@ WORKDIR /src
 ###
 # GStreamer monorepo build with prefix for standalone install.
 ###
-ENV GSTREAMER_VERSION=1.20.5
+ENV GSTREAMER_VERSION=1.22.5
 RUN git clone https://gitlab.freedesktop.org/gstreamer/gstreamer.git && cd gstreamer && git checkout ${GSTREAMER_VERSION}
 RUN cd /src/gstreamer && \
     mkdir -p /opt/gstreamer && \
-    meson --prefix /opt/gstreamer -Dgpl=enabled -Dugly=enabled -Dgst-plugins-ugly:x264=enabled builddir && \
+    meson --prefix /opt/gstreamer -Dgpl=enabled -Dugly=enabled -Dgst-plugins-ugly:x264=enabled -Dgst-plugins-bad:va=enabled builddir && \
     ninja -C builddir && \
     meson install -C builddir
 

docker-compose.yaml

@@ -22,16 +22,13 @@ services:
     # NOTE: export the UBUNTU_RELEASE env var to change the base image during build and run.
     ###
 
-    # TODO: privileged is only required when using Ubuntu 22.04
-    #privileged: true
-
-    image: selkies-gstreamer-example:latest-ubuntu${UBUNTU_RELEASE:-20.04}
+    image: selkies-gstreamer-example:latest-ubuntu${UBUNTU_RELEASE:-22.04}
     entrypoint: ["/tini", "--", "/bin/bash"]
     build:
       context: .
       dockerfile: Dockerfile.example
       args:
-        UBUNTU_RELEASE: ${UBUNTU_RELEASE:-20.04}
+        UBUNTU_RELEASE: ${UBUNTU_RELEASE:-22.04}
 
         # Testing with gstreamer build in-repo:
         # 1. (cd addons/gstreamer && docker build --build-arg UBUNTU_RELEASE=${UBUNTU_RELEASE?} -t gstreamer:latest-ubuntu${UBUNTU_RELEASE?} .)

src/selkies_gstreamer/__main__.py

@@ -609,14 +609,6 @@ def on_resize_handler(res):
                 logger.warning("skipping resize because last resize failed.")
                 return
 
-            # TODO: remove the ximagesrc stop/start after this MR is merged:
-            #   https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1562
-            logger.warning("stopping ximagesrc")
-            app.stop_ximagesrc()
-            logger.warning("resizing display from {} to {}".format(curr_res, new_res))
-            resize_display(res)
-            app.start_ximagesrc()
-
     # Initial binding of enable resize handler.
     if enable_resize:
         webrtc_input.on_resize = on_resize_handler

src/selkies_gstreamer/gstwebrtc_app.py

@@ -306,6 +306,42 @@ def build_video_pipeline(self):
             rtph264pay_capsfilter = Gst.ElementFactory.make("capsfilter")
             rtph264pay_capsfilter.set_property("caps", rtph264pay_caps)
 
+        elif self.encoder in ["vah264enc"]:
+            # color converter
+            videoconvert = Gst.ElementFactory.make("vapostproc")
+            videoconvert_caps = Gst.caps_from_string("video/x-raw,format=NV12")
+            videoconvert_capsfilter = Gst.ElementFactory.make("capsfilter")
+            videoconvert_capsfilter.set_property("caps", videoconvert_caps)
+
+            # encoder
+            vah264enc = Gst.ElementFactory.make("vah264enc", "vah264enc")
+            vah264enc.set_property("i-frames", 0)
+            vah264enc.set_property("b-frames", 0)
+            vah264enc.set_property("key-int-max", 0)
+            vah264enc.set_property("bitrate", self.video_bitrate)
+
+            # capsfilter
+            vah264enc_caps = Gst.caps_from_string("video/x-h264")
+            vah264enc_caps.set_value("stream-format", "byte-stream")
+            vah264enc_caps.set_value("profile", "high")
+            vah264enc_capsfilter = Gst.ElementFactory.make("capsfilter")
+            vah264enc_capsfilter.set_property("caps", vah264enc_caps)
+
+            # RTP payload
+            rtph264pay = Gst.ElementFactory.make("rtph264pay")
+            rtph264pay_caps = Gst.caps_from_string("application/x-rtp")
+            rtph264pay_caps.set_value("media", "video")
+            rtph264pay_caps.set_value("encoding-name", "H264")
+            rtph264pay_caps.set_value("payload", 123)
+            rtph264pay_caps.set_value("rtcp-fb-nack-pli", True)
+            rtph264pay_caps.set_value("rtcp-fb-ccm-fir", True)
+            rtph264pay_caps.set_value("rtcp-fb-x-gstreamer-fir-as-repair", True)
+            rtph264pay_caps.set_value("aggregate-mode", "zero-latency")
+
+            # Create a capability filter for the rtph264pay_caps.
+            rtph264pay_capsfilter = Gst.ElementFactory.make("capsfilter")
+            rtph264pay_capsfilter.set_property("caps", rtph264pay_caps)
+
         elif self.encoder in ["x264enc"]:
             # Videoconvert for colorspace conversion
             videoconvert = Gst.ElementFactory.make("videoconvert")
@@ -405,7 +441,15 @@ def build_video_pipeline(self):
             self.pipeline.add(rtph264pay)
             self.pipeline.add(rtph264pay_capsfilter)
 
-        if self.encoder == "x264enc":
+        elif self.encoder == "vah264enc":
+            self.pipeline.add(videoconvert)
+            self.pipeline.add(videoconvert_capsfilter)
+            self.pipeline.add(vah264enc)
+            self.pipeline.add(vah264enc_capsfilter)
+            self.pipeline.add(rtph264pay)
+            self.pipeline.add(rtph264pay_capsfilter)
+
+        elif self.encoder == "x264enc":
             self.pipeline.add(videoconvert)
             self.pipeline.add(videoconvert_capsfilter)
             self.pipeline.add(x264enc)
@@ -460,6 +504,36 @@ def build_video_pipeline(self):
                 raise GSTWebRTCAppError(
                     "Failed to link rtph264pay_capsfilter -> webrtcbin")
 
+        elif self.encoder == "vah264enc":
+            if not Gst.Element.link(ximagesrc_capsfilter, videoconvert):
+                raise GSTWebRTCAppError(
+                    "Failed to link ximagesrc_capsfilter -> videoconvert")
+
+            if not Gst.Element.link(videoconvert, videoconvert_capsfilter):
+                raise GSTWebRTCAppError(
+                    "Failed to link videoconvert -> videoconvert_capsfilter")
+
+            if not Gst.Element.link(videoconvert_capsfilter, vah264enc):
+                raise GSTWebRTCAppError(
+                    "Failed to link videoconvert_capsfilter -> vah264enc")
+
+            if not Gst.Element.link(vah264enc, vah264enc_capsfilter):
+                raise GSTWebRTCAppError(
+                    "Failed to link vah264enc -> vah264enc_capsfilter")
+
+            if not Gst.Element.link(vah264enc_capsfilter, rtph264pay):
+                raise GSTWebRTCAppError(
+                    "Failed to link vah264enc_capsfilter -> rtph264pay")
+
+            if not Gst.Element.link(rtph264pay, rtph264pay_capsfilter):
+                raise GSTWebRTCAppError(
+                    "Failed to link rtph264pay -> rtph264pay_capsfilter")
+
+            # Link the last element to the webrtcbin
+            if not Gst.Element.link(rtph264pay_capsfilter, self.webrtcbin):
+                raise GSTWebRTCAppError(
+                    "Failed to link rtph264pay_capsfilter -> webrtcbin")
+
         elif self.encoder == "x264enc":
             if not Gst.Element.link(ximagesrc_capsfilter, videoconvert):
                 raise GSTWebRTCAppError(
@@ -638,7 +712,7 @@ def check_plugins(self):
         required = ["opus", "nice", "webrtc", "dtls", "srtp", "rtp", "sctp",
                     "rtpmanager", "ximagesrc"]
 
-        supported = ["nvh264enc", "vp8enc", "vp9enc", "x264enc"]
+        supported = ["nvh264enc", "vp8enc", "vp9enc", "vah264enc", "x264enc"]
         if self.encoder not in supported:
             raise GSTWebRTCAppError('Unsupported encoder, must be one of: ' + ','.join(supported))
 
@@ -770,7 +844,7 @@ def set_pointer_visible(self, visible):
 
     def send_clipboard_data(self, data):
         # TODO: WebRTC DataChannel accepts a maximum length of 65489 (= 65535 - 46 for '{"type": "clipboard", "data": {"content": ""}}'), remove this restriction after implementing DataChannel chunking
-        CLIPBOARD_RESTRICTION = 65488
+        CLIPBOARD_RESTRICTION = 65000
         clipboard_message = base64.b64encode(data.encode()).decode("utf-8")
         clipboard_length = len(clipboard_message)
         if clipboard_length <= CLIPBOARD_RESTRICTION: