#3964 honour the csc modes specified by the client

xpra/codecs/gstreamer/capture.py

@@ -9,22 +9,22 @@
 
 from xpra.os_util import gi_import
 from xpra.util.env import envbool
-from xpra.util.str_fn import csv
 from xpra.util.objects import typedict, AtomicInteger
 from xpra.gstreamer.common import (
     import_gst, GST_FLOW_OK, get_element_str,
-    get_default_appsink_attributes, get_all_plugin_names,
+    get_default_appsink_attributes,
     get_caps_str,
 )
 from xpra.gtk.gobject import n_arg_signal
 from xpra.gstreamer.pipeline import Pipeline
-from xpra.codecs.constants import get_profile, CSC_ALIAS
+from xpra.codecs.constants import get_profile, CSC_ALIAS, VideoSpec
 from xpra.codecs.gstreamer.common import (
     get_version, get_type, get_info,
     init_module, cleanup_module,
     get_video_encoder_caps, get_video_encoder_options,
     get_gst_encoding, get_encoder_info, get_gst_rgb_format,
 )
+from xpra.codecs.video import getVideoHelper
 from xpra.codecs.image import ImageWrapper
 from xpra.log import Logger
 
@@ -47,8 +47,7 @@ class Capture(Pipeline):
     __gsignals__ = Pipeline.__generic_signals__.copy()
     __gsignals__["new-image"] = n_arg_signal(3)
 
-    def __init__(self, element: str = "ximagesrc", pixel_format: str = "BGRX",
-                 width: int = 0, height: int = 0):
+    def __init__(self, element: str = "ximagesrc", pixel_format: str = "BGRX", width: int = 0, height: int = 0):
         super().__init__()
         self.capture_element = element.split(" ")[0]
         self.pixel_format: str = pixel_format
@@ -139,35 +138,39 @@ def get_type(self) -> str:
 GObject.type_register(Capture)
 
 
-ENCODER_ELEMENTS: dict[str, tuple[str, ...]] = {
-    "jpeg": ("jpegenc", ),
-    "h264": ("vah264enc", "vah264lpenc", "openh264enc", "x264enc"),     # "vaapih264enc" no workee?
-    "h265": ("vah265enc", ),
-    "vp8": ("vp8enc", ),
-    "vp9": ("vp9enc", ),
-    "av1": ("av1enc", ),
-}
-
-
-def choose_encoder(plugins: Iterable[str]) -> str:
-    # for now, just use the first one available:
-    for plugin in plugins:
-        if plugin in get_all_plugin_names():
-            return plugin
-    return ""
-
-
-def choose_video_encoder(encodings: Iterable[str]) -> str:
-    log(f"choose_video_encoder({encodings})")
-    for encoding in encodings:
-        plugins = ENCODER_ELEMENTS.get(encoding, ())
-        element = choose_encoder(plugins)
-        if not element:
-            log(f"skipped {encoding!r} due to missing: {csv(plugins)}")
-            continue
-        log(f"selected {encoding!r}")
-        return encoding
-    return ""
+def choose_video_encoder(preferred_encoding: str, full_csc_modes: typedict) -> VideoSpec | None:
+    log(f"choose_video_encoder({preferred_encoding}, {full_csc_modes})")
+    vh = getVideoHelper()
+    scores: dict[int, list[VideoSpec]] = {}
+    for encoding in full_csc_modes.keys():
+        csc_modes = full_csc_modes.strtupleget(encoding)
+        encoder_specs = vh.get_encoder_specs(encoding)
+        # ignore the input colorspace for now,
+        # and assume we can `videoconvert` to this format
+        # TODO: take csc into account for scoring
+        for codec_list in encoder_specs.values():
+            for codec in codec_list:
+                if not codec.codec_type.startswith("gstreamer"):
+                    continue
+                if "*" not in csc_modes:
+                    # verify that the client can decode this csc mode:
+                    matches = tuple(x for x in codec.output_colorspaces if x in csc_modes)
+                    if not matches:
+                        log(f"skipped {codec}: {codec.output_colorspaces} not in {csc_modes}")
+                        continue
+                # prefer encoding matching what the user requested and GPU accelerated:
+                gpu = bool(codec.gpu_cost > codec.cpu_cost)
+                score = int(codec.encoding == preferred_encoding or gpu) * 100 + codec.quality + codec.score_boost
+                log(f"score({codec.codec_type})={score} ({gpu=}, quality={codec.quality}, boost={codec.score_boost})")
+                # (lowest score wins)
+                scores.setdefault(-score, []).append(codec)
+    log(f"choose_video_encoder: scores={scores}")
+    if not scores:
+        return None
+    best_score = sorted(scores)[0]
+    best = scores[best_score][0]
+    log(f"choose_video_encoder({preferred_encoding}, {csc_modes})={best}")
+    return best
 
 
 def choose_csc(modes: Iterable[str], quality=100) -> str:
@@ -177,44 +180,52 @@ def choose_csc(modes: Iterable[str], quality=100) -> str:
     return modes[0]
 
 
+def capture_and_encode(capture_element: str, encoding: str, full_csc_modes: typedict, w: int, h: int):
+    encoder_spec = choose_video_encoder(encoding, full_csc_modes)
+    if not encoder_spec:
+        log(f"unable to find a GStreamer video encoder with csc modes={full_csc_modes}")
+        return None
+    assert encoder_spec.codec_type.startswith("gstreamer-")
+    encoder = encoder_spec.codec_type[len("gstreamer-"):]
+    encoding = encoder_spec.encoding
+    csc_mode = encoder_spec.input_colorspace
+    return CaptureAndEncode(capture_element, encoding, encoder, csc_mode, w, h)
+
+
 class CaptureAndEncode(Capture):
     """
     Uses a GStreamer pipeline to capture the screen
     and encode it to a video stream
     """
 
-    def create_pipeline(self, capture_element: str = "ximagesrc") -> None:
-        # we are overloading "pixel_format" as "encoding":
-        encoding = self.pixel_format
-        elements = ENCODER_ELEMENTS.get(encoding)
-        if not elements:
-            raise ValueError(f"no encoders defined for {encoding!r}")
-        encoder = choose_encoder(elements)
-        if not encoder:
-            raise RuntimeError(f"no encoders found for {encoding!r}")
+    def __init__(self, element: str = "ximagesrc", encoding="vp8", encoder="vp8enc",
+                 pixel_format: str = "YUV420P", width: int = 0, height: int = 0):
+        self.encoding = encoding
         self.encoder = encoder
+        super().__init__(element, pixel_format, width, height)
+
+    def create_pipeline(self, capture_element: str = "ximagesrc") -> None:
         options = typedict({
             "speed": 100,
             "quality": 100,
         })
-        einfo = get_encoder_info(encoder)
-        log(f"{encoder}: {einfo=}")
-        self.csc_mode = choose_csc(einfo.get("format", ()), options.intget("quality", 100))
-        self.profile = get_profile(options, encoding, csc_mode=self.csc_mode,
-                                   default_profile="high" if encoder == "x264enc" else "")
-        eopts = get_video_encoder_options(encoder, self.profile, options)
-        vcaps = get_video_encoder_caps(encoder)
+        einfo = get_encoder_info(self.encoder)
+        log(f"{self.encoder}: {einfo=}")
+        self.profile = get_profile(options, self.encoding, csc_mode=self.pixel_format,
+                                   default_profile="high" if self.encoder == "x264enc" else "")
+        eopts = get_video_encoder_options(self.encoder, self.profile, options)
+        vcaps = get_video_encoder_caps(self.encoder)
         self.extra_client_info: dict[str, Any] = vcaps.copy()
         if self.profile:
             vcaps["profile"] = self.profile
             self.extra_client_info["profile"] = self.profile
-        gst_encoding = get_gst_encoding(encoding)  # ie: "hevc" -> "video/x-h265"
+        gst_encoding = get_gst_encoding(self.encoding)  # ie: "hevc" -> "video/x-h265"
         elements = [
             f"{capture_element} name=capture",  # ie: ximagesrc or pipewiresrc
             "queue leaky=2 max-size-buffers=1",
             "videoconvert",
-            "video/x-raw,format=%s" % get_gst_rgb_format(self.csc_mode),
-            get_element_str(encoder, eopts),
+            "video/x-raw,format=%s" % get_gst_rgb_format(self.pixel_format),
+            get_element_str(self.encoder, eopts),
             get_caps_str(gst_encoding, vcaps),
             get_element_str("appsink", get_default_appsink_attributes()),
         ]
@@ -240,16 +251,15 @@ def on_new_sample(self, _bus) -> int:
             self.frames += 1
             client_info = self.extra_client_info
             client_info["frame"] = self.frames
-            client_info["csc"] = CSC_ALIAS.get(self.csc_mode, self.csc_mode)
+            client_info["csc"] = CSC_ALIAS.get(self.pixel_format, self.pixel_format)
             self.extra_client_info = {}
-            self.emit("new-image", self.pixel_format, data, client_info)
+            self.emit("new-image", self.encoding, data, client_info)
             if SAVE_TO_FILE:
                 if not self.file:
-                    encoding = self.pixel_format
                     gen = generation.increase()
-                    filename = "gstreamer-" + str(gen) + f".{encoding}"
+                    filename = "gstreamer-" + str(gen) + f".{self.encoding}"
                     self.file = open(filename, "wb")
-                    log.info(f"saving gstreamer {encoding} stream to {filename!r}")
+                    log.info(f"saving gstreamer {self.encoding} stream to {filename!r}")
                 self.file.write(data)
         return GST_FLOW_OK
 

xpra/codecs/gstreamer/encoder.py

@@ -73,8 +73,8 @@ class ElementEncoder(Encoder):
     return ElementEncoder
 
 
-def make_spec(element: str, encoding: str, cs_in: str, css_out: tuple[str, ...], cpu_cost: int = 50,
-              gpu_cost: int = 50):
+def make_spec(element: str, encoding: str, cs_in: str, css_out: tuple[str, ...],
+              cpu_cost: int = 50, gpu_cost: int = 50):
     # use a metaclass so all encoders are gstreamer.encoder.Encoder subclasses,
     # each with different pipeline arguments based on the make_spec parameters:
     if cs_in in PACKED_RGB_FORMATS:

xpra/codecs/video.py

@@ -284,15 +284,17 @@ def init(self) -> None:
             self._initialized = True
         log("VideoHelper.init() done")
 
-    def get_gpu_options(self, codec_specs: Vdict) -> dict[str, list[CodecSpec]]:
-        log(f"get_gpu_options({codec_specs})")
-        gpu_fmts: dict[str, list[str]] = {}
+    def get_gpu_options(self, codec_specs: Vdict, out_fmts=("*", )) -> dict[str, list[CodecSpec]]:
+        gpu_fmts: dict[str, list[CodecSpec]] = {}
         for in_fmt, vdict in codec_specs.items():
             for out_fmt, codecs in vdict.items():
-                log(f"get_gpu_options {out_fmt}: {codecs}")
+                if "*" not in out_fmts and out_fmt not in out_fmts:
+                    continue
                 for codec in codecs:
                     if codec.gpu_cost > codec.cpu_cost:
+                        log(f"get_gpu_options {out_fmt}: {codec}")
                         gpu_fmts.setdefault(in_fmt, []).append(codec)
+        log(f"get_gpu_options({codec_specs})={gpu_fmts}")
         return gpu_fmts
 
     def get_gpu_encodings(self) -> dict[str, list[CodecSpec]]:

xpra/platform/posix/fd_portal_shadow.py

@@ -15,7 +15,7 @@
 from xpra.util.env import envbool
 from xpra.common import NotificationID, ConnectionMessage
 from xpra.dbus.helper import dbus_to_native
-from xpra.codecs.gstreamer.capture import Capture, CaptureAndEncode, choose_video_encoder
+from xpra.codecs.gstreamer.capture import Capture, capture_and_encode
 from xpra.gstreamer.common import get_element_str
 from xpra.codecs.image import ImageWrapper
 from xpra.server.shadow.root_window_model import RootWindowModel
@@ -243,7 +243,7 @@ def on_start_response(self, response, results) -> None:
             log.warn(" keyboard and pointer events cannot be forwarded")
 
     def create_capture_pipeline(self, fd: int, node_id: int, w: int, h: int) -> Capture:
-        el = get_element_str("pipewiresrc", {
+        capture_element = get_element_str("pipewiresrc", {
             "fd": fd,
             "path": str(node_id),
             "do-timestamp": True,
@@ -252,21 +252,13 @@ def create_capture_pipeline(self, fd: int, node_id: int, w: int, h: int) -> Capt
         if VIDEO_MODE:
             encoding = getattr(c, "encoding", "")
             encs = getattr(c, "core_encodings", ())
-            options = [encoding] + VIDEO_MODE_ENCODINGS
-            common_video = set(options) & set(encs)
-            log(f"core_encodings({c})={encs}, common video encodings={common_video}")
-            encoding = choose_video_encoder(common_video)
-            if encoding:
-                log(f"will try the following encoding: {encoding}")
-                try:
-                    return CaptureAndEncode(el, encoding, width=w, height=h)
-                except (RuntimeError, ValueError) as e:
-                    log("CaptureAndEncode%s", (el, encoding, w, h), exc_info=True)
-                    log.info(f"cannot use {encoding}: {e}")
-            else:
-                log.warn("no video encoders available")
-            log.warn("Warning: falling back to RGB capture")
-        return Capture(el, pixel_format="BGRX", width=w, height=h)
+            full_csc_modes = getattr(c, "full_csc_modes", {})
+            log(f"create_capture_pipeline() core_encodings={encs}, full_csc_modes={full_csc_modes}")
+            pipeline = capture_and_encode(capture_element, encoding, full_csc_modes, w, h)
+            if pipeline:
+                return pipeline
+            log.warn("Warning: falling back to slow RGB capture")
+        return Capture(capture_element, pixel_format="BGRX", width=w, height=h)
 
     def start_pipewire_capture(self, node_id: int, props: typedict) -> None:
         log(f"start_pipewire_capture({node_id}, {props})")

xpra/server/source/encodings.py

@@ -47,6 +47,7 @@ def init_state(self) -> None:
         self.encoding = ""  # the default encoding for all windows
         self.encodings: tuple[str, ...] = ()  # all the encodings supported by the client
         self.core_encodings: tuple[str, ...] = ()
+        self.full_csc_modes = dict()
         self.window_icon_encodings: tuple[str, ...] = ()
         self.rgb_formats: tuple[str, ...] = ("RGB",)
         self.encoding_options = typedict()
@@ -272,6 +273,7 @@ def parse_batch_int(value, varname):
         # encodings:
         self.encodings = c.strtupleget("encodings")
         self.core_encodings = c.strtupleget("encodings.core", self.encodings)
+        self.full_csc_modes = c.dictget("full_csc_modes") or {}
         log("encodings=%s, core_encodings=%s", self.encodings, self.core_encodings)
         # we can't assume that the window mixin is loaded,
         # or that the ui_client flag exists:

xpra/server/window/video_compress.py

@@ -431,7 +431,7 @@ def update_encoding_selection(self, encoding="", exclude=None, init=False) -> No
         log("wvs.update_encoding_selection(%s, %s, %s) full_csc_modes=%s", encoding, exclude, init, self.full_csc_modes)
         if exclude is None:
             exclude = []
-        log(f"video_encodings={self.video_encodings}, core_encodings={self.core_encodings}")
+        videolog(f"encoding={encoding}, video_encodings={self.video_encodings}, core_encodings={self.core_encodings}")
         for x in self.video_encodings:
             if x not in self.core_encodings:
                 log("video encoding %s not in core encodings", x)
@@ -453,7 +453,7 @@ def update_encoding_selection(self, encoding="", exclude=None, init=False) -> No
                  self.common_video_encodings, self._csc_encoder, self._video_encoder)
         if encoding in ("stream", "auto", "grayscale"):
             vh = self.video_helper
-            if encoding == "auto" and self.content_type in STREAM_CONTENT_TYPES and vh:
+            if encoding in ("auto", "stream") and self.content_type in STREAM_CONTENT_TYPES and vh:
                 accel = vh.get_gpu_encodings()
                 common_accel = preforder(set(self.common_video_encodings) & set(accel.keys()))
                 videolog(f"gpu {accel=} - {common_accel=}")
@@ -675,7 +675,7 @@ def cancel_gstreamer_timer(self) -> None:
 
     def start_gstreamer_pipeline(self) -> bool:
         from xpra.gstreamer.common import plugin_str
-        from xpra.codecs.gstreamer.capture import CaptureAndEncode, choose_video_encoder
+        from xpra.codecs.gstreamer.capture import capture_and_encode
         attrs = {
             "show-pointer": False,
             "do-timestamp": True,
@@ -687,15 +687,11 @@ def start_gstreamer_pipeline(self) -> bool:
             xid = 0
         if xid:
             attrs["xid"] = xid
-        element = plugin_str("ximagesrc", attrs)
-        # for now this doesn't do anything as we require `encoding="stream"`
-        # options = [self.encoding] + list(self.common_encodings)
-        options = self.common_encodings
-        encoding = choose_video_encoder(options)
-        if not encoding:
-            return False
+        capture_element = plugin_str("ximagesrc", attrs)
         w, h = self.window_dimensions
-        self.gstreamer_pipeline = CaptureAndEncode(element, encoding, w, h)
+        self.gstreamer_pipeline = capture_and_encode(capture_element, self.encoding, self.full_csc_modes, w, h)
+        if not self.gstreamer_pipeline:
+            return False
         self.gstreamer_pipeline.connect("new-image", self.new_gstreamer_frame)
         self.gstreamer_pipeline.start()
         gstlog("start_gstreamer_pipeline() %s started", self.gstreamer_pipeline)