* move scaling logic to the window source, the actual implementation …

…is in the encoders - via argb/scale for those that don't have native support,

* add more grayscale options: jpeg has TJSAMP_GRAY, nvjpeg has NVJPEG_CSS_GRAY,
* move rgb encoding to argb/encoder,
* pixels_to_bytes moved out of the server module

xpra/codecs/argb/encoder.py

@@ -0,0 +1,106 @@
+# This file is part of Xpra.
+# Copyright (C) 2021 Antoine Martin <antoine@xpra.org>
+# Xpra is released under the terms of the GNU GPL v2, or, at your option, any
+# later version. See the file COPYING for details.
+
+from xpra.codecs.rgb_transform import rgb_reformat
+from xpra.codecs import rgb_transform
+from xpra.net.compression import Compressed, compressed_wrapper
+from xpra.log import Logger
+
+log = Logger("encoder")
+
+
+def get_version():
+    return 4, 3
+
+def get_type() -> str:
+    return "rgb"
+
+def get_encodings():
+    return "rgb24", "rgb32"
+
+def get_info() -> dict:
+    return  {
+            "version"       : get_version(),
+            "encodings"     : get_encodings(),
+            }
+
+
+def encode(coding : str, image, options : dict):
+    pixel_format = image.get_pixel_format()
+    #log("rgb_encode%s pixel_format=%s, rgb_formats=%s",
+    #    (coding, image, rgb_formats, supports_transparency, speed, rgb_zlib, rgb_lz4), pixel_format, rgb_formats)
+    rgb_formats = options.get("rgb_formats", ("BGRX", "BGRA"))
+    supports_transparency = options.get("alpha", True)
+    if pixel_format not in rgb_formats:
+        log("rgb_encode reformatting because %s not in %s, supports_transparency=%s",
+            pixel_format, rgb_formats, supports_transparency)
+        if not rgb_reformat(image, rgb_formats, supports_transparency):
+            raise Exception("cannot find compatible rgb format to use for %s! (supported: %s)" % (
+                pixel_format, rgb_formats))
+        #get the new format:
+        pixel_format = image.get_pixel_format()
+        #switch encoding if necessary:
+        if len(pixel_format)==4:
+            coding = "rgb32"
+        elif len(pixel_format)==3:
+            coding = "rgb24"
+        else:
+            raise Exception("invalid pixel format %s" % pixel_format)
+    #we may still want to re-stride:
+    image.may_restride()
+    #always tell client which pixel format we are sending:
+    options = {"rgb_format" : pixel_format}
+
+    #compress here and return a wrapper so network code knows it is already zlib compressed:
+    pixels = image.get_pixels()
+    assert pixels, "failed to get pixels from %s" % image
+    width = image.get_width()
+    height = image.get_height()
+    stride = image.get_rowstride()
+    speed = options.get("speed", 50)
+
+    #compression stage:
+    level = 0
+    algo = "not"
+    l = len(pixels)
+    if l>=512 and speed<100:
+        if l>=4096:
+            #speed=99 -> level=1, speed=0 -> level=9
+            level = 1+max(0, min(8, int(100-speed)//12))
+        else:
+            #fewer pixels, make it more likely we won't bother compressing
+            #and use a lower level (max=5)
+            level = max(0, min(5, int(115-speed)//20))
+    if level>0:
+        zlib = options.get("zlib", False)
+        lz4 = options.get("lz4", False)
+        cwrapper = compressed_wrapper(coding, pixels, level=level,
+                                      zlib=zlib, lz4=lz4,
+                                      brotli=False, none=True)
+        algo = cwrapper.algorithm
+        if algo=="none" or len(cwrapper)>=(len(pixels)-32):
+            #no compression is enabled, or compressed is actually bigger!
+            #(fall through to uncompressed)
+            level = 0
+        else:
+            #add compressed marker:
+            options[algo] = level
+            #remove network layer compression marker
+            #so that this data will be decompressed by the decode thread client side:
+            cwrapper.level = 0
+    if level==0:
+        #can't pass a raw buffer to bencode / rencode,
+        #and even if we could, the image containing those pixels may be freed by the time we get to the encoder
+        algo = "not"
+        cwrapper = Compressed(coding, rgb_transform.pixels_to_bytes(pixels), True)
+    if pixel_format.find("A")>=0 or pixel_format.find("X")>=0:
+        bpp = 32
+    else:
+        bpp = 24
+    log("rgb_encode using level=%s for %5i bytes at %3i speed, %s compressed %4sx%-4s in %s/%s: %5s bytes down to %5s",
+        level, l, speed, algo, width, height, coding, pixel_format, len(pixels), len(cwrapper.data))
+    #wrap it using "Compressed" so the network layer receiving it
+    #won't decompress it (leave it to the client's draw thread)
+    return coding, cwrapper, options, width, height, stride, bpp

xpra/codecs/argb/scale.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+# This file is part of Xpra.
+# Copyright (C) 2021 Antoine Martin <antoine@xpra.org>
+# Xpra is released under the terms of the GNU GPL v2, or, at your option, any
+# later version. See the file COPYING for details.
+
+from xpra.log import Logger
+log = Logger("encoding")
+
+#more scaling functions can be added here later
+
+RGB_SCALE_FORMATS = ("BGRX", "BGRA", "RGBA", "RGBX")
+
+def scale_image(image, width, height):
+    try:
+        from xpra.codecs.csc_libyuv.colorspace_converter import argb_scale  #pylint: disable=import-outside-toplevel
+    except ImportError as e:
+        log("cannot downscale: %s", e)
+        return image
+    return argb_scale(image, width, height)

xpra/codecs/jpeg/encoder.pyx

@@ -146,6 +146,7 @@ cdef class Encoder:
     cdef object src_format
     cdef int quality
     cdef int speed
+    cdef int grayscale
     cdef long frames
     cdef object __weakref__
 
@@ -163,6 +164,7 @@ cdef class Encoder:
         self.width = width
         self.height = height
         self.src_format = src_format
+        self.grayscale = options.boolget("grayscale")
         self.scaling = scaling
         self.quality = quality
         self.speed = speed
@@ -204,6 +206,7 @@ cdef class Encoder:
             "height"        : self.height,
             "speed"         : self.speed,
             "quality"       : self.quality,
+            "grayscale"     : bool(self.grayscale),
             })
         return info
 
@@ -218,9 +221,9 @@ cdef class Encoder:
             speed = self.speed
         pfstr = bytestostr(image.get_pixel_format())
         if pfstr in ("YUV420P", "YUV422P", "YUV444P"):
-            cdata = encode_yuv(self.compressor, image, quality, speed)
+            cdata = encode_yuv(self.compressor, image, quality, self.grayscale)
         else:
-            cdata = encode_rgb(self.compressor, image, quality, speed)
+            cdata = encode_rgb(self.compressor, image, quality, self.grayscale)
         if not cdata:
             return None
         self.frames += 1
@@ -231,8 +234,30 @@ def get_error_str():
     cdef char *err = tjGetErrorStr()
     return bytestostr(err)
 
-def encode(coding, image, int quality=50, int speed=50):
+JPEG_INPUT_FORMATS = ("RGB", "RGBX", "BGRX", "XBGR", "XRGB", "RGBA", "BGRA", "ABGR", "ARGB")
+
+def encode(coding, image, options):
     assert coding=="jpeg"
+    cdef int quality = options.get("quality", 50)
+    cdef int grayscale = options.get("grayscale", 0)
+    resize = options.get("resize")
+    log("encode%s", (coding, image, options))
+    rgb_format = image.get_pixel_format()
+    if rgb_format not in JPEG_INPUT_FORMATS:
+        from xpra.codecs.argb.argb import argb_swap         #@UnresolvedImport
+        if not argb_swap(image, JPEG_INPUT_FORMATS):
+            log("jpeg: argb_swap failed to convert %s to a suitable format: %s" % (
+                rgb_format, JPEG_INPUT_FORMATS))
+        log("jpeg converted image: %s", image)
+
+    width = image.get_width()
+    height = image.get_height()
+    if resize:
+        from xpra.codecs.argb.scale import scale_image
+        scaled_width, scaled_height = resize
+        image = scale_image(image, scaled_width, scaled_height)
+        log("jpeg scaled image: %s", image)
+
     #100 would mean lossless, so cap it at 99:
     client_options = {
         "quality"   : min(99, quality),
@@ -243,22 +268,22 @@ def encode(coding, image, int quality=50, int speed=50):
         return None
     cdef int r
     try:
-        cdata = encode_rgb(compressor, image, quality, speed)
+        cdata = encode_rgb(compressor, image, quality, grayscale)
         if not cdata:
             return None
         if SAVE_TO_FILE:    # pragma: no cover
             filename = "./%s.jpeg" % time.time()
             with open(filename, "wb") as f:
                 f.write(cdata)
             log.info("saved %i bytes to %s", len(cdata), filename)
-        return "jpeg", Compressed("jpeg", memoryview(cdata), False), client_options, image.get_width(), image.get_height(), 0, 24
+        return "jpeg", Compressed("jpeg", memoryview(cdata), False), client_options, width, height, 0, 24
     finally:
         r = tjDestroy(compressor)
         if r:
             log.error("Error: failed to destroy the JPEG compressor, code %i:", r)
             log.error(" %s", get_error_str())
 
-cdef encode_rgb(tjhandle compressor, image, int quality, int speed):
+cdef encode_rgb(tjhandle compressor, image, int quality, int grayscale=0):
     cdef int width = image.get_width()
     cdef int height = image.get_height()
     cdef int stride = image.get_rowstride()
@@ -269,7 +294,9 @@ cdef encode_rgb(tjhandle compressor, image, int quality, int speed):
         raise Exception("invalid pixel format %s" % pfstr)
     cdef TJPF tjpf = pf
     cdef TJSAMP subsamp = TJSAMP_444
-    if quality<50:
+    if grayscale:
+        subsamp = TJSAMP_GRAY
+    elif quality<50:
         subsamp = TJSAMP_420
     elif quality<80:
         subsamp = TJSAMP_422
@@ -300,11 +327,13 @@ cdef encode_rgb(tjhandle compressor, image, int quality, int speed):
     assert out_size>0 and out!=NULL, "jpeg compression produced no data"
     return makebuf(out, out_size)
 
-cdef encode_yuv(tjhandle compressor, image, int quality, int speed):
+cdef encode_yuv(tjhandle compressor, image, int quality, int grayscale=0):
     pfstr = bytestostr(image.get_pixel_format())
     assert pfstr in ("YUV420P", "YUV422P"), "invalid yuv pixel format %s" % pfstr
     cdef TJSAMP subsamp
-    if pfstr=="YUV420P":
+    if grayscale:
+        subsamp = TJSAMP_GRAY
+    elif pfstr=="YUV420P":
         subsamp = TJSAMP_420
     elif pfstr=="YUV422P":
         subsamp = TJSAMP_422
@@ -370,5 +399,5 @@ def selftest(full=False):
     from xpra.codecs.codec_checks import make_test_image
     img = make_test_image("BGRA", 32, 32)
     for q in (0, 50, 100):
-        v = encode("jpeg", img, q, 100)
+        v = encode("jpeg", img, {"quality" : q})
         assert v, "encode output was empty!"

xpra/codecs/loader.py

@@ -147,6 +147,7 @@ def xpra_codec_import(name, description, top_module, class_module, classname):
 
 CODEC_OPTIONS = {
     #encoders:
+    "enc_rgb"       : ("RGB encoder",       "argb",         "encoder", "encode"),
     "enc_pillow"    : ("Pillow encoder",    "pillow",       "encoder", "encode"),
     "enc_webp"      : ("webp encoder",      "webp",         "encoder", "encode"),
     "enc_jpeg"      : ("JPEG encoder",      "jpeg",         "encoder", "encode"),
@@ -233,7 +234,7 @@ def has_codec(name) -> bool:
 
 
 CSC_CODECS = "csc_swscale", "csc_cython", "csc_libyuv"
-ENCODER_CODECS = "enc_pillow", "enc_webp", "enc_jpeg", "enc_nvjpeg"
+ENCODER_CODECS = "enc_rgb", "enc_pillow", "enc_webp", "enc_jpeg", "enc_nvjpeg"
 ENCODER_VIDEO_CODECS = "enc_vpx", "enc_x264", "enc_x265", "nvenc", "enc_ffmpeg"
 DECODER_CODECS = "dec_pillow", "dec_webp", "dec_jpeg"
 DECODER_VIDEO_CODECS = "dec_vpx", "dec_avcodec2"

xpra/codecs/nvjpeg/encoder.pyx

@@ -298,9 +298,11 @@ def init_module():
 def cleanup_module():
     log("nvjpeg.cleanup_module()")
 
+NVJPEG_INPUT_FORMATS = ("RGB", "BGR")
+
 def get_input_colorspaces(encoding):
     assert encoding=="jpeg"
-    return ("BGR", "RGB")
+    return NVJPEG_INPUT_FORMATS
 
 def get_output_colorspaces(encoding, input_colorspace):
     assert encoding in get_encodings()
@@ -326,6 +328,7 @@ cdef class Encoder:
     cdef object src_format
     cdef int quality
     cdef int speed
+    cdef int grayscale
     cdef long frames
     cdef nvjpegHandle_t nv_handle
     cdef nvjpegEncoderState_t nv_enc_state
@@ -349,6 +352,7 @@ cdef class Encoder:
         self.src_format = src_format
         self.quality = quality
         self.speed = speed
+        self.grayscale = options.boolget("grayscale", False)
         self.scaling = scaling
         self.init_nvjpeg()
         self.cuda_buffer = None
@@ -362,7 +366,11 @@ cdef class Encoder:
         self.configure_nvjpeg()
 
     def configure_nvjpeg(self):
-        cdef nvjpegChromaSubsampling_t subsampling = get_subsampling(self.quality)
+        cdef nvjpegChromaSubsampling_t subsampling
+        if self.grayscale:
+            subsampling = NVJPEG_CSS_GRAY
+        else:
+            subsampling = get_subsampling(self.quality)
         cdef int r
         r = nvjpegEncoderParamsSetSamplingFactors(self.nv_enc_params, subsampling, self.stream)
         errcheck(r, "nvjpegEncoderParamsSetSamplingFactors %i (%s)",
@@ -524,7 +532,7 @@ cdef nvjpegChromaSubsampling_t get_subsampling(int quality):
     return NVJPEG_CSS_420
 
 
-def encode(coding, image, int quality=50, speed=50):
+def encode(coding, image, options):
     assert coding=="jpeg"
     from xpra.codecs.cuda_common.cuda_context import select_device, cuda_device_context
     cdef double start = monotonic()
@@ -535,25 +543,53 @@ def encode(coding, image, int quality=50, speed=50):
     cuda_context = cuda_device_context(cuda_device_id, cuda_device)
     cdef double end = monotonic()
     log("device init took %.1fms", 1000*(end-start))
-    return device_encode(cuda_context, image, quality, speed)
+    return device_encode(cuda_context, image, options)
 
 errors = []
 def get_errors():
     global errors
     return errors
 
-def device_encode(device_context, image, int quality=50, speed=50):
+def device_encode(device_context, image, options):
     global errors
-    pfstr = bytestostr(image.get_pixel_format())
-    assert pfstr in ("RGB", "BGR"), "invalid pixel format %s" % pfstr
-    options = typedict()
+    pfstr = image.get_pixel_format()
+
     cdef int width = image.get_width()
     cdef int height = image.get_height()
-    cdef int stride = image.get_rowstride()
+    resize = options.get("resize")
+    if resize:
+        #we may need to convert to an RGB format scaling can handle:
+        from xpra.codecs.argb.scale import scale_image, RGB_SCALE_FORMATS
+        if pfstr not in RGB_SCALE_FORMATS:
+            from xpra.codecs.argb.argb import argb_swap         #@UnresolvedImport
+            if not argb_swap(image, RGB_SCALE_FORMATS):
+                log("nvjpeg: argb_swap failed to convert %s to a suitable format for scaling: %s" % (
+                    pfstr, RGB_SCALE_FORMATS))
+            else:
+                pfstr = image.get_pixel_format()
+        if pfstr in RGB_SCALE_FORMATS:
+            w, h = resize
+            image = scale_image(image, w, h)
+            log("nvjpeg scaled image: %s", image)
+
+    if pfstr not in NVJPEG_INPUT_FORMATS:
+        from xpra.codecs.argb.argb import argb_swap         #@UnresolvedImport @Reimport
+        if not argb_swap(image, NVJPEG_INPUT_FORMATS):
+            log("nvjpeg: argb_swap failed to convert %s to a suitable format: %s" % (
+                pfstr, NVJPEG_INPUT_FORMATS))
+            return None
+        log("jpeg converted image: %s", image)
+        pfstr = image.get_pixel_format()
+
+    options = typedict(options)
+    cdef int encode_width = image.get_width()
+    cdef int encode_height = image.get_height()
+    cdef int quality = options.get("quality", 50)
+    cdef int speed = options.get("speed", 50)
     cdef Encoder encoder
     try:
         encoder = Encoder()
-        encoder.init_context(device_context, width, height,
+        encoder.init_context(device_context, encode_width, encode_height,
                        pfstr, (pfstr, ),
                        "jpeg", quality, speed, scaling=(1, 1), options=options)
         r = encoder.compress_image(device_context, image, quality, speed, options)
@@ -579,5 +615,5 @@ def selftest(full=False):
     for size in (32, 256):
         img = make_test_image("BGR", size, size)
         log("testing with %s", img)
-        v = encode("jpeg", img)
+        v = encode("jpeg", img, {})
         assert v, "failed to compress test image"