From 8302076d3e4bc939071d9bbb0a4308f1e4e4521e Mon Sep 17 00:00:00 2001 From: totaam Date: Tue, 23 Nov 2021 16:57:59 +0700 Subject: [PATCH] * move scaling logic to the window source, the actual implementation is in the encoders - via argb/scale for those that don't have native support, * add more grayscale options: jpeg has TJSAMP_GRAY, nvjpeg has NVJPEG_CSS_GRAY, * move rgb encoding to argb/encoder, * pixels_to_bytes moved out of the server module --- xpra/codecs/argb/encoder.py | 106 +++++++++++++ xpra/codecs/argb/scale.py | 20 +++ xpra/codecs/jpeg/encoder.pyx | 49 ++++-- xpra/codecs/loader.py | 3 +- xpra/codecs/nvjpeg/encoder.pyx | 58 +++++-- xpra/codecs/pillow/encoder.py | 25 +-- xpra/codecs/rgb_transform.py | 5 +- xpra/codecs/webp/encoder.pyx | 26 ++-- xpra/platform/darwin/shadow_server.py | 4 +- xpra/server/mixins/encoding_server.py | 1 + xpra/server/picture_encode.py | 97 +----------- xpra/server/window/window_source.py | 179 ++++++++++------------ xpra/server/window/window_video_source.py | 5 + 13 files changed, 342 insertions(+), 236 deletions(-) create mode 100644 xpra/codecs/argb/encoder.py create mode 100755 xpra/codecs/argb/scale.py diff --git a/xpra/codecs/argb/encoder.py b/xpra/codecs/argb/encoder.py new file mode 100644 index 0000000000..edc51a89a5 --- /dev/null +++ b/xpra/codecs/argb/encoder.py @@ -0,0 +1,106 @@ +# This file is part of Xpra. +# Copyright (C) 2021 Antoine Martin +# Xpra is released under the terms of the GNU GPL v2, or, at your option, any +# later version. See the file COPYING for details. + +from xpra.codecs.rgb_transform import rgb_reformat +from xpra.codecs import rgb_transform +from xpra.net.compression import Compressed, compressed_wrapper +from xpra.log import Logger + +log = Logger("encoder") + + +def get_version(): + return 4, 3 + +def get_type() -> str: + return "rgb" + +def get_encodings(): + return "rgb24", "rgb32" + +def get_info() -> dict: + return { + "version" : get_version(), + "encodings" : get_encodings(), + } + + +def encode(coding : str, image, options : dict): + pixel_format = image.get_pixel_format() + #log("rgb_encode%s pixel_format=%s, rgb_formats=%s", + # (coding, image, rgb_formats, supports_transparency, speed, rgb_zlib, rgb_lz4), pixel_format, rgb_formats) + rgb_formats = options.get("rgb_formats", ("BGRX", "BGRA")) + supports_transparency = options.get("alpha", True) + if pixel_format not in rgb_formats: + log("rgb_encode reformatting because %s not in %s, supports_transparency=%s", + pixel_format, rgb_formats, supports_transparency) + if not rgb_reformat(image, rgb_formats, supports_transparency): + raise Exception("cannot find compatible rgb format to use for %s! (supported: %s)" % ( + pixel_format, rgb_formats)) + #get the new format: + pixel_format = image.get_pixel_format() + #switch encoding if necessary: + if len(pixel_format)==4: + coding = "rgb32" + elif len(pixel_format)==3: + coding = "rgb24" + else: + raise Exception("invalid pixel format %s" % pixel_format) + #we may still want to re-stride: + image.may_restride() + #always tell client which pixel format we are sending: + options = {"rgb_format" : pixel_format} + + #compress here and return a wrapper so network code knows it is already zlib compressed: + pixels = image.get_pixels() + assert pixels, "failed to get pixels from %s" % image + width = image.get_width() + height = image.get_height() + stride = image.get_rowstride() + speed = options.get("speed", 50) + + #compression stage: + level = 0 + algo = "not" + l = len(pixels) + if l>=512 and speed<100: + if l>=4096: + #speed=99 -> level=1, speed=0 -> level=9 + level = 1+max(0, min(8, int(100-speed)//12)) + else: + #fewer pixels, make it more likely we won't bother compressing + #and use a lower level (max=5) + level = max(0, min(5, int(115-speed)//20)) + if level>0: + zlib = options.get("zlib", False) + lz4 = options.get("lz4", False) + cwrapper = compressed_wrapper(coding, pixels, level=level, + zlib=zlib, lz4=lz4, + brotli=False, none=True) + algo = cwrapper.algorithm + if algo=="none" or len(cwrapper)>=(len(pixels)-32): + #no compression is enabled, or compressed is actually bigger! + #(fall through to uncompressed) + level = 0 + else: + #add compressed marker: + options[algo] = level + #remove network layer compression marker + #so that this data will be decompressed by the decode thread client side: + cwrapper.level = 0 + if level==0: + #can't pass a raw buffer to bencode / rencode, + #and even if we could, the image containing those pixels may be freed by the time we get to the encoder + algo = "not" + cwrapper = Compressed(coding, rgb_transform.pixels_to_bytes(pixels), True) + if pixel_format.find("A")>=0 or pixel_format.find("X")>=0: + bpp = 32 + else: + bpp = 24 + log("rgb_encode using level=%s for %5i bytes at %3i speed, %s compressed %4sx%-4s in %s/%s: %5s bytes down to %5s", + level, l, speed, algo, width, height, coding, pixel_format, len(pixels), len(cwrapper.data)) + #wrap it using "Compressed" so the network layer receiving it + #won't decompress it (leave it to the client's draw thread) + return coding, cwrapper, options, width, height, stride, bpp diff --git a/xpra/codecs/argb/scale.py b/xpra/codecs/argb/scale.py new file mode 100755 index 0000000000..be7dde3e5a --- /dev/null +++ b/xpra/codecs/argb/scale.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# This file is part of Xpra. +# Copyright (C) 2021 Antoine Martin +# Xpra is released under the terms of the GNU GPL v2, or, at your option, any +# later version. See the file COPYING for details. + +from xpra.log import Logger +log = Logger("encoding") + +#more scaling functions can be added here later + +RGB_SCALE_FORMATS = ("BGRX", "BGRA", "RGBA", "RGBX") + +def scale_image(image, width, height): + try: + from xpra.codecs.csc_libyuv.colorspace_converter import argb_scale #pylint: disable=import-outside-toplevel + except ImportError as e: + log("cannot downscale: %s", e) + return image + return argb_scale(image, width, height) diff --git a/xpra/codecs/jpeg/encoder.pyx b/xpra/codecs/jpeg/encoder.pyx index b8c3c786ab..1e3dec383a 100644 --- a/xpra/codecs/jpeg/encoder.pyx +++ b/xpra/codecs/jpeg/encoder.pyx @@ -146,6 +146,7 @@ cdef class Encoder: cdef object src_format cdef int quality cdef int speed + cdef int grayscale cdef long frames cdef object __weakref__ @@ -163,6 +164,7 @@ cdef class Encoder: self.width = width self.height = height self.src_format = src_format + self.grayscale = options.boolget("grayscale") self.scaling = scaling self.quality = quality self.speed = speed @@ -204,6 +206,7 @@ cdef class Encoder: "height" : self.height, "speed" : self.speed, "quality" : self.quality, + "grayscale" : bool(self.grayscale), }) return info @@ -218,9 +221,9 @@ cdef class Encoder: speed = self.speed pfstr = bytestostr(image.get_pixel_format()) if pfstr in ("YUV420P", "YUV422P", "YUV444P"): - cdata = encode_yuv(self.compressor, image, quality, speed) + cdata = encode_yuv(self.compressor, image, quality, self.grayscale) else: - cdata = encode_rgb(self.compressor, image, quality, speed) + cdata = encode_rgb(self.compressor, image, quality, self.grayscale) if not cdata: return None self.frames += 1 @@ -231,8 +234,30 @@ def get_error_str(): cdef char *err = tjGetErrorStr() return bytestostr(err) -def encode(coding, image, int quality=50, int speed=50): +JPEG_INPUT_FORMATS = ("RGB", "RGBX", "BGRX", "XBGR", "XRGB", "RGBA", "BGRA", "ABGR", "ARGB") + +def encode(coding, image, options): assert coding=="jpeg" + cdef int quality = options.get("quality", 50) + cdef int grayscale = options.get("grayscale", 0) + resize = options.get("resize") + log("encode%s", (coding, image, options)) + rgb_format = image.get_pixel_format() + if rgb_format not in JPEG_INPUT_FORMATS: + from xpra.codecs.argb.argb import argb_swap #@UnresolvedImport + if not argb_swap(image, JPEG_INPUT_FORMATS): + log("jpeg: argb_swap failed to convert %s to a suitable format: %s" % ( + rgb_format, JPEG_INPUT_FORMATS)) + log("jpeg converted image: %s", image) + + width = image.get_width() + height = image.get_height() + if resize: + from xpra.codecs.argb.scale import scale_image + scaled_width, scaled_height = resize + image = scale_image(image, scaled_width, scaled_height) + log("jpeg scaled image: %s", image) + #100 would mean lossless, so cap it at 99: client_options = { "quality" : min(99, quality), @@ -243,7 +268,7 @@ def encode(coding, image, int quality=50, int speed=50): return None cdef int r try: - cdata = encode_rgb(compressor, image, quality, speed) + cdata = encode_rgb(compressor, image, quality, grayscale) if not cdata: return None if SAVE_TO_FILE: # pragma: no cover @@ -251,14 +276,14 @@ def encode(coding, image, int quality=50, int speed=50): with open(filename, "wb") as f: f.write(cdata) log.info("saved %i bytes to %s", len(cdata), filename) - return "jpeg", Compressed("jpeg", memoryview(cdata), False), client_options, image.get_width(), image.get_height(), 0, 24 + return "jpeg", Compressed("jpeg", memoryview(cdata), False), client_options, width, height, 0, 24 finally: r = tjDestroy(compressor) if r: log.error("Error: failed to destroy the JPEG compressor, code %i:", r) log.error(" %s", get_error_str()) -cdef encode_rgb(tjhandle compressor, image, int quality, int speed): +cdef encode_rgb(tjhandle compressor, image, int quality, int grayscale=0): cdef int width = image.get_width() cdef int height = image.get_height() cdef int stride = image.get_rowstride() @@ -269,7 +294,9 @@ cdef encode_rgb(tjhandle compressor, image, int quality, int speed): raise Exception("invalid pixel format %s" % pfstr) cdef TJPF tjpf = pf cdef TJSAMP subsamp = TJSAMP_444 - if quality<50: + if grayscale: + subsamp = TJSAMP_GRAY + elif quality<50: subsamp = TJSAMP_420 elif quality<80: subsamp = TJSAMP_422 @@ -300,11 +327,13 @@ cdef encode_rgb(tjhandle compressor, image, int quality, int speed): assert out_size>0 and out!=NULL, "jpeg compression produced no data" return makebuf(out, out_size) -cdef encode_yuv(tjhandle compressor, image, int quality, int speed): +cdef encode_yuv(tjhandle compressor, image, int quality, int grayscale=0): pfstr = bytestostr(image.get_pixel_format()) assert pfstr in ("YUV420P", "YUV422P"), "invalid yuv pixel format %s" % pfstr cdef TJSAMP subsamp - if pfstr=="YUV420P": + if grayscale: + subsamp = TJSAMP_GRAY + elif pfstr=="YUV420P": subsamp = TJSAMP_420 elif pfstr=="YUV422P": subsamp = TJSAMP_422 @@ -370,5 +399,5 @@ def selftest(full=False): from xpra.codecs.codec_checks import make_test_image img = make_test_image("BGRA", 32, 32) for q in (0, 50, 100): - v = encode("jpeg", img, q, 100) + v = encode("jpeg", img, {"quality" : q}) assert v, "encode output was empty!" diff --git a/xpra/codecs/loader.py b/xpra/codecs/loader.py index 5bf779fab6..bc8d4a5e86 100755 --- a/xpra/codecs/loader.py +++ b/xpra/codecs/loader.py @@ -147,6 +147,7 @@ def xpra_codec_import(name, description, top_module, class_module, classname): CODEC_OPTIONS = { #encoders: + "enc_rgb" : ("RGB encoder", "argb", "encoder", "encode"), "enc_pillow" : ("Pillow encoder", "pillow", "encoder", "encode"), "enc_webp" : ("webp encoder", "webp", "encoder", "encode"), "enc_jpeg" : ("JPEG encoder", "jpeg", "encoder", "encode"), @@ -233,7 +234,7 @@ def has_codec(name) -> bool: CSC_CODECS = "csc_swscale", "csc_cython", "csc_libyuv" -ENCODER_CODECS = "enc_pillow", "enc_webp", "enc_jpeg", "enc_nvjpeg" +ENCODER_CODECS = "enc_rgb", "enc_pillow", "enc_webp", "enc_jpeg", "enc_nvjpeg" ENCODER_VIDEO_CODECS = "enc_vpx", "enc_x264", "enc_x265", "nvenc", "enc_ffmpeg" DECODER_CODECS = "dec_pillow", "dec_webp", "dec_jpeg" DECODER_VIDEO_CODECS = "dec_vpx", "dec_avcodec2" diff --git a/xpra/codecs/nvjpeg/encoder.pyx b/xpra/codecs/nvjpeg/encoder.pyx index c7973f9949..0e8ed6f5f9 100644 --- a/xpra/codecs/nvjpeg/encoder.pyx +++ b/xpra/codecs/nvjpeg/encoder.pyx @@ -298,9 +298,11 @@ def init_module(): def cleanup_module(): log("nvjpeg.cleanup_module()") +NVJPEG_INPUT_FORMATS = ("RGB", "BGR") + def get_input_colorspaces(encoding): assert encoding=="jpeg" - return ("BGR", "RGB") + return NVJPEG_INPUT_FORMATS def get_output_colorspaces(encoding, input_colorspace): assert encoding in get_encodings() @@ -326,6 +328,7 @@ cdef class Encoder: cdef object src_format cdef int quality cdef int speed + cdef int grayscale cdef long frames cdef nvjpegHandle_t nv_handle cdef nvjpegEncoderState_t nv_enc_state @@ -349,6 +352,7 @@ cdef class Encoder: self.src_format = src_format self.quality = quality self.speed = speed + self.grayscale = options.boolget("grayscale", False) self.scaling = scaling self.init_nvjpeg() self.cuda_buffer = None @@ -362,7 +366,11 @@ cdef class Encoder: self.configure_nvjpeg() def configure_nvjpeg(self): - cdef nvjpegChromaSubsampling_t subsampling = get_subsampling(self.quality) + cdef nvjpegChromaSubsampling_t subsampling + if self.grayscale: + subsampling = NVJPEG_CSS_GRAY + else: + subsampling = get_subsampling(self.quality) cdef int r r = nvjpegEncoderParamsSetSamplingFactors(self.nv_enc_params, subsampling, self.stream) errcheck(r, "nvjpegEncoderParamsSetSamplingFactors %i (%s)", @@ -524,7 +532,7 @@ cdef nvjpegChromaSubsampling_t get_subsampling(int quality): return NVJPEG_CSS_420 -def encode(coding, image, int quality=50, speed=50): +def encode(coding, image, options): assert coding=="jpeg" from xpra.codecs.cuda_common.cuda_context import select_device, cuda_device_context cdef double start = monotonic() @@ -535,25 +543,53 @@ def encode(coding, image, int quality=50, speed=50): cuda_context = cuda_device_context(cuda_device_id, cuda_device) cdef double end = monotonic() log("device init took %.1fms", 1000*(end-start)) - return device_encode(cuda_context, image, quality, speed) + return device_encode(cuda_context, image, options) errors = [] def get_errors(): global errors return errors -def device_encode(device_context, image, int quality=50, speed=50): +def device_encode(device_context, image, options): global errors - pfstr = bytestostr(image.get_pixel_format()) - assert pfstr in ("RGB", "BGR"), "invalid pixel format %s" % pfstr - options = typedict() + pfstr = image.get_pixel_format() + cdef int width = image.get_width() cdef int height = image.get_height() - cdef int stride = image.get_rowstride() + resize = options.get("resize") + if resize: + #we may need to convert to an RGB format scaling can handle: + from xpra.codecs.argb.scale import scale_image, RGB_SCALE_FORMATS + if pfstr not in RGB_SCALE_FORMATS: + from xpra.codecs.argb.argb import argb_swap #@UnresolvedImport + if not argb_swap(image, RGB_SCALE_FORMATS): + log("nvjpeg: argb_swap failed to convert %s to a suitable format for scaling: %s" % ( + pfstr, RGB_SCALE_FORMATS)) + else: + pfstr = image.get_pixel_format() + if pfstr in RGB_SCALE_FORMATS: + w, h = resize + image = scale_image(image, w, h) + log("nvjpeg scaled image: %s", image) + + if pfstr not in NVJPEG_INPUT_FORMATS: + from xpra.codecs.argb.argb import argb_swap #@UnresolvedImport @Reimport + if not argb_swap(image, NVJPEG_INPUT_FORMATS): + log("nvjpeg: argb_swap failed to convert %s to a suitable format: %s" % ( + pfstr, NVJPEG_INPUT_FORMATS)) + return None + log("jpeg converted image: %s", image) + pfstr = image.get_pixel_format() + + options = typedict(options) + cdef int encode_width = image.get_width() + cdef int encode_height = image.get_height() + cdef int quality = options.get("quality", 50) + cdef int speed = options.get("speed", 50) cdef Encoder encoder try: encoder = Encoder() - encoder.init_context(device_context, width, height, + encoder.init_context(device_context, encode_width, encode_height, pfstr, (pfstr, ), "jpeg", quality, speed, scaling=(1, 1), options=options) r = encoder.compress_image(device_context, image, quality, speed, options) @@ -579,5 +615,5 @@ def selftest(full=False): for size in (32, 256): img = make_test_image("BGR", size, size) log("testing with %s", img) - v = encode("jpeg", img) + v = encode("jpeg", img, {}) assert v, "failed to compress test image" diff --git a/xpra/codecs/pillow/encoder.py b/xpra/codecs/pillow/encoder.py index 0865f56ffa..9c5081d626 100644 --- a/xpra/codecs/pillow/encoder.py +++ b/xpra/codecs/pillow/encoder.py @@ -10,7 +10,6 @@ from PIL import Image, ImagePalette #@UnresolvedImport from xpra.util import envbool -from xpra.os_util import bytestostr from xpra.net.compression import Compressed from xpra.log import Logger @@ -49,14 +48,15 @@ def get_info() -> dict: } -def encode(coding : str, image, quality : int=50, speed : int=50, - supports_transparency : bool=True, - grayscale : bool=False, - resize=None): - log("pillow.encode%s", (coding, image, quality, speed, supports_transparency, grayscale, resize)) +def encode(coding : str, image, options): assert coding in ("jpeg", "webp", "png", "png/P", "png/L"), "unsupported encoding: %s" % coding - assert image, "no image to encode" - pixel_format = bytestostr(image.get_pixel_format()) + log("pillow.encode%s", (coding, image, options)) + quality = options.get("quality", 50) + speed = options.get("speed", 50) + supports_transparency = options.get("alpha", True) + grayscale = options.get("grayscale", False) + resize = options.get("resize") + pixel_format = image.get_pixel_format() palette = None w = image.get_width() h = image.get_height() @@ -245,7 +245,10 @@ def selftest(full=False): for q in vrange: for s in vrange: for alpha in (True, False): - v = encode(encoding, img, q, s, False, alpha) + v = encode(encoding, img, { + "quality" : q, + "speed" : s, + "alpha" : alpha}) assert v, "encode output was empty!" cdata = v[1].data log("encode(%s)=%s", (encoding, img, q, s, alpha), hexstr(cdata)) @@ -253,4 +256,6 @@ def selftest(full=False): l = log.warn l("Pillow error saving %s with quality=%s, speed=%s, alpha=%s", encoding, q, s, alpha) l(" %s", e, exc_info=True) - ENCODINGS.remove(encoding) + encs = list(ENCODINGS) + encs.remove(encoding) + ENCODINGS = tuple(encs) diff --git a/xpra/codecs/rgb_transform.py b/xpra/codecs/rgb_transform.py index 1dc7eff7ab..2971bd9f45 100644 --- a/xpra/codecs/rgb_transform.py +++ b/xpra/codecs/rgb_transform.py @@ -7,7 +7,7 @@ from time import monotonic from PIL import Image -from xpra.os_util import bytestostr +from xpra.os_util import memoryview_to_bytes, bytestostr from xpra.util import first_time from xpra.log import Logger try: @@ -15,6 +15,9 @@ except ImportError: # pragma: no cover argb_swap = None +#"pixels_to_bytes" gets patched up by the OSX shadow server +pixels_to_bytes = memoryview_to_bytes + log = Logger("encoding") diff --git a/xpra/codecs/webp/encoder.pyx b/xpra/codecs/webp/encoder.pyx index 25ab98afe5..4eff497a6c 100644 --- a/xpra/codecs/webp/encoder.pyx +++ b/xpra/codecs/webp/encoder.pyx @@ -385,25 +385,23 @@ cdef get_config_info(WebPConfig *config): "low_memory" : config.low_memory, } -def encode(coding, image, int quality=50, int speed=50, - supports_alpha=False, - content_type="", - resize=None): +def encode(coding, image, options): + log("webp.encode(%s, %s, %s)", coding, image, options) assert coding=="webp" - log("webp.encode(%s, %i, %i, %s, %s)", image, quality, speed, supports_alpha, content_type) pixel_format = image.get_pixel_format() if pixel_format not in ("RGBX", "RGBA", "BGRX", "BGRA", "RGB", "BGR"): raise Exception("unsupported pixel format %s" % pixel_format) cdef unsigned int width = image.get_width() cdef unsigned int height = image.get_height() assert width<16384 and height<16384, "invalid image dimensions: %ix%i" % (width, height) + resize = options.get("resize", None) cdef unsigned int stride = image.get_rowstride() cdef unsigned int Bpp = len(pixel_format) #ie: "BGRA" -> 4 cdef int size = stride * height - if width>WEBP_MAX_DIMENSION or height>WEBP_MAX_DIMENSION: - raise Exception("this image is too big for webp: %ix%i" % (width, height)) pixels = image.get_pixels() + cdef int supports_alpha = options.get("alpha", False) cdef int alpha_int = supports_alpha and pixel_format.find("A")>=0 + cdef int quality = options.get("quality", 50) cdef int yuv420p = quality=100 + config.lossless = quality>=100 and not resize if config.lossless: #not much to gain from setting a high quality here, #the latency will be higher for a negligible compression gain: @@ -516,17 +516,19 @@ def encode(coding, image, int quality=50, int speed=50, end = monotonic() log("webp %s import took %.1fms", pixel_format, 1000*(end-start)) - cdef int scaled_width, scaled_height + cdef int encode_width, encode_height if resize: - scaled_width, scaled_height = resize + encode_width, encode_height = resize start = monotonic() with nogil: - ret = WebPPictureRescale(&pic, scaled_width, scaled_height) + ret = WebPPictureRescale(&pic, encode_width, encode_height) if not ret: WebPPictureFree(&pic) raise Exception("WebP failed to resize %s to %s" % (image, resize)) end = monotonic() log("webp %s resizing took %.1fms", 1000*(end-start)) + #we could use scaling to fit? + assert encode_width<16384 and encode_height<16384, "invalid image dimensions: %ix%i" % (width, height) if yuv420p: start = monotonic() @@ -583,7 +585,7 @@ def selftest(full=False): for has_alpha in (True, False): img = make_test_image("BGR%s" % ["X", "A"][has_alpha], w, h) for q in (10, 50, 90): - r = encode("webp", img, quality=q, speed=50, supports_alpha=has_alpha) + r = encode("webp", img, {"quality" : q, "speed" : 50, "alpha" : has_alpha}) assert len(r)>0 #import binascii #print("compressed data(%s)=%s" % (has_alpha, binascii.hexlify(r))) diff --git a/xpra/platform/darwin/shadow_server.py b/xpra/platform/darwin/shadow_server.py index d235fd760c..b15d449ec3 100644 --- a/xpra/platform/darwin/shadow_server.py +++ b/xpra/platform/darwin/shadow_server.py @@ -42,8 +42,8 @@ def pixels_to_bytes(v): return memoryview_to_bytes(v) l = CFDataGetLength(v) return CFDataGetBytes(v, (0, l), None) - from xpra.server import picture_encode - picture_encode.pixels_to_bytes = pixels_to_bytes + from xpra.codecs import rgb_transform + rgb_transform.pixels_to_bytes = pixels_to_bytes class OSXRootCapture: diff --git a/xpra/server/mixins/encoding_server.py b/xpra/server/mixins/encoding_server.py index fd149e353f..03b7155140 100644 --- a/xpra/server/mixins/encoding_server.py +++ b/xpra/server/mixins/encoding_server.py @@ -54,6 +54,7 @@ def threaded_setup(self): #load video codecs: getVideoHelper().init() #and load the picture codecs: + load_codec("enc_rgb") load_codec("enc_pillow") ae = self.allowed_encodings if "jpeg" in ae: diff --git a/xpra/server/picture_encode.py b/xpra/server/picture_encode.py index f522f69afe..d0bade0cb3 100644 --- a/xpra/server/picture_encode.py +++ b/xpra/server/picture_encode.py @@ -5,35 +5,23 @@ # later version. See the file COPYING for details. from time import monotonic -from xpra.net import compression from xpra.codecs.loader import get_codec from xpra.util import envbool, first_time -from xpra.codecs.rgb_transform import rgb_reformat -from xpra.os_util import memoryview_to_bytes, bytestostr from xpra.log import Logger -#"pixels_to_bytes" gets patched up by the OSX shadow server -pixels_to_bytes = memoryview_to_bytes -try: - from xpra.net.mmap_pipe import mmap_write -except ImportError: - mmap_write = None #no mmap - log = Logger("window", "encoding") WEBP_PILLOW = envbool("XPRA_WEBP_PILLOW", False) -def webp_encode(coding, image, quality : int=50, speed : int=50, - supports_transparency : bool=True, - content_type=""): +def webp_encode(coding, image, options): stride = image.get_rowstride() pixel_format = image.get_pixel_format() enc_webp = get_codec("enc_webp") log("WEBP_PILLOW=%s, enc_webp=%s, stride=%s, pixel_format=%s", WEBP_PILLOW, enc_webp, stride, pixel_format) if not WEBP_PILLOW and enc_webp and pixel_format in ("BGRA", "BGRX", "RGBA", "RGBX", "RGB", "BGR"): #prefer Cython module: - return enc_webp.encode(coding, image, quality, speed, supports_transparency, content_type) + return enc_webp.encode(coding, image, options) #fallback using Pillow: enc_pillow = get_codec("enc_pillow") if enc_pillow: @@ -42,90 +30,21 @@ def webp_encode(coding, image, quality : int=50, speed : int=50, log.warn(" enc_webp=%s, stride=%s, pixel format=%s", enc_webp, stride, image.get_pixel_format()) for x in ("webp", "png", "jpeg"): if x in enc_pillow.get_encodings(): - return enc_pillow.encode(x, image, quality, speed, supports_transparency) + return enc_pillow.encode(x, image, options) raise Exception("BUG: cannot use 'webp' encoding and none of the PIL fallbacks are available!") -def rgb_encode(coding, image, rgb_formats, supports_transparency, speed, rgb_zlib=True, rgb_lz4=True): - pixel_format = bytestostr(image.get_pixel_format()) - #log("rgb_encode%s pixel_format=%s, rgb_formats=%s", - # (coding, image, rgb_formats, supports_transparency, speed, rgb_zlib, rgb_lz4), pixel_format, rgb_formats) - if pixel_format not in rgb_formats: - log("rgb_encode reformatting because %s not in %s, supports_transparency=%s", - pixel_format, rgb_formats, supports_transparency) - if not rgb_reformat(image, rgb_formats, supports_transparency): - raise Exception("cannot find compatible rgb format to use for %s! (supported: %s)" % ( - pixel_format, rgb_formats)) - #get the new format: - pixel_format = bytestostr(image.get_pixel_format()) - #switch encoding if necessary: - if len(pixel_format)==4: - coding = "rgb32" - elif len(pixel_format)==3: - coding = "rgb24" - else: - raise Exception("invalid pixel format %s" % pixel_format) - #we may still want to re-stride: - image.may_restride() - #always tell client which pixel format we are sending: - options = {"rgb_format" : pixel_format} - - #compress here and return a wrapper so network code knows it is already zlib compressed: - pixels = image.get_pixels() - assert pixels, "failed to get pixels from %s" % image - width = image.get_width() - height = image.get_height() - stride = image.get_rowstride() - - #compression stage: - level = 0 - algo = "not" - l = len(pixels) - if l>=512 and speed<100: - if l>=4096: - #speed=99 -> level=1, speed=0 -> level=9 - level = 1+max(0, min(8, int(100-speed)//12)) - else: - #fewer pixels, make it more likely we won't bother compressing - #and use a lower level (max=5) - level = max(0, min(5, int(115-speed)//20)) - if level>0: - cwrapper = compression.compressed_wrapper(coding, pixels, level=level, - zlib=rgb_zlib, lz4=rgb_lz4, - brotli=False, none=True) - algo = cwrapper.algorithm - if algo=="none" or len(cwrapper)>=(len(pixels)-32): - #no compression is enabled, or compressed is actually bigger! - #(fall through to uncompressed) - level = 0 - else: - #add compressed marker: - options[algo] = level - #remove network layer compression marker - #so that this data will be decompressed by the decode thread client side: - cwrapper.level = 0 - if level==0: - #can't pass a raw buffer to bencode / rencode, - #and even if we could, the image containing those pixels may be freed by the time we get to the encoder - algo = "not" - cwrapper = compression.Compressed(coding, pixels_to_bytes(pixels), True) - if pixel_format.find("A")>=0 or pixel_format.find("X")>=0: - bpp = 32 - else: - bpp = 24 - log("rgb_encode using level=%s for %5i bytes at %3i speed, %s compressed %4sx%-4s in %s/%s: %5s bytes down to %5s", - level, l, speed, algo, width, height, coding, pixel_format, len(pixels), len(cwrapper.data)) - #wrap it using "Compressed" so the network layer receiving it - #won't decompress it (leave it to the client's draw thread) - return coding, cwrapper, options, width, height, stride, bpp - - def mmap_send(mmap, mmap_size, image, rgb_formats, supports_transparency): + try: + from xpra.net.mmap_pipe import mmap_write + except ImportError: + mmap_write = None #no mmap if mmap_write is None: if first_time("mmap_write missing"): log.warn("Warning: cannot use mmap, no write method support") return None if image.get_pixel_format() not in rgb_formats: + from xpra.codecs.rgb_transform import rgb_reformat #pylint: disable=import-outside-toplevel if not rgb_reformat(image, rgb_formats, supports_transparency): warning_key = "mmap_send(%s)" % image.get_pixel_format() if first_time(warning_key): diff --git a/xpra/server/window/window_source.py b/xpra/server/window/window_source.py index b34584fe7a..00b71ef0da 100644 --- a/xpra/server/window/window_source.py +++ b/xpra/server/window/window_source.py @@ -21,13 +21,12 @@ from xpra.server.window.batch_delay_calculator import calculate_batch_delay, get_target_speed, get_target_quality from xpra.server.cystats import time_weighted_average, logp #@UnresolvedImport from xpra.rectangle import rectangle, add_rectangle, remove_rectangle, merge_all #@UnresolvedImport -from xpra.server.picture_encode import rgb_encode, webp_encode, mmap_send +from xpra.server.picture_encode import webp_encode from xpra.simple_stats import get_list_stats -from xpra.codecs.argb.argb import argb_swap #@UnresolvedImport from xpra.codecs.rgb_transform import rgb_reformat from xpra.codecs.loader import get_codec from xpra.codecs.codec_constants import PREFERRED_ENCODING_ORDER, LOSSY_PIXEL_FORMATS -from xpra.net.compression import use, Compressed +from xpra.net.compression import use from xpra.log import Logger log = Logger("window", "encoding") @@ -73,6 +72,7 @@ DAMAGE_STATISTICS = envbool("XPRA_DAMAGE_STATISTICS", False) SCROLL_ALL = envbool("XPRA_SCROLL_ALL", True) +#FIXME: honour it FORCE_PILLOW = envbool("XPRA_FORCE_PILLOW", False) HARDCODED_ENCODING = os.environ.get("XPRA_HARDCODED_ENCODING") @@ -336,8 +336,10 @@ def init_encoders(self): self._all_encoders = {} self._encoders = {} self.full_csc_modes = typedict() - self.add_encoder("rgb24", self.rgb_encode) - self.add_encoder("rgb32", self.rgb_encode) + rgb_encoder = get_codec("enc_rgb") + assert rgb_encoder, "rgb encoder is missing" + self.add_encoder("rgb24", rgb_encoder.encode) + self.add_encoder("rgb32", rgb_encoder.encode) #we need pillow for scaling and grayscale: self.enc_pillow = get_codec("enc_pillow") if self._mmap_size>0: @@ -348,12 +350,12 @@ def add(encoding, encoder): self.add_encoder(encoding, encoder) if self.enc_pillow: for x in self.enc_pillow.get_encodings(): - add(x, self.pillow_encode) + add(x, self.enc_pillow.encode) #prefer these native encoders over the Pillow version: add("webp", self.webp_encode) self.enc_jpeg = get_codec("enc_jpeg") if self.enc_jpeg: - add("jpeg", self.jpeg_encode) + add("jpeg", self.enc_jpeg.encode) #prefer nvjpeg over all the other jpeg encoders: self.enc_nvjpeg = None log("init_encoders() cuda_device_context=%s", self.cuda_device_context) @@ -1004,13 +1006,14 @@ def get_auto_encoding(self, w, h, speed, quality, current_encoding=None): def do_get_auto_encoding(self, w, h, speed, quality, current_encoding, encoding_options): co = encoding_options depth = self.image_depth - if w*h24 and self.client_bit_depth>24 and "rgb32" in co: return "rgb32" if "rgb24" in co: return "rgb24" jpeg = "jpeg" and w>=2 and h>=2 - webp = "webp" in co and 16383>=w>=2 and 16383>=h>=2 + webp = "webp" in co and 16383>=w>=2 and 16383>=h>=2 and not grayscale lossy = quality<100 if depth in (24, 32) and (jpeg or webp): if jpeg and w>=8 and h>=8 and lossy and self.enc_nvjpeg: @@ -1772,8 +1775,19 @@ def assign_sq_options(self, options, speed_delta=0, quality_delta=0): quality += int(elapsed*25) quality = min(100, max(1, self._fixed_min_quality, quality-packets_backlog*20+quality_delta)) eoptions = dict(options) - eoptions["quality"] = quality - eoptions["speed"] = speed + eoptions.update({ + "quality" : quality, + "speed" : speed, + "rgb_formats" : self.rgb_formats, + "zlib" : self.rgb_zlib, + "lz4" : self.rgb_lz4, + }) + if self.encoding=="grayscale": + eoptions["grayscale"] = True + if not self.supports_transparency: + eoptions["alpha"] = False + if self.content_type: + eoptions["content-type"] = self.content_type return eoptions def do_send_delayed_regions(self, damage_time, regions, coding, options, @@ -1954,6 +1968,9 @@ def process_damage_region(self, damage_time, x, y, w, h, coding, options, flush= if self.send_window_size: options["window-size"] = self.window_dimensions + resize = self.scaled_size(image) + if resize: + options["resize"] = resize now = monotonic() item = (w, h, damage_time, now, image, coding, sequence, options, flush) @@ -1962,6 +1979,19 @@ def process_damage_region(self, damage_time, x, y, w, h, coding, options, flush= self.wid, sequence, w, h, coding, 1000*(now-damage_time), 1000*(now-rgb_request_time)) return True + def scaled_size(self, image): + crs = self.client_render_size + if not crs or not DOWNSCALE: + return None + w, h = image.get_width(), image.get_height() + ww, wh = self.window_dimensions + crsw, crsh = crs + #resize if the render size is smaller + if ww-crsw>DOWNSCALE_THRESHOLD and wh-crsh>DOWNSCALE_THRESHOLD: + #keep the same proportions: + return w*crsw//ww, h*crsh//wh + return None + def make_data_packet_cb(self, w, h, damage_time, process_damage_time, image, coding, sequence, options, flush): """ This function is called from the damage data thread! @@ -2540,78 +2570,25 @@ def make_draw_packet(self, x, y, outw, outh, coding, data, outstride, client_opt def webp_encode(self, coding, image, options): - r, image = self.may_scale(coding, image, options) - if r: - return r pixel_format = image.get_pixel_format() #the native webp encoder only takes BGRX / BGRA as input, #but the client may be able to swap channels, #so it may be able to process RGBX / RGBA: - client_rgb_formats = self.full_csc_modes.strtupleget("webp", ("BGRA", "BGRX", )) + client_rgb_formats = self.full_csc_modes.strtupleget("webp", ("BGRA", "BGRX", "BGR", )) if pixel_format not in client_rgb_formats: if not rgb_reformat(image, client_rgb_formats, self.supports_transparency): raise Exception("cannot find compatible rgb format to use for %s! (supported: %s)" % ( pixel_format, self.rgb_formats)) - q = options.get("quality", self._current_quality) - s = options.get("speed", self._current_speed) - return webp_encode(coding, image, q, s, self.supports_transparency, self.content_type) - - def rgb_encode(self, coding, image, options): - s = options.get("speed") or self._current_speed - return rgb_encode(coding, image, self.rgb_formats, self.supports_transparency, s, - self.rgb_zlib, self.rgb_lz4) - - def no_r210(self, image, rgb_formats): - rgb_format = image.get_pixel_format() - if rgb_format=="r210": - argb_swap(image, rgb_formats, self.supports_transparency) - - def jpeg_encode(self, coding, image, options): - r, image = self.may_scale(coding, image, options) - if r: - return r - self.no_r210(image, ["RGB"]) - q = options.get("quality", self._current_quality) - s = options.get("speed", self._current_speed) - return self.enc_jpeg.encode(coding, image, q, s) - - def may_scale(self, coding, image, options): - if self.encoding=="grayscale" or FORCE_PILLOW: - #only pillow can do grayscale at the moment: - return self.pillow_encode(coding, image, options), image - #now check for downscaling: - if not DOWNSCALE or not self.client_render_size: - return None, image - crsw, crsh = self.client_render_size - ww, wh = self.window_dimensions - if ww-crsw80 and self._fixed_quality<=0: - options["quality"] = 80 - crsw, crsh = self.client_render_size - ww, wh = self.window_dimensions - width = image.get_width()*crsw//ww - height = image.get_height()*crsh//wh - return None, argb_scale(image, width, height) + return webp_encode(coding, image, options) def nvjpeg_encode(self, coding, image, options): assert coding=="jpeg" - r, image = self.may_scale(coding, image, options) - if r: - return r def fallback(reason): log("nvjpeg_encode: %s", reason) if self.enc_jpeg: - return self.jpeg_encode(coding, image, options) + return self.enc_jpeg.encode(coding, image, options) if self.enc_pillow: - return self.pillow_encode(coding, image, options) + return self.enc_pillow.encode(coding, image, options) return None cdd = self.cuda_device_context if not cdd: @@ -2620,57 +2597,59 @@ def fallback(reason): h = image.get_height() if w<16 or h<16: return fallback("image size %ix%i is too small" % (w, h)) - NVJPEG_INPUT_FORMATS = ("RGB", "BGR") - self.no_r210(image, NVJPEG_INPUT_FORMATS) - pixel_format = image.get_pixel_format() - if pixel_format not in NVJPEG_INPUT_FORMATS and not argb_swap(image, NVJPEG_INPUT_FORMATS): - return fallback("cannot handle %s" % pixel_format) - q = options.get("quality", self._current_quality) - s = options.get("speed", self._current_speed) log("nvjpeg_encode%s", (coding, image, options)) with cdd: - r = self.enc_nvjpeg.device_encode(cdd, image, q, s) + r = self.enc_nvjpeg.device_encode(cdd, image, options) if r is None: errors = self.enc_nvjpeg.get_errors() MAX_FAILURES = 3 if len(errors)<=MAX_FAILURES: return fallback(errors[-1]) - log.warn("Warning: nvjpeg has failed too many times and is now disabled") + log.warn("Warning: nvjpeg has failed %s times and is now disabled", len(errors)) for e in errors: log(" %s", e) self.enc_nvjpeg = None return fallback("nvjpeg is now disabled") return r - def pillow_encode(self, coding, image, options): - #for more information on pixel formats supported by PIL / Pillow, see: - #https://github.com/python-imaging/Pillow/blob/master/libImaging/Unpack.c - assert coding in self.server_core_encodings - transparency = self.supports_transparency and options.get("transparency", True) - grayscale = self.encoding=="grayscale" - resize = None - w, h = image.get_width(), image.get_height() - ww, wh = self.window_dimensions - crs = self.client_render_size - if crs and DOWNSCALE: - crsw, crsh = crs - #resize if the render size is smaller - if ww-crsw>DOWNSCALE_THRESHOLD and wh-crsh>DOWNSCALE_THRESHOLD: - #keep the same proportions: - resize = w*crsw//ww, h*crsh//wh - q = options.get("quality", self._current_quality) - s = options.get("speed", self._current_speed) - return self.enc_pillow.encode(coding, image, q, s, transparency, grayscale, resize) - def mmap_encode(self, coding, image, _options): assert coding=="mmap" assert self._mmap and self._mmap_size>0 - v = mmap_send(self._mmap, self._mmap_size, image, self.rgb_formats, self.supports_transparency) + v = self.mmap_send(self._mmap, self._mmap_size, image, self.rgb_formats, self.supports_transparency) if v is None: return None mmap_info, mmap_free_size, written = v self.global_statistics.mmap_bytes_sent += written self.global_statistics.mmap_free_size = mmap_free_size #the data we send is the index within the mmap area: - client_options = {"rgb_format" : image.get_pixel_format()} - return "mmap", mmap_info, client_options, image.get_width(), image.get_height(), image.get_rowstride(), 32 + pf = image.get_pixel_format() + return ( + "mmap", mmap_info, {"rgb_format" : pf}, + image.get_width(), image.get_height(), image.get_rowstride(), len(pf)*8, + ) + + def mmap_send(self, mmap, mmap_size, image, rgb_formats, supports_transparency): + try: + from xpra.net.mmap_pipe import mmap_write + except ImportError: + mmap_write = None #no mmap + if mmap_write is None: + if first_time("mmap_write missing"): + log.warn("Warning: cannot use mmap, no write method support") + return None + if image.get_pixel_format() not in rgb_formats: + if not rgb_reformat(image, rgb_formats, supports_transparency): + warning_key = "mmap_send(%s)" % image.get_pixel_format() + if first_time(warning_key): + log.warn("Waening: cannot use mmap to send %s" % image.get_pixel_format()) + return None + start = monotonic() + data = image.get_pixels() + assert data, "failed to get pixels from %s" % image + mmap_data, mmap_free_size = mmap_write(mmap, mmap_size, data) + elapsed = monotonic()-start+0.000000001 #make sure never zero! + log("%s MBytes/s - %s bytes written to mmap in %.1f ms", int(len(data)/elapsed/1024/1024), len(data), 1000*elapsed) + if mmap_data is None: + return None + #replace pixels with mmap info: + return mmap_data, mmap_free_size, len(data) diff --git a/xpra/server/window/window_video_source.py b/xpra/server/window/window_video_source.py index 91f487df78..d7190c5401 100644 --- a/xpra/server/window/window_video_source.py +++ b/xpra/server/window/window_video_source.py @@ -840,6 +840,9 @@ def process_damage_region(self, damage_time, x, y, w, h, coding, options, flush= h = image.get_height() if self.send_window_size: options["window-size"] = self.window_dimensions + resize = self.scaled_size(image) + if resize: + options["resize"] = resize #freeze if: # * we want av-sync @@ -1713,6 +1716,8 @@ def setup_pipeline_option(self, width, height, src_format, ve = encoder_spec.make_instance() options = typedict(self.encoding_options) options.update(self.get_video_encoder_options(encoder_spec.encoding, width, height)) + if self.encoding=="grayscale": + options["grayscale"] = True ve.init_context(self.cuda_device_context, enc_width, enc_height, enc_in_format, dst_formats, encoder_spec.encoding,