From e5e56a2ed9c0d8a528fa1af119049d9c1aad1c47 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Fri, 10 May 2024 23:56:07 -0700 Subject: [PATCH 1/9] add x264 to configure.ac --- configure.ac | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bbab3756b..8b11da514 100644 --- a/configure.ac +++ b/configure.ac @@ -172,7 +172,10 @@ AC_ARG_ENABLE(pixman, AS_HELP_STRING([--enable-pixman], [Use pixman library (default: no)]), [], [enable_pixman=no]) AM_CONDITIONAL(XRDP_PIXMAN, [test x$enable_pixman = xyes]) - +AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264], + [Use x264 library (default: no)]), + [], [enable_x264=no]) +AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes]) AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter], [Do not use included painter library (default: no)]), [], [enable_painter=yes]) @@ -489,6 +492,8 @@ fi AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >= 0.1.0)] ) +AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] ) + # checking for TurboJPEG if test "x$enable_tjpeg" = "xyes" then @@ -662,6 +667,7 @@ echo " fdkaac $enable_fdkaac" echo " jpeg $enable_jpeg" echo " turbo jpeg $enable_tjpeg" echo " rfxcodec $enable_rfxcodec" +echo " x264 $enable_x264" echo " painter $enable_painter" echo " pixman $enable_pixman" echo " fuse $enable_fuse" From ab97002c22b28cebc621b823383a8cf0d5a62cf6 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Sat, 11 May 2024 00:54:58 -0700 Subject: [PATCH 2/9] add xrdp_encoder_x264.c/h --- xrdp/Makefile.am | 11 +- xrdp/xrdp_encoder.c | 4 + xrdp/xrdp_encoder_x264.c | 242 +++++++++++++++++++++++++++++++++++++++ xrdp/xrdp_encoder_x264.h | 38 ++++++ 4 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 xrdp/xrdp_encoder_x264.c create mode 100644 xrdp/xrdp_encoder_x264.h diff --git a/xrdp/Makefile.am b/xrdp/Makefile.am index b771ff8c4..71315ffb1 100644 --- a/xrdp/Makefile.am +++ b/xrdp/Makefile.am @@ -18,6 +18,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/third_party/tomlc99 XRDP_EXTRA_LIBS = +XRDP_EXTRA_SOURCES = if XRDP_RFXCODEC AM_CPPFLAGS += -DXRDP_RFXCODEC @@ -25,6 +26,13 @@ AM_CPPFLAGS += -I$(top_srcdir)/librfxcodec/include XRDP_EXTRA_LIBS += $(top_builddir)/librfxcodec/src/.libs/librfxencode.a endif +if XRDP_X264 +AM_CPPFLAGS += -DXRDP_X264 +AM_CPPFLAGS += $(XRDP_X264_CFLAGS) +XRDP_EXTRA_LIBS += $(XRDP_X264_LIBS) +XRDP_EXTRA_SOURCES += xrdp_encoder_x264.c xrdp_encoder_x264.h +endif + if XRDP_PIXMAN AM_CPPFLAGS += -DXRDP_PIXMAN AM_CPPFLAGS += $(PIXMAN_CFLAGS) @@ -63,7 +71,8 @@ xrdp_SOURCES = \ xrdp_egfx.c \ xrdp_egfx.h \ xrdp_wm.c \ - xrdp_main_utils.c + xrdp_main_utils.c \ + $(XRDP_EXTRA_SOURCES) xrdp_LDADD = \ $(top_builddir)/common/libcommon.la \ diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index cb30f4b13..7666725a9 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -34,6 +34,10 @@ #include "rfxcodec_encode.h" #endif +#ifdef XRDP_X264 +#include "xrdp_encoder_x264.h" +#endif + #define DEFAULT_XRDP_GFX_FRAMES_IN_FLIGHT 2 /* limits used for validate env var XRDP_GFX_FRAMES_IN_FLIGHT */ #define MIN_XRDP_GFX_FRAMES_IN_FLIGHT 1 diff --git a/xrdp/xrdp_encoder_x264.c b/xrdp/xrdp_encoder_x264.c new file mode 100644 index 000000000..ee87586ff --- /dev/null +++ b/xrdp/xrdp_encoder_x264.c @@ -0,0 +1,242 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2016-2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * x264 Encoder + */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include +#include +#include + +#include "xrdp.h" +#include "arch.h" +#include "os_calls.h" +#include "xrdp_encoder_x264.h" + +#define X264_MAX_ENCODERS 16 + +struct x264_encoder +{ + x264_t *x264_enc_han; + char *yuvdata; + x264_param_t x264_params; + int width; + int height; +}; + +struct x264_global +{ + struct x264_encoder encoders[X264_MAX_ENCODERS]; +}; + +/*****************************************************************************/ +void * +xrdp_encoder_x264_create(void) +{ + struct x264_global *xg; + + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_encoder_x264_create:"); + xg = g_new0(struct x264_global, 1); + if (xg == NULL) + { + return NULL; + } + return xg; +} + +/*****************************************************************************/ +int +xrdp_encoder_x264_delete(void *handle) +{ + struct x264_global *xg; + struct x264_encoder *xe; + int index; + + if (handle == NULL) + { + return 0; + } + xg = (struct x264_global *) handle; + for (index = 0; index < 16; index++) + { + xe = &(xg->encoders[index]); + if (xe->x264_enc_han != NULL) + { + x264_encoder_close(xe->x264_enc_han); + } + g_free(xe->yuvdata); + } + g_free(xg); + return 0; +} + +/*****************************************************************************/ +int +xrdp_encoder_x264_encode(void *handle, int session, int left, int top, + int width, int height, int twidth, int theight, + int format, const char *data, + short *crects, int num_crects, + char *cdata, int *cdata_bytes, int *flags_ptr) +{ + struct x264_global *xg; + struct x264_encoder *xe; + const char *src8; + char *dst8; + int index; + x264_nal_t *nals; + int num_nals; + int frame_size; + int x264_width_height; + int flags; + int x; + int y; + int cx; + int cy; + + x264_picture_t pic_in; + x264_picture_t pic_out; + + LOG(LOG_LEVEL_TRACE, "xrdp_encoder_x264_encode:"); + flags = 0; + xg = (struct x264_global *) handle; + xe = &(xg->encoders[session % X264_MAX_ENCODERS]); + if ((xe->x264_enc_han == NULL) || + (xe->width != width) || (xe->height != height)) + { + if (xe->x264_enc_han != NULL) + { + LOG(LOG_LEVEL_INFO, "xrdp_encoder_x264_encode: " + "x264_encoder_close %p", xe->x264_enc_han); + x264_encoder_close(xe->x264_enc_han); + xe->x264_enc_han = NULL; + g_free(xe->yuvdata); + xe->yuvdata = NULL; + flags |= 2; + } + if ((width > 0) && (height > 0)) + { + //x264_param_default_preset(&(xe->x264_params), "superfast", "zerolatency"); + x264_param_default_preset(&(xe->x264_params), "ultrafast", "zerolatency"); + xe->x264_params.i_threads = 1; + xe->x264_params.i_width = width; + xe->x264_params.i_height = height; + xe->x264_params.i_fps_num = 24; + xe->x264_params.i_fps_den = 1; + //xe->x264_params.b_cabac = 1; + //xe->x264_params.i_bframe = 0; + //xe->x264_params.rc.i_rc_method = X264_RC_CQP; + //xe->x264_params.rc.i_qp_constant = 23; + //x264_param_apply_profile(&(xe->x264_params), "high"); + x264_param_apply_profile(&(xe->x264_params), "main"); + //x264_param_apply_profile(&(xe->x264_params), "baseline"); + xe->x264_enc_han = x264_encoder_open(&(xe->x264_params)); + LOG(LOG_LEVEL_INFO, "xrdp_encoder_x264_encode: " + "x264_encoder_open rv %p for width %d height %d", + xe->x264_enc_han, width, height); + if (xe->x264_enc_han == NULL) + { + return 1; + } + xe->yuvdata = g_new(char, width * height * 2); + if (xe->yuvdata == NULL) + { + x264_encoder_close(xe->x264_enc_han); + xe->x264_enc_han = NULL; + return 2; + } + flags |= 1; + } + xe->width = width; + xe->height = height; + } + + if ((data != NULL) && (xe->x264_enc_han != NULL)) + { + x264_width_height = xe->x264_params.i_width * xe->x264_params.i_height; + for (index = 0; index < num_crects; index++) + { + src8 = data; + dst8 = xe->yuvdata; + x = crects[index * 4 + 0]; + y = crects[index * 4 + 1]; + cx = crects[index * 4 + 2]; + cy = crects[index * 4 + 3]; + LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_x264_encode: x %d y %d " + "cx %d cy %d", x, y, cx, cy); + src8 += twidth * y + x; + dst8 += xe->x264_params.i_width * (y - top) + (x - left); + for (; cy > 0; cy -= 1) + { + g_memcpy(dst8, src8, cx); + src8 += twidth; + dst8 += xe->x264_params.i_width; + } + } + for (index = 0; index < num_crects; index++) + { + src8 = data; + src8 += twidth * theight; + dst8 = xe->yuvdata; + dst8 += x264_width_height; + x = crects[index * 4 + 0]; + y = crects[index * 4 + 1]; + cx = crects[index * 4 + 2]; + cy = crects[index * 4 + 3]; + src8 += twidth * (y / 2) + x; + dst8 += xe->x264_params.i_width * ((y - top) / 2) + (x - left); + for (; cy > 0; cy -= 2) + { + g_memcpy(dst8, src8, cx); + src8 += twidth; + dst8 += xe->x264_params.i_width; + } + } + g_memset(&pic_in, 0, sizeof(pic_in)); + pic_in.img.i_csp = X264_CSP_NV12; + pic_in.img.i_plane = 2; + pic_in.img.plane[0] = (unsigned char *) (xe->yuvdata); + pic_in.img.plane[1] = (unsigned char *) + (xe->yuvdata + x264_width_height); + pic_in.img.i_stride[0] = xe->x264_params.i_width; + pic_in.img.i_stride[1] = xe->x264_params.i_width; + num_nals = 0; + frame_size = x264_encoder_encode(xe->x264_enc_han, &nals, &num_nals, + &pic_in, &pic_out); + LOG(LOG_LEVEL_TRACE, "i_type %d", pic_out.i_type); + if (frame_size < 1) + { + return 3; + } + if (*cdata_bytes < frame_size) + { + return 4; + } + g_memcpy(cdata, nals[0].p_payload, frame_size); + *cdata_bytes = frame_size; + } + if (flags_ptr != NULL) + { + *flags_ptr = flags; + } + return 0; +} diff --git a/xrdp/xrdp_encoder_x264.h b/xrdp/xrdp_encoder_x264.h new file mode 100644 index 000000000..0d7ec5ba8 --- /dev/null +++ b/xrdp/xrdp_encoder_x264.h @@ -0,0 +1,38 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2016-2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * libx264 Encoder + */ + +#ifndef _XRDP_ENCODER_X264_H +#define _XRDP_ENCODER_X264_H + +#include "arch.h" + +void * +xrdp_encoder_x264_create(void); +int +xrdp_encoder_x264_delete(void *handle); +int +xrdp_encoder_x264_encode(void *handle, int session, int left, int top, + int width, int height, int twidth, int theight, + int format, const char *data, + short *crects, int num_crects, + char *cdata, int *cdata_bytes, int *flags_ptr); + +#endif + From b583a8d5101dd8df6b51f1ba09abe4ab468381d4 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Tue, 14 May 2024 00:49:10 -0700 Subject: [PATCH 3/9] xrdp_encoder.c changes for x264 --- xrdp/xrdp_encoder.c | 311 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 294 insertions(+), 17 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 7666725a9..b4447122c 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -74,11 +74,13 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); #ifdef XRDP_RFXCODEC static int process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); -static int -process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); #endif +#ifdef XRDP_X264 static int process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); +#endif +static int +process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); /*****************************************************************************/ /* Item destructor for self->fifo_to_proc */ @@ -132,7 +134,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) return NULL; } self->mm = mm; - + self->process_enc = process_enc_egfx; if (client_info->jpeg_codec_id != 0) { LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting jpeg codec session"); @@ -143,6 +145,26 @@ xrdp_encoder_create(struct xrdp_mm *mm) client_info->capture_format = XRDP_a8b8g8r8; self->process_enc = process_enc_jpg; } +#ifdef XRDP_X264 + else if (mm->egfx_flags & XRDP_EGFX_H264) + { + LOG(LOG_LEVEL_INFO, + "xrdp_encoder_create: starting h264 codec session gfx"); + self->in_codec_mode = 1; + client_info->capture_code = 5; + client_info->capture_format = XRDP_nv12_709fr; + self->gfx = 1; + } + else if (client_info->h264_codec_id != 0) + { + LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session"); + self->codec_id = client_info->h264_codec_id; + self->in_codec_mode = 1; + client_info->capture_code = 3; + client_info->capture_format = XRDP_nv12; + self->process_enc = process_enc_h264; + } +#endif #ifdef XRDP_RFXCODEC else if (mm->egfx_flags & XRDP_EGFX_RFX_PRO) { @@ -150,7 +172,6 @@ xrdp_encoder_create(struct xrdp_mm *mm) "xrdp_encoder_create: starting gfx rfx pro codec session"); self->in_codec_mode = 1; client_info->capture_code = 4; - self->process_enc = process_enc_egfx; self->gfx = 1; self->quants = (const char *) g_rfx_quantization_values; self->num_quants = 2; @@ -170,15 +191,6 @@ xrdp_encoder_create(struct xrdp_mm *mm) RFX_FORMAT_YUV, 0); } #endif - else if (client_info->h264_codec_id != 0) - { - LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session"); - self->codec_id = client_info->h264_codec_id; - self->in_codec_mode = 1; - client_info->capture_code = 3; - client_info->capture_format = XRDP_nv12; - self->process_enc = process_enc_h264; - } else { g_free(self); @@ -299,7 +311,7 @@ xrdp_encoder_delete(struct xrdp_encoder *self) { if (self->codec_handle_h264_gfx[index] != NULL) { - rfxcodec_encode_destroy(self->codec_handle_h264_gfx[index]); + xrdp_encoder_x264_delete(self->codec_handle_h264_gfx[index]); } } if (self->codec_handle_h264 != NULL) @@ -567,6 +579,61 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } #endif +#if defined(XRDP_X264) + +/*****************************************************************************/ +static int +out_RFX_AVC420_METABLOCK(struct xrdp_egfx_rect *dst_rect, + struct stream *s, + struct xrdp_egfx_rect *rects, + int num_rects) +{ + struct xrdp_region *reg; + struct xrdp_rect rect; + int index; + int count; + + /* RFX_AVC420_METABLOCK */ + s_push_layer(s, iso_hdr, 4); /* numRegionRects, set later */ + reg = xrdp_region_create(NULL); + if (reg == NULL) + { + return 1; + } + for (index = 0; index < num_rects; index++) + { + rect.left = MAX(0, rects[index].x1 - dst_rect->x1 - 1); + rect.top = MAX(0, rects[index].y1 - dst_rect->y1 - 1); + rect.right = MIN(dst_rect->x2 - dst_rect->x1, + rects[index].x2 - dst_rect->x1 + 1); + rect.bottom = MIN(dst_rect->y2 - dst_rect->y1, + rects[index].y2 - dst_rect->y1 + 1); + xrdp_region_add_rect(reg, &rect); + } + index = 0; + while (xrdp_region_get_rect(reg, index, &rect) == 0) + { + out_uint16_le(s, rect.left); + out_uint16_le(s, rect.top); + out_uint16_le(s, rect.right); + out_uint16_le(s, rect.bottom); + index++; + } + xrdp_region_delete(reg); + count = index; + while (index > 0) + { + out_uint8(s, 23); /* qp */ + out_uint8(s, 100); /* quality level 0..100 */ + index--; + } + s_push_layer(s, mcs_hdr, 0); + s_pop_layer(s, iso_hdr); + out_uint32_le(s, count); /* numRegionRects */ + s_pop_layer(s, mcs_hdr); + return 0; +} + /*****************************************************************************/ /* called from encoder thread */ static int @@ -576,7 +643,7 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) return 0; } -#ifdef XRDP_RFXCODEC +#endif /*****************************************************************************/ static int @@ -618,11 +685,210 @@ gfx_wiretosurface1(struct xrdp_encoder *self, struct xrdp_egfx_bulk *bulk, struct stream *in_s, XRDP_ENC_DATA *enc) { +#ifdef XRDP_X264 + int index; + int surface_id; + int codec_id; + int pixel_format; + int num_rects_d; + int num_rects_c; + struct stream *rv; + short left; + short top; + short width; + short height; + short twidth; + short theight; + int bitmap_data_length; + int flags; + struct xrdp_egfx_rect *d_rects; + struct xrdp_egfx_rect *c_rects; + struct xrdp_egfx_rect dst_rect; + int error; + struct stream ls; + struct stream *s; + short *crects; + struct xrdp_enc_gfx_cmd *enc_gfx_cmd = &(enc->u.gfx); + int mon_index; + + s = &ls; + g_memset(s, 0, sizeof(struct stream)); + s->size = self->max_compressed_bytes; + s->data = g_new(char, s->size); + if (s->data == NULL) + { + return NULL; + } + s->p = s->data; + if (!s_check_rem(in_s, 11)) + { + g_free(s->data); + return NULL; + } + in_uint16_le(in_s, surface_id); + in_uint16_le(in_s, codec_id); + in_uint8(in_s, pixel_format); + in_uint32_le(in_s, flags); + mon_index = (flags >> 28) & 0xF; + in_uint16_le(in_s, num_rects_d); + if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) || + (!s_check_rem(in_s, num_rects_d * 8))) + { + g_free(s->data); + return NULL; + } + d_rects = g_new0(struct xrdp_egfx_rect, num_rects_d); + if (d_rects == NULL) + { + g_free(s->data); + return NULL; + } + for (index = 0; index < num_rects_d; index++) + { + in_uint16_le(in_s, left); + in_uint16_le(in_s, top); + in_uint16_le(in_s, width); + in_uint16_le(in_s, height); + d_rects[index].x1 = left; + d_rects[index].y1 = top; + d_rects[index].x2 = left + width; + d_rects[index].y2 = top + height; + + } + if (!s_check_rem(in_s, 2)) + { + g_free(s->data); + g_free(d_rects); + return NULL; + } + in_uint16_le(in_s, num_rects_c); + if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) || + (!s_check_rem(in_s, num_rects_c * 8))) + { + g_free(s->data); + g_free(d_rects); + return NULL; + } + c_rects = g_new0(struct xrdp_egfx_rect, num_rects_c); + if (c_rects == NULL) + { + g_free(s->data); + g_free(d_rects); + return NULL; + } + crects = g_new(short, num_rects_c * 4); + if (crects == NULL) + { + g_free(s->data); + g_free(c_rects); + g_free(d_rects); + return NULL; + } + g_memcpy(crects, in_s->p, num_rects_c * 2 * 4); + for (index = 0; index < num_rects_c; index++) + { + in_uint16_le(in_s, left); + in_uint16_le(in_s, top); + in_uint16_le(in_s, width); + in_uint16_le(in_s, height); + c_rects[index].x1 = left; + c_rects[index].y1 = top; + c_rects[index].x2 = left + width; + c_rects[index].y2 = top + height; + } + if (!s_check_rem(in_s, 8)) + { + g_free(s->data); + g_free(c_rects); + g_free(d_rects); + g_free(crects); + return NULL; + } + in_uint16_le(in_s, left); + in_uint16_le(in_s, top); + in_uint16_le(in_s, width); + in_uint16_le(in_s, height); + twidth = width; + theight = height; + dst_rect.x1 = left; + dst_rect.y1 = top; + dst_rect.x2 = left + width; + dst_rect.y2 = top + height; + + /* RFX_AVC420_METABLOCK */ + if (out_RFX_AVC420_METABLOCK(&dst_rect, s, d_rects, num_rects_d) != 0) + { + g_free(s->data); + g_free(c_rects); + g_free(d_rects); + g_free(crects); + LOG(LOG_LEVEL_INFO, "10"); + return NULL; + } + + g_free(c_rects); + g_free(d_rects); + + if (ENC_IS_BIT_SET(flags, 0)) + { + /* already compressed */ + out_uint8a(s, enc_gfx_cmd->data, enc_gfx_cmd->data_bytes); + } + else + { + /* assume NV12 format */ + if (twidth * theight * 3 / 2 > enc_gfx_cmd->data_bytes) + { + g_free(s->data); + g_free(crects); + return NULL; + } + bitmap_data_length = s_rem_out(s); + if (self->codec_handle_h264_gfx[mon_index] == NULL) + { + self->codec_handle_h264_gfx[mon_index] = + xrdp_encoder_x264_create(); + if (self->codec_handle_h264_gfx[mon_index] == NULL) + { + g_free(s->data); + g_free(crects); + return NULL; + } + } + error = xrdp_encoder_x264_encode( + self->codec_handle_h264_gfx[mon_index], 0, + left, top, + width, height, twidth, theight, 0, + enc_gfx_cmd->data, + crects, num_rects_c, + s->p, &bitmap_data_length, NULL); + if (error == 0) + { + xstream_seek(s, bitmap_data_length); + } + else + { + g_free(s->data); + g_free(crects); + return NULL; + } + } + s_mark_end(s); + bitmap_data_length = (int) (s->end - s->data); + rv = xrdp_egfx_wire_to_surface1(bulk, surface_id, + codec_id, + pixel_format, &dst_rect, + s->data, bitmap_data_length); + g_free(s->data); + g_free(crects); + return rv; +#else (void)self; (void)bulk; (void)in_s; (void)enc; return NULL; +#endif } /*****************************************************************************/ @@ -631,6 +897,7 @@ gfx_wiretosurface2(struct xrdp_encoder *self, struct xrdp_egfx_bulk *bulk, struct stream *in_s, XRDP_ENC_DATA *enc) { +#ifdef XRDP_RFXCODEC int index; int surface_id; int codec_id; @@ -802,6 +1069,13 @@ gfx_wiretosurface2(struct xrdp_encoder *self, g_free(rfxrects); g_free(bitmap_data); return rv; +#else + (void)self; + (void)bulk; + (void)in_s; + (void)enc; + return NULL; +#endif } /*****************************************************************************/ @@ -1030,6 +1304,7 @@ process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } holdend = in_s.end; in_s.end = holdp + cmd_bytes; + LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_egfx: cmd_id %d", cmd_id); switch (cmd_id) { case XR_RDPGFX_CMDID_WIRETOSURFACE_1: /* 0x0001 */ @@ -1084,12 +1359,14 @@ process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } g_free(s); /* don't call free_stream() here so s->data is valid */ } + else + { + LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_egfx: nil"); + } } return 0; } -#endif - /** * Encoder thread main loop *****************************************************************************/ From 57ed8e9d5274567971863377cce5956e773ee901 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Tue, 14 May 2024 01:32:36 -0700 Subject: [PATCH 4/9] format --- xrdp/xrdp_encoder.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index b4447122c..a1212c4a9 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -856,12 +856,12 @@ gfx_wiretosurface1(struct xrdp_encoder *self, } } error = xrdp_encoder_x264_encode( - self->codec_handle_h264_gfx[mon_index], 0, - left, top, - width, height, twidth, theight, 0, - enc_gfx_cmd->data, - crects, num_rects_c, - s->p, &bitmap_data_length, NULL); + self->codec_handle_h264_gfx[mon_index], 0, + left, top, + width, height, twidth, theight, 0, + enc_gfx_cmd->data, + crects, num_rects_c, + s->p, &bitmap_data_length, NULL); if (error == 0) { xstream_seek(s, bitmap_data_length); From 96ff6fdba819c1739358b2120f21983de65d9da5 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Tue, 14 May 2024 14:19:53 -0700 Subject: [PATCH 5/9] multimon fixes --- xrdp/xrdp_encoder.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index a1212c4a9..e2b8d9c99 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -810,11 +810,13 @@ gfx_wiretosurface1(struct xrdp_encoder *self, in_uint16_le(in_s, height); twidth = width; theight = height; - dst_rect.x1 = left; - dst_rect.y1 = top; - dst_rect.x2 = left + width; - dst_rect.y2 = top + height; - + dst_rect.x1 = 0; + dst_rect.y1 = 0; + dst_rect.x2 = width; + dst_rect.y2 = height; + LOG_DEVEL(LOG_LEVEL_INFO, "gfx_wiretosurface1: left %d top " + "%d width %d height %d mon_index %d", + left, top, width, height, mon_index); /* RFX_AVC420_METABLOCK */ if (out_RFX_AVC420_METABLOCK(&dst_rect, s, d_rects, num_rects_d) != 0) { @@ -857,8 +859,8 @@ gfx_wiretosurface1(struct xrdp_encoder *self, } error = xrdp_encoder_x264_encode( self->codec_handle_h264_gfx[mon_index], 0, - left, top, - width, height, twidth, theight, 0, + 0, 0, + width, height, twidth, theight, 0, enc_gfx_cmd->data, crects, num_rects_c, s->p, &bitmap_data_length, NULL); @@ -994,6 +996,9 @@ gfx_wiretosurface2(struct xrdp_encoder *self, in_uint16_le(in_s, top); in_uint16_le(in_s, width); in_uint16_le(in_s, height); + LOG_DEVEL(LOG_LEVEL_INFO, "gfx_wiretosurface2: left %d top " + "%d width %d height %d mon_index %d", + left, top, width, height, mon_index); if (self->codec_handle_prfx_gfx[mon_index] == NULL) { self->codec_handle_prfx_gfx[mon_index] = rfxcodec_encode_create( From 112a534f4cf79fdde6846d98604c46eec2962a33 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Tue, 14 May 2024 23:10:53 -0700 Subject: [PATCH 6/9] change capture_code to enum --- common/xrdp_client_info.h | 15 +++++++++++++-- xrdp/xrdp_encoder.c | 10 +++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/common/xrdp_client_info.h b/common/xrdp_client_info.h index aa76f0986..648e26815 100644 --- a/common/xrdp_client_info.h +++ b/common/xrdp_client_info.h @@ -74,6 +74,16 @@ enum client_resize_mode CRMODE_MULTI_SCREEN }; +enum xrdp_capture_code +{ + CC_SIMPLE = 0, + CC_SUF_A16 = 1, + CC_SUF_RFX = 2, + CC_SUF_A2 = 3, + CC_GFX_PRO = 4, + CC_GFX_A2 = 5 +}; + /** * Type describing Unicode input state */ @@ -179,7 +189,7 @@ struct xrdp_client_info int mcs_early_capability_flags; int max_fastpath_frag_bytes; - int capture_code; + int old_capture_code; int capture_format; char certificate[1024]; @@ -239,6 +249,7 @@ struct xrdp_client_info enum client_resize_mode client_resize_mode; enum unicode_input_state unicode_input_support; + enum xrdp_capture_code capture_code; }; enum xrdp_encoder_flags @@ -258,6 +269,6 @@ enum xrdp_encoder_flags /* yyyymmdd of last incompatible change to xrdp_client_info */ /* also used for changes to all the xrdp installed headers */ -#define CLIENT_INFO_CURRENT_VERSION 20230425 +#define CLIENT_INFO_CURRENT_VERSION 20240514 #endif diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index e2b8d9c99..2f077b5c8 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -141,7 +141,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) self->codec_id = client_info->jpeg_codec_id; self->in_codec_mode = 1; self->codec_quality = client_info->jpeg_prop[0]; - client_info->capture_code = 0; + client_info->capture_code = CC_SIMPLE; client_info->capture_format = XRDP_a8b8g8r8; self->process_enc = process_enc_jpg; } @@ -151,7 +151,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session gfx"); self->in_codec_mode = 1; - client_info->capture_code = 5; + client_info->capture_code = CC_GFX_A2; client_info->capture_format = XRDP_nv12_709fr; self->gfx = 1; } @@ -160,7 +160,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session"); self->codec_id = client_info->h264_codec_id; self->in_codec_mode = 1; - client_info->capture_code = 3; + client_info->capture_code = CC_SUF_A2; client_info->capture_format = XRDP_nv12; self->process_enc = process_enc_h264; } @@ -171,7 +171,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting gfx rfx pro codec session"); self->in_codec_mode = 1; - client_info->capture_code = 4; + client_info->capture_code = CC_GFX_PRO; self->gfx = 1; self->quants = (const char *) g_rfx_quantization_values; self->num_quants = 2; @@ -184,7 +184,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting rfx codec session"); self->codec_id = client_info->rfx_codec_id; self->in_codec_mode = 1; - client_info->capture_code = 2; + client_info->capture_code = CC_SUF_RFX; self->process_enc = process_enc_rfx; self->codec_handle_rfx = rfxcodec_encode_create(mm->wm->screen->width, mm->wm->screen->height, From 485e64ee76afc919489a6e4591536f6110f4f5d6 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Sun, 19 May 2024 22:20:33 -0700 Subject: [PATCH 7/9] fix for odd width, height --- xrdp/xrdp_encoder_x264.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xrdp/xrdp_encoder_x264.c b/xrdp/xrdp_encoder_x264.c index ee87586ff..6fb3981cc 100644 --- a/xrdp/xrdp_encoder_x264.c +++ b/xrdp/xrdp_encoder_x264.c @@ -138,8 +138,8 @@ xrdp_encoder_x264_encode(void *handle, int session, int left, int top, //x264_param_default_preset(&(xe->x264_params), "superfast", "zerolatency"); x264_param_default_preset(&(xe->x264_params), "ultrafast", "zerolatency"); xe->x264_params.i_threads = 1; - xe->x264_params.i_width = width; - xe->x264_params.i_height = height; + xe->x264_params.i_width = (width + 15) & ~15; + xe->x264_params.i_height = (height + 15) & ~15; xe->x264_params.i_fps_num = 24; xe->x264_params.i_fps_den = 1; //xe->x264_params.b_cabac = 1; From 2bed4f6b9b1e44394c8dfb294b49ccb9ad1e4bbc Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Tue, 21 May 2024 21:02:33 -0700 Subject: [PATCH 8/9] rename old_capture_code to mark unused --- common/xrdp_client_info.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/xrdp_client_info.h b/common/xrdp_client_info.h index 648e26815..43ce2db89 100644 --- a/common/xrdp_client_info.h +++ b/common/xrdp_client_info.h @@ -189,7 +189,7 @@ struct xrdp_client_info int mcs_early_capability_flags; int max_fastpath_frag_bytes; - int old_capture_code; + int pad0; /* unused */ int capture_format; char certificate[1024]; From 4dcf59c8f469256cd68a5ddaef93a7e27644cc52 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Tue, 21 May 2024 21:04:16 -0700 Subject: [PATCH 9/9] remove some unnecessary code --- xrdp/xrdp_encoder_x264.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/xrdp/xrdp_encoder_x264.c b/xrdp/xrdp_encoder_x264.c index 6fb3981cc..81fd71798 100644 --- a/xrdp/xrdp_encoder_x264.c +++ b/xrdp/xrdp_encoder_x264.c @@ -53,15 +53,8 @@ struct x264_global void * xrdp_encoder_x264_create(void) { - struct x264_global *xg; - LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_encoder_x264_create:"); - xg = g_new0(struct x264_global, 1); - if (xg == NULL) - { - return NULL; - } - return xg; + return g_new0(struct x264_global, 1); } /*****************************************************************************/