gfx: work on progressive rfx

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "librfxcodec"]
 	path = librfxcodec
-	url = https://github.com/neutrinolabs/librfxcodec.git
+	url = https://github.com/jsorg71/librfxcodec.git
 	branch = .
 	ignore = untracked
 

xrdp/xrdp_egfx.c

@@ -480,7 +480,7 @@ xrdp_egfx_send_wire_to_surface2(struct xrdp_egfx *egfx, int surface_id,
     char *hold_segment_count;
     char *bitmap_data8;
 
-    LLOGLN(0, ("xrdp_egfx_send_wire_to_surface2:"));
+    LLOGLN(10, ("xrdp_egfx_send_wire_to_surface2:"));
     make_stream(s);
     bytes = bitmap_data_length + 8192;
     bytes += 5 * (bitmap_data_length / 0xFFFF);
@@ -518,7 +518,7 @@ xrdp_egfx_send_wire_to_surface2(struct xrdp_egfx *egfx, int surface_id,
         /* RDP8_BULK_ENCODED_DATA */
         out_uint8(s, 0x04); /* header = PACKET_COMPR_TYPE_RDP8 */
         out_uint8a(s, bitmap_data8 + index, segment_size);
-        LLOGLN(0, ("  segment index %d segment_size %d",
+        LLOGLN(10, ("  segment index %d segment_size %d",
                segment_count, segment_size));
         index += segment_size;
         segment_count++;
@@ -529,7 +529,7 @@ xrdp_egfx_send_wire_to_surface2(struct xrdp_egfx *egfx, int surface_id,
     s->p = hold_segment_count;
     out_uint16_le(s, segment_count);
     error = xrdp_egfx_send_data(egfx, s->data, bytes);
-    LLOGLN(0, ("xrdp_egfx_send_wire_to_surface2: xrdp_egfx_send_data error %d "
+    LLOGLN(10, ("xrdp_egfx_send_wire_to_surface2: xrdp_egfx_send_data error %d "
            "segment_count %d", error, segment_count));
     free_stream(s);
     return error;

xrdp/xrdp_encoder.c

@@ -50,6 +50,13 @@
 
 #define XRDP_SURCMD_PREFIX_BYTES 256
 
+/* LH3 LL3, HH3 HL3, HL2 LH2, LH1 HH2, HH1 HL1 todo check this */
+static const unsigned char g_rfx_quantization_values[] =
+{
+    0x66, 0x66, 0x77, 0x88, 0x98,
+    0x76, 0x77, 0x88, 0x98, 0xA9
+};
+
 /*****************************************************************************/
 static int
 process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
@@ -71,9 +78,13 @@ xrdp_encoder_create(struct xrdp_mm *mm)
 
     client_info = mm->wm->client_info;
 
+    /* RemoteFX 7.1 requires LAN but GFX does not */
     if (client_info->mcs_connection_type != CONNECTION_TYPE_LAN)
     {
-        return 0;
+        if ((mm->egfx_flags & 3) == 0)
+        {
+            return 0;
+        }
     }
     if (client_info->bpp < 24)
     {
@@ -87,10 +98,9 @@ xrdp_encoder_create(struct xrdp_mm *mm)
     }
     self->mm = mm;
 
-    if (client_info->gfx)
+    if (mm->egfx_flags & 1)
     {
         LLOGLN(0, ("xrdp_encoder_create: starting h264 codec session gfx"));
-        self->codec_id = client_info->h264_codec_id;
         self->in_codec_mode = 1;
         client_info->capture_code = 3;
         client_info->capture_format =
@@ -102,6 +112,23 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         self->codec_handle = xrdp_encoder_x264_create();
 #endif
     }
+    else if (mm->egfx_flags & 2)
+    {
+        LLOGLN(0, ("xrdp_encoder_create: starting gfx rfx pro codec session"));
+        self->in_codec_mode = 1;
+        client_info->capture_code = 2;
+        self->process_enc = process_enc_rfx;
+        self->gfx = 1;
+        self->quants = (const char *) g_rfx_quantization_values;
+        self->num_quants = 2;
+        self->quant_idx_y = 0;
+        self->quant_idx_u = 1;
+        self->quant_idx_v = 1;
+        self->codec_handle = rfxcodec_encode_create(mm->wm->screen->width,
+                                                    mm->wm->screen->height,
+                                                    RFX_FORMAT_YUV,
+                                                    RFX_FLAGS_RLGR1 | RFX_FLAGS_PRO1);
+    }
     else if (client_info->jpeg_codec_id != 0)
     {
         LLOGLN(0, ("xrdp_encoder_create: starting jpeg codec session"));
@@ -408,9 +435,9 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                 tiles[index].y = y;
                 tiles[index].cx = cx;
                 tiles[index].cy = cy;
-                tiles[index].quant_y = 0;
-                tiles[index].quant_cb = 0;
-                tiles[index].quant_cr = 0;
+                tiles[index].quant_y = self->quant_idx_y;
+                tiles[index].quant_cb = self->quant_idx_u;
+                tiles[index].quant_cr = self->quant_idx_v;
             }
 
             count = enc->num_drects;
@@ -426,13 +453,17 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                 rfxrects[index].cy = cy;
             }
 
+            LLOGLN(10, ("process_enc_rfx: num_crects %d num_drects %d",
+                   enc->num_crects, enc->num_drects));
+
             out_data_bytes = self->max_compressed_bytes;
             error = rfxcodec_encode(self->codec_handle,
                                     out_data + XRDP_SURCMD_PREFIX_BYTES,
                                     &out_data_bytes, enc->data,
                                     enc->width, enc->height, enc->width * 4,
                                     rfxrects, enc->num_drects,
-                                    tiles, enc->num_crects, 0, 0);
+                                    tiles, enc->num_crects,
+                                    self->quants, self->num_quants);
         }
     }
 
@@ -452,6 +483,10 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
     enc_done->last = 1;
     enc_done->cx = self->mm->wm->screen->width;
     enc_done->cy = self->mm->wm->screen->height;
+    if (self->gfx)
+    {
+        enc_done->flags = 2;
+    }
 
     /* done with msg */
     /* inform main thread done */

xrdp/xrdp_encoder.h

@@ -29,6 +29,11 @@ struct xrdp_encoder
     int frames_in_flight;
     int gfx;
     int gfx_ack_off;
+    const char *quants;
+    int num_quants;
+    int quant_idx_y;
+    int quant_idx_u;
+    int quant_idx_v;
 };
 
 /* used when scheduling tasks in xrdp_encoder.c */

xrdp/xrdp_mm.c

@@ -73,7 +73,13 @@ xrdp_mm_create(struct xrdp_wm *owner)
            self->wm->client_info->rfx_codec_id,
            self->wm->client_info->h264_codec_id));
 
-    self->encoder = xrdp_encoder_create(self);
+    if ((self->wm->client_info->gfx == 0) &&
+        ((self->wm->client_info->h264_codec_id != 0) ||
+         (self->wm->client_info->jpeg_codec_id != 0) ||
+         (self->wm->client_info->rfx_codec_id != 0)))
+    {
+        self->encoder = xrdp_encoder_create(self);
+    }
 
     return self;
 }
@@ -1132,6 +1138,8 @@ xrdp_mm_egfx_caps_advertise(void* user, int caps_count,
     struct xrdp_bitmap *screen;
     int index;
     int best_index;
+    int best_h264_index;
+    int best_pro_index;
     int error;
     int version;
     int flags;
@@ -1140,7 +1148,13 @@ xrdp_mm_egfx_caps_advertise(void* user, int caps_count,
     LLOGLN(0, ("xrdp_mm_egfx_caps_advertise:"));
     self = (struct xrdp_mm *) user;
     screen = self->wm->screen;
+    if (screen->data == NULL)
+    {
+        LLOGLN(0, ("xrdp_mm_egfx_caps_advertise: can not do gfx"));
+    }
     best_index = -1;
+    best_h264_index = -1;
+    best_pro_index = -1;
     for (index = 0; index < caps_count; index++)
     {
         version = versions[index];
@@ -1149,33 +1163,54 @@ xrdp_mm_egfx_caps_advertise(void* user, int caps_count,
         switch (version)
         {
             case XR_RDPGFX_CAPVERSION_8:
+                best_pro_index = index;
                 break;
             case XR_RDPGFX_CAPVERSION_81:
                 if (flags & XR_RDPGFX_CAPS_FLAG_AVC420_ENABLED)
                 {
-                    best_index = index;
+                    best_h264_index = index;
                 }
+                best_pro_index = index;
                 break;
             case XR_RDPGFX_CAPVERSION_10:
+                best_pro_index = index;
                 break;
             case XR_RDPGFX_CAPVERSION_101:
+                best_pro_index = index;
                 break;
             case XR_RDPGFX_CAPVERSION_102:
+                best_pro_index = index;
                 break;
             case XR_RDPGFX_CAPVERSION_103:
+                best_pro_index = index;
                 break;
             case XR_RDPGFX_CAPVERSION_104:
                 if (!(flags & XR_RDPGFX_CAPS_FLAG_AVC_DISABLED))
                 {
-                    best_index = index;
+                    best_h264_index = index;
                 }
+                best_pro_index = index;
                 break;
             case XR_RDPGFX_CAPVERSION_105:
+                best_pro_index = index;
                 break;
             case XR_RDPGFX_CAPVERSION_106:
+                best_pro_index = index;
                 break;
         }
     }
+    if (best_pro_index >= 0)
+    {
+        best_index = best_pro_index;
+        self->egfx_flags = 2;
+    }
+    if (best_h264_index >= 0) /* prefer h264, todo use setting in xrdp.ini for this */
+    {
+#ifdef XRDP_X264
+        best_index = best_h264_index;
+        self->egfx_flags = 1;
+#endif
+    }
     if (best_index >= 0)
     {
         LLOGLN(0, ("  replying version 0x%8.8x flags 0x%8.8x",
@@ -1186,22 +1221,20 @@ xrdp_mm_egfx_caps_advertise(void* user, int caps_count,
         LLOGLN(0, ("xrdp_mm_egfx_caps_advertise: xrdp_egfx_send_capsconfirm "
                "error %d", error));
         self->egfx_up = 1;
-        if (screen->data != NULL)
-        {
-            xrdp_egfx_send_create_surface(self->egfx, 1,
-                                          screen->width, screen->height,
-                                          XR_PIXEL_FORMAT_XRGB_8888);
-            xrdp_egfx_send_map_surface(self->egfx, 1, 0, 0);
-            xr_rect.left = 0;
-            xr_rect.top = 0;
-            xr_rect.right = screen->width;
-            xr_rect.bottom = screen->height;
-            if (self->wm->screen_dirty_region == NULL)
-            {
-                self->wm->screen_dirty_region = xrdp_region_create(self->wm);
-            }
-            xrdp_region_add_rect(self->wm->screen_dirty_region, &xr_rect);
-        }
+        xrdp_egfx_send_create_surface(self->egfx, 1,
+                                      screen->width, screen->height,
+                                      XR_PIXEL_FORMAT_XRGB_8888);
+        xrdp_egfx_send_map_surface(self->egfx, 1, 0, 0);
+        xr_rect.left = 0;
+        xr_rect.top = 0;
+        xr_rect.right = screen->width;
+        xr_rect.bottom = screen->height;
+        if (self->wm->screen_dirty_region == NULL)
+        {
+            self->wm->screen_dirty_region = xrdp_region_create(self->wm);
+        }
+        xrdp_region_add_rect(self->wm->screen_dirty_region, &xr_rect);
+        self->encoder = xrdp_encoder_create(self);
     }
     else
     {
@@ -1234,6 +1267,11 @@ xrdp_mm_egfx_frame_ack(void* user, int queue_depth, int frame_id,
     LLOGLN(10, ("xrdp_mm_egfx_frame_ack:"));
     self = (struct xrdp_mm *) user;
     encoder = self->encoder;
+    if (encoder == NULL)
+    {
+        LLOGLN(0, ("xrdp_mm_egfx_frame_ack: encoder is nil"));
+        return 0;
+    }
     if (queue_depth == XR_SUSPEND_FRAME_ACKNOWLEDGEMENT)
     {
         LLOGLN(0, ("xrdp_mm_egfx_frame_ack: queue_depth %d frame_id %d "
@@ -2864,7 +2902,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
                    "frame_id %d use_frame_acks %d", x, y, cx, cy,
                    enc_done->enc->frame_id,
                    self->wm->client_info->use_frame_acks));
-            if (enc_done->flags & 1) /* gfx */
+            if (enc_done->flags & 1) /* gfx h264 */
             {
                 xrdp_egfx_send_frame_start(self->egfx,
                                            enc_done->enc->frame_id, 0);
@@ -2881,6 +2919,17 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
                                                 enc_done->comp_bytes);
                 xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id);
             }
+            else if (enc_done->flags & 2) /* gfx progressive rfx */
+            {
+                xrdp_egfx_send_frame_start(self->egfx,
+                                           enc_done->enc->frame_id, 0);
+                xrdp_egfx_send_wire_to_surface2(self->egfx, 1, 9, 1,
+                                                XR_PIXEL_FORMAT_XRGB_8888,
+                                                enc_done->comp_pad_data +
+                                                    enc_done->pad_bytes,
+                                                enc_done->comp_bytes);
+                xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id);
+            }
             else
             {
                 libxrdp_fastpath_send_frame_marker(self->wm->session, 0,
@@ -2900,7 +2949,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
         if (enc_done->last)
         {
             LLOGLN(10, ("xrdp_mm_process_enc_done: last set"));
-            if (enc_done->flags & 1) /* gfx */
+            if (enc_done->flags & 3) /* gfx */
             {
                 if (self->encoder->gfx_ack_off)
                 {

xrdp/xrdp_types.h

@@ -301,6 +301,7 @@ struct xrdp_mm
   int xr2cr_cid_map[256];
   struct xrdp_egfx *egfx;
   int egfx_up;
+  int egfx_flags;
 };
 
 struct xrdp_key_info