ui/vnc: fix vmware VGA incompatiblities

this fixes invalid rectangle updates observed after commit 12b316d with the vmware VGA driver. The issues occured because the server and client surface update seems to be out of sync at some points and the max width of the surface is not dividable by VNC_DIRTY_BITS_PER_PIXEL (16). Reported-by: Serge Hallyn <serge.hallyn@ubuntu.com> Signed-off-by: Peter Lieven <pl@kamp.de> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
qemu · Mar 18, 2014 · 2f487a3 · 2f487a3
1 parent 315b593
commit 2f487a3
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/hw/display/vmware_vga.c b/hw/display/vmware_vga.c
@@ -25,6 +25,7 @@
 #include "hw/loader.h"
 #include "trace.h"
 #include "ui/console.h"
+#include "ui/vnc.h"
 #include "hw/pci/pci.h"
 
 #undef VERBOSE
@@ -218,7 +219,7 @@ enum {
 
 /* These values can probably be changed arbitrarily.  */
 #define SVGA_SCRATCH_SIZE               0x8000
-#define SVGA_MAX_WIDTH                  2360
+#define SVGA_MAX_WIDTH                  ROUND_UP(2360, VNC_DIRTY_PIXELS_PER_BIT)
 #define SVGA_MAX_HEIGHT                 1770
 
 #ifdef VERBOSE

diff --git a/ui/vnc.c b/ui/vnc.c
@@ -888,7 +888,7 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
         VncDisplay *vd = vs->vd;
         VncJob *job;
         int y;
-        int height;
+        int height, width;
         int n = 0;
 
         if (vs->output.offset && !vs->audio_cap && !vs->force_update)
@@ -907,6 +907,7 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
         job = vnc_job_new(vs);
 
         height = MIN(pixman_image_get_height(vd->server), vs->client_height);
+        width = MIN(pixman_image_get_width(vd->server), vs->client_width);
 
         y = 0;
         for (;;) {
@@ -925,8 +926,11 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
                                     VNC_DIRTY_BPL(vs), x);
             bitmap_clear(vs->dirty[y], x, x2 - x);
             h = find_and_clear_dirty_height(vs, y, x, x2, height);
-            n += vnc_job_add_rect(job, x * VNC_DIRTY_PIXELS_PER_BIT, y,
-                                  (x2 - x) * VNC_DIRTY_PIXELS_PER_BIT, h);
+            x2 = MIN(x2, width / VNC_DIRTY_PIXELS_PER_BIT);
+            if (x2 > x) {
+                n += vnc_job_add_rect(job, x * VNC_DIRTY_PIXELS_PER_BIT, y,
+                                      (x2 - x) * VNC_DIRTY_PIXELS_PER_BIT, h);
+            }
         }
 
         vnc_job_push(job);