LibVNCClient: Add H.264 encoding for framebuffer updates

This patch implements support in LibVNCClient for framebuffer updates
encoded as H.264 frames. Hardware accelerated decoding is performed
using VA API.

This is experimental support to let the community explore the possibilities
offered by the potential bandwidth and latency reductions that H.264 encoding
allows. This may be particularly useful for use cases such as online gaming,
hosted desktops, hosted set top boxes...

This patch only provides the client side support and is meant to be used
with corresponding server-side support, as provided by an upcoming patch for
qemu ui/vnc module (to view the display of a virtual machine executing under
QEMU).

With this H.264-based encoding, if multiple framebuffer update messages
are generated for a single server framebuffer modification, the H.264
frame data is sent only with the first update message. Subsequent update
framebuffer messages will contain only the coordinates and size of the
additional updated regions.

Instructions/Requirements:
* The patch should be applied on top of the previous patch I submitted with
minor enhancements to the gtkvncviewer application:
http://sourceforge.net/mailarchive/message.php?msg_id=30323804
* Currently only works with libva 1.0: use branch "v1.0-branch" for libva and
intel-driver. Those can be built as follows:
   cd libva
   git checkout v1.0-branch
   ./autogen.sh
   make
   sudo make install
   cd ..
   git clone git://anongit.freedesktop.org/vaapi/intel-driver
   cd intel-driver
   git checkout v1.0-branch
   ./autogen.sh
   make
   sudo make install

Signed-off-by: David Verbeiren <david.verbeiren@intel.com>

client_examples/gtkvncviewer.c

@@ -23,6 +23,10 @@
 #include <gdk/gdkkeysyms.h>
 #include <rfb/rfbclient.h>
 
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+#include <gdk/gdkx.h>
+#endif
+
 static rfbClient *cl;
 static gchar *server_cut_text = NULL;
 static gboolean framebuffer_allocated = FALSE;
@@ -57,6 +61,14 @@ static gboolean expose_event (GtkWidget      *widget,
 		cl->format.greenMax = (1 << image->visual->green_prec) - 1;
 		cl->format.blueMax  = (1 << image->visual->blue_prec) - 1;
 
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+		/* Allow libvncclient to use a more efficient way
+		 * of putting the framebuffer on the screen when
+		 * using the H.264 format.
+		 */
+		cl->outputWindow = GDK_WINDOW_XID(widget->window);
+#endif
+
 		SetFormatAndEncodings (cl);
 
 		framebuffer_allocated = TRUE;
@@ -67,12 +79,14 @@ static gboolean expose_event (GtkWidget      *widget,
 		gdk_cursor_unref( cur );
 	}
 
+#ifndef LIBVNCSERVER_CONFIG_LIBVA
 	gdk_draw_image (GDK_DRAWABLE (widget->window),
 	                widget->style->fg_gc[gtk_widget_get_state(widget)],
 	                image,
 	                event->area.x, event->area.y,
 	                event->area.x, event->area.y,
 	                event->area.width, event->area.height);
+#endif
 
 	return FALSE;
 }
@@ -462,10 +476,12 @@ static void update (rfbClient *cl, int x, int y, int w, int h) {
 		dialog_connecting = NULL;
 	}
 
+#ifndef LIBVNCSERVER_CONFIG_LIBVA
 	GtkWidget *drawing_area = rfbClientGetClientData (cl, gtk_init);
 
 	if (drawing_area != NULL)
 		gtk_widget_queue_draw_area (drawing_area, x, y, w, h);
+#endif
 }
 
 static void kbd_leds (rfbClient *cl, int value, int pad) {

configure.ac

@@ -151,6 +151,20 @@ HAVE_X11="false"
 AC_PATH_XTRA
 AH_TEMPLATE(HAVE_X11, [X11 build environment present])
 
+# See if we want libva support
+# TODO: check if library actually exists
+AH_TEMPLATE(CONFIG_LIBVA, [Build libva support])
+AC_ARG_WITH(libva,
+[  --with-libva                   build libva support],,)
+if test "x$with_libva" != "xno"; then
+    AC_CHECK_LIB(va, vaInitialize,
+        VA_LIBS="-lva -lva-x11"
+        [AC_DEFINE(CONFIG_LIBVA) CONFIG_LIBVA="true"], ,)
+fi
+AC_SUBST(VA_LIBS)
+AM_CONDITIONAL(CONFIG_LIBVA, test ! -z "$VA_LIBS")
+
+
 # See if we are to build x11vnc:
 AH_TEMPLATE(HAVE_SYSTEM_LIBVNCSERVER, [Use the system libvncserver build environment for x11vnc.])
 AC_ARG_WITH(system-libvncserver,

libvncclient/Makefile.am

@@ -14,7 +14,7 @@ endif
 
 
 libvncclient_la_SOURCES=cursor.c listen.c rfbproto.c sockets.c vncviewer.c ../common/minilzo.c $(TLSSRCS)
-libvncclient_la_LIBADD=$(TLSLIBS)
+libvncclient_la_LIBADD=$(TLSLIBS) $(VA_LIBS)
 
 noinst_HEADERS=../common/lzodefs.h ../common/lzoconf.h ../common/minilzo.h tls.h