X11/tsmf: Fixed XShmCreateImage BadMatch crashes

There was not check if the pixformat supplied in the XShmCreateImage function
was actually supported by the X-Video extension. Also reduced the amount of
xf_tsmf_is_format_supported calls in xf_process_tsmf_video_frame_event.

client/X11/xf_tsmf.c

@@ -196,6 +196,8 @@ static void xf_process_tsmf_video_frame_event(xfInfo* xfi, RDP_VIDEO_FRAME_EVENT
 	uint8* data1;
 	uint8* data2;
 	uint32 pixfmt;
+	uint32 xvpixfmt;
+	boolean converti420yv12 = false;
 	XvImage * image;
 	int colorkey = 0;
 	XShmSegmentInfo shminfo;
@@ -230,8 +232,30 @@ static void xf_process_tsmf_video_frame_event(xfInfo* xfi, RDP_VIDEO_FRAME_EVENT
 	}
 
 	pixfmt = vevent->frame_pixfmt;
+
+	if (xf_tsmf_is_format_supported(xv, pixfmt))
+	{
+		xvpixfmt = pixfmt;
+	}
+	else if (pixfmt == RDP_PIXFMT_I420 && xf_tsmf_is_format_supported(xv, RDP_PIXFMT_YV12))
+	{
+		xvpixfmt = RDP_PIXFMT_YV12;
+		converti420yv12 = true;
+	}
+	else if (pixfmt == RDP_PIXFMT_YV12 && xf_tsmf_is_format_supported(xv, RDP_PIXFMT_I420))
+	{
+		xvpixfmt = RDP_PIXFMT_I420;
+		converti420yv12 = true;
+	}
+	else
+	{
+		DEBUG_XV("pixel format 0x%X not supported by hardware.", pixfmt);
+		return;
+	}
+
 	image = XvShmCreateImage(xfi->display, xv->xv_port,
-		pixfmt, 0, vevent->frame_width, vevent->frame_height, &shminfo);
+		xvpixfmt, 0, vevent->frame_width, vevent->frame_height, &shminfo);
+
 	if (xv->xv_image_size != image->data_size)
 	{
 		if (xv->xv_image_size > 0)
@@ -260,12 +284,6 @@ static void xf_process_tsmf_video_frame_event(xfInfo* xfi, RDP_VIDEO_FRAME_EVENT
 	{
 		case RDP_PIXFMT_I420:
 		case RDP_PIXFMT_YV12:
-			if (!xf_tsmf_is_format_supported(xv, RDP_PIXFMT_I420) &&
-				!xf_tsmf_is_format_supported(xv, RDP_PIXFMT_YV12))
-			{
-				DEBUG_XV("pixel format 0x%X not supported by hardware.", pixfmt);
-				break;
-			}
 			/* Y */
 			if (image->pitches[0] == vevent->frame_width)
 			{
@@ -284,7 +302,7 @@ static void xf_process_tsmf_video_frame_event(xfInfo* xfi, RDP_VIDEO_FRAME_EVENT
 			}
 			/* UV */
 			/* Conversion between I420 and YV12 is to simply swap U and V */
-			if (xf_tsmf_is_format_supported(xv, pixfmt))
+			if (converti420yv12 == false)
 			{
 				data1 = vevent->frame_data + vevent->frame_width * vevent->frame_height;
 				data2 = vevent->frame_data + vevent->frame_width * vevent->frame_height +