Permalink
Browse files

X11/tsmf: Fixed XShmCreateImage BadMatch crashes

There was not check if the pixformat supplied in the XShmCreateImage function
was actually supported by the X-Video extension. Also reduced the amount of
xf_tsmf_is_format_supported calls in xf_process_tsmf_video_frame_event.
  • Loading branch information...
nfedera committed Mar 13, 2012
1 parent d1b32a7 commit ba61607fcfbf24928abba3b26af7c50930e62217
Showing with 26 additions and 8 deletions.
  1. +26 −8 client/X11/xf_tsmf.c
View
@@ -196,6 +196,8 @@ static void xf_process_tsmf_video_frame_event(xfInfo* xfi, RDP_VIDEO_FRAME_EVENT
uint8* data1;
uint8* data2;
uint32 pixfmt;
+ uint32 xvpixfmt;
+ boolean converti420yv12 = false;
XvImage * image;
int colorkey = 0;
XShmSegmentInfo shminfo;
@@ -230,8 +232,30 @@ static void xf_process_tsmf_video_frame_event(xfInfo* xfi, RDP_VIDEO_FRAME_EVENT
}
pixfmt = vevent->frame_pixfmt;
+
+ if (xf_tsmf_is_format_supported(xv, pixfmt))
+ {
+ xvpixfmt = pixfmt;
+ }
+ else if (pixfmt == RDP_PIXFMT_I420 && xf_tsmf_is_format_supported(xv, RDP_PIXFMT_YV12))
+ {
+ xvpixfmt = RDP_PIXFMT_YV12;
+ converti420yv12 = true;
+ }
+ else if (pixfmt == RDP_PIXFMT_YV12 && xf_tsmf_is_format_supported(xv, RDP_PIXFMT_I420))
+ {
+ xvpixfmt = RDP_PIXFMT_I420;
+ converti420yv12 = true;
+ }
+ else
+ {
+ DEBUG_XV("pixel format 0x%X not supported by hardware.", pixfmt);
+ return;
+ }
+
image = XvShmCreateImage(xfi->display, xv->xv_port,
- pixfmt, 0, vevent->frame_width, vevent->frame_height, &shminfo);
+ xvpixfmt, 0, vevent->frame_width, vevent->frame_height, &shminfo);
+
if (xv->xv_image_size != image->data_size)
{
if (xv->xv_image_size > 0)
@@ -260,12 +284,6 @@ static void xf_process_tsmf_video_frame_event(xfInfo* xfi, RDP_VIDEO_FRAME_EVENT
{
case RDP_PIXFMT_I420:
case RDP_PIXFMT_YV12:
- if (!xf_tsmf_is_format_supported(xv, RDP_PIXFMT_I420) &&
- !xf_tsmf_is_format_supported(xv, RDP_PIXFMT_YV12))
- {
- DEBUG_XV("pixel format 0x%X not supported by hardware.", pixfmt);
- break;
- }
/* Y */
if (image->pitches[0] == vevent->frame_width)
{
@@ -284,7 +302,7 @@ static void xf_process_tsmf_video_frame_event(xfInfo* xfi, RDP_VIDEO_FRAME_EVENT
}
/* UV */
/* Conversion between I420 and YV12 is to simply swap U and V */
- if (xf_tsmf_is_format_supported(xv, pixfmt))
+ if (converti420yv12 == false)
{
data1 = vevent->frame_data + vevent->frame_width * vevent->frame_height;
data2 = vevent->frame_data + vevent->frame_width * vevent->frame_height +

0 comments on commit ba61607

Please sign in to comment.