Fixed a couple of bugs in the general and X11 shape code, and fixed a…

… bug in testshape that was keeping it from recognizing surfaces without alpha. Thanks to Andreas's bit-bashing tip, X11 shaped windows now work entirely, AFAICT.

src/video/SDL_shape.c

@@ -27,7 +27,7 @@
 #include "SDL_pixels.h"
 #include "SDL_surface.h"
 #include "SDL_shape.h"
-#include "SDL_shape_internals.h"
+#include "../src/video/SDL_shape_internals.h"
 
 SDL_Window* SDL_CreateShapedWindow(const char *title,unsigned int x,unsigned int y,unsigned int w,unsigned int h,Uint32 flags) {
 	SDL_Window *result = SDL_CreateWindow(title,x,y,w,h,SDL_WINDOW_BORDERLESS | flags & !SDL_WINDOW_FULLSCREEN & !SDL_WINDOW_SHOWN);
@@ -57,12 +57,12 @@ SDL_bool SDL_IsShapedWindow(const SDL_Window *window) {
 }
 
 /* REQUIRES that bitmap point to a w-by-h bitmap with ppb pixels-per-byte. */
-void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb,Uint8 value) {
+void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb) {
 	int x = 0;
 	int y = 0;
 	Uint8 r = 0,g = 0,b = 0,alpha = 0;
 	Uint8* pixel = NULL;
-	Uint32 bitmap_pixel,pixel_value = 0;
+	Uint32 bitmap_pixel,pixel_value = 0,mask_value = 0;
 	SDL_Color key;
 	if(SDL_MUSTLOCK(shape))
 		SDL_LockSurface(shape);
@@ -79,6 +79,9 @@ void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8*
 				case(2):
 					pixel_value = *(Uint16*)pixel;
 					break;
+				case(3):
+					pixel_value = *(Uint32*)pixel & (~shape->format->Amask);
+					break;
 				case(4):
 					pixel_value = *(Uint32*)pixel;
 					break;
@@ -87,19 +90,20 @@ void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8*
 			bitmap_pixel = y*shape->w + x;
 			switch(mode.mode) {
 				case(ShapeModeDefault):
-					bitmap[bitmap_pixel / ppb] |= (alpha >= 1 ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
+					mask_value = (alpha >= 1 ? 1 : 0);
 					break;
 				case(ShapeModeBinarizeAlpha):
-					bitmap[bitmap_pixel / ppb] |= (alpha >= mode.parameters.binarizationCutoff ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
+					mask_value = (alpha >= mode.parameters.binarizationCutoff ? 1 : 0);
 					break;
 				case(ShapeModeReverseBinarizeAlpha):
-					bitmap[bitmap_pixel / ppb] |= (alpha <= mode.parameters.binarizationCutoff ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
+					mask_value = (alpha <= mode.parameters.binarizationCutoff ? 1 : 0);
 					break;
 				case(ShapeModeColorKey):
 					key = mode.parameters.colorKey;
-					bitmap[bitmap_pixel / ppb] |= ((key.r == r && key.g == g && key.b == b) ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
+					mask_value = ((key.r != r && key.g != g && key.b != b) ? 1 : 0);
 					break;
 			}
+			bitmap[bitmap_pixel / ppb] |= mask_value << (7 - ((ppb - 1) - (bitmap_pixel % ppb)));
 		}
 	}
 	if(SDL_MUSTLOCK(shape))

src/video/SDL_shape_internals.h

@@ -19,6 +19,7 @@
     Eli Gottlieb
     eligottlieb@gmail.com
 */
+#include "SDL_config.h"
 
 #ifndef _SDL_shape_internals_h
 #define _SDL_shape_internals_h
@@ -51,7 +52,7 @@ typedef struct {
 	SDL_ShapeUnion data;
 } SDL_ShapeTree;
 
-extern void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb,Uint8 value);
+extern void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb);
 extern SDL_ShapeTree* SDL_CalculateShapeTree(SDL_WindowShapeMode mode,SDL_Surface* shape,SDL_bool invert);
 extern void SDL_TraverseShapeTree(SDL_ShapeTree *tree,void(*function)(SDL_ShapeTree*,void*),void* closure);
 extern void SDL_FreeShapeTree(SDL_ShapeTree** shapeTree);

src/video/x11/SDL_x11shape.c

@@ -70,8 +70,7 @@ int X11_ResizeWindowShape(SDL_Window* window) {
 			return -1;
 		}
 	}
-	else
-		memset(data->bitmap,0,data->bitmapsize);
+	memset(data->bitmap,0,data->bitmapsize);
 
 	window->shaper->usershownflag |= window->flags & SDL_WINDOW_SHOWN;
 
@@ -90,7 +89,7 @@ int X11_SetWindowShape(SDL_WindowShaper *shaper,SDL_Surface *shape,SDL_WindowSha
 	SDL_ShapeData *data = shaper->driverdata;
 
 	/* Assume that shaper->alphacutoff already has a value, because SDL_SetWindowShape() should have given it one. */
-	SDL_CalculateShapeBitmap(shaper->mode,shape,data->bitmap,8,1);
+	SDL_CalculateShapeBitmap(shaper->mode,shape,data->bitmap,8);
 
 	SDL_WindowData *windowdata = (SDL_WindowData*)(shaper->window->driverdata);
 	Pixmap shapemask = XCreateBitmapFromData(windowdata->videodata->display,windowdata->xwindow,data->bitmap,shaper->window->w,shaper->window->h);

test/testshape.c

@@ -43,7 +43,7 @@ int main(int argc,char** argv) {
 	Uint8 num_pictures;
 	LoadedPicture* pictures;
 	int i, j;
-	SDL_PixelFormat* format;
+	SDL_PixelFormat* format = NULL;
 	Uint32 format_enum;
 	SDL_Window *window;
 	SDL_Color black = {0,0,0,0xff};
@@ -52,8 +52,8 @@ int main(int argc,char** argv) {
 	int should_exit = 0;
 	unsigned int current_picture;
 	int button_down;
-	Uint32 pixelFormat;
-	int access;
+	Uint32 pixelFormat = 0;
+	int access = 0;
 	SDL_Rect texture_dimensions;;
 
 	if(argc < 2) {
@@ -84,8 +84,7 @@ int main(int argc,char** argv) {
 		}
 
 		format = pictures[i].surface->format;
-		format_enum = SDL_MasksToPixelFormatEnum (format->BitsPerPixel,format->Rmask,format->Gmask, format->Bmask,format->Amask);
-		if(SDL_ISPIXELFORMAT_ALPHA(format_enum)) {
+		if(format->Amask != 0) {
 			pictures[i].mode.mode = ShapeModeBinarizeAlpha;
 			pictures[i].mode.parameters.binarizationCutoff = 1;
 		}
@@ -139,7 +138,6 @@ int main(int argc,char** argv) {
 	event_pending = SDL_PollEvent(&event);
 	current_picture = 0;
 	button_down = 0;
-	format = 0,access = 0;
 	texture_dimensions.h = 0;
 	texture_dimensions.w = 0;
 	texture_dimensions.x = 0;