Fix #386: gdImageGrayScale() may produce colors

We have to make sure to avoid alpha-blending issues by explicitly
switching to `gdEffectReplace` and to restore the old value afterwards.

We also document the algorithm used by `gdImageGrayScale()` and note
its limitations regarding palette images.

src/gd_filter.c

@@ -236,6 +236,12 @@ BGD_DECLARE(int) gdImageNegate(gdImagePtr src)
  *
  * Convert an image to grayscale
  *
+ * The red, green and blue components of each pixel are replaced by their
+ * weighted sum using the same coefficients as the REC.601 luma (Y')
+ * calculation. The alpha components are retained.
+ *
+ * For palette images the result may differ due to palette limitations.
+ *
  * Parameters:
  *   src - The image.
  *
@@ -248,11 +254,15 @@ BGD_DECLARE(int) gdImageGrayScale(gdImagePtr src)
 	int r,g,b,a;
 	int new_pxl, pxl;
 	FuncPtr f;
+	int alpha_blending;
 
 	if (src==NULL) {
 		return 0;
 	}
 
+	alpha_blending = src->alphaBlendingFlag;
+	gdImageAlphaBlending(src, gdEffectReplace);
+
 	f = GET_PIXEL_FUNCTION(src);
 
 	for (y=0; y<src->sy; ++y) {
@@ -271,6 +281,8 @@ BGD_DECLARE(int) gdImageGrayScale(gdImagePtr src)
 			gdImageSetPixel (src, x, y, new_pxl);
 		}
 	}
+	gdImageAlphaBlending(src, alpha_blending);
+
 	return 1;
 }
 

tests/gdimagegrayscale/.gitignore

@@ -1 +1,2 @@
 /basic
+/bug00386

tests/gdimagegrayscale/CMakeLists.txt

@@ -1,6 +1,7 @@
 IF(PNG_FOUND)
 LIST(APPEND TESTS_FILES
 	basic
+	bug00386
 )
 ENDIF(PNG_FOUND)
 

tests/gdimagegrayscale/Makemodule.am

@@ -1,9 +1,11 @@
 if HAVE_LIBPNG
 libgd_test_programs += \
-	gdimagegrayscale/basic
+	gdimagegrayscale/basic \
+	gdimagegrayscale/bug00386
 endif
 
 EXTRA_DIST += \
 	gdimagegrayscale/CMakeLists.txt \
 	gdimagegrayscale/basic.png \
-	gdimagegrayscale/basic_exp.png
+	gdimagegrayscale/basic_exp.png \
+	gdimagegrayscale/bug00386.png

tests/gdimagegrayscale/bug00386.c

@@ -0,0 +1,51 @@
+/**
+ * Test that gdImageGrayScale() produces pure gray-scale images
+ *
+ * We are reading a PNG image with transparency, apply gdImageGrayScale() and
+ * test that each pixel has equal red, blue and green components. To avoid
+ * flooding the test log, we bail out after the first non gray-scale pixel.
+ *
+ * We also make sure that the image is a truecolor image, since otherwise it is
+ * not guaranteed that all pixels are actually gray-scale.
+ *
+ * See also <https://github.com/libgd/libgd/issues/386>.
+ */
+
+
+#include "gd.h"
+#include "gdtest.h"
+
+
+int main()
+{
+    gdImagePtr im;
+    FILE *fp;
+    int res;
+    int i, j;
+
+    fp = gdTestFileOpen2("gdimagegrayscale", "bug00386.png");
+    gdTestAssert(fp != NULL);
+    im = gdImageCreateFromPng(fp);
+    gdTestAssert(im != NULL);
+    fclose(fp);
+
+    res = gdImageTrueColor(im);
+    gdTestAssert(res != 0);
+
+    res = gdImageGrayScale(im);
+    gdTestAssert(res != 0);
+
+    for (i = 0; i < gdImageSX(im); i++) {
+        for (j = 0; j < gdImageSY(im); j++) {
+            int color = gdImageGetTrueColorPixel(im, i, j);
+            int red = gdImageRed(im, color);
+            int green = gdImageGreen(im, color);
+            int blue = gdImageBlue(im, color);
+            if (!gdTestAssert(red == green && green == blue)) {
+                return gdNumFailures();
+            }
+        }
+    }
+
+    return gdNumFailures();
+}