From 35affa093c3e81930e23453e2f092cf631d8e76a Mon Sep 17 00:00:00 2001
From: David Carlier <devnexen@gmail.com>
Date: Tue, 2 Sep 2025 18:20:19 +0100
Subject: [PATCH] Fix GH-19666: imageconvolution() unexpected nan filter value

undefined behavior occurs during cast from float to int when the element
of the filter matrix is -INF.
---
 ext/gd/libgd/gd_filter.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ext/gd/libgd/gd_filter.c b/ext/gd/libgd/gd_filter.c
index db364c923ec60..5a9b627e8cebd 100644
--- a/ext/gd/libgd/gd_filter.c
+++ b/ext/gd/libgd/gd_filter.c
@@ -1,6 +1,7 @@
 #include "gd.h"
 
 #include "gd_intern.h"
+#include "gd_errors.h"
 
 #ifdef _WIN32
 # include <windows.h>
@@ -8,6 +9,7 @@
 # include <unistd.h>
 #endif
 #include <stdlib.h>
+#include <math.h>
 #include <time.h>
 
 /* Filters function added on 2003/12
@@ -383,6 +385,24 @@ int gdImageConvolution(gdImagePtr src, float filter[3][3], float filter_div, flo
 			new_g = (new_g > 255.0f)? 255.0f : ((new_g < 0.0f)? 0.0f:new_g);
 			new_b = (new_b > 255.0f)? 255.0f : ((new_b < 0.0f)? 0.0f:new_b);
 
+			if (isnan(new_r)) {
+				gd_error("Matrix: invalid top-left filter");
+				gdImageDestroy(srcback);
+				return 0;
+			}
+
+			if (isnan(new_g)) {
+				gd_error("Matrix: invalid center filter");
+				gdImageDestroy(srcback);
+				return 0;
+			}
+
+			if (isnan(new_b)) {
+				gd_error("Matrix: invalid bottom-right filter");
+				gdImageDestroy(srcback);
+				return 0;
+			}
+
 			new_pxl = gdImageColorAllocateAlpha(src, (int)new_r, (int)new_g, (int)new_b, new_a);
 			if (new_pxl == -1) {
 				new_pxl = gdImageColorClosestAlpha(src, (int)new_r, (int)new_g, (int)new_b, new_a);