check integer overflow in 'map[pix * width + x] |= (1 << i);'

saitoha · Dec 2, 2019 · bf46a7b · bf46a7b
1 parent 2df6437
commit bf46a7b
Showing 1 changed file with 25 additions and 1 deletion.
diff --git a/src/tosixel.c b/src/tosixel.c
@@ -21,6 +21,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <limits.h>
 
 #if defined(HAVE_INTTYPES_H)
 # include <inttypes.h>
@@ -502,6 +503,7 @@ sixel_encode_body(
     int mx;
     int len;
     int pix;
+    int check_integer_overflow;
     unsigned char *map = NULL;
     sixel_node_t *np, *tp, top;
     int fillable;
@@ -557,8 +559,30 @@ sixel_encode_body(
             fillable = 1;
         }
         for (x = 0; x < width; x++) {
-            pix = pixels[y * width + x];  /* color index */
+            if (y > INT_MAX / width) {
+                /* integer overflow */
+                status = SIXEL_BAD_INTEGER_OVERFLOW;
+                goto end;
+            }
+            check_integer_overflow = y * width;
+            if (check_integer_overflow > INT_MAX - x) {
+                /* integer overflow */
+                status = SIXEL_BAD_INTEGER_OVERFLOW;
+                goto end;
+            }
+            pix = pixels[check_integer_overflow + x];  /* color index */
             if (pix >= 0 && pix < ncolors && pix != keycolor) {
+                if (pix > INT_MAX / width) {
+                    /* integer overflow */
+                    status = SIXEL_BAD_INTEGER_OVERFLOW;
+                    goto end;
+                }
+                check_integer_overflow = pix * width;
+                if (check_integer_overflow > INT_MAX - x) {
+                    /* integer overflow */
+                    status = SIXEL_BAD_INTEGER_OVERFLOW;
+                    goto end;
+                }
                 map[pix * width + x] |= (1 << i);
             }
             else if (!palstate) {