From bdd9b93386c8bbcc093b0d553e47ae2c5596fa92 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Sat, 21 May 2016 07:31:19 -0700 Subject: [PATCH] Replace SIZE_MAX with type specific _MAX --- decode.c | 2 +- encode.c | 2 +- libImaging/Draw.c | 2 +- libImaging/Quant.c | 18 +++++++++++------- libImaging/QuantHeap.c | 3 ++- libImaging/QuantOctree.c | 14 +++++++++++--- libImaging/RankFilter.c | 4 ++-- libImaging/Resample.c | 6 +++--- libImaging/ZipDecode.c | 2 +- libImaging/ZipEncode.c | 2 +- 10 files changed, 34 insertions(+), 21 deletions(-) diff --git a/decode.c b/decode.c index 0c66243cd79..8313c25ad92 100644 --- a/decode.c +++ b/decode.c @@ -189,7 +189,7 @@ _setimage(ImagingDecoderObject* decoder, PyObject* args) /* Allocate memory buffer (if bits field is set) */ if (state->bits > 0) { if (!state->bytes) { - if (state->xsize > ((SIZE_MAX / state->bits)-7)){ + if (state->xsize > ((INT_MAX / state->bits)-7)){ return PyErr_NoMemory(); } state->bytes = (state->bits * state->xsize+7)/8; diff --git a/encode.c b/encode.c index 66ab1616cd9..b9f55306153 100644 --- a/encode.c +++ b/encode.c @@ -234,7 +234,7 @@ _setimage(ImagingEncoderObject* encoder, PyObject* args) /* Allocate memory buffer (if bits field is set) */ if (state->bits > 0) { - if (state->xsize > ((SIZE_MAX / state->bits)-7)) { + if (state->xsize > ((INT_MAX / state->bits)-7)) { return PyErr_NoMemory(); } state->bytes = (state->bits * state->xsize+7)/8; diff --git a/libImaging/Draw.c b/libImaging/Draw.c index 32f024ed0ef..0eedde38773 100644 --- a/libImaging/Draw.c +++ b/libImaging/Draw.c @@ -937,7 +937,7 @@ allocate(ImagingOutline outline, int extra) /* malloc check ok, uses calloc for overflow */ e = calloc(outline->size, sizeof(Edge)); } else { - if (outline->size > SIZE_MAX / sizeof(Edge)) { + if (outline->size > INT_MAX / sizeof(Edge)) { return NULL; } /* malloc check ok, overflow checked above */ diff --git a/libImaging/Quant.c b/libImaging/Quant.c index 5f1ed8cb9ed..769aea7376f 100644 --- a/libImaging/Quant.c +++ b/libImaging/Quant.c @@ -1091,6 +1091,10 @@ k_means(Pixel *pixelData, uint32_t **avgDistSortKey; int changes; int built=0; + + if (nPaletteEntries > UINT32_MAX / (sizeof(uint32_t))) { + return 0; + } /* malloc check ok, using calloc */ if (!(count=calloc(nPaletteEntries, sizeof(uint32_t)))) { return 0; @@ -1106,7 +1110,7 @@ k_means(Pixel *pixelData, } /* this is enough of a check, since the multiplication n*size is done above */ - if (nPaletteEntries > SIZE_MAX / (nPaletteEntries * sizeof(uint32_t))) { + if (nPaletteEntries > UINT32_MAX / (nPaletteEntries * sizeof(uint32_t))) { goto error_1; } /* malloc check ok, using calloc, checking n*n above */ @@ -1265,8 +1269,8 @@ quantize(Pixel *pixelData, qp=calloc(nPixels, sizeof(uint32_t)); if (!qp) { goto error_4; } - if ((nPaletteEntries > SIZE_MAX / nPaletteEntries ) || - (nPaletteEntries > SIZE_MAX / (nPaletteEntries * sizeof(uint32_t)))) { + if ((nPaletteEntries > UINT32_MAX / nPaletteEntries ) || + (nPaletteEntries > UINT32_MAX / (nPaletteEntries * sizeof(uint32_t)))) { goto error_5; } /* malloc check ok, using calloc for overflow, check of n*n above */ @@ -1444,8 +1448,8 @@ quantize2(Pixel *pixelData, qp=calloc(nPixels, sizeof(uint32_t)); if (!qp) { goto error_1; } - if ((nQuantPixels > SIZE_MAX / nQuantPixels ) || - (nQuantPixels > SIZE_MAX / (nQuantPixels * sizeof(uint32_t)))) { + if ((nQuantPixels > UINT32_MAX / nQuantPixels ) || + (nQuantPixels > UINT32_MAX / (nQuantPixels * sizeof(uint32_t)))) { goto error_2; } @@ -1514,8 +1518,8 @@ ImagingQuantize(Imaging im, int colors, int mode, int kmeans) if (!strcmp(im->mode, "RGBA") && mode != 2) return ImagingError_ModeError(); - if ((im->xsize > SIZE_MAX / im->ysize) || - (im->xsize > SIZE_MAX / (im->ysize * sizeof(Pixel)))) { + if ((im->xsize > INT_MAX / im->ysize) || + (im->xsize > INT_MAX / (im->ysize * sizeof(Pixel)))) { return ImagingError_MemoryError(); } /* malloc check ok, using calloc for final overflow, x*y above */ diff --git a/libImaging/QuantHeap.c b/libImaging/QuantHeap.c index 2edf6d9cfe4..2a4a5adc07b 100644 --- a/libImaging/QuantHeap.c +++ b/libImaging/QuantHeap.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "QuantHeap.h" @@ -47,7 +48,7 @@ static int _heap_grow(Heap *h,int newsize) { void *newheap; if (!newsize) newsize=h->heapsize<<1; if (newsizeheapsize) return 0; - if (newsize > ((int)SIZE_MAX) / sizeof(void *)){ + if (newsize > INT_MAX / sizeof(void *)){ return 0; } /* malloc check ok, using calloc for overflow, also checking diff --git a/libImaging/QuantOctree.c b/libImaging/QuantOctree.c index 3f61f072213..ede3ad634d8 100644 --- a/libImaging/QuantOctree.c +++ b/libImaging/QuantOctree.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "QuantOctree.h" @@ -62,6 +63,12 @@ new_color_cube(int r, int g, int b, int a) { cube->bBits = MAX(b, 0); cube->aBits = MAX(a, 0); + /* overflow check for size multiplication below */ + if (cube->rBits + cube->gBits + cube->bBits + cube->aBits > 31) { + free(cube); + return NULL; + } + /* the width of the cube for each dimension */ cube->rWidth = 1<rBits; cube->gWidth = 1<gBits; @@ -77,6 +84,7 @@ new_color_cube(int r, int g, int b, int a) { /* the number of color buckets */ cube->size = cube->rWidth * cube->gWidth * cube->bWidth * cube->aWidth; + /* malloc check ok, overflow checked above */ cube->buckets = calloc(cube->size, sizeof(struct _ColorBucket)); if (!cube->buckets) { @@ -155,7 +163,7 @@ compare_bucket_count(const ColorBucket a, const ColorBucket b) { static ColorBucket create_sorted_color_palette(const ColorCube cube) { ColorBucket buckets; - if (cube->size > SIZE_MAX / sizeof(struct _ColorBucket)) { + if (cube->size > LONG_MAX / sizeof(struct _ColorBucket)) { return NULL; } /* malloc check ok, calloc + overflow check above for memcpy */ @@ -285,8 +293,8 @@ void add_lookup_buckets(ColorCube cube, ColorBucket palette, long nColors, long ColorBucket combined_palette(ColorBucket bucketsA, long nBucketsA, ColorBucket bucketsB, long nBucketsB) { ColorBucket result; - if (nBucketsA > SIZE_MAX - nBucketsB || - (nBucketsA+nBucketsB) > SIZE_MAX / sizeof(struct _ColorBucket)) { + if (nBucketsA > LONG_MAX - nBucketsB || + (nBucketsA+nBucketsB) > LONG_MAX / sizeof(struct _ColorBucket)) { return NULL; } /* malloc check ok, overflow check above */ diff --git a/libImaging/RankFilter.c b/libImaging/RankFilter.c index 51f64611d15..df94b6081ac 100644 --- a/libImaging/RankFilter.c +++ b/libImaging/RankFilter.c @@ -61,8 +61,8 @@ ImagingRankFilter(Imaging im, int size, int rank) return (Imaging) ImagingError_ValueError("bad filter size"); /* malloc check ok, for overflow in the define below */ - if (size > SIZE_MAX / size || - size > SIZE_MAX / (size * sizeof(FLOAT32))) { + if (size > INT_MAX / size || + size > INT_MAX / (size * sizeof(FLOAT32))) { return (Imaging) ImagingError_ValueError("filter size too large"); } diff --git a/libImaging/Resample.c b/libImaging/Resample.c index 579e9101154..bc72d78e11d 100644 --- a/libImaging/Resample.c +++ b/libImaging/Resample.c @@ -139,11 +139,11 @@ ImagingResampleHorizontal(Imaging imIn, int xsize, int filter) kmax = (int) ceil(support) * 2 + 1; // check for overflow - if (kmax > 0 && xsize > SIZE_MAX / kmax) + if (kmax > 0 && xsize > INT_MAX / kmax) return (Imaging) ImagingError_MemoryError(); // sizeof(float) should be greater than 0 - if (xsize * kmax > SIZE_MAX / sizeof(float)) + if (xsize * kmax > INT_MAX / sizeof(float)) return (Imaging) ImagingError_MemoryError(); /* coefficient buffer */ @@ -153,7 +153,7 @@ ImagingResampleHorizontal(Imaging imIn, int xsize, int filter) return (Imaging) ImagingError_MemoryError(); // sizeof(int) should be greater than 0 as well - if (xsize > SIZE_MAX / (2 * sizeof(int))) + if (xsize > INT_MAX / (2 * sizeof(int))) return (Imaging) ImagingError_MemoryError(); /* malloc check ok, overflow check above */ diff --git a/libImaging/ZipDecode.c b/libImaging/ZipDecode.c index dfcfa36bbd9..bbf01114702 100644 --- a/libImaging/ZipDecode.c +++ b/libImaging/ZipDecode.c @@ -57,7 +57,7 @@ ImagingZipDecode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes) context->prefix = 1; /* PNG */ /* overflow check for malloc */ - if (state->bytes > SIZE_MAX - 1) { + if (state->bytes > INT_MAX - 1) { state->errcode = IMAGING_CODEC_MEMORY; return -1; } diff --git a/libImaging/ZipEncode.c b/libImaging/ZipEncode.c index 4a431fb33d2..d642e6da299 100644 --- a/libImaging/ZipEncode.c +++ b/libImaging/ZipEncode.c @@ -38,7 +38,7 @@ ImagingZipEncode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes) /* Valid modes are ZIP_PNG, ZIP_PNG_PALETTE, and ZIP_TIFF */ /* overflow check for malloc */ - if (state->bytes > SIZE_MAX - 1) { + if (state->bytes > INT_MAX - 1) { state->errcode = IMAGING_CODEC_MEMORY; return -1; }