GDALResampleChunk32R_Mode(): avoid potential out-of-memory / integer …

…overflow situation in extreme downsampling situations. Related to https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=26164
OSGeo · Oct 5, 2020 · d6e16a7 · d6e16a7
1 parent ac7d332
commit d6e16a7
Showing 1 changed file with 43 additions and 11 deletions.
diff --git a/gdal/gcore/overview.cpp b/gdal/gcore/overview.cpp
@@ -1123,7 +1123,7 @@ GDALResampleChunk32R_Mode( double dfXRatioDstToSrc, double dfYRatioDstToSrc,
         return CE_Failure;
     }
 
-    GPtrDiff_t nMaxNumPx = 0;
+    size_t nMaxNumPx = 0;
     float *pafVals = nullptr;
     int *panSums = nullptr;
 
@@ -1208,16 +1208,44 @@ GDALResampleChunk32R_Mode( double dfXRatioDstToSrc, double dfYRatioDstToSrc,
                 // filter on floating point data, but here it is for the sake
                 // of compatibility. It won't look right on RGB images by the
                 // nature of the filter.
-                GPtrDiff_t nNumPx = static_cast<GPtrDiff_t>(nSrcYOff2-nSrcYOff)*(nSrcXOff2-nSrcXOff);
-                GPtrDiff_t iMaxInd = 0;
-                GPtrDiff_t iMaxVal = -1;
+
+                if( nSrcYOff2 - nSrcYOff <= 0 ||
+                    nSrcXOff2 - nSrcXOff <= 0 ||
+                    nSrcYOff2 - nSrcYOff > INT_MAX / (nSrcXOff2 - nSrcXOff) ||
+                    static_cast<size_t>(nSrcYOff2-nSrcYOff)*
+                        static_cast<size_t>(nSrcXOff2-nSrcXOff) >
+                            std::numeric_limits<size_t>::max() / sizeof(float) )
+                {
+                    CPLError(CE_Failure, CPLE_NotSupported,
+                             "Too big downsampling factor");
+                    CPLFree( aEntries );
+                    CPLFree( pafVals );
+                    CPLFree( panSums );
+                    return CE_Failure;
+                }
+                const size_t nNumPx = static_cast<size_t>(nSrcYOff2-nSrcYOff)*
+                                      static_cast<size_t>(nSrcXOff2-nSrcXOff);
+                size_t iMaxInd = 0;
+                size_t iMaxVal = 0;
+                bool biMaxValdValid = false;
 
                 if( pafVals == nullptr || nNumPx > nMaxNumPx )
                 {
-                    pafVals = static_cast<float *>(
-                        CPLRealloc(pafVals, nNumPx * sizeof(float)) );
-                    panSums = static_cast<int *>(
-                        CPLRealloc(panSums, nNumPx * sizeof(int)) );
+                    float* pafValsNew = static_cast<float *>(
+                        VSI_REALLOC_VERBOSE(pafVals, nNumPx * sizeof(float)) );
+                    int* panSumsNew = static_cast<int *>(
+                        VSI_REALLOC_VERBOSE(panSums, nNumPx * sizeof(int)) );
+                    if( pafValsNew != nullptr )
+                        pafVals = pafValsNew;
+                    if( panSumsNew != nullptr )
+                        panSums = panSumsNew;
+                    if( pafValsNew == nullptr || panSumsNew == nullptr )
+                    {
+                        CPLFree( aEntries );
+                        CPLFree( pafVals );
+                        CPLFree( panSums );
+                        return CE_Failure;
+                    }
                     nMaxNumPx = nNumPx;
                 }
 
@@ -1230,14 +1258,15 @@ GDALResampleChunk32R_Mode( double dfXRatioDstToSrc, double dfYRatioDstToSrc,
                             pabySrcScanlineNodataMask[iX+iTotYOff] )
                         {
                             const float fVal = pafSrcScanline[iX+iTotYOff];
-                            GPtrDiff_t i = 0;  // Used after for.
+                            size_t i = 0;  // Used after for.
 
                             // Check array for existing entry.
                             for( ; i < iMaxInd; ++i )
                                 if( pafVals[i] == fVal
                                     && ++panSums[i] > panSums[iMaxVal] )
                                 {
                                     iMaxVal = i;
+                                    biMaxValdValid = true;
                                     break;
                                 }
 
@@ -1247,16 +1276,19 @@ GDALResampleChunk32R_Mode( double dfXRatioDstToSrc, double dfYRatioDstToSrc,
                                 pafVals[iMaxInd] = fVal;
                                 panSums[iMaxInd] = 1;
 
-                                if( iMaxVal < 0 )
+                                if( !biMaxValdValid )
+                                {
                                     iMaxVal = iMaxInd;
+                                    biMaxValdValid = true;
+                                }
 
                                 ++iMaxInd;
                             }
                         }
                     }
                 }
 
-                if( iMaxVal == -1 )
+                if( !biMaxValdValid )
                     pafDstScanline[iDstPixel - nDstXOff] = fNoDataValue;
                 else
                     pafDstScanline[iDstPixel - nDstXOff] = pafVals[iMaxVal];