Allow AGG images bigger than 2 GB (#4421)

MapServer · Mar 27, 2014 · 344e7de · 344e7de
1 parent 46a4fea
commit 344e7de
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 7 deletions.
diff --git a/mapagg.cpp b/mapagg.cpp
@@ -807,11 +807,20 @@ imageObj *agg2CreateImage(int width, int height, outputFormatObj *format, colorO
   image = (imageObj *) calloc(1, sizeof (imageObj));
   MS_CHECK_ALLOC(image, sizeof (imageObj), NULL);
   AGG2Renderer *r = new AGG2Renderer();
+
+  /* Compute size on 64bit and check that it is compatible of the platform size_t */
+  AGG_INT64U bufSize64 = (AGG_INT64U)width * height * 4 * sizeof(band_type);
+  size_t bufSize = (size_t)bufSize64;
+  if( (AGG_INT64U)bufSize != bufSize64 ) {
+    msSetError(MS_MEMERR, "%s: %d: Out of memory allocating " AGG_INT64U_FRMT " bytes.\n", "agg2CreateImage()",
+               __FILE__, __LINE__, bufSize64);
+    return NULL;
+  }
 
-  r->buffer = (band_type*)malloc(width * height * 4 * sizeof(band_type));
+  r->buffer = (band_type*)malloc(bufSize);
   if (r->buffer == NULL) {
-    msSetError(MS_MEMERR, "%s: %d: Out of memory allocating %u bytes.\n", "agg2CreateImage()",
-               __FILE__, __LINE__, (unsigned int)(width * height * 4 * sizeof(band_type)));
+    msSetError(MS_MEMERR, "%s: %d: Out of memory allocating " AGG_INT64U_FRMT " bytes.\n", "agg2CreateImage()",
+               __FILE__, __LINE__, bufSize64);
     free(image);
     return NULL;
   }

diff --git a/renderers/agg/include/agg_basics.h b/renderers/agg/include/agg_basics.h
@@ -89,16 +89,20 @@ namespace mapserver
 #ifndef AGG_INT64
 #if defined(_MSC_VER) || defined(__BORLANDC__)
 #define AGG_INT64 signed __int64
+#define AGG_INT64_FRMT "%I64d"
 #else
 #define AGG_INT64 signed long long
+#define AGG_INT64_FRMT "%lld"
 #endif
 #endif
 
 #ifndef AGG_INT64U
 #if defined(_MSC_VER) || defined(__BORLANDC__)
 #define AGG_INT64U unsigned __int64
+#define AGG_INT64U_FRMT "%I64u"
 #else
 #define AGG_INT64U unsigned long long
+#define AGG_INT64U_FRMT "%llu"
 #endif
 #endif
 

diff --git a/renderers/agg/include/agg_rendering_buffer.h b/renderers/agg/include/agg_rendering_buffer.h
@@ -62,7 +62,7 @@ namespace mapserver
 			m_stride = stride;
 			if(stride < 0) 
             { 
-				m_start = m_buf - int(height - 1) * stride;
+				m_start = m_buf - (size_t)(height - 1) * stride;
 			}
         }
 
@@ -80,10 +80,10 @@ namespace mapserver
         //--------------------------------------------------------------------
 		AGG_INLINE       T* row_ptr(int, int y, unsigned) 
         { 
-            return m_start + y * m_stride; 
+            return m_start + (size_t)y * m_stride; 
         }
-		AGG_INLINE       T* row_ptr(int y)       { return m_start + y * m_stride; }
-		AGG_INLINE const T* row_ptr(int y) const { return m_start + y * m_stride; }
+		AGG_INLINE       T* row_ptr(int y)       { return m_start + (size_t)y * m_stride; }
+		AGG_INLINE const T* row_ptr(int y) const { return m_start + (size_t)y * m_stride; }
 		AGG_INLINE row_data row    (int y) const 
         { 
             return row_data(0, m_width-1, row_ptr(y));