reduce size limit for scanline files; prevent large chunkoffset alloc…

…ations (AcademySoftwareFoundation#824)

* reduce size limit for scanline files; protect against large chunkoffset allocations

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>

* bugfix for memory limit changes

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>

* rearrange chunkoffset test to protect bytesperline table too

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>

* remove extraneous function declaration; tidy comments

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>
Signed-off-by: Cary Phillips <cary@ilm.com>

OpenEXR/IlmImf/ImfCompressor.cpp

@@ -176,6 +176,37 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header &hdr)
 }
 
 
+// for a given compression type, return the number of scanlines
+// compressed into a single chunk
+// TODO add to API and move to ImfCompressor.cpp
+int
+numLinesInBuffer(Compression comp)
+{
+    switch(comp)
+    {
+        case NO_COMPRESSION :
+        case RLE_COMPRESSION:
+        case ZIPS_COMPRESSION:
+            return 1;
+        case ZIP_COMPRESSION:
+            return 16;
+        case PIZ_COMPRESSION:
+            return 32;
+        case PXR24_COMPRESSION:
+            return 16;
+        case B44_COMPRESSION:
+        case B44A_COMPRESSION:
+        case DWAA_COMPRESSION:
+            return 32;
+        case DWAB_COMPRESSION:
+            return 256;
+
+        default:
+	        throw IEX_NAMESPACE::ArgExc ("Unknown compression type");
+    }
+}
+
+
 Compressor *
 newTileCompressor (Compression c,
 		   size_t tileLineSize,

OpenEXR/IlmImf/ImfCompressor.h

@@ -268,6 +268,14 @@ Compressor *    newTileCompressor (Compression c,
 				   const Header &hdr);
 
 
+//-----------------------------------------------------------------
+// Return the maximum number of scanlines in each chunk
+// of a scanline image using the given compression scheme
+//-----------------------------------------------------------------
+
+IMF_EXPORT
+int              numLinesInBuffer(Compression comp);
+
 OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT
 
 #endif

OpenEXR/IlmImf/ImfMisc.cpp

@@ -1843,38 +1843,7 @@ usesLongNames (const Header &header)
     return false;
 }
 
-namespace
-{
-// for a given compression type, return the number of scanlines
-// compressed into a single chunk
-// TODO add to API and move to ImfCompressor.cpp
-int
-numLinesInBuffer(Compression comp)
-{
-    switch(comp)
-    {
-        case NO_COMPRESSION :
-        case RLE_COMPRESSION:
-        case ZIPS_COMPRESSION:
-            return 1;
-        case ZIP_COMPRESSION:
-            return 16;
-        case PIZ_COMPRESSION:
-            return 32;
-        case PXR24_COMPRESSION:
-            return 16;
-        case B44_COMPRESSION:
-        case B44A_COMPRESSION:
-        case DWAA_COMPRESSION:
-            return 32;
-        case DWAB_COMPRESSION:
-            return 256;
-
-        default:
-	        throw IEX_NAMESPACE::ArgExc ("Unknown compression type");
-    }
-}
-}
+
 
 int
 getScanlineChunkOffsetTableSize(const Header& header)

OpenEXR/IlmImf/ImfMisc.h

@@ -475,6 +475,7 @@ bool usesLongNames (const Header &header);
 IMF_EXPORT
 int getChunkOffsetTableSize(const Header& header,bool deprecated_attribute=false);
 
+
 OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT
 
 

OpenEXR/IlmImf/ImfMultiPartInputFile.cpp

@@ -724,7 +724,9 @@ MultiPartInputFile::Data::getPart(int partNumber)
     return parts[partNumber];
 }
 
-
+namespace{
+static const int gLargeChunkTableSize = 1024*1024;
+}
 
 void
 MultiPartInputFile::Data::readChunkOffsetTables(bool reconstructChunkOffsetTable)
@@ -734,8 +736,28 @@ MultiPartInputFile::Data::readChunkOffsetTables(bool reconstructChunkOffsetTable
     for (size_t i = 0; i < parts.size(); i++)
     {
         int chunkOffsetTableSize = getChunkOffsetTableSize(parts[i]->header);
+
+        //
+        // avoid allocating excessive memory.
+        // If the chunktablesize claims to be large,
+        // check the file is big enough to contain the table before allocating memory.
+        // Attempt to read the last entry in the table. Either the seekg() or the read()
+        // call will throw an exception if the file is too small to contain the table
+        //
+        if (chunkOffsetTableSize > gLargeChunkTableSize)
+        {
+            Int64 pos = is->tellg();
+            is->seekg(pos + (chunkOffsetTableSize-1)*sizeof(Int64));
+            Int64 temp;
+            OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*is, temp);
+            is->seekg(pos);
+
+        }
+
         parts[i]->chunkOffsets.resize(chunkOffsetTableSize);
 
+
+
         for (int j = 0; j < chunkOffsetTableSize; j++)
             OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*is, parts[i]->chunkOffsets[j]);
 

OpenEXR/IlmImf/ImfScanLineInputFile.cpp

@@ -1090,7 +1090,7 @@ newLineBufferTask (TaskGroup *group,
  }
 
 
-
+static const int gLargeChunkTableSize = 1024*1024;
 
 } // namespace
 
@@ -1117,14 +1117,14 @@ void ScanLineInputFile::initialize(const Header& header)
                               _data->linesInBuffer) / _data->linesInBuffer;
 
         //
-        // avoid allocating excessive memory due to large lineOffsets and bytesPerLine table sizes.
+        // avoid allocating excessive memory due to large lineOffsets table size.
         // If the chunktablesize claims to be large,
-        // check the file is big enough to contain the lineOffsets table before allocating memory
+        // check the file is big enough to contain the table before allocating memory
         // in the bytesPerLineTable and the lineOffsets table.
         // Attempt to read the last entry in the table. Either the seekg() or the read()
         // call will throw an exception if the file is too small to contain the table
         //
-        if (lineOffsetSize * _data->linesInBuffer > gLargeChunkTableSize)
+        if (lineOffsetSize > gLargeChunkTableSize)
         {
             Int64 pos = _streamData->is->tellg();
             _streamData->is->seekg(pos + (lineOffsetSize-1)*sizeof(Int64));
@@ -1137,23 +1137,24 @@ void ScanLineInputFile::initialize(const Header& header)
 
         size_t maxBytesPerLine = bytesPerLineTable (_data->header,
                                                     _data->bytesPerLine);
-        
-        if(maxBytesPerLine > INT_MAX)
+
+        if (maxBytesPerLine*numLinesInBuffer(comp) > INT_MAX)
         {
             throw IEX_NAMESPACE::InputExc("maximum bytes per scanline exceeds maximum permissible size");
         }
 
 
+        //
+        // allocate compressor objects
+        //
         for (size_t i = 0; i < _data->lineBuffers.size(); i++)
         {
-            _data->lineBuffers[i] = new LineBuffer (newCompressor
-                                                (_data->header.compression(),
+            _data->lineBuffers[i] = new LineBuffer (newCompressor(comp,
                                                  maxBytesPerLine,
                                                  _data->header));
         }
 
-        _data->linesInBuffer =
-            numLinesInBuffer (_data->lineBuffers[0]->compressor);
+
 
         _data->lineBufferSize = maxBytesPerLine * _data->linesInBuffer;
 
@@ -1170,8 +1171,6 @@ void ScanLineInputFile::initialize(const Header& header)
                                  _data->linesInBuffer,
                                  _data->offsetInLineBuffer);
 
-        int lineOffsetSize = (dataWindow.max.y - dataWindow.min.y +
-                              _data->linesInBuffer) / _data->linesInBuffer;
 
         _data->lineOffsets.resize (lineOffsetSize);
 }