fix undefined behavior: ignore unused bits in B44 mode detection (Aca…

…demySoftwareFoundation#832)


Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>
Signed-off-by: Cary Phillips <cary@ilm.com>

OpenEXR/IlmImf/ImfB44Compressor.cpp

@@ -381,26 +381,26 @@ unpack14 (const unsigned char b[14], unsigned short s[16])
     s[ 0] = (b[0] << 8) | b[1];
 
     unsigned short shift = (b[ 2] >> 2);
-    unsigned short bias = (0x20 << shift);
+    unsigned short bias = (0x20u << shift);
 
-    s[ 4] = s[ 0] + ((((b[ 2] << 4) | (b[ 3] >> 4)) & 0x3f) << shift) - bias;
-    s[ 8] = s[ 4] + ((((b[ 3] << 2) | (b[ 4] >> 6)) & 0x3f) << shift) - bias;
-    s[12] = s[ 8] +   ((b[ 4]                       & 0x3f) << shift) - bias;
+    s[ 4] = s[ 0] + ((((b[ 2] << 4) | (b[ 3] >> 4)) & 0x3fu) << shift) - bias;
+    s[ 8] = s[ 4] + ((((b[ 3] << 2) | (b[ 4] >> 6)) & 0x3fu) << shift) - bias;
+    s[12] = s[ 8] +   ((b[ 4]                       & 0x3fu) << shift) - bias;
 
-    s[ 1] = s[ 0] +   ((b[ 5] >> 2)                         << shift) - bias;
-    s[ 5] = s[ 4] + ((((b[ 5] << 4) | (b[ 6] >> 4)) & 0x3f) << shift) - bias;
-    s[ 9] = s[ 8] + ((((b[ 6] << 2) | (b[ 7] >> 6)) & 0x3f) << shift) - bias;
-    s[13] = s[12] +   ((b[ 7]                       & 0x3f) << shift) - bias;
+    s[ 1] = s[ 0] +   ((unsigned int) (b[ 5] >> 2)           << shift) - bias;
+    s[ 5] = s[ 4] + ((((b[ 5] << 4) | (b[ 6] >> 4)) & 0x3fu) << shift) - bias;
+    s[ 9] = s[ 8] + ((((b[ 6] << 2) | (b[ 7] >> 6)) & 0x3fu) << shift) - bias;
+    s[13] = s[12] +   ((b[ 7]                       & 0x3fu) << shift) - bias;
 
-    s[ 2] = s[ 1] +   ((b[ 8] >> 2)                         << shift) - bias;
-    s[ 6] = s[ 5] + ((((b[ 8] << 4) | (b[ 9] >> 4)) & 0x3f) << shift) - bias;
-    s[10] = s[ 9] + ((((b[ 9] << 2) | (b[10] >> 6)) & 0x3f) << shift) - bias;
-    s[14] = s[13] +   ((b[10]                       & 0x3f) << shift) - bias;
+    s[ 2] = s[ 1] +   ((unsigned int)(b[ 8] >> 2)            << shift) - bias;
+    s[ 6] = s[ 5] + ((((b[ 8] << 4) | (b[ 9] >> 4)) & 0x3fu) << shift) - bias;
+    s[10] = s[ 9] + ((((b[ 9] << 2) | (b[10] >> 6)) & 0x3fu) << shift) - bias;
+    s[14] = s[13] +   ((b[10]                       & 0x3fu) << shift) - bias;
 
-    s[ 3] = s[ 2] +   ((b[11] >> 2)                         << shift) - bias;
-    s[ 7] = s[ 6] + ((((b[11] << 4) | (b[12] >> 4)) & 0x3f) << shift) - bias;
-    s[11] = s[10] + ((((b[12] << 2) | (b[13] >> 6)) & 0x3f) << shift) - bias;
-    s[15] = s[14] +   ((b[13]                       & 0x3f) << shift) - bias;
+    s[ 3] = s[ 2] +   ((unsigned int)(b[11] >> 2)            << shift) - bias;
+    s[ 7] = s[ 6] + ((((b[11] << 4) | (b[12] >> 4)) & 0x3fu) << shift) - bias;
+    s[11] = s[10] + ((((b[12] << 2) | (b[13] >> 6)) & 0x3fu) << shift) - bias;
+    s[15] = s[14] +   ((b[13]                       & 0x3fu) << shift) - bias;
 
     for (int i = 0; i < 16; ++i)
     {
@@ -951,7 +951,10 @@ B44Compressor::uncompress (const char *inPtr,
 		if (inSize < 3)
 		    notEnoughData();
 
-		if (((const unsigned char *)inPtr)[2] == 0xfc)
+                //
+                // If shift exponent is 63, call unpack14 (ignoring unused bits)
+                //
+		if (((const unsigned char *)inPtr)[2] >= (13<<2) )
 		{
 		    unpack3 ((const unsigned char *)inPtr, s);
 		    inPtr += 3;