DDS: fix crashes and add tests (#3625)

* In the case of DDS cubemap files where a cube face was not present,
  we were filling m_buf with 0 bytes before allocating it.

* Add check for invalid bits per pixel.

* Add regression test for another bug that was perviously fixed in
  2.4.3.

src/dds.imageio/ddsinput.cpp

@@ -501,7 +501,8 @@ DDSInput::open(const std::string& name, ImageSpec& newspec)
     } else
         m_nfaces = 1;
 
-    seek_subimage(0, 0);
+    if (!seek_subimage(0, 0))
+        return false;
     newspec = spec();
     return true;
 }
@@ -675,8 +676,16 @@ DDSInput::seek_subimage(int subimage, int miplevel)
 
     if (m_dds.fmt.bpp
         && (m_dds.fmt.flags
-            & (DDS_PF_RGB | DDS_PF_LUMINANCE | DDS_PF_YUV | DDS_PF_ALPHAONLY)))
+            & (DDS_PF_RGB | DDS_PF_LUMINANCE | DDS_PF_YUV | DDS_PF_ALPHAONLY))) {
+        if (m_dds.fmt.bpp != 8 && m_dds.fmt.bpp != 16 && m_dds.fmt.bpp != 24
+            && m_dds.fmt.bpp != 32) {
+            errorfmt(
+                "Unsupported DDS bit depth: {} (maybe it's a corrupted file?)",
+                m_dds.fmt.bpp);
+            return false;
+        }
         m_spec.attribute("oiio:BitsPerSample", m_dds.fmt.bpp);
+    }
 
     // linear color space for HDR-ish images
     if (basetype == TypeDesc::HALF || basetype == TypeDesc::FLOAT)
@@ -826,8 +835,7 @@ bool
 DDSInput::readimg_tiles()
 {
     // resize destination buffer
-    m_buf.resize(m_spec.tile_bytes());
-
+    OIIO_ASSERT(m_buf.size() >= m_spec.tile_bytes());
     return internal_readimg(&m_buf[0], m_spec.tile_width, m_spec.tile_height,
                             m_spec.tile_depth);
 }
@@ -891,11 +899,11 @@ DDSInput::read_native_tile(int subimage, int miplevel, int x, int y, int z,
         internal_seek_subimage(((x / m_spec.tile_width) << 1)
                                    + y / m_spec.tile_height,
                                m_miplevel, w, h, d);
-#else   // 1x6 layout
+#else                                       // 1x6 layout
         internal_seek_subimage(y / m_spec.tile_height, m_miplevel, w, h, d);
-#endif  // DDS_3X2_CUBE_MAP_LAYOUT
-        if (!w && !h && !d)
-            // face not present in file, black-pad the image
+#endif                                      // DDS_3X2_CUBE_MAP_LAYOUT
+        m_buf.resize(m_spec.tile_bytes());  // resize destination buffer
+        if (!w && !h && !d)  // face not present in file, black-pad the image
             memset(&m_buf[0], 0, m_spec.tile_bytes());
         else
             readimg_tiles();

testsuite/dds/ref/out.txt

@@ -199,3 +199,12 @@ Reading ../oiio-images/dds/broken/dds_bc7_not_enough_data.dds
     channel list: R, G, B, A
     compression: "BC7"
     textureformat: "Plain Texture"
+Reading src/crash-1634.dds
+src/crash-1634.dds   :   16 x    8, 1 channel, uint8 dds
+    SHA-1: CB82FD0F9EF1B69F3F3FF14077564AC2D6C11540
+    channel list: Y
+    textureformat: "Plain Texture"
+    oiio:BitsPerSample: 16
+oiiotool ERROR: read : "src/crash-1635.dds": Unsupported DDS bit depth: 1048576 (maybe it's a corrupted file?)
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-1635.dds

testsuite/dds/run.py

@@ -43,3 +43,7 @@
     "broken/dds_bc7_not_enough_data.dds" ]
 for f in files:
     command += info_command (OIIO_TESTSUITE_IMAGEDIR + "/" + f)
+
+# Test more corrupted files or those that used to crash
+command += info_command ("src/crash-1634.dds", hash=True)
+command += info_command ("src/crash-1635.dds", hash=True)