Set MP4Array size only after successful alloc.

m_numElements was set before checking newSize and trying MP4Realloc. In case the size check or MP4Realloc threw an exception, deallocators would iterate over the never allocated new number of elements and attempt calling delete on them.

Also check size of ftyp atom to avoid the uint32_t underflow that caused the bad alloc request.

Addresses: https://nvd.nist.gov/vuln/detail/CVE-2018-17236

Author: enzo1982

src/atom_ftyp.cpp

@@ -53,6 +53,9 @@ void MP4FtypAtom::Generate()
 
 void MP4FtypAtom::Read()
 {
+    if ( m_size < 8 )
+        throw new EXCEPTION("Invalid ftyp atom size");
+
     compatibleBrands.SetCount( (m_size - 8) / 4 ); // brands array fills rest of atom
     MP4Atom::Read();
 }

src/mp4array.h

@@ -77,9 +77,10 @@ class MP4Array {
                   throw new PLATFORM_EXCEPTION("illegal array index", ERANGE); \
             } \
             if (m_numElements == m_maxNumElements) { \
-                m_maxNumElements = max(m_maxNumElements, (MP4ArrayIndex)1) * 2; \
+                MP4ArrayIndex newSize = max(m_maxNumElements, (MP4ArrayIndex)1) * 2; \
                 m_elements = (type*)MP4Realloc(m_elements, \
-                    m_maxNumElements * sizeof(type)); \
+                    newSize * sizeof(type)); \
+                m_maxNumElements = newSize; \
             } \
             memmove(&m_elements[newIndex + 1], &m_elements[newIndex], \
                 (m_numElements - newIndex) * sizeof(type)); \
@@ -100,12 +101,12 @@ class MP4Array {
             } \
         } \
         void Resize(MP4ArrayIndex newSize) { \
+            if ( (uint64_t) newSize * sizeof(type) > 0xFFFFFFFF ) \
+                throw new PLATFORM_EXCEPTION("requested array size exceeds 4GB", ERANGE); /* prevent overflow */ \
+            m_elements = (type*)MP4Realloc(m_elements, \
+                newSize * sizeof(type)); \
             m_numElements = newSize; \
             m_maxNumElements = newSize; \
-            if ( (uint64_t) m_maxNumElements * sizeof(type) > 0xFFFFFFFF ) \
-               throw new PLATFORM_EXCEPTION("requested array size exceeds 4GB", ERANGE); /* prevent overflow */ \
-            m_elements = (type*)MP4Realloc(m_elements, \
-                m_maxNumElements * sizeof(type)); \
         } \
         \
         type& operator[](MP4ArrayIndex index) { \