SoftAVCEnc: check requested memory size before allocation.

Bug: 20674674

Change-Id: If80186a7b9078e575d389220f3bebe9f7630a956
(cherry picked from commit f6fe434)

Author: Wei Jia

media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp

@@ -34,6 +34,10 @@
 
 #include "SoftAVCEncoder.h"
 
+#ifndef INT32_MAX
+#define INT32_MAX   2147483647
+#endif
+
 namespace android {
 
 template<class T>
@@ -257,6 +261,10 @@ OMX_ERRORTYPE SoftAVCEncoder::initEncParams() {
     if (mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {
         // Color conversion is needed.
         CHECK(mInputFrameData == NULL);
+        if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+            ALOGE("Buffer size is too big.");
+            return OMX_ErrorUndefined;
+        }
         mInputFrameData =
             (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);
         CHECK(mInputFrameData != NULL);
@@ -278,6 +286,10 @@ OMX_ERRORTYPE SoftAVCEncoder::initEncParams() {
     int32_t nMacroBlocks = ((((mVideoWidth + 15) >> 4) << 4) *
             (((mVideoHeight + 15) >> 4) << 4)) >> 8;
     CHECK(mSliceGroup == NULL);
+    if ((size_t)nMacroBlocks > SIZE_MAX / sizeof(uint32_t)) {
+        ALOGE("requested memory size is too big.");
+        return OMX_ErrorUndefined;
+    }
     mSliceGroup = (uint32_t *) malloc(sizeof(uint32_t) * nMacroBlocks);
     CHECK(mSliceGroup != NULL);
     for (int ii = 0, idx = 0; ii < nMacroBlocks; ++ii) {
@@ -698,6 +710,10 @@ OMX_ERRORTYPE SoftAVCEncoder::internalSetParameter(
             if (mStoreMetaDataInBuffers) {
                 mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar;
                 if (mInputFrameData == NULL) {
+                    if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+                        ALOGE("Buffer size is too big.");
+                        return OMX_ErrorUndefined;
+                    }
                     mInputFrameData =
                             (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);
                 }