DO NOT MERGE - libstagefright: check requested memory size before allocation for SoftMPEG4Encoder and SoftVPXEncoder.

Bug: 25812794
Change-Id: I96dc74734380d462583f6efa33d09946f9532809
(cherry picked from commit 87f8cbb)
(cherry picked from commit 6afc659)

Author: Wei Jia

media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp

@@ -33,6 +33,10 @@
 
 #include "SoftMPEG4Encoder.h"
 
+#ifndef INT32_MAX
+#define INT32_MAX   2147483647
+#endif
+
 namespace android {
 
 template<class T>
@@ -149,7 +153,12 @@ OMX_ERRORTYPE SoftMPEG4Encoder::initEncParams() {
 
     if (mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {
         // Color conversion is needed.
-        CHECK(mInputFrameData == NULL);
+        free(mInputFrameData);
+        mInputFrameData = NULL;
+        if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+            ALOGE("b/25812794, Buffer size is too big.");
+            return OMX_ErrorBadParameter;
+        }
         mInputFrameData =
             (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);
         CHECK(mInputFrameData != NULL);

media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp

@@ -25,6 +25,10 @@
 #include <media/stagefright/foundation/ADebug.h>
 #include <media/stagefright/MediaDefs.h>
 
+#ifndef INT32_MAX
+#define INT32_MAX   2147483647
+#endif
+
 namespace android {
 
 
@@ -300,6 +304,10 @@ status_t SoftVPXEncoder::initEncoder() {
 
     if (mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar || mInputDataIsMeta) {
         if (mConversionBuffer == NULL) {
+            if (((uint64_t)mWidth * mHeight) > ((uint64_t)INT32_MAX / 3)) {
+                ALOGE("b/25812794, Buffer size is too big.");
+                return UNKNOWN_ERROR;
+            }
             mConversionBuffer = (uint8_t *)malloc(mWidth * mHeight * 3 / 2);
             if (mConversionBuffer == NULL) {
                 ALOGE("Allocating conversion buffer failed.");