DO NOT MERGE codecs: check OMX buffer size before use in (h263|h264)dec

Bug: 27833616
Change-Id: I0fd599b3da431425d89236ffdd9df423c11947c0

Author: Wonsik Kim

media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp

@@ -209,6 +209,14 @@ void SoftMPEG4::onQueueFilled(OMX_U32 portIndex) {
         int32_t bufferSize = inHeader->nFilledLen;
         int32_t tmp = bufferSize;
 
+        OMX_U32 frameSize = (mWidth * mHeight * 3) / 2;
+        if (outHeader->nAllocLen < frameSize) {
+            android_errorWriteLog(0x534e4554, "27833616");
+            ALOGE("Insufficient output buffer size");
+            notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+            mSignalledError = true;
+            return;
+        }
         // The PV decoder is lying to us, sometimes it'll claim to only have
         // consumed a subset of the buffer when it clearly consumed all of it.
         // ignore whatever it says...
@@ -250,7 +258,7 @@ void SoftMPEG4::onQueueFilled(OMX_U32 portIndex) {
         ++mInputBufferCount;
 
         outHeader->nOffset = 0;
-        outHeader->nFilledLen = (mWidth * mHeight * 3) / 2;
+        outHeader->nFilledLen = frameSize;
 
         List<BufferInfo *>::iterator it = outQueue.begin();
         while ((*it)->mHeader != outHeader) {

media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp

@@ -205,7 +205,12 @@ void SoftAVC::onQueueFilled(OMX_U32 portIndex) {
         }
 
         if (mFirstPicture && !outQueue.empty()) {
-            drainOneOutputBuffer(mFirstPictureId, mFirstPicture);
+            if (!drainOneOutputBuffer(mFirstPictureId, mFirstPicture)) {
+                ALOGE("Drain failed");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
             delete[] mFirstPicture;
             mFirstPicture = NULL;
             mFirstPictureId = -1;
@@ -255,11 +260,15 @@ void SoftAVC::saveFirstOutputBuffer(int32_t picId, uint8_t *data) {
     memcpy(mFirstPicture, data, mPictureSize);
 }
 
-void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) {
+bool SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) {
     List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex);
     BufferInfo *outInfo = *outQueue.begin();
-    outQueue.erase(outQueue.begin());
     OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader;
+    if (outHeader->nAllocLen - outHeader->nOffset < mPictureSize) {
+        android_errorWriteLog(0x534e4554, "27833616");
+        return false;
+    }
+    outQueue.erase(outQueue.begin());
     OMX_BUFFERHEADERTYPE *header = mPicToHeaderMap.valueFor(picId);
     outHeader->nTimeStamp = header->nTimeStamp;
     outHeader->nFlags = header->nFlags;
@@ -270,6 +279,7 @@ void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) {
     delete header;
     outInfo->mOwnedByUs = false;
     notifyFillBufferDone(outHeader);
+    return true;
 }
 
 void SoftAVC::drainAllOutputBuffers(bool eos) {
@@ -282,7 +292,12 @@ void SoftAVC::drainAllOutputBuffers(bool eos) {
                     mHandle, &decodedPicture, eos /* flush */)) {
             int32_t picId = decodedPicture.picId;
             uint8_t *data = (uint8_t *) decodedPicture.pOutputPicture;
-            drainOneOutputBuffer(picId, data);
+            if (!drainOneOutputBuffer(picId, data)) {
+                ALOGE("Drain failed");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
         }
     }
 

media/libstagefright/codecs/on2/h264dec/SoftAVC.h

@@ -73,7 +73,7 @@ struct SoftAVC : public SoftVideoDecoderOMXComponent {
 
     status_t initDecoder();
     void drainAllOutputBuffers(bool eos);
-    void drainOneOutputBuffer(int32_t picId, uint8_t *data);
+    bool drainOneOutputBuffer(int32_t picId, uint8_t *data);
     void saveFirstOutputBuffer(int32_t pidId, uint8_t *data);
     bool handleCropRectEvent(const CropParams* crop);
     bool handlePortSettingChangeEvent(const H264SwDecInfo *info);