hopefully fix out of memory for fmle

source_code/DibHelper.cpp

@@ -295,65 +295,65 @@ void GetRectOfWindowIncludingAero(HWND ofThis, RECT *toHere)
 
 }
 
-
-/* from libvidcap or some odd 
-Based on formulas found at http://en.wikipedia.org/wiki/YUV */
-int rgb32_to_i420(int width, int height, const char * src, char * dst)
-{
-	unsigned char * dst_y_even;
-	unsigned char * dst_y_odd;
-	unsigned char * dst_u;
-	unsigned char * dst_v;
-	const unsigned char *src_even;
-	const unsigned char *src_odd;
-	int i, j;
-
-	src_even = (const unsigned char *)src;
-	src_odd = src_even + width * 4;
-
-	dst_y_even = (unsigned char *)dst;
-	dst_y_odd = dst_y_even + width;
-	dst_u = dst_y_even + width * height;
-	dst_v = dst_u + ((width * height) >> 2);
-
-	for ( i = 0; i < height / 2; ++i )
-	{
-		for ( j = 0; j < width / 2; ++j )
-		{
-			short r, g, b;
-			b = *src_even++;
-			g = *src_even++;
-			r = *src_even++;
-			++src_even;
-			*dst_y_even++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
-
-			*dst_u++ = (( r * -38 - g * 74 + b * 112 + 128 ) >> 8 ) + 128;
-			*dst_v++ = (( r * 112 - g * 94 - b * 18 + 128 ) >> 8 ) + 128;
-
-			b = *src_even++;
-			g = *src_even++;
-			r = *src_even++;
-			++src_even;
-			*dst_y_even++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
-
-			b = *src_odd++;
-			g = *src_odd++;
-			r = *src_odd++;
-			++src_odd;
-			*dst_y_odd++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
-
-			b = *src_odd++;
-			g = *src_odd++;
-			r = *src_odd++;
-			++src_odd;
-			*dst_y_odd++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
-		}
-
-		dst_y_even += width;
-		dst_y_odd += width;
-		src_even += width * 4;
-		src_odd += width * 4;
-	}
-
-	return 0;
-}
+
+/* from libvidcap or some odd 
+Based on formulas found at http://en.wikipedia.org/wiki/YUV */
+int rgb32_to_i420(int width, int height, const char * src, char * dst)
+{
+	unsigned char * dst_y_even;
+	unsigned char * dst_y_odd;
+	unsigned char * dst_u;
+	unsigned char * dst_v;
+	const unsigned char *src_even;
+	const unsigned char *src_odd;
+	int i, j;
+
+	src_even = (const unsigned char *)src;
+	src_odd = src_even + width * 4;
+
+	dst_y_even = (unsigned char *)dst;
+	dst_y_odd = dst_y_even + width;
+	dst_u = dst_y_even + width * height;
+	dst_v = dst_u + ((width * height) >> 2);
+
+	for ( i = 0; i < height / 2; ++i )
+	{
+		for ( j = 0; j < width / 2; ++j )
+		{
+			short r, g, b;
+			b = *src_even++;
+			g = *src_even++;
+			r = *src_even++;
+			++src_even;
+			*dst_y_even++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
+
+			*dst_u++ = (( r * -38 - g * 74 + b * 112 + 128 ) >> 8 ) + 128;
+			*dst_v++ = (( r * 112 - g * 94 - b * 18 + 128 ) >> 8 ) + 128;
+
+			b = *src_even++;
+			g = *src_even++;
+			r = *src_even++;
+			++src_even;
+			*dst_y_even++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
+
+			b = *src_odd++;
+			g = *src_odd++;
+			r = *src_odd++;
+			++src_odd;
+			*dst_y_odd++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
+
+			b = *src_odd++;
+			g = *src_odd++;
+			r = *src_odd++;
+			++src_odd;
+			*dst_y_odd++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
+		}
+
+		dst_y_even += width;
+		dst_y_odd += width;
+		src_even += width * 4;
+		src_odd += width * 4;
+	}
+
+	return 0;
+}

source_code/PushSourceDesktop.cpp

@@ -565,15 +565,15 @@ void CPushPinDesktop::CopyScreenToBitmap(HDC hScrDC, LPRECT lpRect, BYTE *pData,
 		tweakableHeader.bmiHeader.biCompression = BI_RGB;
 		tweakableHeader.bmiHeader.biHeight = -tweakableHeader.bmiHeader.biHeight; // prevent upside down...
 	}
-	doDIBits(hScrDC, hRawBitmap, nHeight, pData, &tweakableHeader); // just copies raw bits to pData, I guess, from an HBITMAP handle. "like" GetObject then, but also does conversions.
 
 	if(m_iConvertToI420) {
+		doDIBits(hScrDC, hRawBitmap, nHeight, pOldData, &tweakableHeader); // just copies raw bits to pData, I guess, from an HBITMAP handle. "like" GetObject then, but also does conversions.
 	    //  memcpy(/* dest */ pOldData, pData, pSample->GetSize()); // 12.8ms for 1920x1080 desktop
 		// TODO smarter conversion/memcpy's around here x2[?]
 		// seems to work to read and write from the same buffer...bit scary :P
-		rgb32_to_i420(nWidth, nHeight, (const char *) pData, (char *) pData);// 31ms for 1920x1080 desktop	
+		rgb32_to_i420(nWidth, nHeight, (const char *) pOldData, (char *) pData);// 31ms for 1920x1080 desktop	
 	} else {
-		// nada :P
+	  doDIBits(hScrDC, hRawBitmap, nHeight, pData, &tweakableHeader); // just copies raw bits to pData, I guess, from an HBITMAP handle. "like" GetObject then, but also does conversions.
 	}
 
     // clean up

source_code/PushSourceDesktopAccessories.cpp

@@ -129,9 +129,8 @@ HRESULT CPushPinDesktop::DecideBufferSize(IMemAllocator *pAlloc,
 	// there may be a windows method that would do this for us...GetBitmapSize(&header); but might be too small...
 	// some pasted code...
 	int bytesPerPixel = (header.biBitCount/8);
-
 	if(m_iConvertToI420) {
-		bytesPerPixel = (32/8); // we may need more space everywhere like 32 bits...TODO pData may not actually need to be this big...yikes
+		bytesPerPixel = 32/8; // we convert from 32 bit
 	}
 
     bytesPerLine = header.biWidth * bytesPerPixel;
@@ -146,9 +145,13 @@ HRESULT CPushPinDesktop::DecideBufferSize(IMemAllocator *pAlloc,
 	ASSERT(header.biWidth > 0); // sanity check
 	// NB that we are adding in space for a final "pixel array" (http://en.wikipedia.org/wiki/BMP_file_format#DIB_Header_.28Bitmap_Information_Header.29) even though we typically don't need it, this seems to fix the segfaults
 	// maybe somehow down the line some VLC thing thinks it might be there...weirder than weird.. LODO debug it LOL.
-	pProperties->cbBuffer = 14 + header.biSize + (long)(bytesPerLine)*(header.biHeight) + bytesPerLine*header.biHeight;
+	int bitmapSize = 14 + header.biSize + (long)(bytesPerLine)*(header.biHeight) + bytesPerLine*header.biHeight;
+	pProperties->cbBuffer = bitmapSize;
 	//pProperties->cbBuffer = max(pProperties->cbBuffer, m_mt.GetSampleSize()); // didn't help anything
-
+	if(m_iConvertToI420) {
+	  pProperties->cbBuffer = header.biHeight * header.biWidth*3/2; // necessary to prevent an "out of memory" error for FMLE. Yikes. Oh wow yikes.
+	}
+
     pProperties->cBuffers = 1; // 2 here doesn't seem to help the crashes...
 
 	// pProperties->cbPrefix = 100; // no sure what a prefix even is...setting this didn't help the VLC segfaults anyway :P
@@ -172,7 +175,7 @@ HRESULT CPushPinDesktop::DecideBufferSize(IMemAllocator *pAlloc,
 		free(pOldData);
 		pOldData = NULL;
 	}
-    pOldData =(BYTE *) malloc(pProperties->cbBuffer*pProperties->cBuffers);
+    pOldData = (BYTE *) malloc(max(pProperties->cbBuffer*pProperties->cBuffers, bitmapSize));
     memset(pOldData, 0, pProperties->cbBuffer*pProperties->cBuffers); // reset it just in case :P	
 
     return NOERROR;