Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

hopefully fix out of memory for fmle

  • Loading branch information...
commit 29f22e6f1c20d3a475653673d5a9c1da0f053980 1 parent d200567
@rdp authored
View
124 source_code/DibHelper.cpp
@@ -295,65 +295,65 @@ void GetRectOfWindowIncludingAero(HWND ofThis, RECT *toHere)
}
-
-/* from libvidcap or some odd
-Based on formulas found at http://en.wikipedia.org/wiki/YUV */
-int rgb32_to_i420(int width, int height, const char * src, char * dst)
-{
- unsigned char * dst_y_even;
- unsigned char * dst_y_odd;
- unsigned char * dst_u;
- unsigned char * dst_v;
- const unsigned char *src_even;
- const unsigned char *src_odd;
- int i, j;
-
- src_even = (const unsigned char *)src;
- src_odd = src_even + width * 4;
-
- dst_y_even = (unsigned char *)dst;
- dst_y_odd = dst_y_even + width;
- dst_u = dst_y_even + width * height;
- dst_v = dst_u + ((width * height) >> 2);
-
- for ( i = 0; i < height / 2; ++i )
- {
- for ( j = 0; j < width / 2; ++j )
- {
- short r, g, b;
- b = *src_even++;
- g = *src_even++;
- r = *src_even++;
- ++src_even;
- *dst_y_even++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
-
- *dst_u++ = (( r * -38 - g * 74 + b * 112 + 128 ) >> 8 ) + 128;
- *dst_v++ = (( r * 112 - g * 94 - b * 18 + 128 ) >> 8 ) + 128;
-
- b = *src_even++;
- g = *src_even++;
- r = *src_even++;
- ++src_even;
- *dst_y_even++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
-
- b = *src_odd++;
- g = *src_odd++;
- r = *src_odd++;
- ++src_odd;
- *dst_y_odd++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
-
- b = *src_odd++;
- g = *src_odd++;
- r = *src_odd++;
- ++src_odd;
- *dst_y_odd++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
- }
-
- dst_y_even += width;
- dst_y_odd += width;
- src_even += width * 4;
- src_odd += width * 4;
- }
-
- return 0;
-}
+
+/* from libvidcap or some odd
+Based on formulas found at http://en.wikipedia.org/wiki/YUV */
+int rgb32_to_i420(int width, int height, const char * src, char * dst)
+{
+ unsigned char * dst_y_even;
+ unsigned char * dst_y_odd;
+ unsigned char * dst_u;
+ unsigned char * dst_v;
+ const unsigned char *src_even;
+ const unsigned char *src_odd;
+ int i, j;
+
+ src_even = (const unsigned char *)src;
+ src_odd = src_even + width * 4;
+
+ dst_y_even = (unsigned char *)dst;
+ dst_y_odd = dst_y_even + width;
+ dst_u = dst_y_even + width * height;
+ dst_v = dst_u + ((width * height) >> 2);
+
+ for ( i = 0; i < height / 2; ++i )
+ {
+ for ( j = 0; j < width / 2; ++j )
+ {
+ short r, g, b;
+ b = *src_even++;
+ g = *src_even++;
+ r = *src_even++;
+ ++src_even;
+ *dst_y_even++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
+
+ *dst_u++ = (( r * -38 - g * 74 + b * 112 + 128 ) >> 8 ) + 128;
+ *dst_v++ = (( r * 112 - g * 94 - b * 18 + 128 ) >> 8 ) + 128;
+
+ b = *src_even++;
+ g = *src_even++;
+ r = *src_even++;
+ ++src_even;
+ *dst_y_even++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
+
+ b = *src_odd++;
+ g = *src_odd++;
+ r = *src_odd++;
+ ++src_odd;
+ *dst_y_odd++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
+
+ b = *src_odd++;
+ g = *src_odd++;
+ r = *src_odd++;
+ ++src_odd;
+ *dst_y_odd++ = (( r * 66 + g * 129 + b * 25 + 128 ) >> 8 ) + 16;
+ }
+
+ dst_y_even += width;
+ dst_y_odd += width;
+ src_even += width * 4;
+ src_odd += width * 4;
+ }
+
+ return 0;
+}
View
6 source_code/PushSourceDesktop.cpp
@@ -565,15 +565,15 @@ void CPushPinDesktop::CopyScreenToBitmap(HDC hScrDC, LPRECT lpRect, BYTE *pData,
tweakableHeader.bmiHeader.biCompression = BI_RGB;
tweakableHeader.bmiHeader.biHeight = -tweakableHeader.bmiHeader.biHeight; // prevent upside down...
}
- doDIBits(hScrDC, hRawBitmap, nHeight, pData, &tweakableHeader); // just copies raw bits to pData, I guess, from an HBITMAP handle. "like" GetObject then, but also does conversions.
if(m_iConvertToI420) {
+ doDIBits(hScrDC, hRawBitmap, nHeight, pOldData, &tweakableHeader); // just copies raw bits to pData, I guess, from an HBITMAP handle. "like" GetObject then, but also does conversions.
// memcpy(/* dest */ pOldData, pData, pSample->GetSize()); // 12.8ms for 1920x1080 desktop
// TODO smarter conversion/memcpy's around here x2[?]
// seems to work to read and write from the same buffer...bit scary :P
- rgb32_to_i420(nWidth, nHeight, (const char *) pData, (char *) pData);// 31ms for 1920x1080 desktop
+ rgb32_to_i420(nWidth, nHeight, (const char *) pOldData, (char *) pData);// 31ms for 1920x1080 desktop
} else {
- // nada :P
+ doDIBits(hScrDC, hRawBitmap, nHeight, pData, &tweakableHeader); // just copies raw bits to pData, I guess, from an HBITMAP handle. "like" GetObject then, but also does conversions.
}
// clean up
View
13 source_code/PushSourceDesktopAccessories.cpp
@@ -129,9 +129,8 @@ HRESULT CPushPinDesktop::DecideBufferSize(IMemAllocator *pAlloc,
// there may be a windows method that would do this for us...GetBitmapSize(&header); but might be too small...
// some pasted code...
int bytesPerPixel = (header.biBitCount/8);
-
if(m_iConvertToI420) {
- bytesPerPixel = (32/8); // we may need more space everywhere like 32 bits...TODO pData may not actually need to be this big...yikes
+ bytesPerPixel = 32/8; // we convert from 32 bit
}
bytesPerLine = header.biWidth * bytesPerPixel;
@@ -146,9 +145,13 @@ HRESULT CPushPinDesktop::DecideBufferSize(IMemAllocator *pAlloc,
ASSERT(header.biWidth > 0); // sanity check
// NB that we are adding in space for a final "pixel array" (http://en.wikipedia.org/wiki/BMP_file_format#DIB_Header_.28Bitmap_Information_Header.29) even though we typically don't need it, this seems to fix the segfaults
// maybe somehow down the line some VLC thing thinks it might be there...weirder than weird.. LODO debug it LOL.
- pProperties->cbBuffer = 14 + header.biSize + (long)(bytesPerLine)*(header.biHeight) + bytesPerLine*header.biHeight;
+ int bitmapSize = 14 + header.biSize + (long)(bytesPerLine)*(header.biHeight) + bytesPerLine*header.biHeight;
+ pProperties->cbBuffer = bitmapSize;
//pProperties->cbBuffer = max(pProperties->cbBuffer, m_mt.GetSampleSize()); // didn't help anything
-
+ if(m_iConvertToI420) {
+ pProperties->cbBuffer = header.biHeight * header.biWidth*3/2; // necessary to prevent an "out of memory" error for FMLE. Yikes. Oh wow yikes.
+ }
+
pProperties->cBuffers = 1; // 2 here doesn't seem to help the crashes...
// pProperties->cbPrefix = 100; // no sure what a prefix even is...setting this didn't help the VLC segfaults anyway :P
@@ -172,7 +175,7 @@ HRESULT CPushPinDesktop::DecideBufferSize(IMemAllocator *pAlloc,
free(pOldData);
pOldData = NULL;
}
- pOldData =(BYTE *) malloc(pProperties->cbBuffer*pProperties->cBuffers);
+ pOldData = (BYTE *) malloc(max(pProperties->cbBuffer*pProperties->cBuffers, bitmapSize));
memset(pOldData, 0, pProperties->cbBuffer*pProperties->cBuffers); // reset it just in case :P
return NOERROR;
Please sign in to comment.
Something went wrong with that request. Please try again.