[VCM] reencode jpg frames with WIC

microsoft · Apr 14, 2021 · 616249e · 616249e · github-actions · Apr 14, 2021
1 parent e540e7a
commit 616249e
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 2 deletions.
diff --git a/src/modules/videoconference/VideoConferenceProxyFilter/ImageLoading.cpp b/src/modules/videoconference/VideoConferenceProxyFilter/ImageLoading.cpp
@@ -42,6 +42,58 @@ IWICImagingFactory* _GetWIC() noexcept
     return s_Factory;
 }
 
+bool ReencodeJPGImage(BYTE* imageBuf, const DWORD imageSize, DWORD& reencodedSize)
+{
+    auto pWIC = _GetWIC();
+    wil::com_ptr_nothrow<IStream> imageStream = SHCreateMemStream(imageBuf, imageSize);
+    if (!imageStream)
+    {
+        return false;
+    }
+
+    // Decode jpg into bitmap
+    wil::com_ptr_nothrow<IWICBitmapDecoder> bitmapDecoder;
+    OK_OR_BAIL(pWIC->CreateDecoderFromStream(imageStream.get(), nullptr, WICDecodeMetadataCacheOnLoad, &bitmapDecoder));
+    wil::com_ptr_nothrow<IWICBitmapFrameDecode> decodedFrame;
+    OK_OR_BAIL(bitmapDecoder->GetFrame(0, &decodedFrame));
+    wil::com_ptr_nothrow<IWICBitmapSource> bitmap;
+    bitmap.attach(decodedFrame.detach());
+    UINT width = 0, height = 0;
+    OK_OR_BAIL(bitmap->GetSize(&width, &height));
+
+    // Initialize jpg encoder
+    wil::com_ptr_nothrow<IWICBitmapEncoder> encoder;
+    OK_OR_BAIL(pWIC->CreateEncoder(GUID_ContainerFormatJpeg, nullptr, &encoder));
+
+    wil::com_ptr_nothrow<IStream> outputStream;
+    OK_OR_BAIL(CreateStreamOnHGlobal(nullptr, true, &outputStream));
+    OK_OR_BAIL(encoder->Initialize(outputStream.get(), WICBitmapEncoderNoCache));
+    wil::com_ptr_nothrow<IWICBitmapFrameEncode> encodedFrame;
+    OK_OR_BAIL(encoder->CreateNewFrame(&encodedFrame, nullptr));
+    OK_OR_BAIL(encodedFrame->Initialize(nullptr));
+    WICPixelFormatGUID intermediateFormat = GUID_WICPixelFormat24bppRGB;
+
+    OK_OR_BAIL(encodedFrame->SetPixelFormat(&intermediateFormat));
+    OK_OR_BAIL(encodedFrame->SetSize(width, height));
+
+    // Commit the image encoding
+    OK_OR_BAIL(encodedFrame->WriteSource(bitmap.get(), nullptr));
+    OK_OR_BAIL(encodedFrame->Commit());
+    OK_OR_BAIL(encoder->Commit());
+
+    STATSTG intermediateStreamStat{};
+    OK_OR_BAIL(outputStream->Stat(&intermediateStreamStat, STATFLAG_NONAME));
+    const ULONGLONG jpgStreamSize = intermediateStreamStat.cbSize.QuadPart;
+    HGLOBAL streamMemoryHandle{};
+    OK_OR_BAIL(GetHGlobalFromStream(outputStream.get(), &streamMemoryHandle));
+
+    auto jpgStreamMemory = static_cast<uint8_t*>(GlobalLock(streamMemoryHandle));
+    std::copy(jpgStreamMemory, jpgStreamMemory + jpgStreamSize, imageBuf);
+    auto unlockJpgStreamMemory = wil::scope_exit([jpgStreamMemory] { GlobalUnlock(jpgStreamMemory); });
+    reencodedSize = (DWORD)jpgStreamSize;
+    return true;
+}
+
 wil::com_ptr_nothrow<IWICBitmapSource> LoadAsRGB24BitmapWithSize(IWICImagingFactory* pWIC,
                                                                  wil::com_ptr_nothrow<IStream> image,
                                                                  const UINT targetWidth,
@@ -105,7 +157,7 @@ wil::com_ptr_nothrow<IStream> EncodeBitmapToContainer(IWICImagingFactory* pWIC,
     // Prepare the encoder output memory stream and encoding params
     wil::com_ptr_nothrow<IStream> encodedBitmap;
     OK_OR_BAIL(CreateStreamOnHGlobal(nullptr, true, &encodedBitmap));
-    encoder->Initialize(encodedBitmap.get(), WICBitmapEncoderNoCache);
+    OK_OR_BAIL(encoder->Initialize(encodedBitmap.get(), WICBitmapEncoderNoCache));
     wil::com_ptr_nothrow<IWICBitmapFrameEncode> encodedFrame;
 
     wil::com_ptr_nothrow<IPropertyBag2> encoderOptions;

diff --git a/src/modules/videoconference/VideoConferenceProxyFilter/VideoCaptureProxyFilter.cpp b/src/modules/videoconference/VideoConferenceProxyFilter/VideoCaptureProxyFilter.cpp
@@ -39,6 +39,7 @@ wil::com_ptr_nothrow<IMemAllocator> VideoCaptureProxyPin::FindAllocator()
 wil::com_ptr_nothrow<IMFSample> LoadImageAsSample(wil::com_ptr_nothrow<IStream> imageStream,
                                                   IMFMediaType* sampleMediaType,
                                                   const float quality) noexcept;
+bool ReencodeJPGImage(BYTE* imageBuf, const DWORD imageSize, DWORD& reencodedSize);
 
 HRESULT VideoCaptureProxyPin::Connect(IPin* pReceivePin, const AM_MEDIA_TYPE*)
 {
@@ -387,6 +388,25 @@ long GetImageSize(wil::com_ptr_nothrow<IMFSample>& image)
     return imageSize;
 }
 
+void ReencodeFrame(IMediaSample* frame)
+{
+    BYTE* frameData = nullptr;
+    frame->GetPointer(&frameData);
+    if (!frameData)
+    {
+        LOG("VideoCaptureProxyPin::ReencodeFrame FAILED frameData");
+        return;
+    }
+    const DWORD frameSize = frame->GetSize();
+    DWORD reencodedSize = 0;
+    if (!ReencodeJPGImage(frameData, frameSize, reencodedSize))
+    {
+        LOG("VideoCaptureProxyPin::ReencodeJPGImage FAILED");
+        return;
+    }
+    frame->SetActualDataLength(reencodedSize);
+}
+
 bool OverwriteFrame(IMediaSample* frame, wil::com_ptr_nothrow<IMFSample>& image)
 {
     if (!image)
@@ -565,9 +585,18 @@ VideoCaptureProxyFilter::VideoCaptureProxyFilter() :
                             OverwriteFrame(_pending_frame, _blankImage);
                         }
 #else
-                      DebugOverwriteFrame(_pending_frame, "R:\\frame.data");
+                        DebugOverwriteFrame(_pending_frame, "R:\\frame.data");
 #endif
                     }
+                    else
+                    {
+                        GUID subtype{};
+                        _targetMediaType->GetGUID(MF_MT_SUBTYPE, &subtype);
+                        if (subtype == MFVideoFormat_MJPG)
+                        {
+                            ReencodeFrame(_pending_frame);
+                        }
+                    }
 
                     _pending_frame = nullptr;
                     input->Receive(sample);