Skip to content

Commit

Permalink
[VCM] use adaptive jpg quality instead of 0.9 and report more video f…
Browse files Browse the repository at this point in the history 
…ormat info
  • Loading branch information
@yuyoyuppe
yuyoyuppe committed Apr 12, 2021
1 parent 81e2bbd commit 8318c20
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 52 deletions.
4 changes: 4 additions & 0 deletions src/modules/videoconference/VideoConferenceProxyFilter/DirectShowUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ unique_media_type_ptr CopyMediaType(const AM_MEDIA_TYPE* source)

wil::com_ptr_nothrow<IMemAllocator> GetPinAllocator(wil::com_ptr_nothrow<IPin>& inputPin)
{
if (!inputPin)
{
return nullptr;
}
wil::com_ptr_nothrow<IMemAllocator> allocator;
if (auto memInput = inputPin.try_query<IMemInputPin>(); memInput)
{
Expand Down
56 changes: 31 additions & 25 deletions src/modules/videoconference/VideoConferenceProxyFilter/ImageLoading.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,31 +21,12 @@
#include <mfidl.h>
#include <mftransform.h>
#include <dshow.h>
#include <Wincodecsdk.h>

#include <shlwapi.h>

#include "Logging.h"

bool failed(HRESULT hr)
{
return hr != S_OK;
}

bool failed(bool val)
{
return val == false;
}

template<typename T>
bool failed(wil::com_ptr_nothrow<T>& ptr)
{
return ptr == nullptr;
}

#define OK_OR_BAIL(expr) \
if (failed(expr)) \
return {};

IWICImagingFactory* _GetWIC() noexcept
{
static IWICImagingFactory* s_Factory = nullptr;
Expand Down Expand Up @@ -110,7 +91,8 @@ wil::com_ptr_nothrow<IStream> EncodeBitmapToContainer(IWICImagingFactory* pWIC,
wil::com_ptr_nothrow<IWICBitmapSource> bitmap,
const GUID& containerGUID,
const UINT width,
const UINT height)
const UINT height,
const float quality)
{
wil::com_ptr_nothrow<IWICBitmapEncoder> encoder;
pWIC->CreateEncoder(containerGUID, nullptr, &encoder);
Expand All @@ -125,8 +107,31 @@ wil::com_ptr_nothrow<IStream> EncodeBitmapToContainer(IWICImagingFactory* pWIC,
OK_OR_BAIL(CreateStreamOnHGlobal(nullptr, true, &encodedBitmap));
encoder->Initialize(encodedBitmap.get(), WICBitmapEncoderNoCache);
wil::com_ptr_nothrow<IWICBitmapFrameEncode> encodedFrame;
OK_OR_BAIL(encoder->CreateNewFrame(&encodedFrame, nullptr));
OK_OR_BAIL(encodedFrame->Initialize(nullptr));

wil::com_ptr_nothrow<IPropertyBag2> encoderOptions;
OK_OR_BAIL(encoder->CreateNewFrame(&encodedFrame, &encoderOptions));

ULONG nProperties = 0;
OK_OR_BAIL(encoderOptions->CountProperties(&nProperties));
for (ULONG propIdx = 0; propIdx < nProperties; ++propIdx)
{
PROPBAG2 propBag{};
ULONG _;
OK_OR_BAIL(encoderOptions->GetPropertyInfo(propIdx, 1, &propBag, &_));
if (propBag.pstrName == std::wstring_view{ L"ImageQuality" })
{
wil::unique_variant variant;
variant.vt = VT_R4;
variant.fltVal = quality;
OK_OR_BAIL(encoderOptions->Write(1, &propBag, &variant));
LOG("Successfully set jpg compression quality");
// skip the rest of the properties
propIdx = nProperties;
}
CoTaskMemFree(propBag.pstrName);
}

OK_OR_BAIL(encodedFrame->Initialize(encoderOptions.get()));

WICPixelFormatGUID intermediateFormat = GUID_WICPixelFormat24bppRGB;
OK_OR_BAIL(encodedFrame->SetPixelFormat(&intermediateFormat));
Expand Down Expand Up @@ -245,7 +250,8 @@ IMFSample* ConvertIMFVideoSample(const MFT_REGISTER_TYPE_INFO& inputType,
}

wil::com_ptr_nothrow<IMFSample> LoadImageAsSample(wil::com_ptr_nothrow<IStream> imageStream,
IMFMediaType* sampleMediaType) noexcept
IMFMediaType* sampleMediaType,
const float quality) noexcept
{
UINT targetWidth = 0;
UINT targetHeight = 0;
Expand Down Expand Up @@ -302,7 +308,7 @@ wil::com_ptr_nothrow<IMFSample> LoadImageAsSample(wil::com_ptr_nothrow<IStream>
{
// Use an intermediate jpg container sample which will be transcoded to the target format
wil::com_ptr_nothrow<IStream> jpgStream =
EncodeBitmapToContainer(pWIC, srcImageBitmap, GUID_ContainerFormatJpeg, targetWidth, targetHeight);
EncodeBitmapToContainer(pWIC, srcImageBitmap, GUID_ContainerFormatJpeg, targetWidth, targetHeight, quality);

// Obtain stream size and lock its memory pointer
STATSTG intermediateStreamStat{};
Expand Down
89 changes: 63 additions & 26 deletions src/modules/videoconference/VideoConferenceProxyFilter/VideoCaptureProxyFilter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ wil::com_ptr_nothrow<IMemAllocator> VideoCaptureProxyPin::FindAllocator()
}

wil::com_ptr_nothrow<IMFSample> LoadImageAsSample(wil::com_ptr_nothrow<IStream> imageStream,
IMFMediaType* sampleMediaType) noexcept;
IMFMediaType* sampleMediaType,
const float quality) noexcept;

HRESULT VideoCaptureProxyPin::Connect(IPin* pReceivePin, const AM_MEDIA_TYPE*)
{
Expand Down Expand Up @@ -69,6 +70,7 @@ HRESULT VideoCaptureProxyPin::Connect(IPin* pReceivePin, const AM_MEDIA_TYPE*)
}

auto allocator = FindAllocator();

memInput->NotifyAllocator(allocator.get(), false);

return S_OK;
Expand Down Expand Up @@ -349,6 +351,20 @@ HRESULT VideoCaptureProxyPin::QuerySupported(REFGUID guidPropSet, DWORD dwPropID
return S_OK;
}

long GetImageSize(wil::com_ptr_nothrow<IMFSample>& image)
{
if (!image)
{
return 0;
}
DWORD imageSize = 0;
wil::com_ptr_nothrow<IMFMediaBuffer> imageBuf;

OK_OR_BAIL(image->GetBufferByIndex(0, &imageBuf));
OK_OR_BAIL(imageBuf->GetCurrentLength(&imageSize));
return imageSize;
}

void OverwriteFrame(IMediaSample* frame, wil::com_ptr_nothrow<IMFSample>& image)
{
if (!image)
Expand All @@ -365,18 +381,24 @@ void OverwriteFrame(IMediaSample* frame, wil::com_ptr_nothrow<IMFSample>& image)
}

wil::com_ptr_nothrow<IMFMediaBuffer> imageBuf;
const long nBytes = frame->GetSize();
const DWORD frameSize = frame->GetSize();

image->GetBufferByIndex(0, &imageBuf);
BYTE* imageData = nullptr;
DWORD maxLength = 0, curLength = 0;
imageBuf->Lock(&imageData, &maxLength, &curLength);
std::copy(imageData, imageData + curLength, frameData);
DWORD _ = 0, imageSize = 0;
imageBuf->Lock(&imageData, &_, &imageSize);

if (imageSize > frameSize)
{
LOG("Error: overlay image size is larger than a frame size - truncated.");
imageSize = frameSize;
}

std::copy(imageData, imageData + imageSize, frameData);
imageBuf->Unlock();
frame->SetActualDataLength(curLength);
frame->SetActualDataLength(imageSize);
}


VideoCaptureProxyFilter::VideoCaptureProxyFilter() :
_worker_thread{ std::thread{ [this]() {
using namespace std::chrono_literals;
Expand Down Expand Up @@ -566,12 +588,18 @@ HRESULT VideoCaptureProxyFilter::EnumPins(IEnumPins** ppEnum)
}

auto& webcam = webcams[*selectedCamIdx];

auto pin = winrt::make_self<VideoCaptureProxyPin>();
pin->_mediaFormat = CopyMediaType(webcam.bestFormat.mediaType);
pin->_owningFilter = this;
_outPin.attach(pin.detach());

auto frameCallback = [this](IMediaSample* sample) {
std::unique_lock<std::mutex> lock{ _worker_mutex };
sample->AddRef();
_pending_frame = sample;
_worker_cv.notify_one();
};

wil::com_ptr_nothrow<IMFMediaType> targetMediaType;
MFCreateMediaType(&targetMediaType);
targetMediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
Expand All @@ -582,28 +610,37 @@ HRESULT VideoCaptureProxyFilter::EnumPins(IEnumPins** ppEnum)
targetMediaType.get(), MF_MT_FRAME_SIZE, webcam.bestFormat.width, webcam.bestFormat.height);
MFSetAttributeRatio(targetMediaType.get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1);

if (!_blankImage)
_captureDevice = VideoCaptureDevice::Create(std::move(webcam), std::move(frameCallback));
if (_captureDevice)
{
wil::com_ptr_nothrow<IStream> blackBMPImage = SHCreateMemStream(bmpPixelData, sizeof(bmpPixelData));
_blankImage = LoadImageAsSample(blackBMPImage, targetMediaType.get());
}
if (!_blankImage)
{
wil::com_ptr_nothrow<IStream> blackBMPImage = SHCreateMemStream(bmpPixelData, sizeof(bmpPixelData));
_blankImage = LoadImageAsSample(blackBMPImage, targetMediaType.get(), false);
}

if (newSettings.overlayImage && !_overlayImage)
{
_overlayImage = LoadImageAsSample(newSettings.overlayImage, targetMediaType.get());
}
if (newSettings.overlayImage && !_overlayImage)
{
long maxFrameSize = 0;
ALLOCATOR_PROPERTIES allocatorProperties{};
if (!failed(_captureDevice->_allocator->GetProperties(&allocatorProperties)))
{
maxFrameSize = allocatorProperties.cbBuffer;
}

LOG("Loaded images");
auto frameCallback = [this](IMediaSample* sample) {
std::unique_lock<std::mutex> lock{ _worker_mutex };
sample->AddRef();
_pending_frame = sample;
_worker_cv.notify_one();
};
size_t selectedModeIdx = 0;
constexpr std::array<float, 3> jpgQualityModes = { 0.5f, 0.25f, 0.1f };
_overlayImage = LoadImageAsSample(newSettings.overlayImage, targetMediaType.get(), jpgQualityModes[selectedModeIdx]);
long imageSize = GetImageSize(_overlayImage);
while (maxFrameSize && maxFrameSize < imageSize && selectedModeIdx < size(jpgQualityModes))
{
_overlayImage = LoadImageAsSample(newSettings.overlayImage, targetMediaType.get(), jpgQualityModes[++selectedModeIdx]);
imageSize = GetImageSize(_overlayImage);
}
}

LOG("Loaded images");

_captureDevice = VideoCaptureDevice::Create(std::move(webcam), std::move(frameCallback));
if (_captureDevice)
{
LOG("Capture device created successfully");
}
else
Expand Down
22 changes: 22 additions & 0 deletions src/modules/videoconference/VideoConferenceShared/Logging.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
#include <guiddef.h>
#include <system_error>

#include <wil/com.h>
#include <Windows.h>

void LogToFile(std::string what, const bool verbose = false);
void LogToFile(std::wstring what, const bool verbose = false);
std::string toMediaTypeString(GUID subtype);
Expand Down Expand Up @@ -38,3 +41,22 @@ std::string toMediaTypeString(GUID subtype);
#define LOG(str) LogToFile(str, false);
#endif

inline bool failed(HRESULT hr)
{
return hr != S_OK;
}

inline bool failed(bool val)
{
return val == false;
}

template<typename T>
inline bool failed(wil::com_ptr_nothrow<T>& ptr)
{
return ptr == nullptr;
}

#define OK_OR_BAIL(expr) \
if (failed(expr)) \
return {};
14 changes: 13 additions & 1 deletion tools/WebcamReportTool/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <Shlobj.h>
#include <Shlobj_core.h>
Expand Down Expand Up @@ -90,6 +91,15 @@ std::string GetMediaSubTypeString(const GUID& guid)
}
}

std::string asString(RECT r)
{
std::ostringstream s;
if (r.left == r.right && r.bottom == r.right && r.top == r.right && r.right == 0)
return "[ZEROES]";
s << '(' << r.left << ", " << r.top << ") - (" << r.right << ", " << r.bottom << ")";
return s.str();
}

void LogMediaTypes(wil::com_ptr_nothrow<IPin>& pin)
{
wil::com_ptr_nothrow<IEnumMediaTypes> mediaTypeEnum;
Expand All @@ -113,7 +123,9 @@ void LogMediaTypes(wil::com_ptr_nothrow<IPin>& pin)
}
const auto formatAvgFPS = 10000000LL / format->AvgTimePerFrame;
log() << GetMediaSubTypeString(mt->subtype) << '\t' << format->bmiHeader.biWidth << "x"
<< format->bmiHeader.biHeight << " - " << formatAvgFPS << "fps\n";
<< format->bmiHeader.biHeight << " - " << formatAvgFPS << "fps src rect: " << asString(format->rcSource)
<< " dst rect: " << asString(format->rcSource)
<< " flipped: " << std::boolalpha << (format->bmiHeader.biHeight < 0) << '\n';
}
log() << '\n';
}
Expand Down

1 comment on commit 8318c20

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Misspellings found, please review:

  • boolalpha
  • flt
  • PROPBAG
  • pstr
  • Wincodecsdk
To accept these changes, run the following commands from this repository on this branch 
pushd $(git rev-parse --show-toplevel)
perl -e '
my @expect_files=qw('".github/actions/spell-check/expect.txt"');
@ARGV=@expect_files;
my @stale=qw('"Expr MJPG TRANSCODE "');
my $re=join "|", @stale;
my $suffix=".".time();
my $previous="";
sub maybe_unlink { unlink($_[0]) if $_[0]; }
while (<>) {
  if ($ARGV ne $old_argv) { maybe_unlink($previous); $previous="$ARGV$suffix"; rename($ARGV, $previous); open(ARGV_OUT, ">$ARGV"); select(ARGV_OUT); $old_argv = $ARGV; }
  next if /^(?:$re)(?:(?:\r|\n)*$| .*)/; print;
}; maybe_unlink($previous);'
perl -e '
my $new_expect_file=".github/actions/spell-check/expect.txt";
use File::Path qw(make_path);
make_path ".github/actions/spell-check";
open FILE, q{<}, $new_expect_file; chomp(my @words = <FILE>); close FILE;
my @add=qw('"boolalpha expr flt mjpg PROPBAG pstr Wincodecsdk "');
my %items; @items{@words} = @words x (1); @items{@add} = @add x (1);
@words = sort {lc($a) cmp lc($b)} keys %items;
open FILE, q{>}, $new_expect_file; for my $word (@words) { print FILE "$word\n" if $word =~ /\w/; };
close FILE;'
popd

Please sign in to comment.