Skip to content
Permalink
Browse files
REGRESSION(r294104): [GStreamer] getUserMedia broken 
https://bugs.webkit.org/show_bug.cgi?id=240420

Patch by Philippe Normand <pnormand@igalia.com> on 2022-05-16
Reviewed by Xabier Rodriguez-Calvar.

The converter handling logic was modified in order to fix getUserMedia negotiated with raw
video and also getDisplayMedia which is always raw video and thus doesn't require decoding.

This patch also introduces a small optimization, reconfiguration is now done once only,
after setting size and framerate. Before this patch it was done twice, so the pipeline was a
taking more time to produce the first frame.

* platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp:
(WebCore::GStreamerVideoCaptureSource::~GStreamerVideoCaptureSource):
(WebCore::GStreamerVideoCaptureSource::settingsDidChange): Trigger capturer reconfiguration after setting both size and framerate.
(WebCore::GStreamerVideoCaptureSource::startProducingData): Ditto.
* platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp:
(WebCore::GStreamerVideoCapturer::createConverter): Do not decode display capture streams, these are always raw anyway.
(WebCore::GStreamerVideoCapturer::setSize): Delay reconfiguration.
(WebCore::GStreamerVideoCapturer::setFrameRate): Ditto.
(WebCore::GStreamerVideoCapturer::reconfigure): Keep track of compatible video format. This
is needed to workaround an issue in pipewiresrc caps negotiation.
(WebCore::GStreamerVideoCapturer::adjustVideoSrcMIMEType): Renamed to reconfigure().
* platform/mediastream/gstreamer/GStreamerVideoCapturer.h:

Canonical link: https://commits.webkit.org/250587@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@294229 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
@philn @webkit-commit-queue
philn authored and webkit-commit-queue committed May 16, 2022
1 parent d4e6307 commit 5e80996639ec4af8fe9a0828feda7f2b374f9106
Showing 4 changed files with 92 additions and 27 deletions.
27 Source/WebCore/ChangeLog
View file
@@ -1,3 +1,30 @@
2022-05-15 Philippe Normand <philn@igalia.com>

REGRESSION(r294104): [GStreamer] getUserMedia broken
https://bugs.webkit.org/show_bug.cgi?id=240420

Reviewed by Xabier Rodriguez-Calvar.

The converter handling logic was modified in order to fix getUserMedia negotiated with raw
video and also getDisplayMedia which is always raw video and thus doesn't require decoding.

This patch also introduces a small optimization, reconfiguration is now done once only,
after setting size and framerate. Before this patch it was done twice, so the pipeline was a
taking more time to produce the first frame.

* platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp:
(WebCore::GStreamerVideoCaptureSource::~GStreamerVideoCaptureSource):
(WebCore::GStreamerVideoCaptureSource::settingsDidChange): Trigger capturer reconfiguration after setting both size and framerate.
(WebCore::GStreamerVideoCaptureSource::startProducingData): Ditto.
* platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp:
(WebCore::GStreamerVideoCapturer::createConverter): Do not decode display capture streams, these are always raw anyway.
(WebCore::GStreamerVideoCapturer::setSize): Delay reconfiguration.
(WebCore::GStreamerVideoCapturer::setFrameRate): Ditto.
(WebCore::GStreamerVideoCapturer::reconfigure): Keep track of compatible video format. This
is needed to workaround an issue in pipewiresrc caps negotiation.
(WebCore::GStreamerVideoCapturer::adjustVideoSrcMIMEType): Renamed to reconfigure().
* platform/mediastream/gstreamer/GStreamerVideoCapturer.h:

2022-05-15 Philippe Normand <philn@igalia.com>

[GStreamer] Add basic video meta handling in sinks
5 Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp
View file
@@ -145,7 +145,7 @@ GStreamerVideoCaptureSource::~GStreamerVideoCaptureSource()
g_signal_handlers_disconnect_by_func(m_capturer->sink(), reinterpret_cast<gpointer>(newSampleCallback), this);
m_capturer->stop();

if (m_capturer->feedingFromPipewire()) {
if (m_capturer->isCapturingDisplay()) {
auto& manager = GStreamerDisplayCaptureDeviceManager::singleton();
manager.stopSource(persistentID());
}
@@ -162,6 +162,8 @@ void GStreamerVideoCaptureSource::settingsDidChange(OptionSet<RealtimeMediaSourc

if (settings.contains(RealtimeMediaSourceSettings::Flag::FrameRate))
m_capturer->setFrameRate(frameRate());

m_capturer->reconfigure();
}

void GStreamerVideoCaptureSource::sourceCapsChanged(const GstCaps* caps)
@@ -186,6 +188,7 @@ void GStreamerVideoCaptureSource::startProducingData()
m_capturer->setSize(size().width(), size().height());

m_capturer->setFrameRate(frameRate());
m_capturer->reconfigure();
g_signal_connect(m_capturer->sink(), "new-sample", G_CALLBACK(newSampleCallback), this);
m_capturer->play();
}
70 Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp
View file
@@ -67,16 +67,43 @@ GstElement* GStreamerVideoCapturer::createSource()

GstElement* GStreamerVideoCapturer::createConverter()
{
auto* bin = gst_bin_new(nullptr);
auto* videoscale = gst_element_factory_make("videoscale", "videoscale");
auto* videoconvert = gst_element_factory_make("videoconvert", nullptr);
auto* videorate = gst_element_factory_make("videorate", "videorate");

// https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/97#note_56575
auto bin = makeGStreamerBin("capsfilter caps=\"video/x-raw\" name=mimetype-filter ! decodebin3 ! videoscale ! videoconvert ! videorate drop-only=1 average-period=1 name=videorate", false);
g_object_set(videorate, "drop-only", 1, "average-period", 1, nullptr);

gst_bin_add_many(GST_BIN_CAST(bin), videoscale, videoconvert, videorate, nullptr);

GstElement* head = videoscale;
if (!isCapturingDisplay()) {
m_videoSrcMIMETypeFilter = gst_element_factory_make("capsfilter", "mimetype-filter");
head = m_videoSrcMIMETypeFilter.get();

m_videoSrcMIMETypeFilter = adoptGRef(gst_bin_get_by_name(GST_BIN(bin), "mimetype-filter"));
auto videorate = adoptGRef(gst_bin_get_by_name(GST_BIN(bin), "videorate"));
auto caps = adoptGRef(gst_caps_new_empty_simple("video/x-raw"));
g_object_set(m_videoSrcMIMETypeFilter.get(), "caps", caps.get(), nullptr);

RELEASE_ASSERT(gst_element_add_pad(bin, gst_ghost_pad_new("sink", GST_PAD(m_videoSrcMIMETypeFilter->sinkpads->data))));
RELEASE_ASSERT(gst_element_add_pad(bin, gst_ghost_pad_new("src", GST_PAD(videorate->srcpads->data))));
auto* decodebin = gst_element_factory_make("decodebin3", nullptr);
gst_bin_add_many(GST_BIN_CAST(bin), m_videoSrcMIMETypeFilter.get(), decodebin, nullptr);
gst_element_link(m_videoSrcMIMETypeFilter.get(), decodebin);

auto sinkPad = adoptGRef(gst_element_get_static_pad(videoscale, "sink"));
g_signal_connect_swapped(decodebin, "pad-added", G_CALLBACK(+[](GstPad* sinkPad, GstPad* srcPad) {
RELEASE_ASSERT(!gst_pad_is_linked(sinkPad));
gst_pad_link(srcPad, sinkPad);
}), sinkPad.get());
}

gst_element_link_many(videoscale, videoconvert, videorate, nullptr);

auto sinkPad = adoptGRef(gst_element_get_static_pad(head, "sink"));
gst_element_add_pad(bin, gst_ghost_pad_new("sink", sinkPad.get()));

auto srcPad = adoptGRef(gst_element_get_static_pad(videorate, "src"));
gst_element_add_pad(bin, gst_ghost_pad_new("src", srcPad.get()));

adjustVideoSrcMIMEType();
return bin;
}

@@ -91,7 +118,7 @@ GstVideoInfo GStreamerVideoCapturer::getBestFormat()

bool GStreamerVideoCapturer::setSize(int width, int height)
{
if (feedingFromPipewire()) {
if (isCapturingDisplay()) {
// Pipewiresrc doesn't seem to support caps re-negotiation and framerate configuration properly.
GST_FIXME_OBJECT(m_pipeline.get(), "Resizing disabled on display capture source");
return true;
@@ -110,8 +137,6 @@ bool GStreamerVideoCapturer::setSize(int width, int height)
m_caps = adoptGRef(gst_caps_copy(m_caps.get()));
gst_caps_set_simple(m_caps.get(), "width", G_TYPE_INT, width, "height", G_TYPE_INT, height, nullptr);

adjustVideoSrcMIMEType();

if (!m_capsfilter)
return false;

@@ -121,7 +146,7 @@ bool GStreamerVideoCapturer::setSize(int width, int height)

bool GStreamerVideoCapturer::setFrameRate(double frameRate)
{
if (feedingFromPipewire()) {
if (isCapturingDisplay()) {
// Pipewiresrc doesn't seem to support caps re-negotiation and framerate configuration properly.
GST_FIXME_OBJECT(m_pipeline.get(), "Framerate override disabled on display capture source");
return true;
@@ -144,8 +169,6 @@ bool GStreamerVideoCapturer::setFrameRate(double frameRate)
m_caps = adoptGRef(gst_caps_copy(m_caps.get()));
gst_caps_set_simple(m_caps.get(), "framerate", GST_TYPE_FRACTION, numerator, denominator, nullptr);

adjustVideoSrcMIMEType();

if (!m_capsfilter)
return false;

@@ -234,14 +257,20 @@ static std::optional<double> getMaxFractionValueFromStructure(const GstStructure
return (maxFraction > -G_MAXDOUBLE) ? std::make_optional<>(maxFraction) : std::nullopt;
}

void GStreamerVideoCapturer::adjustVideoSrcMIMEType()
void GStreamerVideoCapturer::reconfigure()
{
if (isCapturingDisplay()) {
// Pipewiresrc doesn't seem to support caps re-negotiation and framerate configuration properly.
GST_FIXME_OBJECT(m_pipeline.get(), "Caps re-negotiation disabled on display capture source");
return;
}

if (!m_videoSrcMIMETypeFilter)
return;

struct MimeTypeSelector {
const char* mimeType = "video/x-raw";

const char* format = nullptr;
int maxWidth = 0;
int maxHeight = 0;
double maxFrameRate = 0;
@@ -289,6 +318,7 @@ void GStreamerVideoCapturer::adjustVideoSrcMIMEType()
if (*width >= selector->stopCondition.width && *height >= selector->stopCondition.height
&& *frameRate >= selector->stopCondition.frameRate) {
selector->mimeType = gst_structure_get_name(structure);
selector->format = gst_structure_get_string(structure, "format");
return FALSE;
}

@@ -297,15 +327,19 @@ void GStreamerVideoCapturer::adjustVideoSrcMIMEType()
selector->maxHeight = *height;
selector->maxFrameRate = *frameRate;
selector->mimeType = gst_structure_get_name(structure);
selector->format = gst_structure_get_string(structure, "format");
}

return TRUE;
}),
&selector);

GST_INFO_OBJECT(m_pipeline.get(), "Setting video capture device mime type to %s", selector.mimeType);
}), &selector);

auto caps = adoptGRef(gst_caps_new_empty_simple(selector.mimeType));

// Workaround for https://gitlab.freedesktop.org/pipewire/pipewire/-/issues/1793.
if (selector.format)
gst_caps_set_simple(caps.get(), "format", G_TYPE_STRING, selector.format, nullptr);

GST_INFO_OBJECT(m_pipeline.get(), "Setting video capture device caps to %" GST_PTR_FORMAT, caps.get());
g_object_set(m_videoSrcMIMETypeFilter.get(), "caps", caps.get(), nullptr);
}

17 Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.h
View file
@@ -26,33 +26,34 @@

#include "GStreamerCapturer.h"

#include <gst/video/video.h>

namespace WebCore {

class GStreamerVideoCapturer final : public GStreamerCapturer {
friend class GStreamerVideoCaptureSource;
public:
GStreamerVideoCapturer(GStreamerCaptureDevice);
GStreamerVideoCapturer(const char* source_factory, CaptureDevice::DeviceType);
GStreamerVideoCapturer(const char* sourceFactory, CaptureDevice::DeviceType);

GstElement* createSource() final;
GstElement* createConverter() final;
const char* name() final { return "Video"; }

using NodeAndFD = std::pair<uint32_t, int>;

private:
bool setSize(int width, int height);
bool setFrameRate(double);
GstVideoInfo getBestFormat();
void reconfigure();

using NodeAndFD = std::pair<uint32_t, int>;
GstVideoInfo getBestFormat();

void setPipewireNodeAndFD(const NodeAndFD& nodeAndFd) { m_nodeAndFd = nodeAndFd; }
bool feedingFromPipewire() const { return m_nodeAndFd.has_value(); }
bool isCapturingDisplay() const { return m_nodeAndFd.has_value(); }

private:
std::optional<NodeAndFD> m_nodeAndFd;
GRefPtr<GstElement> m_videoSrcMIMETypeFilter;
void adjustVideoSrcMIMEType();
};

} // namespace WebCore

#endif // ENABLE(MEDIA_STREAM) && USE(GSTREAMER)

0 comments on commit 5e80996

Please sign in to comment.