[GTK][WPE][WebRTC] webrtc/h264-baseline.html is crashing since added …

…in r265005

https://bugs.webkit.org/show_bug.cgi?id=215005

Reviewed by Xabier Rodriguez-Calvar.

The test was failing because the outgoing video track size updates were not properly applied to the
capturer. Video size constraints are now properly propagated from the video capture source to the
underlying GStreamerVideoCapturer.

* LayoutTests/platform/glib/TestExpectations:
* Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.cpp:
(WebCore::VideoFrameGStreamer::createFromPixelBuffer):
* Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.cpp:
(WebCore::GStreamerCapturer::setupPipeline):
* Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp:
(WebCore::GStreamerVideoCaptureSource::settingsDidChange):
(WebCore::GStreamerVideoCaptureSource::startProducingData):
(WebCore::GStreamerVideoCaptureSource::setSizeFrameRateAndZoom):
* Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h:
* Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp:
(WebCore::GStreamerVideoCapturer::setSize):
(WebCore::GStreamerVideoCapturer::setFrameRate):
* Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.h:
* Source/WebCore/platform/mediastream/gstreamer/MockDisplayCaptureSourceGStreamer.cpp:
(WebCore::MockDisplayCaptureSourceGStreamer::create):
* Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.cpp:
(WebCore::MockRealtimeVideoSource::create):
(WebCore::MockRealtimeVideoSourceGStreamer::MockRealtimeVideoSourceGStreamer):
(WebCore::MockRealtimeVideoSourceGStreamer::startProducingData):
(WebCore::MockRealtimeVideoSourceGStreamer::updateSampleBuffer):
(WebCore::MockRealtimeVideoSourceGStreamer::setSizeFrameRateAndZoom):
* Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.h:
* Source/WebCore/platform/mock/MockRealtimeVideoSource.h:

Canonical link: https://commits.webkit.org/274812@main

LayoutTests/platform/glib/TestExpectations

@@ -2067,8 +2067,6 @@ webkit.org/b/235885 imported/w3c/web-platform-tests/webrtc/RTCRtpReceiver-getSyn
 webkit.org/b/235885 imported/w3c/web-platform-tests/webrtc/protocol/rtp-clockrate.html [ Skip ]
 webkit.org/b/235885 imported/w3c/web-platform-tests/webrtc-extensions/RTCRtpSynchronizationSource-senderCaptureTimeOffset.html [ Skip ]
 
-# GStreamerRtpSenderBackend::setParameters() unimplemented.
-webkit.org/b/215005 webkit.org/b/218787 webrtc/h264-baseline.html [ Failure Timeout ]
 webkit.org/b/215007 webrtc/h264-high.html [ Failure Timeout ]
 
 # We don't support spatial encoding yet.

Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.cpp

@@ -303,11 +303,13 @@ RefPtr<VideoFrameGStreamer> VideoFrameGStreamer::createFromPixelBuffer(Ref<Pixel
         }
 
         auto outputBuffer = gst_sample_get_buffer(sample.get());
+        gst_buffer_add_video_meta(outputBuffer, GST_VIDEO_FRAME_FLAG_NONE, format, width, height);
         if (metadata)
             webkitGstBufferSetVideoFrameTimeMetadata(outputBuffer, *metadata);
 
         setBufferFields(outputBuffer, presentationTime, frameRate);
     } else {
+        gst_buffer_add_video_meta(buffer.get(), GST_VIDEO_FRAME_FLAG_NONE, format, width, height);
         if (metadata)
             buffer = webkitGstBufferSetVideoFrameTimeMetadata(buffer.get(), *metadata);
 

Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.cpp

@@ -197,6 +197,8 @@ void GStreamerCapturer::setupPipeline()
     m_capsfilter = makeElement("capsfilter");
     m_sink = makeElement("appsink");
 
+    gst_util_set_object_arg(G_OBJECT(m_capsfilter.get()), "caps-change-mode", "delayed");
+
     gst_app_sink_set_emit_signals(GST_APP_SINK(m_sink.get()), TRUE);
     g_object_set(m_sink.get(), "enable-last-sample", FALSE, nullptr);
     g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr);

Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp

@@ -153,7 +153,7 @@ void GStreamerVideoCaptureSource::settingsDidChange(OptionSet<RealtimeMediaSourc
         if (m_deviceType == CaptureDevice::DeviceType::Window || m_deviceType == CaptureDevice::DeviceType::Screen)
             ensureIntrinsicSizeMaintainsAspectRatio();
 
-        if (m_capturer->setSize(size().width(), size().height()))
+        if (m_capturer->setSize(size()))
             reconfigure = true;
     }
 
@@ -186,7 +186,7 @@ void GStreamerVideoCaptureSource::startProducingData()
     m_capturer->setupPipeline();
 
     if (m_deviceType == CaptureDevice::DeviceType::Camera)
-        m_capturer->setSize(size().width(), size().height());
+        m_capturer->setSize(size());
 
     m_capturer->setFrameRate(frameRate());
     m_capturer->reconfigure();
@@ -304,6 +304,15 @@ void GStreamerVideoCaptureSource::generatePresets()
     setSupportedPresets(WTFMove(presets));
 }
 
+void GStreamerVideoCaptureSource::setSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double> frameRate, std::optional<double> zoom)
+{
+    RealtimeVideoCaptureSource::setSizeFrameRateAndZoom(width, height, frameRate, zoom);
+    if (!width || !height)
+        return;
+
+    m_capturer->setSize({ *width, *height });
+}
+
 #undef GST_CAT_DEFAULT
 
 } // namespace WebCore

Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h

@@ -58,7 +58,7 @@ class GStreamerVideoCaptureSource : public RealtimeVideoCaptureSource, GStreamer
     void stopProducingData() override;
     bool canResizeVideoFrames() const final { return true; }
     void generatePresets() override;
-
+    void setSizeFrameRateAndZoom(std::optional<int>, std::optional<int>, std::optional<double>, std::optional<double>) override;
 
     mutable std::optional<RealtimeMediaSourceCapabilities> m_capabilities;
     mutable std::optional<RealtimeMediaSourceSettings> m_currentSettings;

Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp

@@ -124,14 +124,16 @@ GstElement* GStreamerVideoCapturer::createConverter()
     return bin;
 }
 
-bool GStreamerVideoCapturer::setSize(int width, int height)
+bool GStreamerVideoCapturer::setSize(const IntSize& size)
 {
     if (isCapturingDisplay()) {
         // Pipewiresrc doesn't seem to support caps re-negotiation and framerate configuration properly.
         GST_FIXME_OBJECT(m_pipeline.get(), "Resizing disabled on display capture source");
         return true;
     }
 
+    int width = size.width();
+    int height = size.height();
     if (!width || !height)
         return false;
 
@@ -141,13 +143,14 @@ bool GStreamerVideoCapturer::setSize(int width, int height)
         return true;
     }
 
+    if (UNLIKELY(!m_capsfilter))
+        return false;
+
     GST_INFO_OBJECT(m_pipeline.get(), "Setting size to %dx%d", width, height);
+    m_size = size;
     m_caps = adoptGRef(gst_caps_copy(m_caps.get()));
     gst_caps_set_simple(m_caps.get(), "width", G_TYPE_INT, width, "height", G_TYPE_INT, height, nullptr);
 
-    if (!m_capsfilter)
-        return false;
-
     g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr);
     return true;
 }
@@ -161,7 +164,6 @@ bool GStreamerVideoCapturer::setFrameRate(double frameRate)
     }
 
     int numerator, denominator;
-
     gst_util_double_to_fraction(frameRate, &numerator, &denominator);
 
     if (numerator < -G_MAXINT) {
@@ -174,15 +176,14 @@ bool GStreamerVideoCapturer::setFrameRate(double frameRate)
         return false;
     }
 
+    if (UNLIKELY(!m_capsfilter))
+        return false;
+
     m_caps = adoptGRef(gst_caps_copy(m_caps.get()));
     gst_caps_set_simple(m_caps.get(), "framerate", GST_TYPE_FRACTION, numerator, denominator, nullptr);
 
-    if (!m_capsfilter)
-        return false;
-
     GST_INFO_OBJECT(m_pipeline.get(), "Setting framerate to %f fps", frameRate);
     g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr);
-
     return true;
 }
 

Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.h

@@ -48,7 +48,8 @@ class GStreamerVideoCapturer final : public GStreamerCapturer {
     void setSinkVideoFrameCallback(SinkVideoFrameCallback&&);
 
 private:
-    bool setSize(int width, int height);
+    bool setSize(const IntSize&);
+    const IntSize& size() const { return m_size; }
     bool setFrameRate(double);
     void reconfigure();
 
@@ -60,6 +61,7 @@ class GStreamerVideoCapturer final : public GStreamerCapturer {
     std::optional<NodeAndFD> m_nodeAndFd;
     GRefPtr<GstElement> m_videoSrcMIMETypeFilter;
     std::pair<unsigned long, SinkVideoFrameCallback> m_sinkVideoFrameCallback;
+    IntSize m_size;
 };
 
 } // namespace WebCore

Source/WebCore/platform/mediastream/gstreamer/MockDisplayCaptureSourceGStreamer.cpp

@@ -30,7 +30,7 @@ namespace WebCore {
 
 CaptureSourceOrError MockDisplayCaptureSourceGStreamer::create(const CaptureDevice& device, MediaDeviceHashSalts&& hashSalts, const MediaConstraints* constraints, PageIdentifier pageIdentifier)
 {
-    auto mockSource = adoptRef(*new MockRealtimeVideoSourceGStreamer(String { device.persistentId() }, AtomString { device.label() }, MediaDeviceHashSalts { hashSalts }));
+    auto mockSource = adoptRef(*new MockRealtimeVideoSourceGStreamer(String { device.persistentId() }, AtomString { device.label() }, MediaDeviceHashSalts { hashSalts }, pageIdentifier));
 
     if (constraints) {
         if (auto error = mockSource->applyConstraints(*constraints))

Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.cpp

@@ -34,7 +34,7 @@
 
 namespace WebCore {
 
-CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&& hashSalts, const MediaConstraints* constraints, PageIdentifier)
+CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&& hashSalts, const MediaConstraints* constraints, PageIdentifier pageIdentifier)
 {
 #ifndef NDEBUG
     auto device = MockRealtimeMediaSourceCenter::mockDeviceWithPersistentID(deviceID);
@@ -43,7 +43,7 @@ CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, AtomStri
         return CaptureSourceOrError({ "No mock camera device"_s , MediaAccessDenialReason::PermissionDenied });
 #endif
 
-    Ref<RealtimeMediaSource> source = adoptRef(*new MockRealtimeVideoSourceGStreamer(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalts)));
+    Ref<RealtimeMediaSource> source = adoptRef(*new MockRealtimeVideoSourceGStreamer(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalts), pageIdentifier));
     if (constraints) {
         if (auto error = source->applyConstraints(*constraints))
             return CaptureSourceOrError(CaptureSourceError { error->invalidConstraint });
@@ -52,8 +52,8 @@ CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, AtomStri
     return source;
 }
 
-MockRealtimeVideoSourceGStreamer::MockRealtimeVideoSourceGStreamer(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&& hashSalts)
-    : MockRealtimeVideoSource(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalts), { })
+MockRealtimeVideoSourceGStreamer::MockRealtimeVideoSourceGStreamer(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&& hashSalts, PageIdentifier pageIdentifier)
+    : MockRealtimeVideoSource(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalts), pageIdentifier)
 {
     ensureGStreamerInitialized();
     auto& singleton = GStreamerVideoCaptureDeviceManager::singleton();
@@ -86,7 +86,7 @@ MockRealtimeVideoSourceGStreamer::~MockRealtimeVideoSourceGStreamer()
 void MockRealtimeVideoSourceGStreamer::startProducingData()
 {
     if (deviceType() == CaptureDevice::DeviceType::Camera)
-        m_capturer->setSize(size().width(), size().height());
+        m_capturer->setSize(size());
 
     m_capturer->setFrameRate(frameRate());
     m_capturer->start();
@@ -108,7 +108,7 @@ void MockRealtimeVideoSourceGStreamer::captureEnded()
 
 void MockRealtimeVideoSourceGStreamer::updateSampleBuffer()
 {
-    auto imageBuffer = this->imageBufferInternal();
+    RefPtr imageBuffer = this->imageBufferInternal();
     if (!imageBuffer)
         return;
 
@@ -119,7 +119,7 @@ void MockRealtimeVideoSourceGStreamer::updateSampleBuffer()
     VideoFrameTimeMetadata metadata;
     metadata.captureTime = MonotonicTime::now().secondsSinceEpoch();
     auto presentationTime = MediaTime::createWithDouble((elapsedTime()).seconds());
-    auto videoFrame = VideoFrameGStreamer::createFromPixelBuffer(pixelBuffer.releaseNonNull(), VideoFrameGStreamer::CanvasContentType::Canvas2D, videoFrameRotation(), presentationTime, size(), frameRate(), false, WTFMove(metadata));
+    auto videoFrame = VideoFrameGStreamer::createFromPixelBuffer(pixelBuffer.releaseNonNull(), VideoFrameGStreamer::CanvasContentType::Canvas2D, videoFrameRotation(), presentationTime, m_capturer->size(), frameRate(), false, WTFMove(metadata));
     if (!videoFrame)
         return;
 
@@ -128,6 +128,16 @@ void MockRealtimeVideoSourceGStreamer::updateSampleBuffer()
     gst_app_src_push_sample(GST_APP_SRC_CAST(m_capturer->source()), videoFrame->sample());
 }
 
+void MockRealtimeVideoSourceGStreamer::setSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double> frameRate, std::optional<double> zoom)
+{
+    MockRealtimeVideoSource::setSizeFrameRateAndZoom(width, height, frameRate, zoom);
+
+    if (!width || !height)
+        return;
+
+    m_capturer->setSize({ *width, *height });
+}
+
 } // namespace WebCore
 
 #endif // ENABLE(MEDIA_STREAM) && USE(GSTREAMER)

Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.h

@@ -31,7 +31,7 @@ namespace WebCore {
 
 class MockRealtimeVideoSourceGStreamer final : public MockRealtimeVideoSource, GStreamerCapturer::Observer {
 public:
-    MockRealtimeVideoSourceGStreamer(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&&);
+    MockRealtimeVideoSourceGStreamer(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&&, PageIdentifier);
     ~MockRealtimeVideoSourceGStreamer();
 
     // GStreamerCapturer::Observer
@@ -44,6 +44,8 @@ class MockRealtimeVideoSourceGStreamer final : public MockRealtimeVideoSource, G
     void stopProducingData() final;
     void updateSampleBuffer() final;
     bool canResizeVideoFrames() const final { return true; }
+    void setSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double>, std::optional<double>) final;
+
     RefPtr<GStreamerVideoCapturer> m_capturer;
 };
 

Source/WebCore/platform/mock/MockRealtimeVideoSource.h

@@ -86,7 +86,7 @@ class MockRealtimeVideoSource : public RealtimeVideoCaptureSource, private Orien
     bool isCaptureSource() const final { return true; }
     CaptureDevice::DeviceType deviceType() const final { return mockCamera() ? CaptureDevice::DeviceType::Camera : CaptureDevice::DeviceType::Screen; }
     bool supportsSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double>, std::optional<double>) final;
-    void setSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double>, std::optional<double>) final;
+    void setSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double>, std::optional<double>) override;
     void setFrameRateAndZoomWithPreset(double, double, std::optional<VideoPreset>&&) final;
 
     bool isMockSource() const final { return true; }