[GStreamer] add Rialto quirk

https://bugs.webkit.org/show_bug.cgi?id=273812 Reviewed by Philippe Normand. Original patch by Eugene Mutavchi <Ievgen_Mutavchi@comcast.com>. * Source/WebCore/platform/SourcesGStreamer.txt: * Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp: (WebCore::MediaPlayerPrivateGStreamer::createAudioSink): (WebCore::MediaPlayerPrivateGStreamer::createVideoSink): * Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp: Added. (WebCore::GStreamerHolePunchQuirkRialto::createHolePunchVideoSink): (WebCore::GStreamerHolePunchQuirkRialto::setHolePunchVideoRectangle): * Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.h: Added. * Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.cpp: Added. (WebCore::GStreamerQuirkRialto::GStreamerQuirkRialto): (WebCore::GStreamerQuirkRialto::configureElement): (WebCore::GStreamerQuirkRialto::createAudioSink): (WebCore::GStreamerQuirkRialto::createWebAudioSink): (WebCore::GStreamerQuirkRialto::isHardwareAccelerated): * Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.h: Added. * Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp: (WebCore::GStreamerQuirksManager::GStreamerQuirksManager): (WebCore::GStreamerQuirksManager::createAudioSink): * Source/WebCore/platform/gstreamer/GStreamerQuirks.h: (WebCore::GStreamerQuirk::createAudioSink): Canonical link: https://commits.webkit.org/278452@main
WebKit · May 7, 2024 · e1b6b25 · e1b6b25
1 parent eac1ccc
commit e1b6b25
Show file tree

Hide file tree

Showing 8 changed files with 300 additions and 3 deletions.
diff --git a/Source/WebCore/platform/SourcesGStreamer.txt b/Source/WebCore/platform/SourcesGStreamer.txt
@@ -99,11 +99,13 @@ platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp @no-unify
 platform/gstreamer/GStreamerCodecUtilities.cpp
 platform/gstreamer/GStreamerElementHarness.cpp
 platform/gstreamer/GStreamerHolePunchQuirkBcmNexus.cpp
+platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp
 platform/gstreamer/GStreamerHolePunchQuirkWesteros.cpp
 platform/gstreamer/GStreamerQuirkAmLogic.cpp
 platform/gstreamer/GStreamerQuirkBcmNexus.cpp
 platform/gstreamer/GStreamerQuirkBroadcom.cpp
 platform/gstreamer/GStreamerQuirkRealtek.cpp
+platform/gstreamer/GStreamerQuirkRialto.cpp
 platform/gstreamer/GStreamerQuirkWesteros.cpp
 platform/gstreamer/GStreamerQuirks.cpp
 platform/gstreamer/PlatformSpeechSynthesizerGStreamer.cpp

diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp
@@ -1347,7 +1347,7 @@ GstElement* MediaPlayerPrivateGStreamer::createAudioSink()
     // If audio is being controlled by an another pipeline, creating sink here may interfere with
     // audio playback. Instead, check if an audio sink was setup in handleMessage and use it.
     if (quirksManager.isEnabled())
-        return nullptr;
+        return quirksManager.createAudioSink();
 
     RefPtr player = m_player.get();
     if (!player)
@@ -4220,8 +4220,14 @@ GstElement* MediaPlayerPrivateGStreamer::createVideoSink()
             g_value_unset(&value);
         }
 
-        uint64_t maxLateness = 100 * GST_MSECOND;
-        g_object_set(sink, "max-lateness", maxLateness, nullptr);
+        if (gstObjectHasProperty(sink, "max-lateness")) {
+            uint64_t maxLateness = 100 * GST_MSECOND;
+            g_object_set(sink, "max-lateness", maxLateness, nullptr);
+        } else {
+            GST_WARNING_OBJECT(pipeline(), "video sink does not have max-lateness property. This could result in A/V "
+                "desynchronization if it does not discard buffers that are arriving late (for example quality changes "
+                "decoding something again that has already been played)");
+        }
     });
 
     RefPtr player = m_player.get();

diff --git a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2024 RDK Management
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "GStreamerHolePunchQuirkRialto.h"
+
+#if USE(GSTREAMER)
+
+#include "GStreamerCommon.h"
+#include "MediaPlayerPrivateGStreamer.h"
+
+namespace WebCore {
+
+GstElement* GStreamerHolePunchQuirkRialto::createHolePunchVideoSink(bool isLegacyPlaybin, const MediaPlayer* player)
+{
+    AtomString value;
+    bool isPIPRequested = player && player->doesHaveAttribute("pip"_s, &value) && equalLettersIgnoringASCIICase(value, "true"_s);
+    if (isLegacyPlaybin && !isPIPRequested)
+        return nullptr;
+
+    // Rialto using holepunch.
+    GstElement* videoSink = makeGStreamerElement("rialtomsevideosink", nullptr);
+    if (isPIPRequested)
+        g_object_set(G_OBJECT(videoSink), "maxVideoWidth", 640, "maxVideoHeight", 480, "has-drm", FALSE, nullptr);
+    return videoSink;
+}
+
+bool GStreamerHolePunchQuirkRialto::setHolePunchVideoRectangle(GstElement* videoSink, const IntRect& rect)
+{
+    if (UNLIKELY(!gstObjectHasProperty(videoSink, "rectangle")))
+        return false;
+
+    auto rectString = makeString(rect.x(), ',', rect.y(), ',', rect.width(), ',', rect.height());
+    g_object_set(videoSink, "rectangle", rectString.ascii().data(), nullptr);
+    return true;
+}
+
+#undef GST_CAT_DEFAULT
+
+} // namespace WebCore
+
+#endif // USE(GSTREAMER)
diff --git a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.h b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2024 RDK Management
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if USE(GSTREAMER)
+
+#include "GStreamerQuirks.h"
+
+namespace WebCore {
+
+class GStreamerHolePunchQuirkRialto final : public GStreamerHolePunchQuirk {
+public:
+    const char* identifier() final { return "RialtoHolePunch"; }
+
+    GstElement* createHolePunchVideoSink(bool, const MediaPlayer*) final;
+    bool setHolePunchVideoRectangle(GstElement*, const IntRect&) final;
+    bool requiresClockSynchronization() const final { return false; }
+};
+
+} // namespace WebCore
+
+#endif // USE(GSTREAMER)
diff --git a/Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.cpp b/Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.cpp
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2024 RDK Management
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "GStreamerQuirkRialto.h"
+
+#if USE(GSTREAMER)
+
+#include "GStreamerCommon.h"
+#include <wtf/OptionSet.h>
+
+namespace WebCore {
+
+GST_DEBUG_CATEGORY_STATIC(webkit_rialto_quirks_debug);
+#define GST_CAT_DEFAULT webkit_rialto_quirks_debug
+
+GStreamerQuirkRialto::GStreamerQuirkRialto()
+{
+    GST_DEBUG_CATEGORY_INIT(webkit_rialto_quirks_debug, "webkitquirksrialto", 0, "WebKit Rialto Quirks");
+
+    std::array<const char *, 2> rialtoSinks = { "rialtomsevideosink", "rialtomseaudiosink" };
+
+    for (const auto* sink : rialtoSinks) {
+        auto sinkFactory = adoptGRef(gst_element_factory_find(sink));
+        if (UNLIKELY(!sinkFactory))
+            continue;
+
+        gst_object_unref(gst_plugin_feature_load(GST_PLUGIN_FEATURE(sinkFactory.get())));
+        for (auto* padTemplateListElement = gst_element_factory_get_static_pad_templates(sinkFactory.get());
+            padTemplateListElement; padTemplateListElement = g_list_next(padTemplateListElement)) {
+
+            auto* padTemplate = static_cast<GstStaticPadTemplate*>(padTemplateListElement->data);
+            if (padTemplate->direction != GST_PAD_SINK)
+                continue;
+            GRefPtr<GstCaps> templateCaps = adoptGRef(gst_static_caps_get(&padTemplate->static_caps));
+            if (!templateCaps)
+                continue;
+            if (gst_caps_is_empty(templateCaps.get()) || gst_caps_is_any(templateCaps.get()))
+                continue;
+            if (m_sinkCaps)
+                m_sinkCaps = adoptGRef(gst_caps_merge(m_sinkCaps.leakRef(), templateCaps.leakRef()));
+            else
+                m_sinkCaps = WTFMove(templateCaps);
+        }
+    }
+}
+
+void GStreamerQuirkRialto::configureElement(GstElement* element, const OptionSet<ElementRuntimeCharacteristics>&)
+{
+    if (!g_strcmp0(G_OBJECT_TYPE_NAME(G_OBJECT(element)), "GstURIDecodeBin3")) {
+        GRefPtr<GstCaps> defaultCaps;
+        g_object_get(element, "caps", &defaultCaps.outPtr(), nullptr);
+        defaultCaps = adoptGRef(gst_caps_merge(gst_caps_ref(m_sinkCaps.get()), defaultCaps.leakRef()));
+        GST_INFO("Setting stop caps to %" GST_PTR_FORMAT, defaultCaps.get());
+        g_object_set(element, "caps", defaultCaps.get(), nullptr);
+    }
+}
+
+GstElement* GStreamerQuirkRialto::createAudioSink()
+{
+    auto sink = makeGStreamerElement("rialtomseaudiosink", nullptr);
+    RELEASE_ASSERT_WITH_MESSAGE(sink, "rialtomseaudiosink should be available in the system but it is not");
+    return sink;
+}
+
+GstElement* GStreamerQuirkRialto::createWebAudioSink()
+{
+    auto sink = makeGStreamerElement("rialtowebaudiosink", nullptr);
+    RELEASE_ASSERT_WITH_MESSAGE(sink, "rialtowebaudiosink should be available in the system but it is not");
+    return sink;
+}
+
+std::optional<bool> GStreamerQuirkRialto::isHardwareAccelerated(GstElementFactory* factory)
+{
+    if (g_str_has_prefix(GST_OBJECT_NAME(factory), "rialto"))
+        return true;
+
+    return std::nullopt;
+}
+
+#undef GST_CAT_DEFAULT
+
+} // namespace WebCore
+
+#endif // USE(GSTREAMER)
diff --git a/Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.h b/Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2024 RDK Management
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if USE(GSTREAMER)
+
+#include "GStreamerQuirks.h"
+
+namespace WebCore {
+
+class GStreamerQuirkRialto final : public GStreamerQuirk {
+public:
+    GStreamerQuirkRialto();
+    const char* identifier() final { return "Rialto"; }
+
+    void configureElement(GstElement*, const OptionSet<ElementRuntimeCharacteristics>&) final;
+    GstElement* createAudioSink() final;
+    GstElement* createWebAudioSink() final;
+    std::optional<bool> isHardwareAccelerated(GstElementFactory*) final;
+    bool shouldParseIncomingLibWebRTCBitStream() const final { return false; }
+    unsigned getAdditionalPlaybinFlags() const { return getGstPlayFlag("text") | getGstPlayFlag("native-audio") | getGstPlayFlag("native-video"); }
+
+private:
+    GRefPtr<GstCaps> m_sinkCaps;
+};
+
+} // namespace WebCore
+
+#endif // USE(GSTREAMER)
diff --git a/Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp b/Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp
@@ -26,11 +26,13 @@
 #include "GStreamerCommon.h"
 #include "GStreamerHolePunchQuirkBcmNexus.h"
 #include "GStreamerHolePunchQuirkFake.h"
+#include "GStreamerHolePunchQuirkRialto.h"
 #include "GStreamerHolePunchQuirkWesteros.h"
 #include "GStreamerQuirkAmLogic.h"
 #include "GStreamerQuirkBcmNexus.h"
 #include "GStreamerQuirkBroadcom.h"
 #include "GStreamerQuirkRealtek.h"
+#include "GStreamerQuirkRialto.h"
 #include "GStreamerQuirkWesteros.h"
 #include <wtf/NeverDestroyed.h>
 #include <wtf/OptionSet.h>
@@ -84,6 +86,8 @@ GStreamerQuirksManager::GStreamerQuirksManager(bool isForTesting, bool loadQuirk
                 quirk = WTF::makeUnique<GStreamerQuirkBcmNexus>();
             else if (WTF::equalLettersIgnoringASCIICase(identifier, "realtek"_s))
                 quirk = WTF::makeUnique<GStreamerQuirkRealtek>();
+            else if (WTF::equalLettersIgnoringASCIICase(identifier, "rialto"_s))
+                quirk = WTF::makeUnique<GStreamerQuirkRialto>();
             else if (WTF::equalLettersIgnoringASCIICase(identifier, "westeros"_s))
                 quirk = WTF::makeUnique<GStreamerQuirkWesteros>();
             else {
@@ -113,6 +117,8 @@ GStreamerQuirksManager::GStreamerQuirksManager(bool isForTesting, bool loadQuirk
     // TODO: Maybe check this is coherent (somehow) with the quirk(s) selected above.
     if (WTF::equalLettersIgnoringASCIICase(identifier, "bcmnexus"_s))
         m_holePunchQuirk = WTF::makeUnique<GStreamerHolePunchQuirkBcmNexus>();
+    else if (WTF::equalLettersIgnoringASCIICase(identifier, "rialto"_s))
+        m_holePunchQuirk = WTF::makeUnique<GStreamerHolePunchQuirkRialto>();
     else if (WTF::equalLettersIgnoringASCIICase(identifier, "westeros"_s))
         m_holePunchQuirk = WTF::makeUnique<GStreamerHolePunchQuirkWesteros>();
     else if (WTF::equalLettersIgnoringASCIICase(identifier, "fake"_s))
@@ -126,6 +132,19 @@ bool GStreamerQuirksManager::isEnabled() const
     return !m_quirks.isEmpty();
 }
 
+GstElement* GStreamerQuirksManager::createAudioSink()
+{
+    for (const auto& quirk : m_quirks) {
+        auto* sink = quirk->createAudioSink();
+        if (sink) {
+            GST_DEBUG("Using AudioSink from quirk %s : %" GST_PTR_FORMAT, quirk->identifier(), sink);
+            return sink;
+        }
+    }
+
+    return nullptr;
+}
+
 GstElement* GStreamerQuirksManager::createWebAudioSink()
 {
     for (const auto& quirk : m_quirks) {

diff --git a/Source/WebCore/platform/gstreamer/GStreamerQuirks.h b/Source/WebCore/platform/gstreamer/GStreamerQuirks.h
@@ -54,6 +54,7 @@ class GStreamerQuirk : public GStreamerQuirkBase {
     virtual ~GStreamerQuirk() = default;
 
     virtual bool isPlatformSupported() const { return true; }
+    virtual GstElement* createAudioSink() { return nullptr; }
     virtual GstElement* createWebAudioSink() { return nullptr; }
     virtual void configureElement(GstElement*, const OptionSet<ElementRuntimeCharacteristics>&) { }
     virtual std::optional<bool> isHardwareAccelerated(GstElementFactory*) { return std::nullopt; }
@@ -88,6 +89,7 @@ class GStreamerQuirksManager : public RefCounted<GStreamerQuirksManager> {
 
     bool isEnabled() const;
 
+    GstElement* createAudioSink();
     GstElement* createWebAudioSink();
     void configureElement(GstElement*, OptionSet<ElementRuntimeCharacteristics>&&);
     std::optional<bool> isHardwareAccelerated(GstElementFactory*) const;