[GStreamer] Use FakeSink to get a decoded texture from a pipeline

https://bugs.webkit.org/show_bug.cgi?id=153641 Reviewed by Philippe Normand. .: * Source/cmake/FindGStreamer.cmake: Bump gst-gl version to 1.8.0 Source/WebCore: Relying on GstGLImageSink to use GStreamerGL brings a lot of overheads such as window handling, context switching and overlay handling which are not needed in our case. This patch replaces GstGLImageSink with a custom GstBin which has a GstGLUpload, GstGLColorConvert, and GstFakeSink. GstFakeSink sends decoded frames via handoff signal from the vqueue thread of GStreamer. Previously, GstGLImageSink passes frames through GStreamer's GL thread, which adds additional overhead. * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp: (WebCore::GstVideoFrameHolder::~GstVideoFrameHolder): Modified to unmap GstVideoFrame without async call. GstGLMemory will unmap itself in gl-thread. (WebCore::MediaPlayerPrivateGStreamerBase::drawCallback): (WebCore::MediaPlayerPrivateGStreamerBase::createVideoSinkGL): Split out creating of the gst-gl video sink into the separte method. (WebCore::MediaPlayerPrivateGStreamerBase::paint): Remove assertion for the threaded compositor. It can be called by focusing event. * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h: Canonical link: https://commits.webkit.org/175957@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@201076 268f45cc-cd09-0410-ab3c-d52691b4dbfc
WebKit · May 18, 2016 · c01259ceb8d525ebcec525fbf8505aebc32a7140 · c01259c
1 parent f73a66a
commit c01259ceb8d525ebcec525fbf8505aebc32a7140
diff --git a/ChangeLog b/ChangeLog
@@ -1,3 +1,12 @@
+2016-05-18  Gwang Yoon Hwang  <yoon@igalia.com>
+
+        [GStreamer] Use FakeSink to get a decoded texture from a pipeline
+        https://bugs.webkit.org/show_bug.cgi?id=153641
+
+        Reviewed by Philippe Normand.
+
+        * Source/cmake/FindGStreamer.cmake: Bump gst-gl version to 1.8.0
+
 2016-05-17  Dean Jackson  <dino@apple.com>
 
         Remove ES6_GENERATORS flag

diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog
@@ -1,3 +1,32 @@
+2016-05-18  Gwang Yoon Hwang  <yoon@igalia.com>
+
+        [GStreamer] Use FakeSink to get a decoded texture from a pipeline
+        https://bugs.webkit.org/show_bug.cgi?id=153641
+
+        Reviewed by Philippe Normand.
+
+        Relying on GstGLImageSink to use GStreamerGL brings a lot of overheads such as
+        window handling, context switching and overlay handling which are not needed in
+        our case.
+
+        This patch replaces GstGLImageSink with a custom GstBin which has a
+        GstGLUpload, GstGLColorConvert, and GstFakeSink.
+
+        GstFakeSink sends decoded frames via handoff signal from the vqueue thread of
+        GStreamer. Previously,  GstGLImageSink passes frames through GStreamer's GL
+        thread, which adds additional overhead.
+
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
+        (WebCore::GstVideoFrameHolder::~GstVideoFrameHolder): Modified to
+        unmap GstVideoFrame without async call. GstGLMemory will unmap itself
+        in gl-thread.
+        (WebCore::MediaPlayerPrivateGStreamerBase::drawCallback):
+        (WebCore::MediaPlayerPrivateGStreamerBase::createVideoSinkGL): Split
+        out creating of the gst-gl video sink into the separte method.
+        (WebCore::MediaPlayerPrivateGStreamerBase::paint): Remove assertion
+        for the threaded compositor. It can be called by focusing event.
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h:
+
 2016-05-18  Antti Koivisto  <antti@apple.com>
 
         Resolve !important properties from different shadow trees in a single pass.

diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp
@@ -115,13 +115,10 @@ class GstVideoFrameHolder : public TextureMapperPlatformLayerBuffer::UnmanagedBu
         m_flags = GST_VIDEO_INFO_HAS_ALPHA(&videoInfo) ? TextureMapperGL::ShouldBlend : 0;
 
         GstBuffer* buffer = gst_sample_get_buffer(sample);
-        m_videoFrame = new GstVideoFrame();
-        if (UNLIKELY(!gst_video_frame_map(m_videoFrame, &videoInfo, buffer, static_cast<GstMapFlags>(GST_MAP_READ | GST_MAP_GL)))) {
-            delete m_videoFrame;
+        if (UNLIKELY(!gst_video_frame_map(&m_videoFrame, &videoInfo, buffer, static_cast<GstMapFlags>(GST_MAP_READ | GST_MAP_GL))))
             return;
-        }
 
-        m_textureID = *reinterpret_cast<GLuint*>(m_videoFrame->data[0]);
+        m_textureID = *reinterpret_cast<GLuint*>(m_videoFrame.data[0]);
         m_isValid = true;
     }
 
@@ -130,21 +127,7 @@ class GstVideoFrameHolder : public TextureMapperPlatformLayerBuffer::UnmanagedBu
         if (UNLIKELY(!m_isValid))
             return;
 
-        GstMapInfo* info = &m_videoFrame->map[0];
-        GstGLBaseBuffer* mem = (GstGLBaseBuffer*)info->memory;
-        GstGLWindow* gstWindow = gst_gl_context_get_window(mem->context);
-
-        gst_gl_window_send_message_async(gstWindow, (GstGLWindowCB)unmapVideoFrameCallback, m_videoFrame, (GDestroyNotify)freeVideoFrameCallback);
-    }
-
-    static void unmapVideoFrameCallback(GstVideoFrame* videoFrame)
-    {
-        gst_video_frame_unmap(videoFrame);
-    }
-
-    static void freeVideoFrameCallback(GstVideoFrame* videoFrame)
-    {
-        delete videoFrame;
+        gst_video_frame_unmap(&m_videoFrame);
     }
 
     const IntSize& size() const { return m_size; }
@@ -153,7 +136,7 @@ class GstVideoFrameHolder : public TextureMapperPlatformLayerBuffer::UnmanagedBu
     bool isValid() const { return m_isValid; }
 
 private:
-    GstVideoFrame* m_videoFrame;
+    GstVideoFrame m_videoFrame;
     IntSize m_size;
     TextureMapperGL::Flags m_flags;
     GLuint m_textureID;
@@ -577,9 +560,13 @@ void MediaPlayerPrivateGStreamerBase::repaintCallback(MediaPlayerPrivateGStreame
 }
 
 #if USE(GSTREAMER_GL)
-gboolean MediaPlayerPrivateGStreamerBase::drawCallback(MediaPlayerPrivateGStreamerBase* player, GstGLContext*, GstSample* sample)
+gboolean MediaPlayerPrivateGStreamerBase::drawCallback(MediaPlayerPrivateGStreamerBase* player, GstBuffer* buffer, GstPad* pad, GstBaseSink* sink)
 {
-    player->triggerRepaint(sample);
+    GST_PAD_STREAM_LOCK(pad);
+    GRefPtr<GstCaps> caps = adoptGRef(gst_pad_get_current_caps(pad));
+    GRefPtr<GstSample> sample = adoptGRef(gst_sample_new(buffer, caps.get(), &sink->segment, NULL));
+    GST_PAD_STREAM_UNLOCK(pad);
+    player->triggerRepaint(sample.get());
     return TRUE;
 }
 #endif
@@ -592,7 +579,6 @@ void MediaPlayerPrivateGStreamerBase::setSize(const IntSize& size)
 void MediaPlayerPrivateGStreamerBase::paint(GraphicsContext& context, const FloatRect& rect)
 {
 #if USE(COORDINATED_GRAPHICS_THREADED)
-    ASSERT_NOT_REACHED();
     return;
 #elif USE(TEXTURE_MAPPER_GL) && !USE(COORDINATED_GRAPHICS)
     if (client())
@@ -719,17 +705,58 @@ MediaPlayer::MovieLoadType MediaPlayerPrivateGStreamerBase::movieLoadType() cons
     return MediaPlayer::Download;
 }
 
+#if USE(GSTREAMER_GL)
+GstElement* MediaPlayerPrivateGStreamerBase::createVideoSinkGL()
+{
+    if (!webkitGstCheckVersion(1, 8, 0))
+        return nullptr;
+
+    gboolean result = TRUE;
+    GstElement* videoSink = gst_bin_new(nullptr);
+    GstElement* upload = gst_element_factory_make("glupload", nullptr);
+    GstElement* colorconvert = gst_element_factory_make("glcolorconvert", nullptr);
+
+    if (!upload || !colorconvert) {
+        WARN_MEDIA_MESSAGE("Failed to create GstGL elements");
+        gst_object_unref(videoSink);
+
+        if (upload)
+            gst_object_unref(upload);
+        if (colorconvert)
+            gst_object_unref(colorconvert);
+
+        return nullptr;
+    }
+
+    GstElement* fakesink = gst_element_factory_make("fakesink", nullptr);
+
+    gst_bin_add_many(GST_BIN(videoSink), upload, colorconvert, fakesink, nullptr);
+
+    GRefPtr<GstCaps> caps = adoptGRef(gst_caps_from_string("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), format = (string) { RGBA }"));
+
+    result &= gst_element_link_pads(upload, "src", colorconvert, "sink");
+    result &= gst_element_link_pads_filtered(colorconvert, "src", fakesink, "sink", caps.get());
+
+    GRefPtr<GstPad> pad = adoptGRef(gst_element_get_static_pad(upload, "sink"));
+    gst_element_add_pad(videoSink, gst_ghost_pad_new("sink", pad.get()));
+
+    g_object_set(fakesink, "enable-last-sample", FALSE, "signal-handoffs", TRUE, "silent", TRUE, "sync", TRUE, nullptr);
+
+    if (result)
+        g_signal_connect_swapped(fakesink, "handoff", G_CALLBACK(drawCallback), this);
+    else {
+        WARN_MEDIA_MESSAGE("Failed to link GstGL elements");
+        gst_object_unref(videoSink);
+        videoSink = nullptr;
+    }
+    return videoSink;
+}
+#endif
+
 GstElement* MediaPlayerPrivateGStreamerBase::createVideoSink()
 {
-    GstElement* videoSink = nullptr;
 #if USE(GSTREAMER_GL)
-    if (webkitGstCheckVersion(1, 5, 0)) {
-        m_videoSink = gst_element_factory_make("glimagesink", nullptr);
-        if (m_videoSink) {
-            g_signal_connect_swapped(m_videoSink.get(), "client-draw", G_CALLBACK(drawCallback), this);
-            videoSink = m_videoSink.get();
-        }
-    }
+    m_videoSink = createVideoSinkGL();
 #endif
 
     if (!m_videoSink) {
@@ -738,6 +765,7 @@ GstElement* MediaPlayerPrivateGStreamerBase::createVideoSink()
         g_signal_connect_swapped(m_videoSink.get(), "repaint-requested", G_CALLBACK(repaintCallback), this);
     }
 
+    GstElement* videoSink = nullptr;
     m_fpsSink = gst_element_factory_make("fpsdisplaysink", "sink");
     if (m_fpsSink) {
         g_object_set(m_fpsSink.get(), "silent", TRUE , nullptr);

diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h
@@ -35,6 +35,7 @@
 #include <wtf/Forward.h>
 #include <wtf/RunLoop.h>
 
+typedef struct _GstBaseSink GstBaseSink;
 typedef struct _GstMessage GstMessage;
 typedef struct _GstStreamVolume GstStreamVolume;
 typedef struct _GstVideoInfo GstVideoInfo;
@@ -120,6 +121,10 @@ class MediaPlayerPrivateGStreamerBase : public MediaPlayerPrivateInterface
     MediaPlayerPrivateGStreamerBase(MediaPlayer*);
     virtual GstElement* createVideoSink();
 
+#if USE(GSTREAMER_GL)
+    GstElement* createVideoSinkGL();
+#endif
+
     void setStreamVolumeElement(GstStreamVolume*);
     virtual GstElement* createAudioSink() { return 0; }
     virtual GstElement* audioSink() const { return 0; }
@@ -133,7 +138,7 @@ class MediaPlayerPrivateGStreamerBase : public MediaPlayerPrivateInterface
 
     static void repaintCallback(MediaPlayerPrivateGStreamerBase*, GstSample*);
 #if USE(GSTREAMER_GL)
-    static gboolean drawCallback(MediaPlayerPrivateGStreamerBase*, GstGLContext*, GstSample*);
+    static gboolean drawCallback(MediaPlayerPrivateGStreamerBase*, GstBuffer*, GstPad*, GstBaseSink*);
 #endif
 
     void notifyPlayerOfVolumeChange();

diff --git a/Source/cmake/FindGStreamer.cmake b/Source/cmake/FindGStreamer.cmake
@@ -85,7 +85,7 @@ FIND_GSTREAMER_COMPONENT(GSTREAMER_BASE gstreamer-base-1.0 gstbase-1.0)
 FIND_GSTREAMER_COMPONENT(GSTREAMER_APP gstreamer-app-1.0 gstapp-1.0)
 FIND_GSTREAMER_COMPONENT(GSTREAMER_AUDIO gstreamer-audio-1.0 gstaudio-1.0)
 FIND_GSTREAMER_COMPONENT(GSTREAMER_FFT gstreamer-fft-1.0 gstfft-1.0)
-FIND_GSTREAMER_COMPONENT(GSTREAMER_GL gstreamer-gl-1.0>=1.6.0 gstgl-1.0)
+FIND_GSTREAMER_COMPONENT(GSTREAMER_GL gstreamer-gl-1.0>=1.8.0 gstgl-1.0)
 FIND_GSTREAMER_COMPONENT(GSTREAMER_MPEGTS gstreamer-mpegts-1.0>=1.4.0 gstmpegts-1.0)
 FIND_GSTREAMER_COMPONENT(GSTREAMER_PBUTILS gstreamer-pbutils-1.0 gstpbutils-1.0)
 FIND_GSTREAMER_COMPONENT(GSTREAMER_TAG gstreamer-tag-1.0 gsttag-1.0)