Merge r168060 - [GStreamer] Use GstMetaVideo

https://bugs.webkit.org/show_bug.cgi?id=132247 Reviewed by Philippe Normand. In WebKitVideoSink we announce the usage of GstMetaVideo, but we do not use it when handling the video frames. This might break some decoders and filters that rely on buffer's meta, rather that in the caps structures. This patch enables the use of GstMetaVideo through the GstVideoFrame API. And it is used everywhere the buffer mapping is required. Also this patch changes to nullptr where zeros were used. Also, compile conditionally the video buffer conversion when it is ARGB/BGRA, since it is only required for the Cairo backend. No new tests, already covered by current tests. * platform/graphics/gstreamer/GStreamerUtilities.cpp: (WebCore::getVideoSizeAndFormatFromCaps): init the GstVideoInfo before used and remove caps fixate check since it is done by gst_video_info_from_caps(). * platform/graphics/gstreamer/ImageGStreamer.h: * platform/graphics/gstreamer/ImageGStreamerCairo.cpp: (ImageGStreamer::ImageGStreamer): use GstVideoFrame for buffer mapping and unmapping. (ImageGStreamer::~ImageGStreamer): ditto. * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp: (WebCore::MediaPlayerPrivateGStreamerBase::updateTexture): ditto. (WebCore::MediaPlayerPrivateGStreamerBase::currentVideoSinkCaps): return nullptr if failed. * platform/graphics/gstreamer/VideoSinkGStreamer.cpp: (webkitVideoSinkRender): rely on GstVideoInfo rather than on the caps. Use GstVideoFrame for buffer mapping and unmapping. Add guards for buffer transformation, since it's only used by Cairo. (webkitVideoSinkDispose): remove glib version guards. (webkitVideoSinkSetCaps): update the value of the private GstVideoInfo.
WebKit · May 22, 2014 · 1772a19 · 1772a19
1 parent de63573
commit 1772a19
Show file tree

Hide file tree

Showing 6 changed files with 118 additions and 54 deletions.
diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog
@@ -1,3 +1,46 @@
+2014-04-30  Víctor Manuel Jáquez Leal  <vjaquez@igalia.com>
+
+        [GStreamer] Use GstMetaVideo
+        https://bugs.webkit.org/show_bug.cgi?id=132247
+
+        Reviewed by Philippe Normand.
+
+        In WebKitVideoSink we announce the usage of GstMetaVideo, but we do
+        not use it when handling the video frames. This might break
+        some decoders and filters that rely on buffer's meta, rather
+        that in the caps structures.
+
+        This patch enables the use of GstMetaVideo through the GstVideoFrame
+        API. And it is used everywhere the buffer mapping is required.
+
+        Also this patch changes to nullptr where zeros were used.
+
+        Also, compile conditionally the video buffer conversion when it is
+        ARGB/BGRA, since it is only required for the Cairo backend.
+
+        No new tests, already covered by current tests.
+
+        * platform/graphics/gstreamer/GStreamerUtilities.cpp:
+        (WebCore::getVideoSizeAndFormatFromCaps): init the GstVideoInfo before
+        used and remove caps fixate check since it is done by
+        gst_video_info_from_caps().
+        * platform/graphics/gstreamer/ImageGStreamer.h:
+        * platform/graphics/gstreamer/ImageGStreamerCairo.cpp:
+        (ImageGStreamer::ImageGStreamer): use GstVideoFrame for buffer mapping
+        and unmapping.
+        (ImageGStreamer::~ImageGStreamer): ditto.
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
+        (WebCore::MediaPlayerPrivateGStreamerBase::updateTexture): ditto.
+        (WebCore::MediaPlayerPrivateGStreamerBase::currentVideoSinkCaps):
+        return nullptr if failed.
+        * platform/graphics/gstreamer/VideoSinkGStreamer.cpp:
+        (webkitVideoSinkRender): rely on GstVideoInfo rather than on the
+        caps. Use GstVideoFrame for buffer mapping and unmapping. Add guards
+        for buffer transformation, since it's only used by Cairo.
+        (webkitVideoSinkDispose): remove glib version guards.
+        (webkitVideoSinkSetCaps): update the value of the private
+        GstVideoInfo.
+
 2014-05-06  Myles C. Maxfield  <mmaxfield@apple.com>
 
         Dragging text from one paragraph to another does not render as expected

diff --git a/Source/WebCore/platform/graphics/gstreamer/GStreamerUtilities.cpp b/Source/WebCore/platform/graphics/gstreamer/GStreamerUtilities.cpp
@@ -52,7 +52,8 @@ bool getVideoSizeAndFormatFromCaps(GstCaps* caps, WebCore::IntSize& size, GstVid
 {
     GstVideoInfo info;
 
-    if (!gst_caps_is_fixed(caps) || !gst_video_info_from_caps(&info, caps))
+    gst_video_info_init(&info);
+    if (!gst_video_info_from_caps(&info, caps))
         return false;
 
     format = GST_VIDEO_INFO_FORMAT(&info);

diff --git a/Source/WebCore/platform/graphics/gstreamer/ImageGStreamer.h b/Source/WebCore/platform/graphics/gstreamer/ImageGStreamer.h
@@ -27,6 +27,7 @@
 #include "GRefPtrGStreamer.h"
 
 #include <gst/gst.h>
+#include <gst/video/video.h>
 
 #include <wtf/PassRefPtr.h>
 #include <wtf/RefCounted.h>
@@ -64,8 +65,7 @@ class ImageGStreamer : public RefCounted<ImageGStreamer> {
         FloatRect m_cropRect;
 
 #if USE(CAIRO)
-        GRefPtr<GstBuffer> m_buffer;
-        GstMapInfo m_mapInfo;
+        GstVideoFrame m_videoFrame;
 #endif
     };
 }

diff --git a/Source/WebCore/platform/graphics/gstreamer/ImageGStreamerCairo.cpp b/Source/WebCore/platform/graphics/gstreamer/ImageGStreamerCairo.cpp
@@ -33,24 +33,32 @@ using namespace std;
 using namespace WebCore;
 
 ImageGStreamer::ImageGStreamer(GstBuffer* buffer, GstCaps* caps)
-    : m_buffer(buffer)
 {
-    GstVideoFormat format;
-    IntSize size;
-    int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride;
-    getVideoSizeAndFormatFromCaps(caps, size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride);
+    GstVideoInfo videoInfo;
+    gst_video_info_init(&videoInfo);
+    if (!gst_video_info_from_caps(&videoInfo, caps))
+        return;
 
-    gst_buffer_map(buffer, &m_mapInfo, GST_MAP_READ);
-    unsigned char* bufferData = reinterpret_cast<unsigned char*>(m_mapInfo.data);
+    // Right now the TextureMapper only supports chromas with one plane
+    ASSERT(GST_VIDEO_INFO_N_PLANES(&videoInfo) == 1);
+
+    if (!gst_video_frame_map(&m_videoFrame, &videoInfo, buffer, GST_MAP_READ))
+        return;
+
+    unsigned char* bufferData = reinterpret_cast<unsigned char*>(GST_VIDEO_FRAME_PLANE_DATA(&m_videoFrame, 0));
 
     cairo_format_t cairoFormat;
 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
-    cairoFormat = (format == GST_VIDEO_FORMAT_BGRA) ? CAIRO_FORMAT_ARGB32 : CAIRO_FORMAT_RGB24;
+    cairoFormat = (GST_VIDEO_FRAME_FORMAT(&m_videoFrame) == GST_VIDEO_FORMAT_BGRA) ? CAIRO_FORMAT_ARGB32 : CAIRO_FORMAT_RGB24;
 #else
-    cairoFormat = (format == GST_VIDEO_FORMAT_ARGB) ? CAIRO_FORMAT_ARGB32 : CAIRO_FORMAT_RGB24;
+    cairoFormat = (GST_VIDEO_FRAME_FORMAT(&m_videoFrame) == GST_VIDEO_FORMAT_ARGB) ? CAIRO_FORMAT_ARGB32 : CAIRO_FORMAT_RGB24;
 #endif
 
-    RefPtr<cairo_surface_t> surface = adoptRef(cairo_image_surface_create_for_data(bufferData, cairoFormat, size.width(), size.height(), stride));
+    int stride = GST_VIDEO_FRAME_PLANE_STRIDE(&m_videoFrame, 0);
+    int width = GST_VIDEO_FRAME_WIDTH(&m_videoFrame);
+    int height = GST_VIDEO_FRAME_HEIGHT(&m_videoFrame);
+
+    RefPtr<cairo_surface_t> surface = adoptRef(cairo_image_surface_create_for_data(bufferData, cairoFormat, width, height, stride));
     ASSERT(cairo_surface_status(surface.get()) == CAIRO_STATUS_SUCCESS);
     m_image = BitmapImage::create(surface.release());
 
@@ -67,6 +75,6 @@ ImageGStreamer::~ImageGStreamer()
 
     // We keep the buffer memory mapped until the image is destroyed because the internal
     // cairo_surface_t was created using cairo_image_surface_create_for_data().
-    gst_buffer_unmap(m_buffer.get(), &m_mapInfo);
+    gst_video_frame_unmap(&m_videoFrame);
 }
 #endif // USE(GSTREAMER)
diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp
@@ -42,6 +42,7 @@
 #include <wtf/text/CString.h>
 
 #include <gst/audio/streamvolume.h>
+#include <gst/video/gstvideometa.h>
 
 #if GST_CHECK_VERSION(1, 1, 0) && USE(ACCELERATED_COMPOSITING) && USE(TEXTURE_MAPPER_GL)
 #include "TextureMapperGL.h"
@@ -309,22 +310,19 @@ PassRefPtr<BitmapTexture> MediaPlayerPrivateGStreamerBase::updateTexture(Texture
 {
     GMutexLocker lock(m_bufferMutex);
     if (!m_buffer)
-        return 0;
+        return nullptr;
 
-    const void* srcData = 0;
     GRefPtr<GstCaps> caps = currentVideoSinkCaps();
     if (!caps)
-        return 0;
-
-    IntSize size;
-    GstVideoFormat format;
-    int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride;
-    if (!getVideoSizeAndFormatFromCaps(caps.get(), size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride))
-        return 0;
+        return nullptr;
 
-    const GstVideoFormatInfo* formatInfo = gst_video_format_get_info(format);
+    GstVideoInfo videoInfo;
+    gst_video_info_init(&videoInfo);
+    if (!gst_video_info_from_caps(&videoInfo, caps.get()))
+        return nullptr;
 
-    RefPtr<BitmapTexture> texture = textureMapper->acquireTextureFromPool(size, GST_VIDEO_FORMAT_INFO_HAS_ALPHA(formatInfo) ? BitmapTexture::SupportsAlpha : BitmapTexture::NoFlag);
+    IntSize size = IntSize(GST_VIDEO_INFO_WIDTH(&videoInfo), GST_VIDEO_INFO_HEIGHT(&videoInfo));
+    RefPtr<BitmapTexture> texture = textureMapper->acquireTextureFromPool(size, GST_VIDEO_INFO_HAS_ALPHA(&videoInfo) ? BitmapTexture::SupportsAlpha : BitmapTexture::NoFlag);
 
 #if GST_CHECK_VERSION(1, 1, 0)
     GstVideoGLTextureUploadMeta* meta;
@@ -339,13 +337,18 @@ PassRefPtr<BitmapTexture> MediaPlayerPrivateGStreamerBase::updateTexture(Texture
     }
 #endif
 
-    GstMapInfo srcInfo;
-    gst_buffer_map(m_buffer, &srcInfo, GST_MAP_READ);
-    srcData = srcInfo.data;
+    // Right now the TextureMapper only supports chromas with one plane
+    ASSERT(GST_VIDEO_INFO_N_PLANES(&videoInfo) == 1);
 
+    GstVideoFrame videoFrame;
+    if (!gst_video_frame_map(&videoFrame, &videoInfo, m_buffer, GST_MAP_READ))
+        return nullptr;
+
+    int stride = GST_VIDEO_FRAME_PLANE_STRIDE(&videoFrame, 0);
+    const void* srcData = GST_VIDEO_FRAME_PLANE_DATA(&videoFrame, 0);
     texture->updateContents(srcData, WebCore::IntRect(WebCore::IntPoint(0, 0), size), WebCore::IntPoint(0, 0), stride, BitmapTexture::UpdateCannotModifyOriginalImageData);
+    gst_video_frame_unmap(&videoFrame);
 
-    gst_buffer_unmap(m_buffer, &srcInfo);
     return texture;
 }
 #endif
@@ -442,7 +445,7 @@ MediaPlayer::MovieLoadType MediaPlayerPrivateGStreamerBase::movieLoadType() cons
 GRefPtr<GstCaps> MediaPlayerPrivateGStreamerBase::currentVideoSinkCaps() const
 {
     if (!m_webkitVideoSink)
-        return 0;
+        return nullptr;
 
     GRefPtr<GstCaps> currentCaps;
     g_object_get(G_OBJECT(m_webkitVideoSink.get()), "current-caps", &currentCaps.outPtr(), NULL);

diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoSinkGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/VideoSinkGStreamer.cpp
@@ -147,21 +147,14 @@ static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buf
 
     priv->buffer = gst_buffer_ref(buffer);
 
-    GRefPtr<GstCaps> caps;
     // The video info structure is valid only if the sink handled an allocation query.
-    if (GST_VIDEO_INFO_FORMAT(&priv->info) != GST_VIDEO_FORMAT_UNKNOWN)
-        caps = adoptGRef(gst_video_info_to_caps(&priv->info));
-    else
-        caps = priv->currentCaps;
-
-    GstVideoFormat format;
-    WebCore::IntSize size;
-    int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride;
-    if (!getVideoSizeAndFormatFromCaps(caps.get(), size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride)) {
+    GstVideoFormat format = GST_VIDEO_INFO_FORMAT(&priv->info);
+    if (format == GST_VIDEO_FORMAT_UNKNOWN) {
         gst_buffer_unref(buffer);
         return GST_FLOW_ERROR;
     }
 
+#if !(USE(TEXTURE_MAPPER_GL) && !USE(COORDINATED_GRAPHICS))
     // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't.
     // Here we convert to Cairo's ARGB.
     if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) {
@@ -173,22 +166,35 @@ static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buf
         GstBuffer* newBuffer = WebCore::createGstBuffer(buffer);
 
         // Check if allocation failed.
-        if (UNLIKELY(!newBuffer))
+        if (UNLIKELY(!newBuffer)) {
+            gst_buffer_unref(buffer);
             return GST_FLOW_ERROR;
+        }
 
         // We don't use Color::premultipliedARGBFromColor() here because
         // one function call per video pixel is just too expensive:
         // For 720p/PAL for example this means 1280*720*25=23040000
         // function calls per second!
-        GstMapInfo sourceInfo;
-        GstMapInfo destinationInfo;
-        gst_buffer_map(buffer, &sourceInfo, GST_MAP_READ);
-        const guint8* source = const_cast<guint8*>(sourceInfo.data);
-        gst_buffer_map(newBuffer, &destinationInfo, GST_MAP_WRITE);
-        guint8* destination = static_cast<guint8*>(destinationInfo.data);
-
-        for (int x = 0; x < size.height(); x++) {
-            for (int y = 0; y < size.width(); y++) {
+        GstVideoFrame sourceFrame;
+        GstVideoFrame destinationFrame;
+
+        if (!gst_video_frame_map(&sourceFrame, &priv->info, buffer, GST_MAP_READ)) {
+            gst_buffer_unref(buffer);
+            gst_buffer_unref(newBuffer);
+            return GST_FLOW_ERROR;
+        }
+        if (!gst_video_frame_map(&destinationFrame, &priv->info, newBuffer, GST_MAP_WRITE)) {
+            gst_video_frame_unmap(&sourceFrame);
+            gst_buffer_unref(buffer);
+            gst_buffer_unref(newBuffer);
+            return GST_FLOW_ERROR;
+        }
+
+        const guint8* source = static_cast<guint8*>(GST_VIDEO_FRAME_PLANE_DATA(&sourceFrame, 0));
+        guint8* destination = static_cast<guint8*>(GST_VIDEO_FRAME_PLANE_DATA(&destinationFrame, 0));
+
+        for (int x = 0; x < GST_VIDEO_FRAME_HEIGHT(&sourceFrame); x++) {
+            for (int y = 0; y < GST_VIDEO_FRAME_WIDTH(&sourceFrame); y++) {
 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
                 unsigned short alpha = source[3];
                 destination[0] = (source[0] * alpha + 128) / 255;
@@ -207,11 +213,12 @@ static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buf
             }
         }
 
-        gst_buffer_unmap(buffer, &sourceInfo);
-        gst_buffer_unmap(newBuffer, &destinationInfo);
+        gst_video_frame_unmap(&sourceFrame);
+        gst_video_frame_unmap(&destinationFrame);
         gst_buffer_unref(buffer);
         buffer = priv->buffer = newBuffer;
     }
+#endif
 
     // This should likely use a lower priority, but glib currently starves
     // lower priority sources.
@@ -335,12 +342,14 @@ static gboolean webkitVideoSinkSetCaps(GstBaseSink* baseSink, GstCaps* caps)
 
     GST_DEBUG_OBJECT(sink, "Current caps %" GST_PTR_FORMAT ", setting caps %" GST_PTR_FORMAT, priv->currentCaps, caps);
 
-    GstVideoInfo info;
-    if (!gst_video_info_from_caps(&info, caps)) {
+    GstVideoInfo videoInfo;
+    gst_video_info_init(&videoInfo);
+    if (!gst_video_info_from_caps(&videoInfo, caps)) {
         GST_ERROR_OBJECT(sink, "Invalid caps %" GST_PTR_FORMAT, caps);
         return FALSE;
     }
 
+    priv->info = videoInfo;
     gst_caps_replace(&priv->currentCaps, caps);
     return TRUE;
 }