From 1772a19657e07bb264c97cca1bbfb836c2e94bf3 Mon Sep 17 00:00:00 2001 From: Victor Jaquez Date: Thu, 22 May 2014 12:16:49 +0000 Subject: [PATCH] Merge r168060 - [GStreamer] Use GstMetaVideo https://bugs.webkit.org/show_bug.cgi?id=132247 Reviewed by Philippe Normand. In WebKitVideoSink we announce the usage of GstMetaVideo, but we do not use it when handling the video frames. This might break some decoders and filters that rely on buffer's meta, rather that in the caps structures. This patch enables the use of GstMetaVideo through the GstVideoFrame API. And it is used everywhere the buffer mapping is required. Also this patch changes to nullptr where zeros were used. Also, compile conditionally the video buffer conversion when it is ARGB/BGRA, since it is only required for the Cairo backend. No new tests, already covered by current tests. * platform/graphics/gstreamer/GStreamerUtilities.cpp: (WebCore::getVideoSizeAndFormatFromCaps): init the GstVideoInfo before used and remove caps fixate check since it is done by gst_video_info_from_caps(). * platform/graphics/gstreamer/ImageGStreamer.h: * platform/graphics/gstreamer/ImageGStreamerCairo.cpp: (ImageGStreamer::ImageGStreamer): use GstVideoFrame for buffer mapping and unmapping. (ImageGStreamer::~ImageGStreamer): ditto. * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp: (WebCore::MediaPlayerPrivateGStreamerBase::updateTexture): ditto. (WebCore::MediaPlayerPrivateGStreamerBase::currentVideoSinkCaps): return nullptr if failed. * platform/graphics/gstreamer/VideoSinkGStreamer.cpp: (webkitVideoSinkRender): rely on GstVideoInfo rather than on the caps. Use GstVideoFrame for buffer mapping and unmapping. Add guards for buffer transformation, since it's only used by Cairo. (webkitVideoSinkDispose): remove glib version guards. (webkitVideoSinkSetCaps): update the value of the private GstVideoInfo. --- Source/WebCore/ChangeLog | 43 ++++++++++++++ .../graphics/gstreamer/GStreamerUtilities.cpp | 3 +- .../graphics/gstreamer/ImageGStreamer.h | 4 +- .../gstreamer/ImageGStreamerCairo.cpp | 30 ++++++---- .../MediaPlayerPrivateGStreamerBase.cpp | 35 ++++++------ .../graphics/gstreamer/VideoSinkGStreamer.cpp | 57 +++++++++++-------- 6 files changed, 118 insertions(+), 54 deletions(-) diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog index 90749c6efe76c..ebf07e17d9e4b 100644 --- a/Source/WebCore/ChangeLog +++ b/Source/WebCore/ChangeLog @@ -1,3 +1,46 @@ +2014-04-30 Víctor Manuel Jáquez Leal + + [GStreamer] Use GstMetaVideo + https://bugs.webkit.org/show_bug.cgi?id=132247 + + Reviewed by Philippe Normand. + + In WebKitVideoSink we announce the usage of GstMetaVideo, but we do + not use it when handling the video frames. This might break + some decoders and filters that rely on buffer's meta, rather + that in the caps structures. + + This patch enables the use of GstMetaVideo through the GstVideoFrame + API. And it is used everywhere the buffer mapping is required. + + Also this patch changes to nullptr where zeros were used. + + Also, compile conditionally the video buffer conversion when it is + ARGB/BGRA, since it is only required for the Cairo backend. + + No new tests, already covered by current tests. + + * platform/graphics/gstreamer/GStreamerUtilities.cpp: + (WebCore::getVideoSizeAndFormatFromCaps): init the GstVideoInfo before + used and remove caps fixate check since it is done by + gst_video_info_from_caps(). + * platform/graphics/gstreamer/ImageGStreamer.h: + * platform/graphics/gstreamer/ImageGStreamerCairo.cpp: + (ImageGStreamer::ImageGStreamer): use GstVideoFrame for buffer mapping + and unmapping. + (ImageGStreamer::~ImageGStreamer): ditto. + * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp: + (WebCore::MediaPlayerPrivateGStreamerBase::updateTexture): ditto. + (WebCore::MediaPlayerPrivateGStreamerBase::currentVideoSinkCaps): + return nullptr if failed. + * platform/graphics/gstreamer/VideoSinkGStreamer.cpp: + (webkitVideoSinkRender): rely on GstVideoInfo rather than on the + caps. Use GstVideoFrame for buffer mapping and unmapping. Add guards + for buffer transformation, since it's only used by Cairo. + (webkitVideoSinkDispose): remove glib version guards. + (webkitVideoSinkSetCaps): update the value of the private + GstVideoInfo. + 2014-05-06 Myles C. Maxfield Dragging text from one paragraph to another does not render as expected diff --git a/Source/WebCore/platform/graphics/gstreamer/GStreamerUtilities.cpp b/Source/WebCore/platform/graphics/gstreamer/GStreamerUtilities.cpp index a6aae02297be2..7b666944ecfc1 100644 --- a/Source/WebCore/platform/graphics/gstreamer/GStreamerUtilities.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/GStreamerUtilities.cpp @@ -52,7 +52,8 @@ bool getVideoSizeAndFormatFromCaps(GstCaps* caps, WebCore::IntSize& size, GstVid { GstVideoInfo info; - if (!gst_caps_is_fixed(caps) || !gst_video_info_from_caps(&info, caps)) + gst_video_info_init(&info); + if (!gst_video_info_from_caps(&info, caps)) return false; format = GST_VIDEO_INFO_FORMAT(&info); diff --git a/Source/WebCore/platform/graphics/gstreamer/ImageGStreamer.h b/Source/WebCore/platform/graphics/gstreamer/ImageGStreamer.h index 38bf3b4d1ad13..c9d2fb4e91574 100644 --- a/Source/WebCore/platform/graphics/gstreamer/ImageGStreamer.h +++ b/Source/WebCore/platform/graphics/gstreamer/ImageGStreamer.h @@ -27,6 +27,7 @@ #include "GRefPtrGStreamer.h" #include +#include #include #include @@ -64,8 +65,7 @@ class ImageGStreamer : public RefCounted { FloatRect m_cropRect; #if USE(CAIRO) - GRefPtr m_buffer; - GstMapInfo m_mapInfo; + GstVideoFrame m_videoFrame; #endif }; } diff --git a/Source/WebCore/platform/graphics/gstreamer/ImageGStreamerCairo.cpp b/Source/WebCore/platform/graphics/gstreamer/ImageGStreamerCairo.cpp index b21aca760c90b..b153f09ec19bd 100644 --- a/Source/WebCore/platform/graphics/gstreamer/ImageGStreamerCairo.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/ImageGStreamerCairo.cpp @@ -33,24 +33,32 @@ using namespace std; using namespace WebCore; ImageGStreamer::ImageGStreamer(GstBuffer* buffer, GstCaps* caps) - : m_buffer(buffer) { - GstVideoFormat format; - IntSize size; - int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride; - getVideoSizeAndFormatFromCaps(caps, size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride); + GstVideoInfo videoInfo; + gst_video_info_init(&videoInfo); + if (!gst_video_info_from_caps(&videoInfo, caps)) + return; - gst_buffer_map(buffer, &m_mapInfo, GST_MAP_READ); - unsigned char* bufferData = reinterpret_cast(m_mapInfo.data); + // Right now the TextureMapper only supports chromas with one plane + ASSERT(GST_VIDEO_INFO_N_PLANES(&videoInfo) == 1); + + if (!gst_video_frame_map(&m_videoFrame, &videoInfo, buffer, GST_MAP_READ)) + return; + + unsigned char* bufferData = reinterpret_cast(GST_VIDEO_FRAME_PLANE_DATA(&m_videoFrame, 0)); cairo_format_t cairoFormat; #if G_BYTE_ORDER == G_LITTLE_ENDIAN - cairoFormat = (format == GST_VIDEO_FORMAT_BGRA) ? CAIRO_FORMAT_ARGB32 : CAIRO_FORMAT_RGB24; + cairoFormat = (GST_VIDEO_FRAME_FORMAT(&m_videoFrame) == GST_VIDEO_FORMAT_BGRA) ? CAIRO_FORMAT_ARGB32 : CAIRO_FORMAT_RGB24; #else - cairoFormat = (format == GST_VIDEO_FORMAT_ARGB) ? CAIRO_FORMAT_ARGB32 : CAIRO_FORMAT_RGB24; + cairoFormat = (GST_VIDEO_FRAME_FORMAT(&m_videoFrame) == GST_VIDEO_FORMAT_ARGB) ? CAIRO_FORMAT_ARGB32 : CAIRO_FORMAT_RGB24; #endif - RefPtr surface = adoptRef(cairo_image_surface_create_for_data(bufferData, cairoFormat, size.width(), size.height(), stride)); + int stride = GST_VIDEO_FRAME_PLANE_STRIDE(&m_videoFrame, 0); + int width = GST_VIDEO_FRAME_WIDTH(&m_videoFrame); + int height = GST_VIDEO_FRAME_HEIGHT(&m_videoFrame); + + RefPtr surface = adoptRef(cairo_image_surface_create_for_data(bufferData, cairoFormat, width, height, stride)); ASSERT(cairo_surface_status(surface.get()) == CAIRO_STATUS_SUCCESS); m_image = BitmapImage::create(surface.release()); @@ -67,6 +75,6 @@ ImageGStreamer::~ImageGStreamer() // We keep the buffer memory mapped until the image is destroyed because the internal // cairo_surface_t was created using cairo_image_surface_create_for_data(). - gst_buffer_unmap(m_buffer.get(), &m_mapInfo); + gst_video_frame_unmap(&m_videoFrame); } #endif // USE(GSTREAMER) diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp index 4d12e3abd085e..b663a1b78c565 100644 --- a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp @@ -42,6 +42,7 @@ #include #include +#include #if GST_CHECK_VERSION(1, 1, 0) && USE(ACCELERATED_COMPOSITING) && USE(TEXTURE_MAPPER_GL) #include "TextureMapperGL.h" @@ -309,22 +310,19 @@ PassRefPtr MediaPlayerPrivateGStreamerBase::updateTexture(Texture { GMutexLocker lock(m_bufferMutex); if (!m_buffer) - return 0; + return nullptr; - const void* srcData = 0; GRefPtr caps = currentVideoSinkCaps(); if (!caps) - return 0; - - IntSize size; - GstVideoFormat format; - int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride; - if (!getVideoSizeAndFormatFromCaps(caps.get(), size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride)) - return 0; + return nullptr; - const GstVideoFormatInfo* formatInfo = gst_video_format_get_info(format); + GstVideoInfo videoInfo; + gst_video_info_init(&videoInfo); + if (!gst_video_info_from_caps(&videoInfo, caps.get())) + return nullptr; - RefPtr texture = textureMapper->acquireTextureFromPool(size, GST_VIDEO_FORMAT_INFO_HAS_ALPHA(formatInfo) ? BitmapTexture::SupportsAlpha : BitmapTexture::NoFlag); + IntSize size = IntSize(GST_VIDEO_INFO_WIDTH(&videoInfo), GST_VIDEO_INFO_HEIGHT(&videoInfo)); + RefPtr texture = textureMapper->acquireTextureFromPool(size, GST_VIDEO_INFO_HAS_ALPHA(&videoInfo) ? BitmapTexture::SupportsAlpha : BitmapTexture::NoFlag); #if GST_CHECK_VERSION(1, 1, 0) GstVideoGLTextureUploadMeta* meta; @@ -339,13 +337,18 @@ PassRefPtr MediaPlayerPrivateGStreamerBase::updateTexture(Texture } #endif - GstMapInfo srcInfo; - gst_buffer_map(m_buffer, &srcInfo, GST_MAP_READ); - srcData = srcInfo.data; + // Right now the TextureMapper only supports chromas with one plane + ASSERT(GST_VIDEO_INFO_N_PLANES(&videoInfo) == 1); + GstVideoFrame videoFrame; + if (!gst_video_frame_map(&videoFrame, &videoInfo, m_buffer, GST_MAP_READ)) + return nullptr; + + int stride = GST_VIDEO_FRAME_PLANE_STRIDE(&videoFrame, 0); + const void* srcData = GST_VIDEO_FRAME_PLANE_DATA(&videoFrame, 0); texture->updateContents(srcData, WebCore::IntRect(WebCore::IntPoint(0, 0), size), WebCore::IntPoint(0, 0), stride, BitmapTexture::UpdateCannotModifyOriginalImageData); + gst_video_frame_unmap(&videoFrame); - gst_buffer_unmap(m_buffer, &srcInfo); return texture; } #endif @@ -442,7 +445,7 @@ MediaPlayer::MovieLoadType MediaPlayerPrivateGStreamerBase::movieLoadType() cons GRefPtr MediaPlayerPrivateGStreamerBase::currentVideoSinkCaps() const { if (!m_webkitVideoSink) - return 0; + return nullptr; GRefPtr currentCaps; g_object_get(G_OBJECT(m_webkitVideoSink.get()), "current-caps", ¤tCaps.outPtr(), NULL); diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoSinkGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/VideoSinkGStreamer.cpp index 17d0aef7d9eea..651c7f4d3caee 100644 --- a/Source/WebCore/platform/graphics/gstreamer/VideoSinkGStreamer.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/VideoSinkGStreamer.cpp @@ -147,21 +147,14 @@ static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buf priv->buffer = gst_buffer_ref(buffer); - GRefPtr caps; // The video info structure is valid only if the sink handled an allocation query. - if (GST_VIDEO_INFO_FORMAT(&priv->info) != GST_VIDEO_FORMAT_UNKNOWN) - caps = adoptGRef(gst_video_info_to_caps(&priv->info)); - else - caps = priv->currentCaps; - - GstVideoFormat format; - WebCore::IntSize size; - int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride; - if (!getVideoSizeAndFormatFromCaps(caps.get(), size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride)) { + GstVideoFormat format = GST_VIDEO_INFO_FORMAT(&priv->info); + if (format == GST_VIDEO_FORMAT_UNKNOWN) { gst_buffer_unref(buffer); return GST_FLOW_ERROR; } +#if !(USE(TEXTURE_MAPPER_GL) && !USE(COORDINATED_GRAPHICS)) // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't. // Here we convert to Cairo's ARGB. if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) { @@ -173,22 +166,35 @@ static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buf GstBuffer* newBuffer = WebCore::createGstBuffer(buffer); // Check if allocation failed. - if (UNLIKELY(!newBuffer)) + if (UNLIKELY(!newBuffer)) { + gst_buffer_unref(buffer); return GST_FLOW_ERROR; + } // We don't use Color::premultipliedARGBFromColor() here because // one function call per video pixel is just too expensive: // For 720p/PAL for example this means 1280*720*25=23040000 // function calls per second! - GstMapInfo sourceInfo; - GstMapInfo destinationInfo; - gst_buffer_map(buffer, &sourceInfo, GST_MAP_READ); - const guint8* source = const_cast(sourceInfo.data); - gst_buffer_map(newBuffer, &destinationInfo, GST_MAP_WRITE); - guint8* destination = static_cast(destinationInfo.data); - - for (int x = 0; x < size.height(); x++) { - for (int y = 0; y < size.width(); y++) { + GstVideoFrame sourceFrame; + GstVideoFrame destinationFrame; + + if (!gst_video_frame_map(&sourceFrame, &priv->info, buffer, GST_MAP_READ)) { + gst_buffer_unref(buffer); + gst_buffer_unref(newBuffer); + return GST_FLOW_ERROR; + } + if (!gst_video_frame_map(&destinationFrame, &priv->info, newBuffer, GST_MAP_WRITE)) { + gst_video_frame_unmap(&sourceFrame); + gst_buffer_unref(buffer); + gst_buffer_unref(newBuffer); + return GST_FLOW_ERROR; + } + + const guint8* source = static_cast(GST_VIDEO_FRAME_PLANE_DATA(&sourceFrame, 0)); + guint8* destination = static_cast(GST_VIDEO_FRAME_PLANE_DATA(&destinationFrame, 0)); + + for (int x = 0; x < GST_VIDEO_FRAME_HEIGHT(&sourceFrame); x++) { + for (int y = 0; y < GST_VIDEO_FRAME_WIDTH(&sourceFrame); y++) { #if G_BYTE_ORDER == G_LITTLE_ENDIAN unsigned short alpha = source[3]; destination[0] = (source[0] * alpha + 128) / 255; @@ -207,11 +213,12 @@ static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buf } } - gst_buffer_unmap(buffer, &sourceInfo); - gst_buffer_unmap(newBuffer, &destinationInfo); + gst_video_frame_unmap(&sourceFrame); + gst_video_frame_unmap(&destinationFrame); gst_buffer_unref(buffer); buffer = priv->buffer = newBuffer; } +#endif // This should likely use a lower priority, but glib currently starves // lower priority sources. @@ -335,12 +342,14 @@ static gboolean webkitVideoSinkSetCaps(GstBaseSink* baseSink, GstCaps* caps) GST_DEBUG_OBJECT(sink, "Current caps %" GST_PTR_FORMAT ", setting caps %" GST_PTR_FORMAT, priv->currentCaps, caps); - GstVideoInfo info; - if (!gst_video_info_from_caps(&info, caps)) { + GstVideoInfo videoInfo; + gst_video_info_init(&videoInfo); + if (!gst_video_info_from_caps(&videoInfo, caps)) { GST_ERROR_OBJECT(sink, "Invalid caps %" GST_PTR_FORMAT, caps); return FALSE; } + priv->info = videoInfo; gst_caps_replace(&priv->currentCaps, caps); return TRUE; }