Skip to content
Permalink
Browse files
[GStreamer] Add support to copy YUV video textures into platform text…
…ures

https://bugs.webkit.org/show_bug.cgi?id=200914

Patch by Chris Lord <clord@igalia.com> on 2019-09-03
Reviewed by Xabier Rodriguez-Calvar and Miguel Gomez.

Enable YUV (including planar and semi-planar) video texture to platform
texture copy in VideoTextureCopierGStreamer.

No new tests, not changing behavior.

* platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
(WebCore::GstVideoFrameHolder::platformLayerBuffer):
(WebCore::MediaPlayerPrivateGStreamerBase::pushTextureToCompositor):
(WebCore::MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture):
(WebCore::MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime):
* platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp:
(WebCore::VideoTextureCopierGStreamer::VideoTextureCopierGStreamer):
(WebCore::VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture):
* platform/graphics/gstreamer/VideoTextureCopierGStreamer.h:
* platform/graphics/texmap/TextureMapperPlatformLayerBuffer.h:
(WebCore::TextureMapperPlatformLayerBuffer::textureVariant):

Canonical link: https://commits.webkit.org/215059@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@249428 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
Chris Lord authored and webkit-commit-queue committed Sep 3, 2019
1 parent 6095642 commit 1b0d8276c6c544a006b654a0b153f89d0543dad2
@@ -1,3 +1,27 @@
2019-09-03 Chris Lord <clord@igalia.com>

[GStreamer] Add support to copy YUV video textures into platform textures
https://bugs.webkit.org/show_bug.cgi?id=200914

Reviewed by Xabier Rodriguez-Calvar and Miguel Gomez.

Enable YUV (including planar and semi-planar) video texture to platform
texture copy in VideoTextureCopierGStreamer.

No new tests, not changing behavior.

* platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
(WebCore::GstVideoFrameHolder::platformLayerBuffer):
(WebCore::MediaPlayerPrivateGStreamerBase::pushTextureToCompositor):
(WebCore::MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture):
(WebCore::MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime):
* platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp:
(WebCore::VideoTextureCopierGStreamer::VideoTextureCopierGStreamer):
(WebCore::VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture):
* platform/graphics/gstreamer/VideoTextureCopierGStreamer.h:
* platform/graphics/texmap/TextureMapperPlatformLayerBuffer.h:
(WebCore::TextureMapperPlatformLayerBuffer::textureVariant):

2019-09-03 Zan Dobersek <zdobersek@igalia.com> and Chris Lord <clord@igalia.com>

[Texmap][GStreamer] Add support to upload more color formats into the texture
@@ -216,6 +216,53 @@ class GstVideoFrameHolder : public TextureMapperPlatformLayerBuffer::UnmanagedBu
texture.updateContents(srcData, WebCore::IntRect(0, 0, m_size.width(), m_size.height()), WebCore::IntPoint(0, 0), stride);
}

std::unique_ptr<TextureMapperPlatformLayerBuffer> platformLayerBuffer()
{
if (!m_hasMappedTextures)
return nullptr;

using Buffer = TextureMapperPlatformLayerBuffer;

if ((GST_VIDEO_INFO_IS_RGB(&m_videoFrame.info) && GST_VIDEO_INFO_N_PLANES(&m_videoFrame.info) == 1))
return makeUnique<Buffer>(Buffer::TextureVariant { Buffer::RGBTexture { *static_cast<GLuint*>(m_videoFrame.data[0]) } }, m_size, m_flags, GraphicsContext3D::RGBA);

if (GST_VIDEO_INFO_IS_YUV(&m_videoFrame.info)) {
if (GST_VIDEO_INFO_N_COMPONENTS(&m_videoFrame.info) < 3 || GST_VIDEO_INFO_N_PLANES(&m_videoFrame.info) > 3)
return nullptr;

unsigned numberOfPlanes = GST_VIDEO_INFO_N_PLANES(&m_videoFrame.info);
std::array<GLuint, 3> planes;
std::array<unsigned, 3> yuvPlane;
std::array<unsigned, 3> yuvPlaneOffset;
for (unsigned i = 0; i < numberOfPlanes; ++i)
planes[i] = *static_cast<GLuint*>(m_videoFrame.data[i]);
for (unsigned i = 0; i < 3; ++i) {
yuvPlane[i] = GST_VIDEO_INFO_COMP_PLANE(&m_videoFrame.info, i);
yuvPlaneOffset[i] = GST_VIDEO_INFO_COMP_POFFSET(&m_videoFrame.info, i);
}

std::array<GLfloat, 9> yuvToRgb;
if (gst_video_colorimetry_matches(&GST_VIDEO_INFO_COLORIMETRY(&m_videoFrame.info), GST_VIDEO_COLORIMETRY_BT709)) {
yuvToRgb = {
1.164f, 0.0f, 1.787f,
1.164f, -0.213f, -0.531f,
1.164f, 2.112f, 0.0f
};
} else {
// Default to bt601. This is the same behaviour as GStreamer's glcolorconvert element.
yuvToRgb = {
1.164f, 0.0f, 1.596f,
1.164f, -0.391f, -0.813f,
1.164f, 2.018f, 0.0f
};
}

return makeUnique<Buffer>( Buffer::TextureVariant { Buffer::YUVTexture { numberOfPlanes, planes, yuvPlane, yuvPlaneOffset, yuvToRgb } }, m_size, m_flags, GraphicsContext3D::RGBA);
}

return nullptr;
}

private:
GstBuffer* m_buffer;
GstVideoFrame m_videoFrame { };
@@ -732,57 +779,11 @@ void MediaPlayerPrivateGStreamerBase::pushTextureToCompositor()

std::unique_ptr<GstVideoFrameHolder> frameHolder = makeUnique<GstVideoFrameHolder>(m_sample.get(), m_textureMapperFlags, !m_usingFallbackVideoSink);

using Buffer = TextureMapperPlatformLayerBuffer;
std::unique_ptr<Buffer> layerBuffer;
std::unique_ptr<TextureMapperPlatformLayerBuffer> layerBuffer;
if (frameHolder->hasMappedTextures()) {
auto& videoFrame = frameHolder->videoFrame();
[&] {
if ((GST_VIDEO_INFO_IS_RGB(&videoFrame.info) && GST_VIDEO_INFO_N_PLANES(&videoFrame.info) == 1)) {
layerBuffer = makeUnique<Buffer>(
Buffer::TextureVariant { Buffer::RGBTexture { *static_cast<GLuint*>(videoFrame.data[0]) } },
frameHolder->size(), frameHolder->flags(), GraphicsContext3D::RGBA);
return;
}

if (GST_VIDEO_INFO_IS_YUV(&videoFrame.info)) {
if (GST_VIDEO_INFO_N_COMPONENTS(&videoFrame.info) < 3 || GST_VIDEO_INFO_N_PLANES(&videoFrame.info) > 3)
return;

unsigned numberOfPlanes = GST_VIDEO_INFO_N_PLANES(&videoFrame.info);
std::array<GLuint, 3> planes;
std::array<unsigned, 3> yuvPlane;
std::array<unsigned, 3> yuvPlaneOffset;
for (unsigned i = 0; i < numberOfPlanes; ++i)
planes[i] = *static_cast<GLuint*>(videoFrame.data[i]);
for (unsigned i = 0; i < 3; ++i) {
yuvPlane[i] = GST_VIDEO_INFO_COMP_PLANE(&videoFrame.info, i);
yuvPlaneOffset[i] = GST_VIDEO_INFO_COMP_POFFSET(&videoFrame.info, i);
}

std::array<GLfloat, 9> yuvToRgb;
if (gst_video_colorimetry_matches(&GST_VIDEO_INFO_COLORIMETRY(&videoFrame.info), GST_VIDEO_COLORIMETRY_BT709)) {
yuvToRgb = {
1.164f, 0.0f, 1.787f,
1.164f, -0.213f, -0.531f,
1.164f, 2.112f, 0.0f
};
} else {
// Default to bt601. This is the same behaviour as GStreamer's glcolorconvert element.
yuvToRgb = {
1.164f, 0.0f, 1.596f,
1.164f, -0.391f, -0.813f,
1.164f, 2.018f, 0.0f
};
}

layerBuffer = makeUnique<Buffer>(Buffer::TextureVariant { Buffer::YUVTexture { numberOfPlanes, planes, yuvPlane, yuvPlaneOffset, yuvToRgb } },
frameHolder->size(), frameHolder->flags(), GraphicsContext3D::RGBA);
}
}();

layerBuffer = frameHolder->platformLayerBuffer();
if (!layerBuffer)
return;

layerBuffer->setUnmanagedBufferDataHolder(WTFMove(frameHolder));
} else {
layerBuffer = proxy.getAvailableBuffer(frameHolder->size(), GL_DONT_CARE);
@@ -993,8 +994,8 @@ bool MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture(Graphics

std::unique_ptr<GstVideoFrameHolder> frameHolder = makeUnique<GstVideoFrameHolder>(m_sample.get(), m_textureMapperFlags, true);

auto textureID = frameHolder->textureID();
if (!textureID)
std::unique_ptr<TextureMapperPlatformLayerBuffer> layerBuffer = frameHolder->platformLayerBuffer();
if (!layerBuffer)
return false;

auto size = frameHolder->size();
@@ -1004,7 +1005,9 @@ bool MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture(Graphics
if (!m_videoTextureCopier)
m_videoTextureCopier = makeUnique<VideoTextureCopierGStreamer>(TEXTURE_COPIER_COLOR_CONVERT_FLAG);

return m_videoTextureCopier->copyVideoTextureToPlatformTexture(textureID, size, outputTexture, outputTarget, level, internalFormat, format, type, flipY, m_videoSourceOrientation);
frameHolder->waitForCPUSync();

return m_videoTextureCopier->copyVideoTextureToPlatformTexture(*layerBuffer.get(), size, outputTexture, outputTarget, level, internalFormat, format, type, flipY, m_videoSourceOrientation);
}

NativeImagePtr MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime()
@@ -1020,8 +1023,8 @@ NativeImagePtr MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime()

std::unique_ptr<GstVideoFrameHolder> frameHolder = makeUnique<GstVideoFrameHolder>(m_sample.get(), m_textureMapperFlags, true);

auto textureID = frameHolder->textureID();
if (!textureID)
std::unique_ptr<TextureMapperPlatformLayerBuffer> layerBuffer = frameHolder->platformLayerBuffer();
if (!layerBuffer)
return nullptr;

auto size = frameHolder->size();
@@ -1034,7 +1037,9 @@ NativeImagePtr MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime()
if (!m_videoTextureCopier)
m_videoTextureCopier = makeUnique<VideoTextureCopierGStreamer>(TEXTURE_COPIER_COLOR_CONVERT_FLAG);

if (!m_videoTextureCopier->copyVideoTextureToPlatformTexture(textureID, size, 0, GraphicsContext3D::TEXTURE_2D, 0, GraphicsContext3D::RGBA, GraphicsContext3D::RGBA, GraphicsContext3D::UNSIGNED_BYTE, false, m_videoSourceOrientation))
frameHolder->waitForCPUSync();

if (!m_videoTextureCopier->copyVideoTextureToPlatformTexture(*layerBuffer.get(), size, 0, GraphicsContext3D::TEXTURE_2D, 0, GraphicsContext3D::RGBA, GraphicsContext3D::RGBA, GraphicsContext3D::UNSIGNED_BYTE, false, m_videoSourceOrientation))
return nullptr;

return adoptRef(cairo_gl_surface_create_for_texture(context->cairoDevice(), CAIRO_CONTENT_COLOR_ALPHA, m_videoTextureCopier->resultTexture(), size.width(), size.height()));
@@ -35,8 +35,6 @@ VideoTextureCopierGStreamer::VideoTextureCopierGStreamer(ColorConversion colorCo
ASSERT(previousContext);
PlatformDisplay::sharedDisplayForCompositing().sharingGLContext()->makeContextCurrent();

m_shaderProgram = TextureMapperShaderProgram::create(TextureMapperShaderProgram::TextureRGB);

glGenFramebuffers(1, &m_framebuffer);
glGenTextures(1, &m_resultTexture);

@@ -137,9 +135,9 @@ void VideoTextureCopierGStreamer::updateTransformationMatrix()
-1, 1, -(farValue + nearValue) / (farValue - nearValue), 1);
}

bool VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture(GLuint inputTexture, IntSize& frameSize, GLuint outputTexture, GLenum outputTarget, GLint level, GLenum internalFormat, GLenum format, GLenum type, bool flipY, ImageOrientation sourceOrientation)
bool VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture(TextureMapperPlatformLayerBuffer& inputTexture, IntSize& frameSize, GLuint outputTexture, GLenum outputTarget, GLint level, GLenum internalFormat, GLenum format, GLenum type, bool flipY, ImageOrientation sourceOrientation)
{
if (!m_shaderProgram || !m_framebuffer || !m_vbo || frameSize.isEmpty())
if (!m_framebuffer || !m_vbo || frameSize.isEmpty())
return false;

if (m_size != frameSize) {
@@ -158,6 +156,38 @@ bool VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture(GLuint input
ASSERT(previousContext);
PlatformDisplay::sharedDisplayForCompositing().sharingGLContext()->makeContextCurrent();

// Determine what shader program to use and create it if necessary.
using Buffer = TextureMapperPlatformLayerBuffer;
TextureMapperShaderProgram::Options options;
WTF::switchOn(inputTexture.textureVariant(),
[&](const Buffer::RGBTexture&) { options = TextureMapperShaderProgram::TextureRGB; },
[&](const Buffer::YUVTexture& texture) {
switch (texture.numberOfPlanes) {
case 1:
ASSERT(texture.yuvPlane[0] == texture.yuvPlane[1] && texture.yuvPlane[1] == texture.yuvPlane[2]);
ASSERT(texture.yuvPlaneOffset[0] == 2 && texture.yuvPlaneOffset[1] == 1 && !texture.yuvPlaneOffset[2]);
options = TextureMapperShaderProgram::TexturePackedYUV;
break;
case 2:
ASSERT(!texture.yuvPlaneOffset[0]);
options = texture.yuvPlaneOffset[1] ? TextureMapperShaderProgram::TextureNV21 : TextureMapperShaderProgram::TextureNV12;
break;
case 3:
ASSERT(!texture.yuvPlaneOffset[0] && !texture.yuvPlaneOffset[1] && !texture.yuvPlaneOffset[2]);
options = TextureMapperShaderProgram::TextureYUV;
break;
}
});

if (options != m_shaderOptions) {
m_shaderProgram = TextureMapperShaderProgram::create(options);
m_shaderOptions = options;
}
if (!m_shaderProgram) {
previousContext->makeContextCurrent();
return false;
}

// Save previous bound framebuffer, texture and viewport.
GLint boundFramebuffer = 0;
GLint boundTexture = 0;
@@ -181,18 +211,48 @@ bool VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture(GLuint input
glBindFramebuffer(GL_FRAMEBUFFER, m_framebuffer);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, outputTexture, 0);

// Set proper wrap parameter to the source texture.
glBindTexture(GL_TEXTURE_2D, inputTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

// Set the viewport.
glViewport(0, 0, m_size.width(), m_size.height());

// Set program parameters.
glUseProgram(m_shaderProgram->programID());
glUniform1i(m_shaderProgram->samplerLocation(), 0);

WTF::switchOn(inputTexture.textureVariant(),
[&](const Buffer::RGBTexture& texture) {
glUniform1i(m_shaderProgram->samplerLocation(), 0);

glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, texture.id);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
},
[&](const Buffer::YUVTexture& texture) {
switch (texture.numberOfPlanes) {
case 1:
glUniform1i(m_shaderProgram->samplerLocation(), texture.yuvPlane[0]);
break;
case 2:
glUniform1i(m_shaderProgram->samplerYLocation(), texture.yuvPlane[0]);
glUniform1i(m_shaderProgram->samplerULocation(), texture.yuvPlane[1]);
break;
case 3:
glUniform1i(m_shaderProgram->samplerYLocation(), texture.yuvPlane[0]);
glUniform1i(m_shaderProgram->samplerULocation(), texture.yuvPlane[1]);
glUniform1i(m_shaderProgram->samplerVLocation(), texture.yuvPlane[2]);
break;
}
glUniformMatrix3fv(m_shaderProgram->yuvToRgbLocation(), 1, GL_FALSE, static_cast<const GLfloat *>(&texture.yuvToRgbMatrix[0]));

for (int i = texture.numberOfPlanes - 1; i >= 0; --i) {
glActiveTexture(GL_TEXTURE0 + i);
glBindTexture(GL_TEXTURE_2D, texture.planes[i]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
});

m_shaderProgram->setMatrix(m_shaderProgram->modelViewMatrixLocation(), m_modelViewMatrix);
m_shaderProgram->setMatrix(m_shaderProgram->projectionMatrixLocation(), m_projectionMatrix);
m_shaderProgram->setMatrix(m_shaderProgram->textureSpaceMatrixLocation(), m_textureSpaceMatrix);
@@ -24,6 +24,7 @@

#include "ImageOrientation.h"
#include "TextureMapperGLHeaders.h"
#include "TextureMapperPlatformLayerBuffer.h"
#include "TransformationMatrix.h"
#include <wtf/RefPtr.h>

@@ -44,14 +45,15 @@ class VideoTextureCopierGStreamer {
VideoTextureCopierGStreamer(ColorConversion);
~VideoTextureCopierGStreamer();

bool copyVideoTextureToPlatformTexture(GLuint inputTexture, IntSize& frameSize, GLuint outputTexture, GLenum outputTarget, GLint level, GLenum internalFormat, GLenum format, GLenum type, bool flipY, ImageOrientation sourceOrientation);
bool copyVideoTextureToPlatformTexture(TextureMapperPlatformLayerBuffer& inputTexture, IntSize& frameSize, GLuint outputTexture, GLenum outputTarget, GLint level, GLenum internalFormat, GLenum format, GLenum type, bool flipY, ImageOrientation sourceOrientation);
void updateColorConversionMatrix(ColorConversion);
void updateTextureSpaceMatrix();
void updateTransformationMatrix();
GLuint resultTexture() { return m_resultTexture; }

private:
RefPtr<TextureMapperShaderProgram> m_shaderProgram;
unsigned m_shaderOptions { 0 };
GLuint m_framebuffer { 0 };
GLuint m_vbo { 0 };
#if !USE(OPENGL_ES)
@@ -92,6 +92,8 @@ class TextureMapperPlatformLayerBuffer : public TextureMapperPlatformLayer {

void setHolePunchClient(std::unique_ptr<HolePunchClient>&& client) { m_holePunchClient = WTFMove(client); }

const TextureVariant& textureVariant() { return m_variant; }

private:

RefPtr<BitmapTexture> m_texture;

0 comments on commit 1b0d827

Please sign in to comment.