Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Painting an in-process VideoFrame in a remotely-backed canvas does no…
…t work

https://bugs.webkit.org/show_bug.cgi?id=250002
rdar://problem/103812253

Reviewed by Eric Carlson.

In-process rendering of a VideoFrame in a canvas is based on VideoFrame::paintInContext using a PixelBufferConformerCV.
This does not work in WebProcess once sandbox is tigthened/using remote rendering tree.
To fix this, we reuse the existing RemoteDisplayListRecorder infrastructure.
We use a SharedVideoFrameWriter/SharedVideoFrameReader in RemoteDisplayListRecorderProxy/RemoteDisplayListRecorder.
We enable ARGB since VideoFrameCV uses this type.

Covered by WPT webcodecs tests with remote rendering tree enabled.

* Source/WebCore/platform/graphics/BifurcatedGraphicsContext.cpp:
(WebCore::BifurcatedGraphicsContext::drawNativeImage):
* Source/WebCore/platform/graphics/cg/GraphicsContextCG.cpp:
(WebCore::GraphicsContextCG::drawNativeImage):
* Source/WebCore/platform/graphics/displaylists/DisplayListRecorder.cpp:
(WebCore::DisplayList::Recorder::drawNativeImage):
(WebCore::DisplayList::Recorder::paintVideoFrame):
* Source/WebCore/platform/graphics/displaylists/DisplayListRecorder.h:
* Source/WebCore/platform/graphics/displaylists/DisplayListRecorderImpl.cpp:
(WebCore::DisplayList::RecorderImpl::recordPaintFrameForMedia):
(WebCore::DisplayList::RecorderImpl::recordPaintVideoFrame):
* Source/WebCore/platform/graphics/displaylists/DisplayListRecorderImpl.h:
* Source/WebKit/GPUProcess/graphics/RemoteDisplayListRecorder.cpp:
(WebKit::RemoteDisplayListRecorder::drawNativeImage):
(WebKit::RemoteDisplayListRecorder::paintVideoFrame):
(WebKit::RemoteDisplayListRecorder::setSharedVideoFrameSemaphore):
(WebKit::RemoteDisplayListRecorder::setSharedVideoFrameMemory):
* Source/WebKit/GPUProcess/graphics/RemoteDisplayListRecorder.h:
* Source/WebKit/GPUProcess/graphics/RemoteDisplayListRecorder.messages.in:
* Source/WebKit/WebProcess/GPU/graphics/RemoteDisplayListRecorderProxy.cpp:
(WebKit::RemoteDisplayListRecorderProxy::recordPaintVideoFrame):
(WebKit::RemoteDisplayListRecorderProxy::disconnect):
* Source/WebKit/WebProcess/GPU/graphics/RemoteDisplayListRecorderProxy.h:
* Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.cpp:
* Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.h:

Canonical link: https://commits.webkit.org/258479@main
  • Loading branch information
youennf committed Jan 5, 2023
1 parent f78fd48 commit 2818008
Show file tree
Hide file tree
Showing 15 changed files with 80 additions and 14 deletions.
3 changes: 2 additions & 1 deletion Source/WebCore/platform/cocoa/SharedVideoFrameInfo.mm
Expand Up @@ -55,14 +55,15 @@
SharedVideoFrameInfo SharedVideoFrameInfo::fromCVPixelBuffer(CVPixelBufferRef pixelBuffer)
{
auto type = CVPixelBufferGetPixelFormatType(pixelBuffer);
if (type == kCVPixelFormatType_32BGRA)
if (type == kCVPixelFormatType_32BGRA || type == kCVPixelFormatType_32ARGB)
return { type, static_cast<uint32_t>(CVPixelBufferGetWidth(pixelBuffer)), static_cast<uint32_t>(CVPixelBufferGetHeight(pixelBuffer)), static_cast<uint32_t>(CVPixelBufferGetBytesPerRow(pixelBuffer)) };
return { type, static_cast<uint32_t>(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0)), static_cast<uint32_t>(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)), static_cast<uint32_t>(CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0)), static_cast<uint32_t>(CVPixelBufferGetWidthOfPlane(pixelBuffer, 1)), static_cast<uint32_t>(CVPixelBufferGetHeightOfPlane(pixelBuffer, 1)), static_cast<uint32_t>(CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1)) };
}

bool SharedVideoFrameInfo::isReadWriteSupported() const
{
return m_bufferType == kCVPixelFormatType_32BGRA
|| m_bufferType == kCVPixelFormatType_32ARGB
|| m_bufferType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
|| m_bufferType == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
|| m_bufferType == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange
Expand Down
2 changes: 0 additions & 2 deletions Source/WebCore/platform/graphics/GraphicsContext.cpp
Expand Up @@ -610,9 +610,7 @@ void GraphicsContext::paintFrameForMedia(MediaPlayer& player, const FloatRect& d
{
player.playerPrivate()->paintCurrentFrameInContext(*this, destination);
}
#endif

#if ENABLE(WEB_CODECS)
void GraphicsContext::paintVideoFrame(VideoFrame& frame, const FloatRect& destination, bool shouldDiscardAlpha)
{
frame.paintInContext(*this, destination, ImageOrientation::None, shouldDiscardAlpha);
Expand Down
2 changes: 0 additions & 2 deletions Source/WebCore/platform/graphics/GraphicsContext.h
Expand Up @@ -274,8 +274,6 @@ class GraphicsContext {

#if ENABLE(VIDEO)
WEBCORE_EXPORT virtual void paintFrameForMedia(MediaPlayer&, const FloatRect& destination);
#endif
#if ENABLE(WEB_CODECS)
WEBCORE_EXPORT virtual void paintVideoFrame(VideoFrame&, const FloatRect& destination, bool shouldDiscardAlpha);
#endif

Expand Down
Expand Up @@ -582,16 +582,10 @@ void Recorder::paintFrameForMedia(MediaPlayer& player, const FloatRect& destinat
ASSERT(player.identifier());
recordPaintFrameForMedia(player, destination);
}
#endif

#if ENABLE(WEB_CODECS)
void Recorder::paintVideoFrame(VideoFrame& frame, const FloatRect& destination, bool shouldDiscardAlpha)
{
if (!frame.isRemoteProxy()) {
GraphicsContext::paintVideoFrame(frame, destination, shouldDiscardAlpha);
return;
}
// FIXME: Implement recording.
recordPaintVideoFrame(frame, destination, shouldDiscardAlpha);
}
#endif

Expand Down
Expand Up @@ -119,6 +119,7 @@ class Recorder : public GraphicsContext {
virtual void recordFillEllipse(const FloatRect&) = 0;
#if ENABLE(VIDEO)
virtual void recordPaintFrameForMedia(MediaPlayer&, const FloatRect& destination) = 0;
virtual void recordPaintVideoFrame(VideoFrame&, const FloatRect& destination, bool shouldDiscardAlpha) = 0;
#endif
virtual void recordStrokeRect(const FloatRect&, float) = 0;
#if ENABLE(INLINE_PATH_DATA)
Expand Down Expand Up @@ -273,8 +274,6 @@ class Recorder : public GraphicsContext {

#if ENABLE(VIDEO)
WEBCORE_EXPORT void paintFrameForMedia(MediaPlayer&, const FloatRect& destination) final;
#endif
#if ENABLE(WEB_CODECS)
WEBCORE_EXPORT void paintVideoFrame(VideoFrame&, const FloatRect&, bool shouldDiscardAlpha) final;
#endif

Expand Down
Expand Up @@ -316,6 +316,11 @@ void RecorderImpl::recordPaintFrameForMedia(MediaPlayer& player, const FloatRect
{
append<PaintFrameForMedia>(player, destination);
}

void RecorderImpl::recordPaintVideoFrame(VideoFrame&, const FloatRect&, bool /* shouldDiscardAlpha */)
{
// FIXME: TODO
}
#endif // ENABLE(VIDEO)

void RecorderImpl::recordStrokeRect(const FloatRect& rect, float width)
Expand Down
Expand Up @@ -98,6 +98,7 @@ class RecorderImpl : public Recorder {
void recordFillEllipse(const FloatRect&) final;
#if ENABLE(VIDEO)
void recordPaintFrameForMedia(MediaPlayer&, const FloatRect& destination) final;
void recordPaintVideoFrame(VideoFrame&, const FloatRect& destination, bool shouldDiscardAlpha) final;
#endif
void recordStrokeRect(const FloatRect&, float) final;
#if ENABLE(INLINE_PATH_DATA)
Expand Down
21 changes: 21 additions & 0 deletions Source/WebKit/GPUProcess/graphics/RemoteDisplayListRecorder.cpp
Expand Up @@ -46,6 +46,9 @@ RemoteDisplayListRecorder::RemoteDisplayListRecorder(ImageBuffer& imageBuffer, Q
, m_imageBufferIdentifier(imageBufferIdentifier)
, m_webProcessIdentifier(webProcessIdentifier)
, m_renderingBackend(&renderingBackend)
#if PLATFORM(COCOA) && ENABLE(VIDEO)
, m_sharedVideoFrameReader(Ref { renderingBackend.gpuConnectionToWebProcess().videoFrameObjectHeap() }, renderingBackend.gpuConnectionToWebProcess().webProcessIdentity())
#endif
{
}

Expand Down Expand Up @@ -484,6 +487,24 @@ void RemoteDisplayListRecorder::paintFrameForMedia(MediaPlayerIdentifier identif
});
}

#if PLATFORM(COCOA) && ENABLE(VIDEO)
void RemoteDisplayListRecorder::paintVideoFrame(SharedVideoFrame&& frame, const WebCore::FloatRect& destination, bool shouldDiscardAlpha)
{
if (auto videoFrame = m_sharedVideoFrameReader.read(WTFMove(frame)))
drawingContext().paintVideoFrame(*videoFrame, destination, shouldDiscardAlpha);
}

void RemoteDisplayListRecorder::setSharedVideoFrameSemaphore(IPC::Semaphore&& semaphore)
{
m_sharedVideoFrameReader.setSemaphore(WTFMove(semaphore));
}

void RemoteDisplayListRecorder::setSharedVideoFrameMemory(const SharedMemory::Handle& handle)
{
m_sharedVideoFrameReader.setSharedMemory(handle);
}
#endif // PLATFORM(COCOA) && ENABLE(VIDEO)

void RemoteDisplayListRecorder::strokeRect(const FloatRect& rect, float lineWidth)
{
handleItem(DisplayList::StrokeRect(rect, lineWidth));
Expand Down
10 changes: 10 additions & 0 deletions Source/WebKit/GPUProcess/graphics/RemoteDisplayListRecorder.h
Expand Up @@ -31,6 +31,7 @@
#include "DisplayListRecorderFlushIdentifier.h"
#include "QualifiedRenderingResourceIdentifier.h"
#include "RemoteRenderingBackend.h"
#include "SharedVideoFrame.h"
#include "StreamMessageReceiver.h"
#include "StreamServerConnection.h"
#include <WebCore/ControlFactory.h>
Expand Down Expand Up @@ -160,11 +161,20 @@ class RemoteDisplayListRecorder : public IPC::StreamMessageReceiver, public CanM
void startListeningForIPC();
void didReceiveStreamMessage(IPC::StreamServerConnection&, IPC::Decoder&) final;

#if PLATFORM(COCOA) && ENABLE(VIDEO)
void paintVideoFrame(SharedVideoFrame&&, const WebCore::FloatRect&, bool shouldDiscardAlpha);
void setSharedVideoFrameSemaphore(IPC::Semaphore&&);
void setSharedVideoFrameMemory(const SharedMemory::Handle&);
#endif

WeakPtr<WebCore::ImageBuffer> m_imageBuffer;
QualifiedRenderingResourceIdentifier m_imageBufferIdentifier;
WebCore::ProcessIdentifier m_webProcessIdentifier;
RefPtr<RemoteRenderingBackend> m_renderingBackend;
std::unique_ptr<WebCore::ControlFactory> m_controlFactory;
#if PLATFORM(COCOA) && ENABLE(VIDEO)
SharedVideoFrameReader m_sharedVideoFrameReader;
#endif
};

} // namespace WebKit
Expand Down
Expand Up @@ -96,6 +96,12 @@ messages -> RemoteDisplayListRecorder NotRefCounted Stream {
#endif
ApplyDeviceScaleFactor(float scaleFactor)
FlushContext(WebKit::DisplayListRecorderFlushIdentifier identifier)

#if PLATFORM(COCOA) && ENABLE(VIDEO)
PaintVideoFrame(struct WebKit::SharedVideoFrame frame, WebCore::FloatRect rect, bool shouldDiscardAlpha) NotStreamEncodable
SetSharedVideoFrameSemaphore(IPC::Semaphore semaphore) NotStreamEncodable
SetSharedVideoFrameMemory(WebKit::SharedMemory::Handle storageHandle) NotStreamEncodable
#endif
}

#endif // ENABLE(GPU_PROCESS)
3 changes: 3 additions & 0 deletions Source/WebKit/GPUProcess/graphics/RemoteRenderingBackend.h
Expand Up @@ -103,6 +103,9 @@ class RemoteRenderingBackend : private IPC::MessageSender, public IPC::StreamMes
const WebCore::ProcessIdentity& resourceOwner() const { return m_resourceOwner; }

void releaseResource(WebCore::RenderingResourceIdentifier);

GPUConnectionToWebProcess& gpuConnectionToWebProcess() { return m_gpuConnectionToWebProcess.get(); }

private:
RemoteRenderingBackend(GPUConnectionToWebProcess&, RemoteRenderingBackendCreationParameters&&, IPC::StreamServerConnection::Handle&&);
void startListeningForIPC();
Expand Down
Expand Up @@ -337,11 +337,27 @@ void RemoteDisplayListRecorderProxy::recordFillEllipse(const FloatRect& rect)
send(Messages::RemoteDisplayListRecorder::FillEllipse(rect));
}

#if ENABLE(VIDEO)
void RemoteDisplayListRecorderProxy::recordPaintFrameForMedia(MediaPlayer& player, const FloatRect& destination)
{
send(Messages::RemoteDisplayListRecorder::PaintFrameForMedia(player.identifier(), destination));
}

void RemoteDisplayListRecorderProxy::recordPaintVideoFrame(VideoFrame& frame, const FloatRect& destination, bool shouldDiscardAlpha)
{
#if PLATFORM(COCOA)
auto sharedVideoFrame = m_sharedVideoFrameWriter.write(frame, [&](auto& semaphore) {
send(Messages::RemoteDisplayListRecorder::SetSharedVideoFrameSemaphore { semaphore });
}, [&](auto& handle) {
send(Messages::RemoteDisplayListRecorder::SetSharedVideoFrameMemory { handle });
});
if (!sharedVideoFrame)
return;
send(Messages::RemoteDisplayListRecorder::PaintVideoFrame(*sharedVideoFrame, destination, shouldDiscardAlpha));
#endif
}
#endif

void RemoteDisplayListRecorderProxy::recordStrokeRect(const FloatRect& rect, float width)
{
send(Messages::RemoteDisplayListRecorder::StrokeRect(rect, width));
Expand Down Expand Up @@ -508,6 +524,9 @@ RefPtr<ImageBuffer> RemoteDisplayListRecorderProxy::createAlignedImageBuffer(con
void RemoteDisplayListRecorderProxy::disconnect()
{
m_renderingBackend = nullptr;
#if PLATFORM(COCOA) && ENABLE(VIDEO)
m_sharedVideoFrameWriter.disable();
#endif
}

} // namespace WebCore
Expand Down
Expand Up @@ -28,6 +28,7 @@
#if ENABLE(GPU_PROCESS)

#include "DisplayListRecorderFlushIdentifier.h"
#include "SharedVideoFrame.h"
#include <WebCore/DisplayListRecorder.h>
#include <WebCore/DrawGlyphsRecorder.h>
#include <WebCore/GraphicsContext.h>
Expand Down Expand Up @@ -111,7 +112,10 @@ class RemoteDisplayListRecorderProxy : public WebCore::DisplayList::Recorder {
#endif
void recordFillPath(const WebCore::Path&) final;
void recordFillEllipse(const WebCore::FloatRect&) final;
#if ENABLE(VIDEO)
void recordPaintFrameForMedia(WebCore::MediaPlayer&, const WebCore::FloatRect& destination) final;
void recordPaintVideoFrame(WebCore::VideoFrame&, const WebCore::FloatRect&, bool shouldDiscardAlpha) final;
#endif
void recordStrokeRect(const WebCore::FloatRect&, float) final;
#if ENABLE(INLINE_PATH_DATA)
void recordStrokeLine(const WebCore::LineData&) final;
Expand Down Expand Up @@ -144,6 +148,9 @@ class RemoteDisplayListRecorderProxy : public WebCore::DisplayList::Recorder {
WebCore::RenderingResourceIdentifier m_destinationBufferIdentifier;
WeakPtr<RemoteImageBufferProxy> m_imageBuffer;
WeakPtr<RemoteRenderingBackendProxy> m_renderingBackend;
#if PLATFORM(COCOA) && ENABLE(VIDEO)
SharedVideoFrameWriter m_sharedVideoFrameWriter;
#endif
};

} // namespace WebKit
Expand Down
2 changes: 2 additions & 0 deletions Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.cpp
Expand Up @@ -201,6 +201,8 @@ SharedVideoFrameReader::SharedVideoFrameReader(RefPtr<RemoteVideoFrameObjectHeap

SharedVideoFrameReader::SharedVideoFrameReader() = default;

SharedVideoFrameReader::~SharedVideoFrameReader() = default;

RetainPtr<CVPixelBufferRef> SharedVideoFrameReader::readBufferFromSharedMemory()
{
if (!m_storage) {
Expand Down
2 changes: 2 additions & 0 deletions Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.h
Expand Up @@ -100,6 +100,8 @@ class SharedVideoFrameWriter {
class SharedVideoFrameReader {
WTF_MAKE_FAST_ALLOCATED;
public:
~SharedVideoFrameReader();

enum class UseIOSurfaceBufferPool { No, Yes };
explicit SharedVideoFrameReader(RefPtr<RemoteVideoFrameObjectHeap>&&, const WebCore::ProcessIdentity& = { }, UseIOSurfaceBufferPool = UseIOSurfaceBufferPool::Yes);
SharedVideoFrameReader();
Expand Down

0 comments on commit 2818008

Please sign in to comment.