Skip to content
Permalink
Browse files
Implement GPU based WebCodecsVideoDecoder flush
https://bugs.webkit.org/show_bug.cgi?id=246328

Reviewed by Eric Carlson.

Add IPC to send flush order from WebProcess to GPUProcess.
Call VTDecompressionSessionWaitForAsynchronousFrames in GPUProcess to flush all frames.
This might block the codec queue which is for the whole GPU Process.
We might in a follow-up have a queue per web process.

Covered by rebasd test.

* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker_h264_annexb-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any_h264_annexb-expected.txt:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitDecoder.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitDecoder.mm:
(-[WK_RTCLocalVideoH264H265VP9Decoder flush]):
(webrtc::flushLocalDecoder):
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm:
(-[RTCVideoDecoderH264 flush]):
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm:
(-[RTCVideoDecoderH265 flush]):
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderVTBVP9.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderVTBVP9.mm:
(-[RTCVideoDecoderVTBVP9 flush]):
* Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h:
* Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in:
* Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm:
(WebKit::LibWebRTCCodecsProxy::createDecoderCallback):
(WebKit::LibWebRTCCodecsProxy::createLocalDecoder):
(WebKit::LibWebRTCCodecsProxy::flushDecoder):
* Source/WebKit/WebProcess/GPU/media/RemoteVideoCodecFactory.cpp:
(WebKit::RemoteVideoDecoder::decode):
(WebKit::RemoteVideoDecoder::flush):
(WebKit::RemoteVideoDecoder::reset):
(WebKit::RemoteVideoDecoderCallbacks::notifyVideoFrame):
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp:
(WebKit::LibWebRTCCodecs::flushDecoder):
(WebKit::LibWebRTCCodecs::flushDecoderCompleted):
(WebKit::LibWebRTCCodecs::gpuProcessConnectionDidClose):
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h:
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.messages.in:

Canonical link: https://commits.webkit.org/255388@main
  • Loading branch information
youennf committed Oct 11, 2022
1 parent 0b9c1e6 commit 38b1154457d6e4d9ab9548da02a233eb1a68c00c
Show file tree
Hide file tree
Showing 20 changed files with 108 additions and 12 deletions.
@@ -4,17 +4,17 @@ PASS Test isConfigSupported() with 1080p crop
PASS Test that isConfigSupported() returns a parsed configuration
PASS Test invalid configs
PASS Test configure()
FAIL Decode a key frame assert_equals: outputs expected 1 but got 0
PASS Decode a key frame
PASS Decode a non key frame first fails
FAIL Verify reset() suppresses outputs assert_equals: expected 5 but got 1
PASS Verify reset() suppresses outputs
PASS Test unconfigured VideoDecoder operations
PASS Test closed VideoDecoder operations
FAIL Decode empty frame assert_unreached: Should have rejected: undefined Reached unreachable code
FAIL Decode corrupt frame assert_unreached: Should have rejected: undefined Reached unreachable code
PASS Close while decoding corrupt frame
FAIL Test decoding after flush assert_equals: outputs expected 1 but got 0
FAIL Test decoding a with negative timestamp assert_equals: outputs expected 1 but got 0
FAIL Test reset during flush assert_unreached: Should have rejected: undefined Reached unreachable code
PASS Test decoding after flush
FAIL Test decoding a with negative timestamp assert_equals: timestamp expected -42 but got 4294967254
PASS Test reset during flush
PASS Test low-latency decoding
PASS VideoDecoder decodeQueueSize test

@@ -4,17 +4,17 @@ PASS Test isConfigSupported() with 1080p crop
PASS Test that isConfigSupported() returns a parsed configuration
PASS Test invalid configs
PASS Test configure()
FAIL Decode a key frame assert_equals: outputs expected 1 but got 0
PASS Decode a key frame
PASS Decode a non key frame first fails
FAIL Verify reset() suppresses outputs assert_equals: expected 5 but got 1
PASS Verify reset() suppresses outputs
PASS Test unconfigured VideoDecoder operations
PASS Test closed VideoDecoder operations
FAIL Decode empty frame assert_unreached: Should have rejected: undefined Reached unreachable code
FAIL Decode corrupt frame assert_unreached: Should have rejected: undefined Reached unreachable code
PASS Close while decoding corrupt frame
FAIL Test decoding after flush assert_equals: outputs expected 1 but got 0
FAIL Test decoding a with negative timestamp assert_equals: outputs expected 1 but got 0
FAIL Test reset during flush assert_unreached: Should have rejected: undefined Reached unreachable code
PASS Test decoding after flush
FAIL Test decoding a with negative timestamp assert_equals: timestamp expected -42 but got 4294967254
PASS Test reset during flush
PASS Test low-latency decoding
PASS VideoDecoder decodeQueueSize test

@@ -199,6 +199,7 @@ __ZN6webrtc22createLocalH264DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc22createLocalH265DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc21createLocalVP9DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc19releaseLocalDecoderEPv
__ZN6webrtc17flushLocalDecoderEPv
__ZN6webrtc11decodeFrameEPvjPKhm
__ZN6webrtc24videoDecoderTaskCompleteEPvjjP10__CVBuffer
__ZN6webrtc24videoDecoderTaskCompleteEPvjjS0_PFP10__CVBufferS0_EPFvS0_Eii
@@ -199,6 +199,7 @@ __ZN6webrtc22createLocalH264DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc22createLocalH265DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc21createLocalVP9DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc19releaseLocalDecoderEPv
__ZN6webrtc17flushLocalDecoderEPv
__ZN6webrtc11decodeFrameEPvjPKhm
__ZN6webrtc24videoDecoderTaskCompleteEPvjjP10__CVBuffer
__ZN6webrtc24videoDecoderTaskCompleteEPvjjS0_PFP10__CVBufferS0_EPFvS0_Eii
@@ -199,6 +199,7 @@ __ZN6webrtc22createLocalH264DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc22createLocalH265DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc21createLocalVP9DecoderEU13block_pointerFvP10__CVBufferjjE
__ZN6webrtc19releaseLocalDecoderEPv
__ZN6webrtc17flushLocalDecoderEPv
__ZN6webrtc11decodeFrameEPvjPKhm
__ZN6webrtc24videoDecoderTaskCompleteEPvjjP10__CVBuffer
__ZN6webrtc24videoDecoderTaskCompleteEPvjjS0_PFP10__CVBufferS0_EPFvS0_Eii
@@ -63,6 +63,7 @@ void* createLocalH264Decoder(LocalDecoderCallback);
void* createLocalH265Decoder(LocalDecoderCallback);
void* createLocalVP9Decoder(LocalDecoderCallback);
void releaseLocalDecoder(LocalDecoder);
void flushLocalDecoder(LocalDecoder);
int32_t decodeFrame(LocalDecoder, uint32_t timeStamp, const uint8_t*, size_t);
void setDecoderFrameSize(LocalDecoder, uint16_t width, uint16_t height);

@@ -44,6 +44,7 @@ - (instancetype)initH264DecoderWithCallback:(webrtc::LocalDecoderCallback)callba
- (instancetype)initH265DecoderWithCallback:(webrtc::LocalDecoderCallback)callback;
- (NSInteger)decodeData:(const uint8_t *)data size:(size_t)size timeStamp:(uint32_t)timeStamp;
- (NSInteger)releaseDecoder;
- (void)flush;
@end

@implementation WK_RTCLocalVideoH264H265VP9Decoder {
@@ -108,6 +109,19 @@ - (NSInteger)releaseDecoder {
return [m_h265Decoder releaseDecoder];
return [m_vp9Decoder releaseDecoder];
}

- (void)flush {
if (m_h264Decoder) {
[m_h264Decoder flush];
return;
}
if (m_h265Decoder) {
[m_h265Decoder flush];
return;
}
[m_vp9Decoder flush];
}

@end

namespace webrtc {
@@ -285,6 +299,12 @@ void releaseLocalDecoder(LocalDecoder localDecoder)
[decoder releaseDecoder];
}

void flushLocalDecoder(LocalDecoder localDecoder)
{
auto* decoder = (__bridge WK_RTCLocalVideoH264H265VP9Decoder *)(localDecoder);
[decoder flush];
}

int32_t decodeFrame(LocalDecoder localDecoder, uint32_t timeStamp, const uint8_t* data, size_t size)
{
auto* decoder = (__bridge WK_RTCLocalVideoH264H265VP9Decoder *)(localDecoder);
@@ -19,4 +19,5 @@ __attribute__((objc_runtime_name("WK_RTCVideoDecoderH264")))
- (NSInteger)decodeData:(const uint8_t *)data
size:(size_t)size
timeStamp:(uint32_t)timeStamp;
- (void)flush;
@end
@@ -283,6 +283,11 @@ - (void)destroyDecompressionSession {
}
}

- (void)flush {
if (_decompressionSession)
VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
}

- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
if (_videoFormat == videoFormat) {
return;
@@ -19,4 +19,5 @@ __attribute__((objc_runtime_name("WK_RTCVideoDecoderH265")))
- (NSInteger)decodeData:(const uint8_t *)data
size:(size_t)size
timeStamp:(uint32_t)timeStamp;
- (void)flush;
@end
@@ -274,6 +274,11 @@ - (void)destroyDecompressionSession {
}
}

- (void)flush {
if (_decompressionSession)
VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
}

- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
if (_videoFormat == videoFormat) {
return;
@@ -35,4 +35,5 @@ __attribute__((objc_runtime_name("WK_RTCVideoDecoderVTBVP9")))
- (NSInteger)decodeData:(const uint8_t *)data
size:(size_t)size
timeStamp:(uint32_t)timeStamp;
- (void)flush;
@end
@@ -370,6 +370,11 @@ - (void)destroyDecompressionSession {
}
}

- (void)flush {
if (_decompressionSession)
VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
}

- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
if (_videoFormat == videoFormat) {
return;
@@ -82,6 +82,7 @@ class LibWebRTCCodecsProxy final : public IPC::WorkQueueMessageReceiver {

void createDecoder(VideoDecoderIdentifier, VideoCodecType, bool useRemoteFrames);
void releaseDecoder(VideoDecoderIdentifier);
void flushDecoder(VideoDecoderIdentifier);
void decodeFrame(VideoDecoderIdentifier, uint32_t timeStamp, const IPC::DataReference&);
void setFrameSize(VideoDecoderIdentifier, uint16_t width, uint16_t height);

@@ -26,6 +26,7 @@
messages -> LibWebRTCCodecsProxy NotRefCounted {
CreateDecoder(WebKit::VideoDecoderIdentifier id, enum:uint8_t WebKit::VideoCodecType codecType, bool useRemoteFrames)
ReleaseDecoder(WebKit::VideoDecoderIdentifier id)
FlushDecoder(WebKit::VideoDecoderIdentifier id)
DecodeFrame(WebKit::VideoDecoderIdentifier id, uint32_t timeStamp, IPC::DataReference data)
SetFrameSize(WebKit::VideoDecoderIdentifier id, uint16_t width, uint16_t height)

@@ -147,6 +147,18 @@
m_hasEncodersOrDecoders = !m_encoders.isEmpty() || !m_decoders.isEmpty();
}

void LibWebRTCCodecsProxy::flushDecoder(VideoDecoderIdentifier identifier)
{
assertIsCurrent(workQueue());
auto decoder = m_decoders.get(identifier);
if (!decoder) {
ASSERT_IS_TESTING_IPC();
return;
}
webrtc::flushLocalDecoder(decoder);
m_connection->send(Messages::LibWebRTCCodecs::FlushDecoderCompleted { identifier }, 0);
}

void LibWebRTCCodecsProxy::decodeFrame(VideoDecoderIdentifier identifier, uint32_t timeStamp, const IPC::DataReference& data) WTF_IGNORES_THREAD_SAFETY_ANALYSIS
{
assertIsCurrent(workQueue());
@@ -191,8 +191,11 @@ void RemoteVideoDecoder::decode(EncodedFrame&& frame, DecodeCallback&& callback)

void RemoteVideoDecoder::flush(Function<void()>&& callback)
{
// FIXME: Implement this.
callback();
WebProcess::singleton().libWebRTCCodecs().flushDecoder(m_internalDecoder, [callback = WTFMove(callback), callbacks = m_callbacks]() mutable {
callbacks->postTask([callback = WTFMove(callback)]() mutable {
callback();
});
});
}

void RemoteVideoDecoder::reset()
@@ -307,6 +307,19 @@ int32_t LibWebRTCCodecs::releaseDecoder(Decoder& decoder)
}

// May be called on any thread.
void LibWebRTCCodecs::flushDecoder(Decoder& decoder, Function<void()>&& callback)
{
Locker locker { m_connectionLock };
if (!decoder.connection || decoder.hasError) {
callback();
return;
}

decoder.connection->send(Messages::LibWebRTCCodecsProxy::FlushDecoder { decoder.identifier }, 0);
Locker flushLocker { decoder.flushCallbacksLock };
decoder.flushCallbacks.append(WTFMove(callback));
}

int32_t LibWebRTCCodecs::decodeFrame(Decoder& decoder, uint32_t timeStamp, const uint8_t* data, size_t size, uint16_t width, uint16_t height)
{
Locker locker { m_connectionLock };
@@ -347,6 +360,19 @@ void LibWebRTCCodecs::failedDecoding(VideoDecoderIdentifier decoderIdentifier)
decoder->hasError = true;
}

void LibWebRTCCodecs::flushDecoderCompleted(VideoDecoderIdentifier decoderIdentifier)
{
assertIsCurrent(workQueue());

auto* decoder = m_decoders.get(decoderIdentifier);
if (!decoder)
return;

Locker locker { decoder->flushCallbacksLock };
if (!decoder->flushCallbacks.isEmpty())
decoder->flushCallbacks.takeFirst()();
}

void LibWebRTCCodecs::completedDecoding(VideoDecoderIdentifier decoderIdentifier, uint32_t timeStamp, uint32_t timeStampNs, RemoteVideoFrameProxy::Properties&& properties)
{
assertIsCurrent(workQueue());
@@ -629,6 +655,11 @@ void LibWebRTCCodecs::gpuProcessConnectionDidClose(GPUProcessConnection&)
{
Locker locker { m_connectionLock };
for (auto& decoder : m_decoders.values()) {
{
Locker locker { decoder->flushCallbacksLock };
while (!decoder->flushCallbacks.isEmpty())
decoder->flushCallbacks.takeFirst()();
}
createRemoteDecoder(*decoder, *connection, m_useRemoteFrames);
setDecoderConnection(*decoder, connection.get());
}
@@ -40,6 +40,7 @@
#include "VideoEncoderIdentifier.h"
#include "WorkQueueMessageReceiver.h"
#include <map>
#include <wtf/Deque.h>
#include <wtf/HashMap.h>
#include <wtf/HashSet.h>
#include <wtf/Lock.h>
@@ -81,12 +82,15 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC
Lock decodedImageCallbackLock;
bool hasError { false };
RefPtr<IPC::Connection> connection;
Deque<Function<void()>> flushCallbacks WTF_GUARDED_BY_LOCK(flushCallbacksLock);
Lock flushCallbacksLock;
};

Decoder* createDecoder(VideoCodecType);
void createDecoderAndWaitUntilReady(VideoCodecType, Function<void(Decoder&)>&&);

int32_t releaseDecoder(Decoder&);
void flushDecoder(Decoder&, Function<void()>&&);
int32_t decodeFrame(Decoder&, uint32_t timeStamp, const uint8_t*, size_t, uint16_t width, uint16_t height);
void registerDecodeFrameCallback(Decoder&, void* decodedImageCallback);
void registerDecodedVideoFrameCallback(Decoder&, DecoderCallback&&);
@@ -140,6 +144,7 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC
void gpuProcessConnectionMayNoLongerBeNeeded();

void failedDecoding(VideoDecoderIdentifier);
void flushDecoderCompleted(VideoDecoderIdentifier);
void completedDecoding(VideoDecoderIdentifier, uint32_t timeStamp, uint32_t timeStampNs, RemoteVideoFrameProxy::Properties&&);
// FIXME: Will be removed once RemoteVideoFrameProxy providers are the only ones sending data.
void completedDecodingCV(VideoDecoderIdentifier, uint32_t timeStamp, uint32_t timeStampNs, RetainPtr<CVPixelBufferRef>&&);
@@ -24,6 +24,7 @@

messages -> LibWebRTCCodecs NotRefCounted {
FailedDecoding(WebKit::VideoDecoderIdentifier identifier)
FlushDecoderCompleted(WebKit::VideoDecoderIdentifier identifier)
CompletedDecoding(WebKit::VideoDecoderIdentifier identifier, uint32_t timeStamp, uint32_t timeStampNs, WebKit::RemoteVideoFrameProxy::Properties frame)
CompletedDecodingCV(WebKit::VideoDecoderIdentifier identifier, uint32_t timeStamp, uint32_t timeStampNs, RetainPtr<CVPixelBufferRef> pixelBuffer)
CompletedEncoding(WebKit::VideoEncoderIdentifier identifier, IPC::DataReference data, struct webrtc::WebKitEncodedFrameInfo info);

0 comments on commit 38b1154

Please sign in to comment.