Implement GPU based WebCodecsVideoDecoder flush

https://bugs.webkit.org/show_bug.cgi?id=246328

Reviewed by Eric Carlson.

Add IPC to send flush order from WebProcess to GPUProcess.
Call VTDecompressionSessionWaitForAsynchronousFrames in GPUProcess to flush all frames.
This might block the codec queue which is for the whole GPU Process.
We might in a follow-up have a queue per web process.

Covered by rebasd test.

* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker_h264_annexb-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any_h264_annexb-expected.txt:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitDecoder.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitDecoder.mm:
(-[WK_RTCLocalVideoH264H265VP9Decoder flush]):
(webrtc::flushLocalDecoder):
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm:
(-[RTCVideoDecoderH264 flush]):
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm:
(-[RTCVideoDecoderH265 flush]):
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderVTBVP9.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderVTBVP9.mm:
(-[RTCVideoDecoderVTBVP9 flush]):
* Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h:
* Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in:
* Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm:
(WebKit::LibWebRTCCodecsProxy::createDecoderCallback):
(WebKit::LibWebRTCCodecsProxy::createLocalDecoder):
(WebKit::LibWebRTCCodecsProxy::flushDecoder):
* Source/WebKit/WebProcess/GPU/media/RemoteVideoCodecFactory.cpp:
(WebKit::RemoteVideoDecoder::decode):
(WebKit::RemoteVideoDecoder::flush):
(WebKit::RemoteVideoDecoder::reset):
(WebKit::RemoteVideoDecoderCallbacks::notifyVideoFrame):
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp:
(WebKit::LibWebRTCCodecs::flushDecoder):
(WebKit::LibWebRTCCodecs::flushDecoderCompleted):
(WebKit::LibWebRTCCodecs::gpuProcessConnectionDidClose):
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h:
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.messages.in:

Canonical link: https://commits.webkit.org/255388@main

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker_h264_annexb-expected.txt

@@ -4,17 +4,17 @@ PASS Test isConfigSupported() with 1080p crop
 PASS Test that isConfigSupported() returns a parsed configuration
 PASS Test invalid configs
 PASS Test configure()
-FAIL Decode a key frame assert_equals: outputs expected 1 but got 0
+PASS Decode a key frame
 PASS Decode a non key frame first fails
-FAIL Verify reset() suppresses outputs assert_equals: expected 5 but got 1
+PASS Verify reset() suppresses outputs
 PASS Test unconfigured VideoDecoder operations
 PASS Test closed VideoDecoder operations
 FAIL Decode empty frame assert_unreached: Should have rejected: undefined Reached unreachable code
 FAIL Decode corrupt frame assert_unreached: Should have rejected: undefined Reached unreachable code
 PASS Close while decoding corrupt frame
-FAIL Test decoding after flush assert_equals: outputs expected 1 but got 0
-FAIL Test decoding a with negative timestamp assert_equals: outputs expected 1 but got 0
-FAIL Test reset during flush assert_unreached: Should have rejected: undefined Reached unreachable code
+PASS Test decoding after flush
+FAIL Test decoding a with negative timestamp assert_equals: timestamp expected -42 but got 4294967254
+PASS Test reset during flush
 PASS Test low-latency decoding
 PASS VideoDecoder decodeQueueSize test
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any_h264_annexb-expected.txt

@@ -4,17 +4,17 @@ PASS Test isConfigSupported() with 1080p crop
 PASS Test that isConfigSupported() returns a parsed configuration
 PASS Test invalid configs
 PASS Test configure()
-FAIL Decode a key frame assert_equals: outputs expected 1 but got 0
+PASS Decode a key frame
 PASS Decode a non key frame first fails
-FAIL Verify reset() suppresses outputs assert_equals: expected 5 but got 1
+PASS Verify reset() suppresses outputs
 PASS Test unconfigured VideoDecoder operations
 PASS Test closed VideoDecoder operations
 FAIL Decode empty frame assert_unreached: Should have rejected: undefined Reached unreachable code
 FAIL Decode corrupt frame assert_unreached: Should have rejected: undefined Reached unreachable code
 PASS Close while decoding corrupt frame
-FAIL Test decoding after flush assert_equals: outputs expected 1 but got 0
-FAIL Test decoding a with negative timestamp assert_equals: outputs expected 1 but got 0
-FAIL Test reset during flush assert_unreached: Should have rejected: undefined Reached unreachable code
+PASS Test decoding after flush
+FAIL Test decoding a with negative timestamp assert_equals: timestamp expected -42 but got 4294967254
+PASS Test reset during flush
 PASS Test low-latency decoding
 PASS VideoDecoder decodeQueueSize test
 

Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp

@@ -199,6 +199,7 @@ __ZN6webrtc22createLocalH264DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc22createLocalH265DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc21createLocalVP9DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc19releaseLocalDecoderEPv
+__ZN6webrtc17flushLocalDecoderEPv
 __ZN6webrtc11decodeFrameEPvjPKhm
 __ZN6webrtc24videoDecoderTaskCompleteEPvjjP10__CVBuffer
 __ZN6webrtc24videoDecoderTaskCompleteEPvjjS0_PFP10__CVBufferS0_EPFvS0_Eii

Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp

@@ -199,6 +199,7 @@ __ZN6webrtc22createLocalH264DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc22createLocalH265DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc21createLocalVP9DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc19releaseLocalDecoderEPv
+__ZN6webrtc17flushLocalDecoderEPv
 __ZN6webrtc11decodeFrameEPvjPKhm
 __ZN6webrtc24videoDecoderTaskCompleteEPvjjP10__CVBuffer
 __ZN6webrtc24videoDecoderTaskCompleteEPvjjS0_PFP10__CVBufferS0_EPFvS0_Eii

Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp

@@ -199,6 +199,7 @@ __ZN6webrtc22createLocalH264DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc22createLocalH265DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc21createLocalVP9DecoderEU13block_pointerFvP10__CVBufferjjE
 __ZN6webrtc19releaseLocalDecoderEPv
+__ZN6webrtc17flushLocalDecoderEPv
 __ZN6webrtc11decodeFrameEPvjPKhm
 __ZN6webrtc24videoDecoderTaskCompleteEPvjjP10__CVBuffer
 __ZN6webrtc24videoDecoderTaskCompleteEPvjjS0_PFP10__CVBufferS0_EPFvS0_Eii

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitDecoder.h

@@ -63,6 +63,7 @@ void* createLocalH264Decoder(LocalDecoderCallback);
 void* createLocalH265Decoder(LocalDecoderCallback);
 void* createLocalVP9Decoder(LocalDecoderCallback);
 void releaseLocalDecoder(LocalDecoder);
+void flushLocalDecoder(LocalDecoder);
 int32_t decodeFrame(LocalDecoder, uint32_t timeStamp, const uint8_t*, size_t);
 void setDecoderFrameSize(LocalDecoder, uint16_t width, uint16_t height);
 

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitDecoder.mm

@@ -44,6 +44,7 @@ - (instancetype)initH264DecoderWithCallback:(webrtc::LocalDecoderCallback)callba
 - (instancetype)initH265DecoderWithCallback:(webrtc::LocalDecoderCallback)callback;
 - (NSInteger)decodeData:(const uint8_t *)data size:(size_t)size timeStamp:(uint32_t)timeStamp;
 - (NSInteger)releaseDecoder;
+- (void)flush;
 @end
 
 @implementation WK_RTCLocalVideoH264H265VP9Decoder {
@@ -108,6 +109,19 @@ - (NSInteger)releaseDecoder {
         return [m_h265Decoder releaseDecoder];
     return [m_vp9Decoder releaseDecoder];
 }
+
+- (void)flush {
+    if (m_h264Decoder) {
+        [m_h264Decoder flush];
+        return;
+    }
+    if (m_h265Decoder) {
+        [m_h265Decoder flush];
+        return;
+    }
+    [m_vp9Decoder flush];
+}
+
 @end
 
 namespace webrtc {
@@ -285,6 +299,12 @@ void releaseLocalDecoder(LocalDecoder localDecoder)
     [decoder releaseDecoder];
 }
 
+void flushLocalDecoder(LocalDecoder localDecoder)
+{
+    auto* decoder = (__bridge WK_RTCLocalVideoH264H265VP9Decoder *)(localDecoder);
+    [decoder flush];
+}
+
 int32_t decodeFrame(LocalDecoder localDecoder, uint32_t timeStamp, const uint8_t* data, size_t size)
 {
     auto* decoder = (__bridge WK_RTCLocalVideoH264H265VP9Decoder *)(localDecoder);

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.h

@@ -19,4 +19,5 @@ __attribute__((objc_runtime_name("WK_RTCVideoDecoderH264")))
 - (NSInteger)decodeData:(const uint8_t *)data
     size:(size_t)size
     timeStamp:(uint32_t)timeStamp;
+- (void)flush;
 @end

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm

@@ -283,6 +283,11 @@ - (void)destroyDecompressionSession {
   }
 }
 
+- (void)flush {
+  if (_decompressionSession)
+    VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
+}
+
 - (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
   if (_videoFormat == videoFormat) {
     return;

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.h

@@ -19,4 +19,5 @@ __attribute__((objc_runtime_name("WK_RTCVideoDecoderH265")))
 - (NSInteger)decodeData:(const uint8_t *)data
     size:(size_t)size
     timeStamp:(uint32_t)timeStamp;
+- (void)flush;
 @end

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm

@@ -274,6 +274,11 @@ - (void)destroyDecompressionSession {
   }
 }
 
+- (void)flush {
+  if (_decompressionSession)
+    VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
+}
+
 - (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
   if (_videoFormat == videoFormat) {
     return;

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderVTBVP9.h

@@ -35,4 +35,5 @@ __attribute__((objc_runtime_name("WK_RTCVideoDecoderVTBVP9")))
 - (NSInteger)decodeData:(const uint8_t *)data
     size:(size_t)size
     timeStamp:(uint32_t)timeStamp;
+- (void)flush;
 @end

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderVTBVP9.mm

@@ -370,6 +370,11 @@ - (void)destroyDecompressionSession {
   }
 }
 
+- (void)flush {
+  if (_decompressionSession)
+    VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
+}
+
 - (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
   if (_videoFormat == videoFormat) {
     return;

Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h

@@ -82,6 +82,7 @@ class LibWebRTCCodecsProxy final : public IPC::WorkQueueMessageReceiver {
 
     void createDecoder(VideoDecoderIdentifier, VideoCodecType, bool useRemoteFrames);
     void releaseDecoder(VideoDecoderIdentifier);
+    void flushDecoder(VideoDecoderIdentifier);
     void decodeFrame(VideoDecoderIdentifier, uint32_t timeStamp, const IPC::DataReference&);
     void setFrameSize(VideoDecoderIdentifier, uint16_t width, uint16_t height);
 

Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in

@@ -26,6 +26,7 @@
 messages -> LibWebRTCCodecsProxy NotRefCounted {
     CreateDecoder(WebKit::VideoDecoderIdentifier id, enum:uint8_t WebKit::VideoCodecType codecType, bool useRemoteFrames)
     ReleaseDecoder(WebKit::VideoDecoderIdentifier id)
+    FlushDecoder(WebKit::VideoDecoderIdentifier id)
     DecodeFrame(WebKit::VideoDecoderIdentifier id, uint32_t timeStamp, IPC::DataReference data)
     SetFrameSize(WebKit::VideoDecoderIdentifier id, uint16_t width, uint16_t height)
 

Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm

@@ -147,6 +147,18 @@
     m_hasEncodersOrDecoders = !m_encoders.isEmpty() || !m_decoders.isEmpty();
 }
 
+void LibWebRTCCodecsProxy::flushDecoder(VideoDecoderIdentifier identifier)
+{
+    assertIsCurrent(workQueue());
+    auto decoder = m_decoders.get(identifier);
+    if (!decoder) {
+        ASSERT_IS_TESTING_IPC();
+        return;
+    }
+    webrtc::flushLocalDecoder(decoder);
+    m_connection->send(Messages::LibWebRTCCodecs::FlushDecoderCompleted { identifier }, 0);
+}
+
 void LibWebRTCCodecsProxy::decodeFrame(VideoDecoderIdentifier identifier, uint32_t timeStamp, const IPC::DataReference& data) WTF_IGNORES_THREAD_SAFETY_ANALYSIS
 {
     assertIsCurrent(workQueue());

Source/WebKit/WebProcess/GPU/media/RemoteVideoCodecFactory.cpp

@@ -191,8 +191,11 @@ void RemoteVideoDecoder::decode(EncodedFrame&& frame, DecodeCallback&& callback)
 
 void RemoteVideoDecoder::flush(Function<void()>&& callback)
 {
-    // FIXME: Implement this.
-    callback();
+    WebProcess::singleton().libWebRTCCodecs().flushDecoder(m_internalDecoder, [callback = WTFMove(callback), callbacks = m_callbacks]() mutable {
+        callbacks->postTask([callback = WTFMove(callback)]() mutable {
+            callback();
+        });
+    });
 }
 
 void RemoteVideoDecoder::reset()

Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp

@@ -307,6 +307,19 @@ int32_t LibWebRTCCodecs::releaseDecoder(Decoder& decoder)
 }
 
 // May be called on any thread.
+void LibWebRTCCodecs::flushDecoder(Decoder& decoder, Function<void()>&& callback)
+{
+    Locker locker { m_connectionLock };
+    if (!decoder.connection || decoder.hasError) {
+        callback();
+        return;
+    }
+
+    decoder.connection->send(Messages::LibWebRTCCodecsProxy::FlushDecoder { decoder.identifier }, 0);
+    Locker flushLocker { decoder.flushCallbacksLock };
+    decoder.flushCallbacks.append(WTFMove(callback));
+}
+
 int32_t LibWebRTCCodecs::decodeFrame(Decoder& decoder, uint32_t timeStamp, const uint8_t* data, size_t size, uint16_t width, uint16_t height)
 {
     Locker locker { m_connectionLock };
@@ -347,6 +360,19 @@ void LibWebRTCCodecs::failedDecoding(VideoDecoderIdentifier decoderIdentifier)
         decoder->hasError = true;
 }
 
+void LibWebRTCCodecs::flushDecoderCompleted(VideoDecoderIdentifier decoderIdentifier)
+{
+    assertIsCurrent(workQueue());
+
+    auto* decoder = m_decoders.get(decoderIdentifier);
+    if (!decoder)
+        return;
+
+    Locker locker { decoder->flushCallbacksLock };
+    if (!decoder->flushCallbacks.isEmpty())
+        decoder->flushCallbacks.takeFirst()();
+}
+
 void LibWebRTCCodecs::completedDecoding(VideoDecoderIdentifier decoderIdentifier, uint32_t timeStamp, uint32_t timeStampNs, RemoteVideoFrameProxy::Properties&& properties)
 {
     assertIsCurrent(workQueue());
@@ -629,6 +655,11 @@ void LibWebRTCCodecs::gpuProcessConnectionDidClose(GPUProcessConnection&)
         {
             Locker locker { m_connectionLock };
             for (auto& decoder : m_decoders.values()) {
+                {
+                    Locker locker { decoder->flushCallbacksLock };
+                    while (!decoder->flushCallbacks.isEmpty())
+                        decoder->flushCallbacks.takeFirst()();
+                }
                 createRemoteDecoder(*decoder, *connection, m_useRemoteFrames);
                 setDecoderConnection(*decoder, connection.get());
             }

Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h

@@ -40,6 +40,7 @@
 #include "VideoEncoderIdentifier.h"
 #include "WorkQueueMessageReceiver.h"
 #include <map>
+#include <wtf/Deque.h>
 #include <wtf/HashMap.h>
 #include <wtf/HashSet.h>
 #include <wtf/Lock.h>
@@ -81,12 +82,15 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC
         Lock decodedImageCallbackLock;
         bool hasError { false };
         RefPtr<IPC::Connection> connection;
+        Deque<Function<void()>> flushCallbacks WTF_GUARDED_BY_LOCK(flushCallbacksLock);
+        Lock flushCallbacksLock;
     };
 
     Decoder* createDecoder(VideoCodecType);
     void createDecoderAndWaitUntilReady(VideoCodecType, Function<void(Decoder&)>&&);
 
     int32_t releaseDecoder(Decoder&);
+    void flushDecoder(Decoder&, Function<void()>&&);
     int32_t decodeFrame(Decoder&, uint32_t timeStamp, const uint8_t*, size_t, uint16_t width, uint16_t height);
     void registerDecodeFrameCallback(Decoder&, void* decodedImageCallback);
     void registerDecodedVideoFrameCallback(Decoder&, DecoderCallback&&);
@@ -140,6 +144,7 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC
     void gpuProcessConnectionMayNoLongerBeNeeded();
 
     void failedDecoding(VideoDecoderIdentifier);
+    void flushDecoderCompleted(VideoDecoderIdentifier);
     void completedDecoding(VideoDecoderIdentifier, uint32_t timeStamp, uint32_t timeStampNs, RemoteVideoFrameProxy::Properties&&);
     // FIXME: Will be removed once RemoteVideoFrameProxy providers are the only ones sending data.
     void completedDecodingCV(VideoDecoderIdentifier, uint32_t timeStamp, uint32_t timeStampNs, RetainPtr<CVPixelBufferRef>&&);

Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.messages.in

@@ -24,6 +24,7 @@
 
 messages -> LibWebRTCCodecs NotRefCounted {
     FailedDecoding(WebKit::VideoDecoderIdentifier identifier)
+    FlushDecoderCompleted(WebKit::VideoDecoderIdentifier identifier)
     CompletedDecoding(WebKit::VideoDecoderIdentifier identifier, uint32_t timeStamp, uint32_t timeStampNs, WebKit::RemoteVideoFrameProxy::Properties frame)
     CompletedDecodingCV(WebKit::VideoDecoderIdentifier identifier, uint32_t timeStamp, uint32_t timeStampNs, RetainPtr<CVPixelBufferRef> pixelBuffer)
     CompletedEncoding(WebKit::VideoEncoderIdentifier identifier, IPC::DataReference data, struct webrtc::WebKitEncodedFrameInfo info);