Timestamps should be preserved between VideoTrackGenerator and MediaS…

…treamTrackProcessor

https://bugs.webkit.org/show_bug.cgi?id=267929
rdar://121462248

Reviewed by Eric Carlson.

Add support for setting the VideoFrame presentationTime based on the WebCodecsVideoFrame timestamp.
In the future, the WebCodecsVideoFrame timestamp will be removed in favour of the VideoFrame presentationTime.

Make sure to have MediaStreamTrackProcessor correctly create the WebCodecsVideoFrame with the presentationTime.

* LayoutTests/http/wpt/mediastream/worker-mediastreamtrack.worker.js:
(makeOffscreenCanvasVideoFrame):
(promise_test.async t):
* Source/WebCore/Modules/mediastream/MediaStreamTrackProcessor.cpp:
(WebCore::MediaStreamTrackProcessor::VideoFrameObserver::takeVideoFrame):
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp:
(WebCore::WebCodecsVideoFrame::create):
(WebCore::WebCodecsVideoFrame::initializeFrameFromOtherFrame):
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.h:

Canonical link: https://commits.webkit.org/273404@main

LayoutTests/http/wpt/mediastream/worker-mediastreamtrack.worker.js

@@ -3,7 +3,7 @@ importScripts("/resources/testharness.js");
 
 function makeOffscreenCanvasVideoFrame(width, height, timestamp) {
     if (!timestamp)
-      timestamp = 1;
+        timestamp = 1;
     let canvas = new OffscreenCanvas(width, height);
     let ctx = canvas.getContext('2d');
     ctx.fillStyle = 'rgba(50, 100, 150, 255)';
@@ -162,10 +162,10 @@ promise_test(async (t) => {
 }, "Track gets muted based on VideoTrackGenerator.muted");
 
 promise_test(async (t) => {
-    const frame1 = makeOffscreenCanvasVideoFrame(100, 100);
+    const frame1 = makeOffscreenCanvasVideoFrame(100, 100, 1);
     t.add_cleanup(() => frame1.close());
 
-    const frame2 = makeOffscreenCanvasVideoFrame(200, 200);
+    const frame2 = makeOffscreenCanvasVideoFrame(200, 200, 2);
     t.add_cleanup(() => frame2.close());
 
     const generator = new VideoTrackGenerator();
@@ -176,13 +176,17 @@ promise_test(async (t) => {
 
     writer.write(frame1);
 
-    let chunk = await reader.read();
-    assert_equals(chunk.value.codedWidth, 100);
+    const chunk1 = await reader.read();
+    assert_equals(chunk1.value.codedWidth, 100);
+    assert_equals(chunk1.value.timestamp, 1);
+    t.add_cleanup(() => chunk1.value.close());
 
     writer.write(frame2);
 
-    chunk = await reader.read();
-    assert_equals(chunk.value.codedWidth, 200);
+    const chunk2 = await reader.read();
+    assert_equals(chunk2.value.codedWidth, 200);
+    assert_equals(chunk2.value.timestamp, 2);
+    t.add_cleanup(() => chunk2.value.close());
 
     const endedPromise = new Promise(resolve => generator.track.onended = resolve);
 

Source/WebCore/Modules/mediastream/MediaStreamTrackProcessor.cpp

@@ -29,6 +29,7 @@
 
 #include "JSWebCodecsVideoFrame.h"
 #include "ReadableStream.h"
+#include <wtf/Seconds.h>
 
 namespace WebCore {
 
@@ -158,6 +159,7 @@ RefPtr<WebCodecsVideoFrame> MediaStreamTrackProcessor::VideoFrameObserver::takeV
     init.codedWidth = videoFrame->presentationSize().width();
     init.codedHeight = videoFrame->presentationSize().height();
     init.colorSpace = videoFrame->colorSpace();
+    init.timestamp = Seconds(videoFrame->presentationTime().toDouble()).microseconds();
 
     return WebCodecsVideoFrame::create(context, videoFrame.releaseNonNull(), WTFMove(init));
 }

Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp

@@ -45,6 +45,7 @@
 #include "SecurityOrigin.h"
 #include "VideoColorSpace.h"
 #include "WebCodecsVideoFrameAlgorithms.h"
+#include <wtf/Seconds.h>
 
 #if PLATFORM(COCOA)
 #include "VideoFrameCV.h"
@@ -191,7 +192,7 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::create(ScriptExecutio
         RefPtr<VideoFrame> videoFrame = video->player() ? video->player()->videoFrameForCurrentTime() : nullptr;
         if (!videoFrame)
             return Exception { ExceptionCode::InvalidStateError,  "Video element has no video frame"_s };
-        return initializeFrameFromOtherFrame(context, videoFrame.releaseNonNull(), WTFMove(init));
+        return initializeFrameFromOtherFrame(context, videoFrame.releaseNonNull(), WTFMove(init), CanUpdateVideoFrameTimestamp::No);
     },
 #endif
     [&] (RefPtr<HTMLCanvasElement>& canvas) -> ExceptionOr<Ref<WebCodecsVideoFrame>> {
@@ -204,7 +205,7 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::create(ScriptExecutio
         auto videoFrame = canvas->toVideoFrame();
         if (!videoFrame)
             return Exception { ExceptionCode::InvalidStateError,  "Canvas has no frame"_s };
-        return initializeFrameFromOtherFrame(context, videoFrame.releaseNonNull(), WTFMove(init));
+        return initializeFrameFromOtherFrame(context, videoFrame.releaseNonNull(), WTFMove(init), CanUpdateVideoFrameTimestamp::Yes);
     },
 #if ENABLE(OFFSCREEN_CANVAS)
     [&] (RefPtr<OffscreenCanvas>& canvas) -> ExceptionOr<Ref<WebCodecsVideoFrame>> {
@@ -251,14 +252,14 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::create(ScriptExecutio
     if (!videoFrame)
         return Exception { ExceptionCode::InvalidStateError,  "Unable to create frame from buffer"_s };
 
-    return WebCodecsVideoFrame::initializeFrameFromOtherFrame(context, videoFrame.releaseNonNull(), WTFMove(init));
+    return WebCodecsVideoFrame::initializeFrameFromOtherFrame(context, videoFrame.releaseNonNull(), WTFMove(init), CanUpdateVideoFrameTimestamp::Yes);
 }
 
 ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::create(ScriptExecutionContext& context, Ref<WebCodecsVideoFrame>&& initFrame, Init&& init)
 {
     if (initFrame->isDetached())
         return Exception { ExceptionCode::InvalidStateError,  "VideoFrame is detached"_s };
-    return initializeFrameFromOtherFrame(context, WTFMove(initFrame), WTFMove(init));
+    return initializeFrameFromOtherFrame(context, WTFMove(initFrame), WTFMove(init), CanUpdateVideoFrameTimestamp::No);
 }
 
 static std::optional<Exception> validateI420Sizes(const WebCodecsVideoFrame::BufferInit& init)
@@ -315,7 +316,7 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::create(ScriptExecutio
 
     if (!videoFrame)
         return Exception { ExceptionCode::TypeError, "Unable to create internal resource from data"_s };
-    
+
     return WebCodecsVideoFrame::create(context, videoFrame.releaseNonNull(), WTFMove(init));
 }
 
@@ -346,6 +347,7 @@ Ref<WebCodecsVideoFrame> WebCodecsVideoFrame::create(ScriptExecutionContext& con
 
     result->m_data.duration = init.duration;
     result->m_data.timestamp = init.timestamp;
+    result->m_data.internalFrame->initializeCharacteristics(MediaTime::createWithDouble(Seconds::fromMicroseconds(init.timestamp).value()), false, VideoFrameRotation::None);
 
     return result;
 }
@@ -373,7 +375,7 @@ static VideoPixelFormat computeVideoPixelFormat(VideoPixelFormat baseFormat, boo
 }
 
 // https://w3c.github.io/webcodecs/#videoframe-initialize-frame-from-other-frame
-ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameFromOtherFrame(ScriptExecutionContext& context, Ref<WebCodecsVideoFrame>&& videoFrame, Init&& init)
+ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameFromOtherFrame(ScriptExecutionContext& context, Ref<WebCodecsVideoFrame>&& videoFrame, Init&& init, CanUpdateVideoFrameTimestamp canUpdateVideoFrameTimestamp)
 {
     auto codedWidth = videoFrame->m_data.codedWidth;
     auto codedHeight = videoFrame->m_data.codedHeight;
@@ -393,11 +395,14 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameFromOt
 
     result->m_data.duration = init.duration ? init.duration : videoFrame->m_data.duration;
     result->m_data.timestamp = init.timestamp.value_or(videoFrame->m_data.timestamp);
+    // FIXME: Handle CanUpdateVideoFrameTimestamp::No case.
+    if (canUpdateVideoFrameTimestamp == CanUpdateVideoFrameTimestamp::Yes && init.timestamp)
+        result->m_data.internalFrame->initializeCharacteristics(MediaTime::createWithDouble(Seconds::fromMicroseconds(*init.timestamp).value()), false, VideoFrameRotation::None);
 
     return result;
 }
 
-ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameFromOtherFrame(ScriptExecutionContext& context, Ref<VideoFrame>&& internalVideoFrame, Init&& init)
+ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameFromOtherFrame(ScriptExecutionContext& context, Ref<VideoFrame>&& internalVideoFrame, Init&& init, CanUpdateVideoFrameTimestamp canUpdateVideoFrameTimestamp)
 {
     auto codedWidth = internalVideoFrame->presentationSize().width();
     auto codedHeight = internalVideoFrame->presentationSize().height();
@@ -416,6 +421,9 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameFromOt
     result->m_data.duration = init.duration;
     // FIXME: Use internalVideoFrame timestamp if available and init has no timestamp.
     result->m_data.timestamp = init.timestamp.value_or(0);
+    // FIXME: Handle CanUpdateVideoFrameTimestamp::No case.
+    if (canUpdateVideoFrameTimestamp == CanUpdateVideoFrameTimestamp::Yes && init.timestamp)
+        result->m_data.internalFrame->initializeCharacteristics(MediaTime::createWithDouble(Seconds::fromMicroseconds(*init.timestamp).value()), false, VideoFrameRotation::None);
 
     return result;
 }

Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.h

@@ -143,8 +143,9 @@ class WebCodecsVideoFrame : public RefCounted<WebCodecsVideoFrame>, public Conte
     explicit WebCodecsVideoFrame(ScriptExecutionContext&);
     WebCodecsVideoFrame(ScriptExecutionContext&, WebCodecsVideoFrameData&&);
 
-    static ExceptionOr<Ref<WebCodecsVideoFrame>> initializeFrameFromOtherFrame(ScriptExecutionContext&, Ref<WebCodecsVideoFrame>&&, Init&&);
-    static ExceptionOr<Ref<WebCodecsVideoFrame>> initializeFrameFromOtherFrame(ScriptExecutionContext&, Ref<VideoFrame>&&, Init&&);
+    enum class CanUpdateVideoFrameTimestamp : bool { No, Yes };
+    static ExceptionOr<Ref<WebCodecsVideoFrame>> initializeFrameFromOtherFrame(ScriptExecutionContext&, Ref<WebCodecsVideoFrame>&&, Init&&, CanUpdateVideoFrameTimestamp);
+    static ExceptionOr<Ref<WebCodecsVideoFrame>> initializeFrameFromOtherFrame(ScriptExecutionContext&, Ref<VideoFrame>&&, Init&&, CanUpdateVideoFrameTimestamp);
     static ExceptionOr<Ref<WebCodecsVideoFrame>> initializeFrameWithResourceAndSize(ScriptExecutionContext&, Ref<NativeImage>&&, Init&&);
 
     WebCodecsVideoFrameData m_data;