Implement WebCodecs VideoFrame copyTo

https://bugs.webkit.org/show_bug.cgi?id=246210
rdar://problem/100893780

Reviewed by Eric Carlson.

Add support for copyTo for RGB, NV12 and I420 formats for macOS/iOS ports.
Make use of libyuv for I420 and direct copy for RGB and NV12.

* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any.js:
(makeNV12_4x2):
(promise_test.async t):
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any.worker-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any.worker-expected.txt:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm:
(webrtc::copyI420BufferToPixelBuffer):
(webrtc::pixelBufferFromI420Buffer):
(webrtc::copyPixelBufferInI420Buffer):
(webrtc::copyBufferToPixelBuffer): Deleted.
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.cpp:
(WebCore::sampleCountPerPixel):
(WebCore::computeLayoutAndAllocationSize):
* Source/WebCore/platform/graphics/cv/VideoFrameCV.mm:
(WebCore::VideoFrame::createRGBA):
(WebCore::VideoFrame::createI420):
(WebCore::copyRGBData):
(WebCore::copyNV12):
(WebCore::VideoFrame::copyTo):

Canonical link: https://commits.webkit.org/255429@main

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any-expected.txt

@@ -1,17 +1,18 @@
 
 PASS Test closed frame.
-FAIL Test copying I420 frame to a non-shared ArrayBuffer promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
-FAIL Test copying I420 frame to a non-shared ArrayBufferView promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
-FAIL Test RGBA frame. assert_object_equals: plane 0 layout property "stride" expected 8 got 0
+PASS Test copying I420 frame to a non-shared ArrayBuffer
+PASS Test copying I420 frame to a non-shared ArrayBufferView
+PASS Test RGBA frame.
+PASS Test NV12 frame.
 PASS Test undersized buffer.
 PASS Test incorrect plane count.
-FAIL Test stride and offset work. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
-FAIL Test stride and offset with padding. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
+PASS Test stride and offset work.
+PASS Test stride and offset with padding.
 PASS Test invalid stride.
 PASS Test address overflow.
-FAIL Test codedRect. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
+PASS Test codedRect.
 PASS Test empty rect.
 PASS Test unaligned rect.
-FAIL Test left crop. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
+FAIL Test left crop. assert_equals: buffer contents at index 0 expected 3 but got 1
 PASS Test invalid rect.
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any.js

@@ -15,6 +15,22 @@ function makeRGBA_2x2() {
   return new VideoFrame(data, init);
 }
 
+const NV12_DATA = new Uint8Array([
+      1, 2, 3, 4,   // y
+      5, 6, 7, 8,
+      9, 10, 11, 12 // uv
+  ]);
+
+function makeNV12_4x2() {
+  const init = {
+      format: 'NV12',
+      timestamp: 0,
+      codedWidth: 4,
+      codedHeight: 2,
+  };
+  return new VideoFrame(NV12_DATA, init);
+}
+
 promise_test(async t => {
   const frame = makeI420_4x2();
   frame.close();
@@ -51,6 +67,24 @@ promise_test(async t => {
   assert_buffer_equals(data, expectedData);
 }, 'Test RGBA frame.');
 
+promise_test(async t => {
+  const frame = makeNV12_4x2();
+  const expectedLayout = [
+      {offset: 0, stride: 4},
+      {offset: 8, stride: 4},
+  ];
+  const expectedData = new Uint8Array([
+      1,2,3,4,
+      5,6,7,8,
+      9,10,11,12
+  ]);
+  assert_equals(frame.allocationSize(), expectedData.length, 'allocationSize()');
+  const data = new Uint8Array(expectedData.length);
+  const layout = await frame.copyTo(data);
+  assert_layout_equals(layout, expectedLayout);
+  assert_buffer_equals(data, expectedData);
+}, 'Test NV12 frame.');
+
 promise_test(async t => {
   const frame = makeI420_4x2();
   const data = new Uint8Array(11);

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any.worker-expected.txt

@@ -1,17 +1,18 @@
 
 PASS Test closed frame.
-FAIL Test copying I420 frame to a non-shared ArrayBuffer promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
-FAIL Test copying I420 frame to a non-shared ArrayBufferView promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
-FAIL Test RGBA frame. assert_object_equals: plane 0 layout property "stride" expected 8 got 0
+PASS Test copying I420 frame to a non-shared ArrayBuffer
+PASS Test copying I420 frame to a non-shared ArrayBufferView
+PASS Test RGBA frame.
+PASS Test NV12 frame.
 PASS Test undersized buffer.
 PASS Test incorrect plane count.
-FAIL Test stride and offset work. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
-FAIL Test stride and offset with padding. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
+PASS Test stride and offset work.
+PASS Test stride and offset with padding.
 PASS Test invalid stride.
 PASS Test address overflow.
-FAIL Test codedRect. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
+PASS Test codedRect.
 PASS Test empty rect.
 PASS Test unaligned rect.
-FAIL Test left crop. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
+FAIL Test left crop. assert_equals: buffer contents at index 0 expected 3 but got 1
 PASS Test invalid rect.
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any-expected.txt

@@ -1,4 +1,4 @@
 
-FAIL Test copying I420 frame to SharedArrayBuffer. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
-FAIL Test copying I420 frame to shared ArrayBufferView. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
+PASS Test copying I420 frame to SharedArrayBuffer.
+PASS Test copying I420 frame to shared ArrayBufferView.
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any.worker-expected.txt

@@ -1,4 +1,4 @@
 
-FAIL Test copying I420 frame to SharedArrayBuffer. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
-FAIL Test copying I420 frame to shared ArrayBufferView. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
+PASS Test copying I420 frame to SharedArrayBuffer.
+PASS Test copying I420 frame to shared ArrayBufferView.
 

Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp

@@ -339,6 +339,7 @@ __ZN3rtc18NetworkManagerBaseC2EPKN6webrtc15FieldTrialsViewE
 __ZN3rtc13SocketAddressC1EN4absl11string_viewEi
 __ZN3rtc17AsyncPacketSocket14SubscribeCloseEPKvNSt3__18functionIFvPS0_iEEE
 __ZN6webrtc25pixelBufferFromI420BufferEPKhmmmNS_16I420BufferLayoutE
+__ZN6webrtc27copyPixelBufferInI420BufferEPhmP10__CVBufferNS_16I420BufferLayoutE
 __ZN6webrtc10VP8Decoder6CreateEv
 __ZN6webrtc10VP9Decoder6CreateEv
 __ZN6webrtc12EncodedImageC1Ev

Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp

@@ -339,6 +339,7 @@ __ZN3rtc18NetworkManagerBaseC2EPKN6webrtc15FieldTrialsViewE
 __ZN3rtc13SocketAddressC1EN4absl11string_viewEi
 __ZN3rtc17AsyncPacketSocket14SubscribeCloseEPKvNSt3__18functionIFvPS0_iEEE
 __ZN6webrtc25pixelBufferFromI420BufferEPKhmmmNS_16I420BufferLayoutE
+__ZN6webrtc27copyPixelBufferInI420BufferEPhmP10__CVBufferNS_16I420BufferLayoutE
 __ZN6webrtc10VP8Decoder6CreateEv
 __ZN6webrtc10VP9Decoder6CreateEv
 __ZN6webrtc12EncodedImageC1Ev

Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp

@@ -339,6 +339,7 @@ __ZN3rtc18NetworkManagerBaseC2EPKN6webrtc15FieldTrialsViewE
 __ZN3rtc13SocketAddressC1EN4absl11string_viewEi
 __ZN3rtc17AsyncPacketSocket14SubscribeCloseEPKvNSt3__18functionIFvPS0_iEEE
 __ZN6webrtc25pixelBufferFromI420BufferEPKhmmmNS_16I420BufferLayoutE
+__ZN6webrtc27copyPixelBufferInI420BufferEPhmP10__CVBufferNS_16I420BufferLayoutE
 __ZN6webrtc10VP8Decoder6CreateEv
 __ZN6webrtc10VP9Decoder6CreateEv
 __ZN6webrtc12EncodedImageC1Ev

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h

@@ -71,5 +71,6 @@ struct I420BufferLayout {
 };
 
 CVPixelBufferRef pixelBufferFromI420Buffer(const uint8_t* buffer, size_t length, size_t width, size_t height, I420BufferLayout) CF_RETURNS_RETAINED;
+bool copyPixelBufferInI420Buffer(uint8_t* buffer, size_t length, CVPixelBufferRef, I420BufferLayout);
 
 }

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm

@@ -68,29 +68,29 @@ void setApplicationStatus(bool isActive)
     return static_cast<ObjCFrameBuffer*>(buffer.get())->frame_buffer_provider();
 }
 
-static bool copyBufferToPixelBuffer(CVPixelBufferRef pxielBuffer, const uint8_t* buffer, size_t length, size_t width, size_t height, I420BufferLayout layout)
+static bool copyI420BufferToPixelBuffer(CVPixelBufferRef pixelBuffer, const uint8_t* buffer, size_t length, size_t width, size_t height, I420BufferLayout layout)
 {
     auto sourceWidthY = width;
     auto sourceHeightY = height;
     auto sourceWidthUV = (width + 1) / 2;
     auto sourceHeightUV = (height + 1) / 2;
 
-    auto destinationWidthY = CVPixelBufferGetWidthOfPlane(pxielBuffer, 0);
-    auto destinationHeightY = CVPixelBufferGetHeightOfPlane(pxielBuffer, 0);
+    auto destinationWidthY = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
+    auto destinationHeightY = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
 
-    auto destinationWidthUV = CVPixelBufferGetWidthOfPlane(pxielBuffer, 1);
-    auto destinationHeightUV = CVPixelBufferGetHeightOfPlane(pxielBuffer, 1);
+    auto destinationWidthUV = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1);
+    auto destinationHeightUV = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1);
 
     if (sourceWidthY != destinationWidthY
         || sourceHeightY != destinationHeightY
         || sourceWidthUV != destinationWidthUV
         || sourceHeightUV != destinationHeightUV)
         return false;
 
-    uint8_t* destinationY = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pxielBuffer, 0));
-    int destinationStrideY = CVPixelBufferGetBytesPerRowOfPlane(pxielBuffer, 0);
-    uint8_t* destinationUV = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pxielBuffer, 1));
-    int destinationStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pxielBuffer, 1);
+    uint8_t* destinationY = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
+    int destinationStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
+    uint8_t* destinationUV = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
+    int destinationStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
 
     auto* sourceY = buffer + layout.offsetY;
     int sourceStrideY = layout.strideY;
@@ -119,7 +119,7 @@ CVPixelBufferRef pixelBufferFromI420Buffer(const uint8_t* buffer, size_t length,
     if (CVPixelBufferLockBaseAddress(pixelBuffer, 0) != kCVReturnSuccess)
         return nullptr;
 
-    bool result = copyBufferToPixelBuffer(pixelBuffer, buffer, length, width, height, layout);
+    bool result = copyI420BufferToPixelBuffer(pixelBuffer, buffer, length, width, height, layout);
     CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
 
     if (!result) {
@@ -129,6 +129,35 @@ CVPixelBufferRef pixelBufferFromI420Buffer(const uint8_t* buffer, size_t length,
     return pixelBuffer;
 }
 
+bool copyPixelBufferInI420Buffer(uint8_t* buffer, size_t length, CVPixelBufferRef pixelBuffer, I420BufferLayout layout)
+{
+    if (CVPixelBufferGetPixelFormatType(pixelBuffer) != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
+        return false;
+
+    if (CVPixelBufferLockBaseAddress(pixelBuffer, 0) != kCVReturnSuccess)
+        return false;
+
+    size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
+    size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
+
+    uint8_t* sourceY = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
+    int sourceStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
+
+    uint8_t* sourceUV = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
+    int sourceStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
+
+    bool result = !libyuv::NV12ToI420(
+        sourceY, sourceStrideY,
+        sourceUV, sourceStrideUV,
+        buffer + layout.offsetY, layout.strideY,
+        buffer + layout.offsetU, layout.strideU,
+        buffer + layout.offsetV, layout.strideV,
+        width, height);
+
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
+    return result;
+}
+
 static bool CopyVideoFrameToPixelBuffer(const webrtc::I420BufferInterface* frame, CVPixelBufferRef pixel_buffer) {
     RTC_DCHECK(pixel_buffer);
     RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || CVPixelBufferGetPixelFormatType(pixel_buffer) == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);

Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.cpp

@@ -147,15 +147,16 @@ size_t videoPixelFormatToSampleByteSizePerPlane()
     return 1;
 }
 
-static inline size_t sampleCountPerPixel(VideoPixelFormat format)
+static inline size_t sampleCountPerPixel(VideoPixelFormat format, size_t planeNumber)
 {
     switch (format) {
     case VideoPixelFormat::I420:
     case VideoPixelFormat::I420A:
     case VideoPixelFormat::I444:
     case VideoPixelFormat::I422:
-    case VideoPixelFormat::NV12:
         return 1;
+    case VideoPixelFormat::NV12:
+        return planeNumber ? 2 : 1;
     case VideoPixelFormat::RGBA:
     case VideoPixelFormat::RGBX:
     case VideoPixelFormat::BGRA:
@@ -190,13 +191,14 @@ ExceptionOr<CombinedPlaneLayout> computeLayoutAndAllocationSize(const DOMRectIni
     if (layout && layout->size() != planeCount)
         return Exception { TypeError, "layout size is invalid"_s };
 
-    size_t pixelSampleCount = sampleCountPerPixel(format);
     size_t minAllocationSize = 0;
     Vector<ComputedPlaneLayout> computedLayouts;
     computedLayouts.reserveInitialCapacity(planeCount);
     Vector<size_t> endOffsets;
     endOffsets.reserveInitialCapacity(planeCount);
     for (size_t i = 0; i < planeCount; ++i) {
+        size_t pixelSampleCount = sampleCountPerPixel(format, i);
+
         auto sampleBytes = videoPixelFormatToSampleByteSizePerPlane();
         auto sampleWidth = videoPixelFormatToSubSampling(format, i);
         auto sampleHeight = videoPixelFormatToSubSampling(format, i);