Skip to content
Permalink
Browse files
Implement WebCodecs VideoFrame copyTo
https://bugs.webkit.org/show_bug.cgi?id=246210
rdar://problem/100893780

Reviewed by Eric Carlson.

Add support for copyTo for RGB, NV12 and I420 formats for macOS/iOS ports.
Make use of libyuv for I420 and direct copy for RGB and NV12.

* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any.js:
(makeNV12_4x2):
(promise_test.async t):
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.any.worker-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-copyTo.crossOriginIsolated.https.any.worker-expected.txt:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm:
(webrtc::copyI420BufferToPixelBuffer):
(webrtc::pixelBufferFromI420Buffer):
(webrtc::copyPixelBufferInI420Buffer):
(webrtc::copyBufferToPixelBuffer): Deleted.
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.cpp:
(WebCore::sampleCountPerPixel):
(WebCore::computeLayoutAndAllocationSize):
* Source/WebCore/platform/graphics/cv/VideoFrameCV.mm:
(WebCore::VideoFrame::createRGBA):
(WebCore::VideoFrame::createI420):
(WebCore::copyRGBData):
(WebCore::copyNV12):
(WebCore::VideoFrame::copyTo):

Canonical link: https://commits.webkit.org/255429@main
  • Loading branch information
youennf committed Oct 12, 2022
1 parent b94dffe commit 13105f5f6a4520b29b20c7f2ac44e892b29ba61e
Show file tree
Hide file tree
Showing 12 changed files with 239 additions and 91 deletions.
@@ -1,17 +1,18 @@

PASS Test closed frame.
FAIL Test copying I420 frame to a non-shared ArrayBuffer promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test copying I420 frame to a non-shared ArrayBufferView promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test RGBA frame. assert_object_equals: plane 0 layout property "stride" expected 8 got 0
PASS Test copying I420 frame to a non-shared ArrayBuffer
PASS Test copying I420 frame to a non-shared ArrayBufferView
PASS Test RGBA frame.
PASS Test NV12 frame.
PASS Test undersized buffer.
PASS Test incorrect plane count.
FAIL Test stride and offset work. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test stride and offset with padding. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
PASS Test stride and offset work.
PASS Test stride and offset with padding.
PASS Test invalid stride.
PASS Test address overflow.
FAIL Test codedRect. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
PASS Test codedRect.
PASS Test empty rect.
PASS Test unaligned rect.
FAIL Test left crop. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test left crop. assert_equals: buffer contents at index 0 expected 3 but got 1
PASS Test invalid rect.

@@ -15,6 +15,22 @@ function makeRGBA_2x2() {
return new VideoFrame(data, init);
}

const NV12_DATA = new Uint8Array([
1, 2, 3, 4, // y
5, 6, 7, 8,
9, 10, 11, 12 // uv
]);

function makeNV12_4x2() {
const init = {
format: 'NV12',
timestamp: 0,
codedWidth: 4,
codedHeight: 2,
};
return new VideoFrame(NV12_DATA, init);
}

promise_test(async t => {
const frame = makeI420_4x2();
frame.close();
@@ -51,6 +67,24 @@ promise_test(async t => {
assert_buffer_equals(data, expectedData);
}, 'Test RGBA frame.');

promise_test(async t => {
const frame = makeNV12_4x2();
const expectedLayout = [
{offset: 0, stride: 4},
{offset: 8, stride: 4},
];
const expectedData = new Uint8Array([
1,2,3,4,
5,6,7,8,
9,10,11,12
]);
assert_equals(frame.allocationSize(), expectedData.length, 'allocationSize()');
const data = new Uint8Array(expectedData.length);
const layout = await frame.copyTo(data);
assert_layout_equals(layout, expectedLayout);
assert_buffer_equals(data, expectedData);
}, 'Test NV12 frame.');

promise_test(async t => {
const frame = makeI420_4x2();
const data = new Uint8Array(11);
@@ -1,17 +1,18 @@

PASS Test closed frame.
FAIL Test copying I420 frame to a non-shared ArrayBuffer promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test copying I420 frame to a non-shared ArrayBufferView promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test RGBA frame. assert_object_equals: plane 0 layout property "stride" expected 8 got 0
PASS Test copying I420 frame to a non-shared ArrayBuffer
PASS Test copying I420 frame to a non-shared ArrayBufferView
PASS Test RGBA frame.
PASS Test NV12 frame.
PASS Test undersized buffer.
PASS Test incorrect plane count.
FAIL Test stride and offset work. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test stride and offset with padding. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
PASS Test stride and offset work.
PASS Test stride and offset with padding.
PASS Test invalid stride.
PASS Test address overflow.
FAIL Test codedRect. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
PASS Test codedRect.
PASS Test empty rect.
PASS Test unaligned rect.
FAIL Test left crop. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test left crop. assert_equals: buffer contents at index 0 expected 3 but got 1
PASS Test invalid rect.

@@ -1,4 +1,4 @@

FAIL Test copying I420 frame to SharedArrayBuffer. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test copying I420 frame to shared ArrayBufferView. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
PASS Test copying I420 frame to SharedArrayBuffer.
PASS Test copying I420 frame to shared ArrayBufferView.

@@ -1,4 +1,4 @@

FAIL Test copying I420 frame to SharedArrayBuffer. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
FAIL Test copying I420 frame to shared ArrayBufferView. promise_test: Unhandled rejection with value: object "TypeError: Unable to copy data"
PASS Test copying I420 frame to SharedArrayBuffer.
PASS Test copying I420 frame to shared ArrayBufferView.

@@ -339,6 +339,7 @@ __ZN3rtc18NetworkManagerBaseC2EPKN6webrtc15FieldTrialsViewE
__ZN3rtc13SocketAddressC1EN4absl11string_viewEi
__ZN3rtc17AsyncPacketSocket14SubscribeCloseEPKvNSt3__18functionIFvPS0_iEEE
__ZN6webrtc25pixelBufferFromI420BufferEPKhmmmNS_16I420BufferLayoutE
__ZN6webrtc27copyPixelBufferInI420BufferEPhmP10__CVBufferNS_16I420BufferLayoutE
__ZN6webrtc10VP8Decoder6CreateEv
__ZN6webrtc10VP9Decoder6CreateEv
__ZN6webrtc12EncodedImageC1Ev
@@ -339,6 +339,7 @@ __ZN3rtc18NetworkManagerBaseC2EPKN6webrtc15FieldTrialsViewE
__ZN3rtc13SocketAddressC1EN4absl11string_viewEi
__ZN3rtc17AsyncPacketSocket14SubscribeCloseEPKvNSt3__18functionIFvPS0_iEEE
__ZN6webrtc25pixelBufferFromI420BufferEPKhmmmNS_16I420BufferLayoutE
__ZN6webrtc27copyPixelBufferInI420BufferEPhmP10__CVBufferNS_16I420BufferLayoutE
__ZN6webrtc10VP8Decoder6CreateEv
__ZN6webrtc10VP9Decoder6CreateEv
__ZN6webrtc12EncodedImageC1Ev
@@ -339,6 +339,7 @@ __ZN3rtc18NetworkManagerBaseC2EPKN6webrtc15FieldTrialsViewE
__ZN3rtc13SocketAddressC1EN4absl11string_viewEi
__ZN3rtc17AsyncPacketSocket14SubscribeCloseEPKvNSt3__18functionIFvPS0_iEEE
__ZN6webrtc25pixelBufferFromI420BufferEPKhmmmNS_16I420BufferLayoutE
__ZN6webrtc27copyPixelBufferInI420BufferEPhmP10__CVBufferNS_16I420BufferLayoutE
__ZN6webrtc10VP8Decoder6CreateEv
__ZN6webrtc10VP9Decoder6CreateEv
__ZN6webrtc12EncodedImageC1Ev
@@ -71,5 +71,6 @@ struct I420BufferLayout {
};

CVPixelBufferRef pixelBufferFromI420Buffer(const uint8_t* buffer, size_t length, size_t width, size_t height, I420BufferLayout) CF_RETURNS_RETAINED;
bool copyPixelBufferInI420Buffer(uint8_t* buffer, size_t length, CVPixelBufferRef, I420BufferLayout);

}
@@ -68,29 +68,29 @@ void setApplicationStatus(bool isActive)
return static_cast<ObjCFrameBuffer*>(buffer.get())->frame_buffer_provider();
}

static bool copyBufferToPixelBuffer(CVPixelBufferRef pxielBuffer, const uint8_t* buffer, size_t length, size_t width, size_t height, I420BufferLayout layout)
static bool copyI420BufferToPixelBuffer(CVPixelBufferRef pixelBuffer, const uint8_t* buffer, size_t length, size_t width, size_t height, I420BufferLayout layout)
{
auto sourceWidthY = width;
auto sourceHeightY = height;
auto sourceWidthUV = (width + 1) / 2;
auto sourceHeightUV = (height + 1) / 2;

auto destinationWidthY = CVPixelBufferGetWidthOfPlane(pxielBuffer, 0);
auto destinationHeightY = CVPixelBufferGetHeightOfPlane(pxielBuffer, 0);
auto destinationWidthY = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
auto destinationHeightY = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);

auto destinationWidthUV = CVPixelBufferGetWidthOfPlane(pxielBuffer, 1);
auto destinationHeightUV = CVPixelBufferGetHeightOfPlane(pxielBuffer, 1);
auto destinationWidthUV = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1);
auto destinationHeightUV = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1);

if (sourceWidthY != destinationWidthY
|| sourceHeightY != destinationHeightY
|| sourceWidthUV != destinationWidthUV
|| sourceHeightUV != destinationHeightUV)
return false;

uint8_t* destinationY = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pxielBuffer, 0));
int destinationStrideY = CVPixelBufferGetBytesPerRowOfPlane(pxielBuffer, 0);
uint8_t* destinationUV = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pxielBuffer, 1));
int destinationStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pxielBuffer, 1);
uint8_t* destinationY = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
int destinationStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
uint8_t* destinationUV = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
int destinationStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);

auto* sourceY = buffer + layout.offsetY;
int sourceStrideY = layout.strideY;
@@ -119,7 +119,7 @@ CVPixelBufferRef pixelBufferFromI420Buffer(const uint8_t* buffer, size_t length,
if (CVPixelBufferLockBaseAddress(pixelBuffer, 0) != kCVReturnSuccess)
return nullptr;

bool result = copyBufferToPixelBuffer(pixelBuffer, buffer, length, width, height, layout);
bool result = copyI420BufferToPixelBuffer(pixelBuffer, buffer, length, width, height, layout);
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);

if (!result) {
@@ -129,6 +129,35 @@ CVPixelBufferRef pixelBufferFromI420Buffer(const uint8_t* buffer, size_t length,
return pixelBuffer;
}

bool copyPixelBufferInI420Buffer(uint8_t* buffer, size_t length, CVPixelBufferRef pixelBuffer, I420BufferLayout layout)
{
if (CVPixelBufferGetPixelFormatType(pixelBuffer) != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
return false;

if (CVPixelBufferLockBaseAddress(pixelBuffer, 0) != kCVReturnSuccess)
return false;

size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);

uint8_t* sourceY = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
int sourceStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);

uint8_t* sourceUV = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
int sourceStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);

bool result = !libyuv::NV12ToI420(
sourceY, sourceStrideY,
sourceUV, sourceStrideUV,
buffer + layout.offsetY, layout.strideY,
buffer + layout.offsetU, layout.strideU,
buffer + layout.offsetV, layout.strideV,
width, height);

CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
return result;
}

static bool CopyVideoFrameToPixelBuffer(const webrtc::I420BufferInterface* frame, CVPixelBufferRef pixel_buffer) {
RTC_DCHECK(pixel_buffer);
RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || CVPixelBufferGetPixelFormatType(pixel_buffer) == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
@@ -147,15 +147,16 @@ size_t videoPixelFormatToSampleByteSizePerPlane()
return 1;
}

static inline size_t sampleCountPerPixel(VideoPixelFormat format)
static inline size_t sampleCountPerPixel(VideoPixelFormat format, size_t planeNumber)
{
switch (format) {
case VideoPixelFormat::I420:
case VideoPixelFormat::I420A:
case VideoPixelFormat::I444:
case VideoPixelFormat::I422:
case VideoPixelFormat::NV12:
return 1;
case VideoPixelFormat::NV12:
return planeNumber ? 2 : 1;
case VideoPixelFormat::RGBA:
case VideoPixelFormat::RGBX:
case VideoPixelFormat::BGRA:
@@ -190,13 +191,14 @@ ExceptionOr<CombinedPlaneLayout> computeLayoutAndAllocationSize(const DOMRectIni
if (layout && layout->size() != planeCount)
return Exception { TypeError, "layout size is invalid"_s };

size_t pixelSampleCount = sampleCountPerPixel(format);
size_t minAllocationSize = 0;
Vector<ComputedPlaneLayout> computedLayouts;
computedLayouts.reserveInitialCapacity(planeCount);
Vector<size_t> endOffsets;
endOffsets.reserveInitialCapacity(planeCount);
for (size_t i = 0; i < planeCount; ++i) {
size_t pixelSampleCount = sampleCountPerPixel(format, i);

auto sampleBytes = videoPixelFormatToSampleByteSizePerPlane();
auto sampleWidth = videoPixelFormatToSubSampling(format, i);
auto sampleHeight = videoPixelFormatToSubSampling(format, i);

0 comments on commit 13105f5

Please sign in to comment.