From 88d7c41f6d42a67e5236ea7f8c358c1934ee8ee3 Mon Sep 17 00:00:00 2001 From: Dan Glastonbury Date: Thu, 16 Nov 2023 14:57:30 -0800 Subject: [PATCH] [WebXR] Correct WebGL content over camera feed https://bugs.webkit.org/show_bug.cgi?id=264925 rdar://118494318 Reviewed by Dean Jackson. Bug 262774 introduced rendering for WebGL content over ARKit provided camera feed using a CAMetalLayer hosted swap chain. It was intended that the MTLSharedEvent active.completionEvent would be used to pace the frames and delay the present until the frame has been rendered in the GPUP. This didn't work and would lead to Metal preventing from processing our content for misbehaving. This patch removes blocking on completionEvent and replaces it with blocking on a semaphore. This is the same approach present in the visionOS WebXR system. While this means that we could be racing the rendering, resulting in glitches, we no long lock up. In addition, set frame views and origin correctly to make the device behave as an "interactive camera" in the WebXR scene. * Source/WebCore/platform/xr/cocoa/PlatformXRPose.cpp: (PlatformXRPose::toColumnMajorFloatArray const): * Source/WebCore/platform/xr/cocoa/PlatformXRPose.h: helper toColumnMajorFloatArray() converts simd_float4x4 into a flat float[16] for serialization. * Source/WebKit/UIProcess/XR/ios/PlatformXRARKit.mm: (WebKit::ARKitCoordinator::scheduleAnimationFrame): (WebKit::ARKitCoordinator::submitFrame): (WebKit::ARKitCoordinator::renderLoop): * Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.h: * Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.mm: (-[_WKARPresentationSession currentFrame]): (-[_WKARPresentationSession startFrame]): (-[_WKARPresentationSession viewDidLoad]): Vertically flip the CAMetalLayer so the WebGL is displayed the correct way up. (-[_WKARPresentationSession present]): (-[_WKARPresentationSession _loadMetal]): Remote Metal command queue and command buffer. Canonical link: https://commits.webkit.org/270848@main --- .../platform/xr/cocoa/PlatformXRPose.cpp | 11 +++++++ .../platform/xr/cocoa/PlatformXRPose.h | 3 ++ .../UIProcess/XR/ios/PlatformXRARKit.mm | 30 +++++++++++++++---- .../XR/ios/WKARPresentationSession.h | 2 ++ .../XR/ios/WKARPresentationSession.mm | 19 ++++-------- 5 files changed, 46 insertions(+), 19 deletions(-) diff --git a/Source/WebCore/platform/xr/cocoa/PlatformXRPose.cpp b/Source/WebCore/platform/xr/cocoa/PlatformXRPose.cpp index bc04ec348c03..f0f9fce4c56e 100644 --- a/Source/WebCore/platform/xr/cocoa/PlatformXRPose.cpp +++ b/Source/WebCore/platform/xr/cocoa/PlatformXRPose.cpp @@ -27,6 +27,17 @@ PlatformXR::Device::FrameData::Pose PlatformXRPose::pose() const return pose; } +PlatformXRPose::FloatMatrix4 PlatformXRPose::toColumnMajorFloatArray() const +{ + const simd_float4 (&columns)[4] = m_simdTransform.columns; + return { { + columns[0][0], columns[0][1], columns[0][2], columns[0][3], + columns[1][0], columns[1][1], columns[1][2], columns[1][3], + columns[2][0], columns[2][1], columns[2][2], columns[2][3], + columns[3][0], columns[3][1], columns[3][2], columns[3][3], + } }; +} + float PlatformXRPose::distanceToPose(const PlatformXRPose& otherPose) const { simd_float3 position = simdPosition(); diff --git a/Source/WebCore/platform/xr/cocoa/PlatformXRPose.h b/Source/WebCore/platform/xr/cocoa/PlatformXRPose.h index a2c2da8016c0..4b44f4dddb5c 100644 --- a/Source/WebCore/platform/xr/cocoa/PlatformXRPose.h +++ b/Source/WebCore/platform/xr/cocoa/PlatformXRPose.h @@ -20,6 +20,9 @@ class PlatformXRPose { WEBCORE_EXPORT PlatformXR::Device::FrameData::FloatQuaternion orientation() const; WEBCORE_EXPORT PlatformXR::Device::FrameData::Pose pose() const; + using FloatMatrix4 = std::array; + WEBCORE_EXPORT FloatMatrix4 toColumnMajorFloatArray() const; + WEBCORE_EXPORT float distanceToPose(const PlatformXRPose&) const; WEBCORE_EXPORT PlatformXRPose verticalTransformPose() const; diff --git a/Source/WebKit/UIProcess/XR/ios/PlatformXRARKit.mm b/Source/WebKit/UIProcess/XR/ios/PlatformXRARKit.mm index 7e5ed2f92331..84af650670b2 100644 --- a/Source/WebKit/UIProcess/XR/ios/PlatformXRARKit.mm +++ b/Source/WebKit/UIProcess/XR/ios/PlatformXRARKit.mm @@ -34,6 +34,7 @@ #import #import +#import #import "ARKitSoftLink.h" @@ -172,7 +173,6 @@ } active.onFrameUpdate = WTFMove(onFrameUpdateCallback); - active.renderSemaphore->signal(); }, [&](Terminating&) { RELEASE_LOG(XR, "ARKitCoordinator: trying to schedule frame for terminating session"); @@ -194,8 +194,10 @@ return; } - // FIXME: What to do here? The frame is submitted on the signalling of - // of active.presentationSession.completionEvent + // FIXME: rdar://118492973 (Re-enable MTLSharedEvent completion sync) + // Replace frame presentation to depend on + // active.presentationSession.completionEvent + active.renderSemaphore->signal(); }, [&](Terminating&) { RELEASE_LOG(XR, "ARKitCoordinator: trying to submit frame update for a terminating session"); @@ -236,15 +238,29 @@ break; auto& active = *maybeActive; - active.renderSemaphore->wait(); if (!active.onFrameUpdate) - break; + continue; @autoreleasepool { [active.presentationSession startFrame]; - PlatformXR::Device::FrameData frameData = { }; + ARFrame* frame = [active.presentationSession currentFrame]; + ARCamera* camera = frame.camera; + PlatformXR::Device::FrameData frameData = { }; + // FIXME: Use ARSession state to calculate correct values. + frameData.isTrackingValid = true; + frameData.isPositionValid = true; + frameData.predictedDisplayTime = frame.timestamp; + frameData.origin = PlatformXRPose(camera.transform).pose(); + + // Only one view + frameData.views.append({ + .offset = { }, + .projection = { + PlatformXRPose(frame.camera.projectionMatrix).toColumnMajorFloatArray() + }, + }); auto colorTexture = makeMachSendRight([active.presentationSession colorTexture]); auto renderingFrameIndex = [active.presentationSession renderingFrameIndex]; // FIXME: Send this event once at setup time, not every frame. @@ -263,6 +279,8 @@ callback(WTFMove(frameData)); }); + active.renderSemaphore->wait(); + [active.presentationSession present]; } } diff --git a/Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.h b/Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.h index 8c2cf07dba72..79aa99a82539 100644 --- a/Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.h +++ b/Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.h @@ -31,6 +31,7 @@ NS_ASSUME_NONNULL_BEGIN +@class ARFrame; @class ARSession; @interface WKARPresentationSessionDescriptor : NSObject @@ -41,6 +42,7 @@ NS_ASSUME_NONNULL_BEGIN @end @protocol WKARPresentationSession +@property (nonatomic, retain, readonly) ARFrame *currentFrame; @property (nonatomic, retain, readonly) ARSession *session; @property (nonatomic, nonnull, retain, readonly) id completionEvent; @property (nonatomic, nullable, retain, readonly) id colorTexture; diff --git a/Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.mm b/Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.mm index 73f1231c0d6f..c990e9c1f904 100644 --- a/Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.mm +++ b/Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.mm @@ -97,7 +97,6 @@ @implementation _WKARPresentationSession { // Metal state RetainPtr> _device; - RetainPtr> _commandQueue; RetainPtr> _completionEvent; NSUInteger _renderingFrameIndex; @@ -123,6 +122,10 @@ - (nullable instancetype)initWithSession:(ARSession *)session descriptor:(WKARPr #pragma mark - WKARPresentationSession +-(ARFrame *)currentFrame { + return [_session currentFrame]; +} + -(ARSession *)session { return (ARSession *) _session; } @@ -144,6 +147,7 @@ - (NSUInteger)startFrame } _capturedImage = currentFrame.capturedImage; + _currentDrawable = [_metalLayer nextDrawable]; _renderingFrameIndex += 1; return 1; @@ -151,17 +155,6 @@ - (NSUInteger)startFrame - (void)present { - RetainPtr> currentCommandBuffer = [_commandQueue commandBuffer]; - if (!currentCommandBuffer) { - RELEASE_LOG_ERROR(XR, "WKARPresentationSession: failed to obtain command buffer"); - return; - } - - [currentCommandBuffer setLabel:@"WKARPresentationSession"]; - [currentCommandBuffer encodeWaitForEvent:_completionEvent.get() value:_renderingFrameIndex]; - [currentCommandBuffer commit]; - [currentCommandBuffer waitUntilScheduled]; - [CATransaction begin]; { [_cameraLayer setContents:(id) _capturedImage.get()]; @@ -207,6 +200,7 @@ - (void)viewDidLoad [_metalLayer setPixelFormat:MTLPixelFormatBGRA8Unorm_sRGB]; [_metalLayer setPresentsWithTransaction:YES]; [_metalLayer setFrame:[_cameraLayer bounds]]; + [_metalLayer setAffineTransform:CGAffineTransformMakeScale(1, -1)]; } #if !PLATFORM(VISION) @@ -247,7 +241,6 @@ - (void)_enterFullscreen - (void)_loadMetal { - _commandQueue = [_device newCommandQueue]; _completionEvent = adoptNS([_device newSharedEvent]); _renderingFrameIndex = 0; }