[WebXR] Correct WebGL content over camera feed

https://bugs.webkit.org/show_bug.cgi?id=264925
rdar://118494318

Reviewed by Dean Jackson.

Bug 262774 introduced rendering for WebGL content over ARKit provided camera
feed using a CAMetalLayer hosted swap chain. It was intended that the
MTLSharedEvent active.completionEvent would be used to pace the frames and delay
the present until the frame has been rendered in the GPUP. This didn't work and
would lead to Metal preventing from processing our content for misbehaving.

This patch removes blocking on completionEvent and replaces it with blocking on
a semaphore. This is the same approach present in the visionOS WebXR
system. While this means that we could be racing the rendering, resulting in
glitches, we no long lock up.

In addition, set frame views and origin correctly to make the device behave as
an "interactive camera" in the WebXR scene.

* Source/WebCore/platform/xr/cocoa/PlatformXRPose.cpp:
(PlatformXRPose::toColumnMajorFloatArray const):
* Source/WebCore/platform/xr/cocoa/PlatformXRPose.h:
helper toColumnMajorFloatArray() converts simd_float4x4 into a flat float[16]
for serialization.

* Source/WebKit/UIProcess/XR/ios/PlatformXRARKit.mm:
(WebKit::ARKitCoordinator::scheduleAnimationFrame):
(WebKit::ARKitCoordinator::submitFrame):
(WebKit::ARKitCoordinator::renderLoop):
* Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.h:
* Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.mm:
(-[_WKARPresentationSession currentFrame]):
(-[_WKARPresentationSession startFrame]):
(-[_WKARPresentationSession viewDidLoad]):
Vertically flip the CAMetalLayer so the WebGL is displayed the correct way up.

(-[_WKARPresentationSession present]):
(-[_WKARPresentationSession _loadMetal]):
Remote Metal command queue and command buffer.

Canonical link: https://commits.webkit.org/270848@main

Source/WebCore/platform/xr/cocoa/PlatformXRPose.cpp

@@ -27,6 +27,17 @@ PlatformXR::Device::FrameData::Pose PlatformXRPose::pose() const
     return pose;
 }
 
+PlatformXRPose::FloatMatrix4 PlatformXRPose::toColumnMajorFloatArray() const
+{
+    const simd_float4 (&columns)[4] = m_simdTransform.columns;
+    return { {
+        columns[0][0], columns[0][1], columns[0][2], columns[0][3],
+        columns[1][0], columns[1][1], columns[1][2], columns[1][3],
+        columns[2][0], columns[2][1], columns[2][2], columns[2][3],
+        columns[3][0], columns[3][1], columns[3][2], columns[3][3],
+    } };
+}
+
 float PlatformXRPose::distanceToPose(const PlatformXRPose& otherPose) const
 {
     simd_float3 position = simdPosition();

Source/WebCore/platform/xr/cocoa/PlatformXRPose.h

@@ -20,6 +20,9 @@ class PlatformXRPose {
     WEBCORE_EXPORT PlatformXR::Device::FrameData::FloatQuaternion orientation() const;
     WEBCORE_EXPORT PlatformXR::Device::FrameData::Pose pose() const;
 
+    using FloatMatrix4 = std::array<float, 16>;
+    WEBCORE_EXPORT FloatMatrix4 toColumnMajorFloatArray() const;
+
     WEBCORE_EXPORT float distanceToPose(const PlatformXRPose&) const;
     WEBCORE_EXPORT PlatformXRPose verticalTransformPose() const;
 

Source/WebKit/UIProcess/XR/ios/PlatformXRARKit.mm

@@ -34,6 +34,7 @@
 
 #import <Metal/MTLEvent_Private.h>
 #import <Metal/MTLTexture_Private.h>
+#import <WebCore/PlatformXRPose.h>
 
 #import "ARKitSoftLink.h"
 
@@ -172,7 +173,6 @@
             }
 
             active.onFrameUpdate = WTFMove(onFrameUpdateCallback);
-            active.renderSemaphore->signal();
         },
         [&](Terminating&) {
             RELEASE_LOG(XR, "ARKitCoordinator: trying to schedule frame for terminating session");
@@ -194,8 +194,10 @@
                 return;
             }
 
-            // FIXME: What to do here? The frame is submitted on the signalling of
-            // of active.presentationSession.completionEvent
+            // FIXME: rdar://118492973 (Re-enable MTLSharedEvent completion sync)
+            // Replace frame presentation to depend on
+            // active.presentationSession.completionEvent
+            active.renderSemaphore->signal();
         },
         [&](Terminating&) {
             RELEASE_LOG(XR, "ARKitCoordinator: trying to submit frame update for a terminating session");
@@ -236,15 +238,29 @@
             break;
 
         auto& active = *maybeActive;
-        active.renderSemaphore->wait();
         if (!active.onFrameUpdate)
-            break;
+            continue;
 
         @autoreleasepool {
             [active.presentationSession startFrame];
 
-            PlatformXR::Device::FrameData frameData = { };
+            ARFrame* frame = [active.presentationSession currentFrame];
+            ARCamera* camera = frame.camera;
 
+            PlatformXR::Device::FrameData frameData = { };
+            // FIXME: Use ARSession state to calculate correct values.
+            frameData.isTrackingValid = true;
+            frameData.isPositionValid = true;
+            frameData.predictedDisplayTime = frame.timestamp;
+            frameData.origin = PlatformXRPose(camera.transform).pose();
+
+            // Only one view
+            frameData.views.append({
+                .offset = { },
+                .projection = {
+                    PlatformXRPose(frame.camera.projectionMatrix).toColumnMajorFloatArray()
+                },
+            });
             auto colorTexture = makeMachSendRight([active.presentationSession colorTexture]);
             auto renderingFrameIndex = [active.presentationSession renderingFrameIndex];
             // FIXME: Send this event once at setup time, not every frame.
@@ -263,6 +279,8 @@
                 callback(WTFMove(frameData));
             });
 
+            active.renderSemaphore->wait();
+
             [active.presentationSession present];
         }
     }

Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.h

@@ -31,6 +31,7 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
+@class ARFrame;
 @class ARSession;
 
 @interface WKARPresentationSessionDescriptor : NSObject <NSCopying>
@@ -41,6 +42,7 @@ NS_ASSUME_NONNULL_BEGIN
 @end
 
 @protocol WKARPresentationSession <NSObject>
+@property (nonatomic, retain, readonly) ARFrame *currentFrame;
 @property (nonatomic, retain, readonly) ARSession *session;
 @property (nonatomic, nonnull, retain, readonly) id<MTLSharedEvent> completionEvent;
 @property (nonatomic, nullable, retain, readonly) id<MTLTexture> colorTexture;

Source/WebKit/UIProcess/XR/ios/WKARPresentationSession.mm

@@ -97,7 +97,6 @@ @implementation _WKARPresentationSession {
 
     // Metal state
     RetainPtr<id<MTLDevice>> _device;
-    RetainPtr<id<MTLCommandQueue>> _commandQueue;
     RetainPtr<id<MTLSharedEvent>> _completionEvent;
     NSUInteger _renderingFrameIndex;
 
@@ -123,6 +122,10 @@ - (nullable instancetype)initWithSession:(ARSession *)session descriptor:(WKARPr
 
 #pragma mark - WKARPresentationSession
 
+-(ARFrame *)currentFrame {
+    return [_session currentFrame];
+}
+
 -(ARSession *)session {
     return (ARSession *) _session;
 }
@@ -144,24 +147,14 @@ - (NSUInteger)startFrame
     }
 
     _capturedImage = currentFrame.capturedImage;
+    _currentDrawable = [_metalLayer nextDrawable];
     _renderingFrameIndex += 1;
 
     return 1;
 }
 
 - (void)present
 {
-    RetainPtr<id<MTLCommandBuffer>> currentCommandBuffer = [_commandQueue commandBuffer];
-    if (!currentCommandBuffer) {
-        RELEASE_LOG_ERROR(XR, "WKARPresentationSession: failed to obtain command buffer");
-        return;
-    }
-
-    [currentCommandBuffer setLabel:@"WKARPresentationSession"];
-    [currentCommandBuffer encodeWaitForEvent:_completionEvent.get() value:_renderingFrameIndex];
-    [currentCommandBuffer commit];
-    [currentCommandBuffer waitUntilScheduled];
-
     [CATransaction begin];
     {
         [_cameraLayer setContents:(id) _capturedImage.get()];
@@ -207,6 +200,7 @@ - (void)viewDidLoad
     [_metalLayer setPixelFormat:MTLPixelFormatBGRA8Unorm_sRGB];
     [_metalLayer setPresentsWithTransaction:YES];
     [_metalLayer setFrame:[_cameraLayer bounds]];
+    [_metalLayer setAffineTransform:CGAffineTransformMakeScale(1, -1)];
 }
 
 #if !PLATFORM(VISION)
@@ -247,7 +241,6 @@ - (void)_enterFullscreen
 
 - (void)_loadMetal
 {
-    _commandQueue = [_device newCommandQueue];
     _completionEvent = adoptNS([_device newSharedEvent]);
     _renderingFrameIndex = 0;
 }