[WebXR] Service video frame callbacks in WebXRSession if page backgro…

…unds during immersive WebXR session

https://bugs.webkit.org/show_bug.cgi?id=273901
rdar://124236441

Reviewed by Simon Fraser and Jer Noble.

Instead of not freezing the layer tree if the page gets backgrounded
during an immersive WebXR session so video frame callbacks can continue
to be serviced, just service the video frame callbacks in the XR session.

* Source/WebCore/Modules/webxr/WebXRSession.cpp:
(WebCore::WebXRSession::minimalUpdateRendering):
We'll special case servicing the video frame callbacks as the minimum render
updates we'll do in XR session's rAF if the page is backgrounded.
(WebCore::WebXRSession::onFrame):
* Source/WebCore/Modules/webxr/WebXRSession.h:
* Source/WebCore/Modules/webxr/WebXRSystem.cpp:
(WebCore::WebXRSystem::activeImmersiveSession const):
(WebCore::WebXRSystem::hasActiveImmersiveSession const): Deleted.
* Source/WebCore/Modules/webxr/WebXRSystem.h:
* Source/WebCore/page/Page.cpp:
(WebCore::Page::applicationDidEnterBackground):
(WebCore::Page::applicationWillEnterForeground):
(WebCore::Page::hasActiveImmersiveSession const):
(WebCore::Page::activeImmersiveXRSession const):
(WebCore::Page::shouldBlockLayerTreeFreezingForVideo): Deleted.
* Source/WebCore/page/Page.h:
* Source/WebKit/WebProcess/WebPage/WebPage.cpp:
(WebKit::WebPage::updateDrawingAreaLayerTreeFreezeState):
Remove the logic to prevent the freezing of the layer tree if the
page is backgrounded due to an active immersive session. It's not good
for performance to continue render updates for the page when it's
backgrounded and not visible.

Canonical link: https://commits.webkit.org/279146@main

Source/WebCore/Modules/webxr/WebXRSession.cpp

@@ -33,6 +33,7 @@
 #include "EventNames.h"
 #include "JSDOMPromiseDeferred.h"
 #include "JSWebXRReferenceSpace.h"
+#include "Page.h"
 #include "SecurityOrigin.h"
 #include "WebCoreOpaqueRoot.h"
 #include "WebXRBoundedReferenceSpace.h"
@@ -509,6 +510,24 @@ void WebXRSession::applyPendingRenderState()
     }
 }
 
+void WebXRSession::minimalUpdateRendering()
+{
+    // Spec is unclear about this, but we have to execute any scheduled video frame updates for
+    // videos to work in WebXR if the page is backgrounded
+    if (!m_shouldServiceRequestVideoFrameCallbacks)
+        return;
+
+    RefPtr sessionDocument = downcast<Document>(scriptExecutionContext());
+    if (!sessionDocument)
+        return;
+
+    if (auto* page = sessionDocument->page()) {
+        page->forEachDocument([&] (Document& document) {
+            document.serviceRequestVideoFrameCallbacks();
+        });
+    }
+}
+
 // https://immersive-web.github.io/webxr/#should-be-rendered
 bool WebXRSession::frameShouldBeRendered() const
 {
@@ -597,6 +616,7 @@ void WebXRSession::onFrame(PlatformXR::FrameData&& frameData)
                 m_inputSources->update(now, m_frameData.inputSources);
 
             tracePoint(WebXRSessionFrameCallbacksStart);
+            minimalUpdateRendering();
             // 6.5.For each entry in session’s list of currently running animation frame callbacks, in order:
             for (auto& callback : callbacks) {
                 //  6.6.If the entry’s cancelled boolean is true, continue to the next entry.

Source/WebCore/Modules/webxr/WebXRSession.h

@@ -90,6 +90,12 @@ class WebXRSession final : public RefCounted<WebXRSession>, public EventTarget,
     bool supportsViewportScaling() const; 
     bool isPositionEmulated() const;
 
+    // If the immersive session obscures the HTML document (for example, in standalone devices),
+    // Page::updateRendering() won't be called and the WebXRSession needs to take over the
+    // responsibility to service requestVideoFrameCallbacks.
+    void applicationDidEnterBackground() { m_shouldServiceRequestVideoFrameCallbacks = true; }
+    void applicationWillEnterForeground() { m_shouldServiceRequestVideoFrameCallbacks = false; }
+
     // EventTarget.
     ScriptExecutionContext* scriptExecutionContext() const final { return ActiveDOMObject::scriptExecutionContext(); }
 
@@ -133,12 +139,14 @@ class WebXRSession final : public RefCounted<WebXRSession>, public EventTarget,
     void requestFrameIfNeeded();
     void onFrame(PlatformXR::FrameData&&);
     void applyPendingRenderState();
+    void minimalUpdateRendering();
 
     XREnvironmentBlendMode m_environmentBlendMode { XREnvironmentBlendMode::Opaque };
     XRInteractionMode m_interactionMode { XRInteractionMode::WorldSpace };
     XRVisibilityState m_visibilityState { XRVisibilityState::Visible };
     UniqueRef<WebXRInputSourceArray> m_inputSources;
     bool m_ended { false };
+    bool m_shouldServiceRequestVideoFrameCallbacks { false };
     std::unique_ptr<EndPromise> m_endPromise;
 
     WebXRSystem& m_xrSystem;

Source/WebCore/Modules/webxr/WebXRSystem.cpp

@@ -609,9 +609,9 @@ void WebXRSystem::sessionEnded(WebXRSession& session)
     m_inlineSessions.remove(session);
 }
 
-bool WebXRSystem::hasActiveImmersiveSession() const
+RefPtr<WebXRSession> WebXRSystem::activeImmersiveSession() const
 {
-    return !!m_activeImmersiveSession;
+    return m_activeImmersiveSession;
 }
 
 class InlineRequestAnimationFrameCallback final: public RequestAnimationFrameCallback {

Source/WebCore/Modules/webxr/WebXRSystem.h

@@ -76,7 +76,7 @@ class WebXRSystem final : public RefCounted<WebXRSystem>, public EventTarget, pu
     void ensureImmersiveXRDeviceIsSelected(CompletionHandler<void()>&&);
     bool hasActiveImmersiveXRDevice() const { return !!m_activeImmersiveDevice.get(); }
 
-    bool hasActiveImmersiveSession() const;
+    RefPtr<WebXRSession> activeImmersiveSession() const;
     void sessionEnded(WebXRSession&);
 
     // For testing purpouses only.

Source/WebCore/page/Page.cpp

@@ -214,8 +214,9 @@
 #include "AccessibilityRootAtspi.h"
 #endif
 
-#if PLATFORM(IOS_FAMILY) && ENABLE(WEBXR)
+#if ENABLE(WEBXR)
 #include "NavigatorWebXR.h"
+#include "WebXRSession.h"
 #include "WebXRSystem.h"
 #endif
 
@@ -2722,20 +2723,6 @@ void Page::setMuted(MediaProducerMutedStateFlags muted)
     });
 }
 
-bool Page::shouldBlockLayerTreeFreezingForVideo()
-{
-    bool shouldBlockLayerTreeFreezingForVideo = false;
-    forEachMediaElement([&shouldBlockLayerTreeFreezingForVideo] (HTMLMediaElement& element) {
-        // FIXME: Consider only returning true when `element.readyState >=
-        // HTMLMediaElementEnums::HAVE_METADATA` and forcing an update to the layer tree
-        // freeze state when an element's readyState gets to HAVE_METADATA in
-        // `HTMLMediaElement::setReadyState`
-        if (element.isVideo())
-            shouldBlockLayerTreeFreezingForVideo = true;
-    });
-    return shouldBlockLayerTreeFreezingForVideo;
-}
-
 void Page::stopMediaCapture(MediaProducerMediaCaptureKind kind)
 {
     UNUSED_PARAM(kind);
@@ -4069,11 +4056,21 @@ void Page::applicationWillResignActive()
 void Page::applicationDidEnterBackground()
 {
     m_webRTCProvider->setActive(false);
+
+#if ENABLE(WEBXR)
+    if (auto session = this->activeImmersiveXRSession())
+        session->applicationDidEnterBackground();
+#endif
 }
 
 void Page::applicationWillEnterForeground()
 {
     m_webRTCProvider->setActive(true);
+
+#if ENABLE(WEBXR)
+    if (auto session = this->activeImmersiveXRSession())
+        session->applicationWillEnterForeground();
+#endif
 }
 
 void Page::applicationDidBecomeActive()
@@ -4774,8 +4771,15 @@ void Page::setAccessibilityRootObject(AccessibilityRootAtspi* rootObject)
 }
 #endif // USE(ATSPI)
 
-#if PLATFORM(IOS_FAMILY) && ENABLE(WEBXR)
+#if ENABLE(WEBXR)
+#if PLATFORM(IOS_FAMILY)
 bool Page::hasActiveImmersiveSession() const
+{
+    return !!activeImmersiveXRSession();
+}
+#endif // PLATFORM(IOS_FAMILY)
+
+RefPtr<WebXRSession> Page::activeImmersiveXRSession() const
 {
     for (RefPtr frame = &m_mainFrame.get(); frame; frame = frame->tree().traverseNext()) {
         RefPtr localFrame = dynamicDowncast<LocalFrame>(frame);
@@ -4787,13 +4791,13 @@ bool Page::hasActiveImmersiveSession() const
         if (!navigator)
             continue;
 
-        auto* xrSystem = NavigatorWebXR::xrIfExists(*navigator);
-        if (xrSystem && xrSystem->hasActiveImmersiveSession())
-            return true;
+        if (auto xrSystem = NavigatorWebXR::xrIfExists(*navigator))
+            return xrSystem->activeImmersiveSession();
     }
-    return false;
+
+    return nullptr;
 }
-#endif // PLATFORM(IOS_FAMILY) && ENABLE(WEBXR)
+#endif //  ENABLE(WEBXR)
 
 #if HAVE(SPATIAL_TRACKING_LABEL)
 void Page::setDefaultSpatialTrackingLabel(const String& label)

Source/WebCore/page/Page.h

@@ -174,6 +174,10 @@ class WheelEventDeltaFilter;
 class WheelEventTestMonitor;
 class WindowEventLoop;
 
+#if ENABLE(WEBXR)
+class WebXRSession;
+#endif
+
 struct AXTreeData;
 struct ApplePayAMSUIRequest;
 struct SimpleRange;
@@ -894,7 +898,6 @@ class Page : public RefCounted<Page>, public Supplementable<Page>, public CanMak
     void playbackControlsMediaEngineChanged();
 #endif
     WEBCORE_EXPORT void setMuted(MediaProducerMutedStateFlags);
-    WEBCORE_EXPORT bool shouldBlockLayerTreeFreezingForVideo();
 
     WEBCORE_EXPORT void stopMediaCapture(MediaProducerMediaCaptureKind);
 
@@ -1182,6 +1185,10 @@ class Page : public RefCounted<Page>, public Supplementable<Page>, public CanMak
     void updateElementsWithTextRecognitionResults();
 #endif
 
+#if ENABLE(WEBXR)
+    RefPtr<WebXRSession> activeImmersiveXRSession() const;
+#endif
+
     std::optional<PageIdentifier> m_identifier;
     UniqueRef<Chrome> m_chrome;
     UniqueRef<DragCaretController> m_dragCaretController;

Source/WebKit/WebProcess/WebPage/WebPage.cpp

@@ -3319,26 +3319,14 @@ void WebPage::updateDrawingAreaLayerTreeFreezeState()
     if (!m_drawingArea)
         return;
 
-    if (m_layerTreeFreezeReasons.hasExactlyOneBitSet() && m_layerTreeFreezeReasons.contains(LayerTreeFreezeReason::BackgroundApplication)) {
-        // When the browser is in the background, we should not freeze the layer tree
-        // if the page has a video playing in picture-in-picture or if the page is in an
-        // active immersive session.
-        bool shouldSkipFreezingLayerTreeOnBackgrounding = false;
 #if ENABLE(VIDEO_PRESENTATION_MODE)
-        if (m_videoPresentationManager && m_videoPresentationManager->hasVideoPlayingInPictureInPicture())
-            shouldSkipFreezingLayerTreeOnBackgrounding = true;
-#endif
-#if PLATFORM(VISION) && ENABLE(WEBXR)
-        if (RefPtr page = m_page) {
-            if (page->hasActiveImmersiveSession() && page->shouldBlockLayerTreeFreezingForVideo())
-                shouldSkipFreezingLayerTreeOnBackgrounding = true;
-        }
-#endif
-        if (shouldSkipFreezingLayerTreeOnBackgrounding) {
-            m_drawingArea->setLayerTreeStateIsFrozen(false);
-            return;
-        }
+    // When the browser is in the background, we should not freeze the layer tree
+    // if the page has a video playing in picture-in-picture.
+    if (m_videoPresentationManager && m_videoPresentationManager->hasVideoPlayingInPictureInPicture() && m_layerTreeFreezeReasons.hasExactlyOneBitSet() && m_layerTreeFreezeReasons.contains(LayerTreeFreezeReason::BackgroundApplication)) {
+        m_drawingArea->setLayerTreeStateIsFrozen(false);
+        return;
     }
+#endif
 
     m_drawingArea->setLayerTreeStateIsFrozen(!!m_layerTreeFreezeReasons);
 }