WKWebView specific bug: WKWebView as the contents of a SceneKit/ARKit…

… node doesn't work (UIWebView works)

https://bugs.webkit.org/show_bug.cgi?id=203060
<rdar://problem/82899923>

Reviewed by Dean Jackson.

SceneKit's snapshotter cannot handle CAMachPort-as-layer-contents.

If we get parented in a SceneKit snapshotting window, map all existing
surfaces, and prefer using actual IOSurfaces instead of CAMachPort from then on.

* UIProcess/Cocoa/PageClientImplCocoa.h:
* UIProcess/Cocoa/PageClientImplCocoa.mm:
(WebKit::PageClientImplCocoa::windowKind):
* UIProcess/WebPageProxy.cpp:
(WebKit::WebPageProxy::webViewDidMoveToWindow):
* UIProcess/WebPageProxy.h:
* UIProcess/WindowKind.h: Added.
Keep track of the kind of window we're currently in (the only special
case at the moment is the SceneKit snapshotting window, but you could
easily imagine us caring about others).

* UIProcess/DrawingAreaProxy.h:
(WebKit::DrawingAreaProxy::windowKindDidChange):
* UIProcess/PageClient.h:
(WebKit::PageClient::windowKind):
* UIProcess/RemoteLayerTree/RemoteLayerTreeDrawingAreaProxy.h:
* UIProcess/RemoteLayerTree/RemoteLayerTreeDrawingAreaProxy.mm:
(WebKit::RemoteLayerTreeDrawingAreaProxy::windowKindDidChange):
Map CAMachPorts to IOSurfaces when we move into a SceneKit snapshotting window.

* UIProcess/RemoteLayerTree/RemoteLayerTreeHost.h:
* UIProcess/RemoteLayerTree/RemoteLayerTreeHost.mm:
(WebKit::RemoteLayerTreeHost::layerContentsType const):
(WebKit::RemoteLayerTreeHost::updateLayerTree):
Prefer IOSurface if we're already in a SceneKit snapshotting window.
Also factor this code out and improve the comments.


Canonical link: https://commits.webkit.org/241476@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@282189 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebKit/ChangeLog

@@ -1,3 +1,43 @@
+2021-09-08  Tim Horton  <timothy_horton@apple.com>
+
+        WKWebView specific bug: WKWebView as the contents of a SceneKit/ARKit node doesn't work (UIWebView works)
+        https://bugs.webkit.org/show_bug.cgi?id=203060
+        <rdar://problem/82899923>
+
+        Reviewed by Dean Jackson.
+
+        SceneKit's snapshotter cannot handle CAMachPort-as-layer-contents.
+
+        If we get parented in a SceneKit snapshotting window, map all existing
+        surfaces, and prefer using actual IOSurfaces instead of CAMachPort from then on.
+
+        * UIProcess/Cocoa/PageClientImplCocoa.h:
+        * UIProcess/Cocoa/PageClientImplCocoa.mm:
+        (WebKit::PageClientImplCocoa::windowKind):
+        * UIProcess/WebPageProxy.cpp:
+        (WebKit::WebPageProxy::webViewDidMoveToWindow):
+        * UIProcess/WebPageProxy.h:
+        * UIProcess/WindowKind.h: Added.
+        Keep track of the kind of window we're currently in (the only special
+        case at the moment is the SceneKit snapshotting window, but you could
+        easily imagine us caring about others).
+
+        * UIProcess/DrawingAreaProxy.h:
+        (WebKit::DrawingAreaProxy::windowKindDidChange):
+        * UIProcess/PageClient.h:
+        (WebKit::PageClient::windowKind):
+        * UIProcess/RemoteLayerTree/RemoteLayerTreeDrawingAreaProxy.h:
+        * UIProcess/RemoteLayerTree/RemoteLayerTreeDrawingAreaProxy.mm:
+        (WebKit::RemoteLayerTreeDrawingAreaProxy::windowKindDidChange):
+        Map CAMachPorts to IOSurfaces when we move into a SceneKit snapshotting window.
+
+        * UIProcess/RemoteLayerTree/RemoteLayerTreeHost.h:
+        * UIProcess/RemoteLayerTree/RemoteLayerTreeHost.mm:
+        (WebKit::RemoteLayerTreeHost::layerContentsType const):
+        (WebKit::RemoteLayerTreeHost::updateLayerTree):
+        Prefer IOSurface if we're already in a SceneKit snapshotting window.
+        Also factor this code out and improve the comments.
+
 2021-09-08  Fujii Hironori  <Hironori.Fujii@sony.com>
 
         REGRESSION(r282115): undefined reference to `IPC::messageArgumentDescriptions(IPC::MessageName)' in Debug build

Source/WebKit/UIProcess/Cocoa/PageClientImplCocoa.h

@@ -87,6 +87,8 @@ class PageClientImplCocoa : public PageClient {
     void microphoneCaptureChanged() final;
     void cameraCaptureChanged() final;
 
+    WindowKind windowKind() final;
+
 protected:
     WeakObjCPtr<WKWebView> m_webView;
     std::unique_ptr<WebCore::AlternativeTextUIController> m_alternativeTextUIController;

Source/WebKit/UIProcess/Cocoa/PageClientImplCocoa.mm

@@ -193,4 +193,14 @@
     [m_webView didChangeValueForKey:@"cameraCaptureState"];
 }
 
+WindowKind PageClientImplCocoa::windowKind()
+{
+    auto window = [m_webView window];
+    if (!window)
+        return WindowKind::Unparented;
+    if ([window isKindOfClass:NSClassFromString(@"_SCNSnapshotWindow")])
+        return WindowKind::InProcessSnapshotting;
+    return WindowKind::Normal;
+}
+
 }

Source/WebKit/UIProcess/DrawingAreaProxy.h

@@ -84,6 +84,7 @@ class DrawingAreaProxy : public IPC::MessageReceiver, protected IPC::MessageSend
     virtual void colorSpaceDidChange() { }
     virtual void minimumSizeForAutoLayoutDidChange() { }
     virtual void sizeToContentAutoSizeMaximumSizeDidChange() { }
+    virtual void windowKindDidChange() { }
 
     virtual void adjustTransientZoom(double, WebCore::FloatPoint) { }
     virtual void commitTransientZoom(double, WebCore::FloatPoint) { }

Source/WebKit/UIProcess/PageClient.h

@@ -36,6 +36,7 @@
 #include "WebDataListSuggestionsDropdown.h"
 #include "WebDateTimePicker.h"
 #include "WebPopupMenuProxy.h"
+#include "WindowKind.h"
 #include <WebCore/ActivityState.h>
 #include <WebCore/AlternativeTextClient.h>
 #include <WebCore/ContactInfo.h>
@@ -245,6 +246,8 @@ class PageClient : public CanMakeWeakPtr<PageClient> {
     // Return the layer hosting mode for the view.
     virtual LayerHostingMode viewLayerHostingMode() { return LayerHostingMode::InProcess; }
 
+    virtual WindowKind windowKind() { return isViewInWindow() ? WindowKind::Normal : WindowKind::Unparented; }
+
     virtual void processDidExit() = 0;
     virtual void processWillSwap() { processDidExit(); }
     virtual void didRelaunchProcess() = 0;

Source/WebKit/UIProcess/RemoteLayerTree/RemoteLayerTreeDrawingAreaProxy.h

@@ -61,39 +61,40 @@ class RemoteLayerTreeDrawingAreaProxy final : public DrawingAreaProxy {
     CALayer *layerWithIDForTesting(uint64_t) const;
 
 private:
-    void sizeDidChange() override;
-    void deviceScaleFactorDidChange() override;
-    void didUpdateGeometry() override;
+    void sizeDidChange() final;
+    void deviceScaleFactorDidChange() final;
+    void windowKindDidChange() final;
+    void didUpdateGeometry() final;
 
     // For now, all callbacks are called before committing changes, because that's what the only client requires.
     // Once we have other callbacks, it may make sense to have a before-commit/after-commit option.
-    void dispatchAfterEnsuringDrawing(WTF::Function<void (CallbackBase::Error)>&&) override;
+    void dispatchAfterEnsuringDrawing(WTF::Function<void (CallbackBase::Error)>&&) final;
 
 #if PLATFORM(MAC)
-    void didChangeViewExposedRect() override;
+    void didChangeViewExposedRect() final;
 #endif
 
 #if PLATFORM(IOS_FAMILY)
     WKOneShotDisplayLinkHandler *displayLinkHandler();
 #endif
 
     float indicatorScale(WebCore::IntSize contentsSize) const;
-    void updateDebugIndicator() override;
+    void updateDebugIndicator() final;
     void updateDebugIndicator(WebCore::IntSize contentsSize, bool rootLayerChanged, float scale, const WebCore::IntPoint& scrollPosition);
     void updateDebugIndicatorPosition();
     void initializeDebugIndicator();
 
-    void waitForDidUpdateActivityState(ActivityStateChangeID) override;
-    void hideContentUntilPendingUpdate() override;
-    void hideContentUntilAnyUpdate() override;
-    bool hasVisibleContent() const override;
+    void waitForDidUpdateActivityState(ActivityStateChangeID) final;
+    void hideContentUntilPendingUpdate() final;
+    void hideContentUntilAnyUpdate() final;
+    bool hasVisibleContent() const final;
 
     void prepareForAppSuspension() final;
 
     WebCore::FloatPoint indicatorLocation() const;
 
     // IPC::MessageReceiver
-    void didReceiveMessage(IPC::Connection&, IPC::Decoder&) override;
+    void didReceiveMessage(IPC::Connection&, IPC::Decoder&) final;
 
     // Message handlers
     void setPreferredFramesPerSecond(WebCore::FramesPerSecond);

Source/WebKit/UIProcess/RemoteLayerTree/RemoteLayerTreeDrawingAreaProxy.mm

@@ -507,4 +507,10 @@ - (void)pause
     return m_remoteLayerTreeHost->layerWithIDForTesting(layerID);
 }
 
+void RemoteLayerTreeDrawingAreaProxy::windowKindDidChange()
+{
+    if (m_webPageProxy.windowKind() == WindowKind::InProcessSnapshotting)
+        m_remoteLayerTreeHost->mapAllIOSurfaceBackingStore();
+}
+
 } // namespace WebKit

Source/WebKit/UIProcess/RemoteLayerTree/RemoteLayerTreeHost.h

@@ -81,6 +81,8 @@ class RemoteLayerTreeHost {
 
     void layerWillBeRemoved(WebCore::GraphicsLayer::PlatformLayerID);
 
+    RemoteLayerBackingStore::LayerContentsType layerContentsType() const;
+
     RemoteLayerTreeDrawingAreaProxy* m_drawingArea { nullptr };
     RemoteLayerTreeNode* m_rootNode { nullptr };
     HashMap<WebCore::GraphicsLayer::PlatformLayerID, std::unique_ptr<RemoteLayerTreeNode>> m_nodes;

Source/WebKit/UIProcess/RemoteLayerTree/RemoteLayerTreeHost.mm

@@ -64,6 +64,24 @@
     clearLayers();
 }
 
+RemoteLayerBackingStore::LayerContentsType RemoteLayerTreeHost::layerContentsType() const
+{
+#if PLATFORM(MAC) || PLATFORM(MACCATALYST)
+    // CAMachPort currently does not work on macOS (or macCatalyst): rdar://problem/31247730
+    return RemoteLayerBackingStore::LayerContentsType::IOSurface;
+#else
+    // If a surface will be referenced by multiple layers (as in the tile debug indicator), CAMachPort cannot be used.
+    if (m_drawingArea->hasDebugIndicator())
+        return RemoteLayerBackingStore::LayerContentsType::IOSurface;
+
+    // If e.g. SceneKit will be doing an in-process snapshot of the layer tree, CAMachPort cannot be used: rdar://problem/47481972
+    if (m_drawingArea->page().windowKind() == WindowKind::InProcessSnapshotting)
+        return RemoteLayerBackingStore::LayerContentsType::IOSurface;
+
+    return RemoteLayerBackingStore::LayerContentsType::CAMachPort;
+#endif
+}
+
 bool RemoteLayerTreeHost::updateLayerTree(const RemoteLayerTreeTransaction& transaction, float indicatorScaleFactor)
 {
     if (!m_drawingArea)
@@ -89,13 +107,7 @@
     };
     Vector<LayerAndClone> clonesToUpdate;
 
-#if PLATFORM(MAC) || PLATFORM(MACCATALYST)
-    // Can't use the iOS code on macOS yet: rdar://problem/31247730
-    auto layerContentsType = RemoteLayerBackingStore::LayerContentsType::IOSurface;
-#else
-    auto layerContentsType = m_drawingArea->hasDebugIndicator() ? RemoteLayerBackingStore::LayerContentsType::IOSurface : RemoteLayerBackingStore::LayerContentsType::CAMachPort;
-#endif
-
+    auto layerContentsType = this->layerContentsType();
     for (auto& [layerID, propertiesPointer] : transaction.changedLayerProperties()) {
         const RemoteLayerTreeTransaction::LayerProperties& properties = *propertiesPointer;
 

Source/WebKit/UIProcess/WebPageProxy.cpp

@@ -10576,6 +10576,13 @@ void WebPageProxy::webViewDidMoveToWindow()
             continue;
         observer.value->webViewDidMoveToWindow();
     }
+
+    auto newWindowKind = pageClient().windowKind();
+    if (m_windowKind != newWindowKind) {
+        m_windowKind = newWindowKind;
+        if (m_drawingArea)
+            m_drawingArea->windowKindDidChange();    
+    }
 }
 
 void WebPageProxy::setCanShowPlaceholder(const WebCore::ElementContext& context, bool canShowPlaceholder)

Source/WebKit/UIProcess/WebPageProxy.h

@@ -79,6 +79,7 @@
 #include "WebPreferences.h"
 #include "WebUndoStepID.h"
 #include "WebsitePoliciesData.h"
+#include "WindowKind.h"
 #include <WebCore/ActivityState.h>
 #include <WebCore/AutoplayEvent.h>
 #include <WebCore/Color.h>
@@ -729,6 +730,8 @@ class WebPageProxy final : public API::ObjectImpl<API::Object::Type::Page>
     bool isViewFocused() const { return m_activityState.contains(WebCore::ActivityState::IsFocused); }
     bool isViewWindowActive() const { return m_activityState.contains(WebCore::ActivityState::WindowIsActive); }
 
+    WindowKind windowKind() const { return m_windowKind; };
+
     void addMIMETypeWithCustomContentProvider(const String& mimeType);
 
     void selectAll();
@@ -3098,6 +3101,8 @@ class WebPageProxy final : public API::ObjectImpl<API::Object::Type::Page>
 #endif
 
     bool m_needsSiteSpecificViewportQuirks { true };
+
+    WindowKind m_windowKind { WindowKind::Unparented };
 };
 
 #ifdef __OBJC__

Source/WebKit/UIProcess/WindowKind.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+namespace WebKit {
+
+enum class WindowKind : uint8_t {
+    Unparented,
+    Normal,
+    InProcessSnapshotting,
+};
+
+} // namespace WebKit

Source/WebKit/WebKit.xcodeproj/project.pbxproj

@@ -3465,6 +3465,7 @@
 		2D8786221BDB58FF00D02ABB /* APIUserStyleSheet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = APIUserStyleSheet.h; sourceTree = "<group>"; };
 		2D8949EE182044F600E898AA /* PlatformCALayerRemoteTiledBacking.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PlatformCALayerRemoteTiledBacking.cpp; sourceTree = "<group>"; };
 		2D8949EF182044F600E898AA /* PlatformCALayerRemoteTiledBacking.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PlatformCALayerRemoteTiledBacking.h; sourceTree = "<group>"; };
+		2D922D8A26E990FC00EC1A59 /* WindowKind.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WindowKind.h; sourceTree = "<group>"; };
 		2D9CD5EB21FA503F0029ACFA /* TextCheckingControllerProxy.messages.in */ = {isa = PBXFileReference; lastKnownFileType = text; path = TextCheckingControllerProxy.messages.in; sourceTree = "<group>"; };
 		2D9CD5EC21FA503F0029ACFA /* TextCheckingControllerProxy.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = TextCheckingControllerProxy.mm; sourceTree = "<group>"; };
 		2D9CD5ED21FA503F0029ACFA /* TextCheckingControllerProxy.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = TextCheckingControllerProxy.h; sourceTree = "<group>"; };
@@ -10289,6 +10290,7 @@
 				51E8B68D1E712873001B7132 /* WebURLSchemeTask.cpp */,
 				51D124271E6D3F1F002B2820 /* WebURLSchemeTask.h */,
 				572FD44122265CE200A1ECC3 /* WebViewDidMoveToWindowObserver.h */,
+				2D922D8A26E990FC00EC1A59 /* WindowKind.h */,
 			);
 			path = UIProcess;
 			sourceTree = "<group>";