Expose model entity transform to JS

rdar://124986396
https://bugs.webkit.org/show_bug.cgi?id=271208

Reviewed by Alex Christensen.

Exposes the root entity transform matrix to JS side, so that developers
can move the 3D object around within the <model> tag portal.

HTMLModelTag has an m_entityTransform "truth" that's used to serve JS
getter synchronously.

ModelPlayer now has the ability to determine whether a certain transform
matrix is supported. If not, HTMLModelElement::setEntityTransform will
raise an Unsupported Exception.

Upon model load, Model Process will send an IPC message to Web Process
to update the m_entityTransform state of the HTMLModelElement to make
it aware of the default applied transform.

* Source/WebCore/Modules/model-element/HTMLModelElement.cpp:
(WebCore::HTMLModelElement::modelPlayer const):
(WebCore::HTMLModelElement::entityTransform const):
* Source/WebCore/Modules/model-element/HTMLModelElement.h:
* Source/WebCore/Modules/model-element/HTMLModelElement.idl:
* Source/WebCore/Modules/model-element/ModelPlayer.cpp:
(WebCore::ModelPlayer::entityTransform):
(WebCore::ModelPlayer::setEntityTransform):
(WebCore::ModelPlayer::supportsTransform):
* Source/WebCore/Modules/model-element/ModelPlayer.h:
* Source/WebCore/SourcesCocoa.txt:
* Source/WebCore/WebCore.xcodeproj/project.pbxproj:

* Source/WebCore/platform/graphics/cocoa/TransformationMatrixCocoa.cpp:
(WebCore::TransformationMatrix::TransformationMatrix):
(WebCore::TransformationMatrix::operator simd_float4x4 const):
Added convenience converter methods to/from simd_float4x4, which is the commonly used transform matrix data type on visionOS.
Applicable only on Cocoa platforms.

* Source/WebCore/platform/graphics/transforms/TransformationMatrix.h:
* Source/WebKit/ModelProcess/cocoa/ModelProcessModelPlayerProxy.h:
* Source/WebKit/ModelProcess/cocoa/ModelProcessModelPlayerProxy.messages.in:
* Source/WebKit/ModelProcess/cocoa/ModelProcessModelPlayerProxy.mm:

(WebKit::ModelProcessModelPlayerProxy::transformSupported):
Require the transform to be a non-shear, uniform-scaling matrix, for now.

(WebKit::ModelProcessModelPlayerProxy::computeTransform):
(WebKit::ModelProcessModelPlayerProxy::updateTransform):
(WebKit::ModelProcessModelPlayerProxy::didFinishLoading):
(WebKit::ModelProcessModelPlayerProxy::entityTransform):
(WebKit::ModelProcessModelPlayerProxy::setEntityTransform):
* Source/WebKit/WebProcess/Model/ModelProcessModelPlayer.cpp:
(WebKit::ModelProcessModelPlayer::didUpdateEntityTransform):
(WebKit::ModelProcessModelPlayer::entityTransform):
(WebKit::ModelProcessModelPlayer::setEntityTransform):
(WebKit::ModelProcessModelPlayer::supportsTransform):
* Source/WebKit/WebProcess/Model/ModelProcessModelPlayer.h:
* Source/WebKit/WebProcess/Model/ModelProcessModelPlayer.messages.in:

Canonical link: https://commits.webkit.org/276661@main

Source/WebCore/Modules/model-element/HTMLModelElement.cpp

@@ -29,6 +29,7 @@
 #if ENABLE(MODEL_ELEMENT)
 
 #include "CachedResourceLoader.h"
+#include "DOMMatrixReadOnly.h"
 #include "DOMPromiseProxy.h"
 #include "Document.h"
 #include "DocumentInlines.h"
@@ -75,6 +76,9 @@ HTMLModelElement::HTMLModelElement(const QualifiedName& tagName, Document& docum
     : HTMLElement(tagName, document, { TypeFlag::HasCustomStyleResolveCallbacks, TypeFlag::HasDidMoveToNewDocument })
     , ActiveDOMObject(document)
     , m_readyPromise { makeUniqueRef<ReadyPromise>(*this, &HTMLModelElement::readyPromiseResolve) }
+#if ENABLE(MODEL_PROCESS)
+    , m_entityTransform(DOMMatrixReadOnly::create(TransformationMatrix::identity, DOMMatrixReadOnly::Is2D::No))
+#endif
 {
 }
 
@@ -152,6 +156,10 @@ void HTMLModelElement::setSourceURL(const URL& url)
     if (m_modelPlayer)
         m_modelPlayer = nullptr;
 
+#if ENABLE(MODEL_PROCESS)
+    m_entityTransform = DOMMatrixReadOnly::create(TransformationMatrix::identity, DOMMatrixReadOnly::Is2D::No);
+#endif
+
     if (!m_readyPromise->isFulfilled())
         m_readyPromise->reject(Exception { ExceptionCode::AbortError });
 
@@ -284,6 +292,9 @@ void HTMLModelElement::createModelPlayer()
         return;
 
     ASSERT(document().page());
+#if ENABLE(MODEL_PROCESS)
+    m_entityTransform = DOMMatrixReadOnly::create(TransformationMatrix::identity, DOMMatrixReadOnly::Is2D::No);
+#endif
     m_modelPlayer = document().page()->modelPlayerProvider().createModelPlayer(*this);
     if (!m_modelPlayer) {
         if (!m_readyPromise->isFulfilled())
@@ -377,6 +388,37 @@ void HTMLModelElement::applyBackgroundColor(Color color)
         m_modelPlayer->setBackgroundColor(color);
 }
 
+#if ENABLE(MODEL_PROCESS)
+const DOMMatrixReadOnly& HTMLModelElement::entityTransform() const
+{
+    return m_entityTransform.get();
+}
+
+ExceptionOr<void> HTMLModelElement::setEntityTransform(const DOMMatrixReadOnly& transform)
+{
+    auto player = m_modelPlayer;
+    if (!player) {
+        ASSERT_NOT_REACHED();
+        return Exception { ExceptionCode::UnknownError };
+    }
+
+    TransformationMatrix matrix = transform.transformationMatrix();
+
+    if (!player->supportsTransform(matrix))
+        return Exception { ExceptionCode::NotSupportedError };
+
+    m_entityTransform = DOMMatrixReadOnly::create(matrix, DOMMatrixReadOnly::Is2D::No);
+    player->setEntityTransform(matrix);
+
+    return { };
+}
+
+void HTMLModelElement::didUpdateEntityTransform(ModelPlayer&, const TransformationMatrix& transform)
+{
+    m_entityTransform = DOMMatrixReadOnly::create(transform, DOMMatrixReadOnly::Is2D::No);
+}
+#endif // ENABLE(MODEL_PROCESS)
+
 // MARK: - Fullscreen support.
 
 void HTMLModelElement::enterFullscreen()

Source/WebCore/Modules/model-element/HTMLModelElement.h

@@ -31,6 +31,7 @@
 #include "CachedRawResource.h"
 #include "CachedRawResourceClient.h"
 #include "CachedResourceHandle.h"
+#include "ExceptionOr.h"
 #include "HTMLElement.h"
 #include "HTMLModelElementCamera.h"
 #include "IDLTypes.h"
@@ -43,6 +44,7 @@
 
 namespace WebCore {
 
+class DOMMatrixReadOnly;
 class Event;
 class LayoutSize;
 class Model;
@@ -80,6 +82,11 @@ class HTMLModelElement final : public HTMLElement, private CachedRawResourceClie
 
     void applyBackgroundColor(Color);
 
+#if ENABLE(MODEL_PROCESS)
+    const DOMMatrixReadOnly& entityTransform() const;
+    ExceptionOr<void> setEntityTransform(const DOMMatrixReadOnly&);
+#endif
+
     void enterFullscreen();
 
     using CameraPromise = DOMPromiseDeferred<IDLDictionary<HTMLModelElementCamera>>;
@@ -157,6 +164,9 @@ class HTMLModelElement final : public HTMLElement, private CachedRawResourceClie
     void didUpdateLayerHostingContextIdentifier(ModelPlayer&, LayerHostingContextIdentifier) final;
     void didFinishLoading(ModelPlayer&) final;
     void didFailLoading(ModelPlayer&, const ResourceError&) final;
+#if ENABLE(MODEL_PROCESS)
+    void didUpdateEntityTransform(ModelPlayer&, const TransformationMatrix&) final;
+#endif
     PlatformLayerIdentifier platformLayerID() final;
 
     void defaultEventHandler(Event&) final;
@@ -180,6 +190,9 @@ class HTMLModelElement final : public HTMLElement, private CachedRawResourceClie
     bool m_shouldCreateModelPlayerUponRendererAttachment { false };
 
     RefPtr<ModelPlayer> m_modelPlayer;
+#if ENABLE(MODEL_PROCESS)
+    Ref<DOMMatrixReadOnly> m_entityTransform;
+#endif
 };
 
 } // namespace WebCore

Source/WebCore/Modules/model-element/HTMLModelElement.idl

@@ -40,6 +40,8 @@
 
     [Reflect] attribute boolean interactive;
 
+    [Conditional=MODEL_PROCESS, EnabledBySetting=ModelProcessEnabled, CEReactions=NotNeeded] attribute DOMMatrixReadOnly entityTransform;
+
     undefined enterFullscreen();
 
     [Conditional=ARKIT_INLINE_PREVIEW_CAMERA_TRANSFORM] Promise<HTMLModelElementCamera> getCamera();

Source/WebCore/Modules/model-element/ModelPlayer.cpp

@@ -27,6 +27,7 @@
 #include "ModelPlayer.h"
 
 #include "Color.h"
+#include "TransformationMatrix.h"
 
 namespace WebCore {
 
@@ -36,6 +37,10 @@ void ModelPlayer::setBackgroundColor(Color)
 {
 }
 
+void ModelPlayer::setEntityTransform(TransformationMatrix)
+{
+}
+
 bool ModelPlayer::supportsMouseInteraction()
 {
     return false;
@@ -46,6 +51,11 @@ bool ModelPlayer::supportsDragging()
     return true;
 }
 
+bool ModelPlayer::supportsTransform(TransformationMatrix)
+{
+    return false;
+}
+
 void ModelPlayer::setInteractionEnabled(bool)
 {
 }

Source/WebCore/Modules/model-element/ModelPlayer.h

@@ -39,6 +39,7 @@ namespace WebCore {
 
 class Color;
 class Model;
+class TransformationMatrix;
 
 class WEBCORE_EXPORT ModelPlayer : public RefCounted<ModelPlayer> {
     WTF_MAKE_FAST_ALLOCATED;
@@ -50,9 +51,11 @@ class WEBCORE_EXPORT ModelPlayer : public RefCounted<ModelPlayer> {
     virtual PlatformLayer* layer() = 0;
     virtual std::optional<LayerHostingContextIdentifier> layerHostingContextIdentifier() = 0;
     virtual void setBackgroundColor(Color);
+    virtual void setEntityTransform(TransformationMatrix);
     virtual void enterFullscreen() = 0;
     virtual bool supportsMouseInteraction();
     virtual bool supportsDragging();
+    virtual bool supportsTransform(TransformationMatrix);
     virtual void setInteractionEnabled(bool);
     virtual void handleMouseDown(const LayoutPoint&, MonotonicTime) = 0;
     virtual void handleMouseMove(const LayoutPoint&, MonotonicTime) = 0;

Source/WebCore/Modules/model-element/ModelPlayerClient.h

@@ -42,6 +42,9 @@ class WEBCORE_EXPORT ModelPlayerClient : public CanMakeWeakPtr<ModelPlayerClient
     virtual void didUpdateLayerHostingContextIdentifier(ModelPlayer&, LayerHostingContextIdentifier) = 0;
     virtual void didFinishLoading(ModelPlayer&) = 0;
     virtual void didFailLoading(ModelPlayer&, const ResourceError&) = 0;
+#if ENABLE(MODEL_PROCESS)
+    virtual void didUpdateEntityTransform(ModelPlayer&, const TransformationMatrix&) = 0;
+#endif
     virtual PlatformLayerIdentifier platformLayerID() = 0;
 };
 

Source/WebCore/SourcesCocoa.txt

@@ -449,6 +449,7 @@ platform/graphics/cocoa/SourceBufferParserWebM.cpp
 platform/graphics/cocoa/SystemFontDatabaseCocoa.mm
 platform/graphics/cocoa/SystemFontDatabaseCoreText.cpp
 platform/graphics/cocoa/TextTrackRepresentationCocoa.mm
+platform/graphics/cocoa/TransformationMatrixCocoa.cpp
 platform/graphics/cocoa/UnrealizedCoreTextFont.cpp
 platform/graphics/cocoa/VP9UtilitiesCocoa.mm @no-unify
 platform/graphics/cocoa/WebActionDisablingCALayerDelegate.mm

Source/WebCore/WebCore.xcodeproj/project.pbxproj

@@ -11831,6 +11831,7 @@
 		53E29E5D167A8A1900586D3D /* InternalSettingsGenerated.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = InternalSettingsGenerated.h; sourceTree = "<group>"; };
 		53ED3FDC167A88E7006762E6 /* JSInternalSettingsGenerated.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JSInternalSettingsGenerated.cpp; sourceTree = "<group>"; };
 		53ED3FDD167A88E7006762E6 /* JSInternalSettingsGenerated.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = JSInternalSettingsGenerated.h; sourceTree = "<group>"; };
+		5456A17B2BA8A8B000C8611F /* TransformationMatrixCocoa.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = TransformationMatrixCocoa.cpp; sourceTree = "<group>"; };
 		54BBB5F92B7C41A900BF1A8A /* ModelPlayerIdentifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ModelPlayerIdentifier.h; sourceTree = "<group>"; };
 		550A0BC7085F6039007353D6 /* QualifiedName.cpp */ = {isa = PBXFileReference; fileEncoding = 30; indentWidth = 4; lastKnownFileType = sourcecode.cpp.cpp; path = QualifiedName.cpp; sourceTree = "<group>"; tabWidth = 8; usesTabs = 0; };
 		550A0BC8085F6039007353D6 /* QualifiedName.h */ = {isa = PBXFileReference; fileEncoding = 30; indentWidth = 4; lastKnownFileType = sourcecode.c.h; path = QualifiedName.h; sourceTree = "<group>"; tabWidth = 8; usesTabs = 0; };
@@ -33081,6 +33082,7 @@
 				526724F21CB2FDF60075974D /* TextTrackRepresentationCocoa.h */,
 				526724F11CB2FDF60075974D /* TextTrackRepresentationCocoa.mm */,
 				BE39136F29B266D800FA5D4F /* TextTransformCocoa.cpp */,
+				5456A17B2BA8A8B000C8611F /* TransformationMatrixCocoa.cpp */,
 				1C436D112999889400C59A5E /* UnrealizedCoreTextFont.cpp */,
 				1C436D122999889400C59A5E /* UnrealizedCoreTextFont.h */,
 				512FF6D42B312DAD00899005 /* VideoMediaSampleRenderer.h */,

Source/WebCore/platform/graphics/cocoa/TransformationMatrixCocoa.cpp

@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2024 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "TransformationMatrix.h"
+
+#if PLATFORM(COCOA)
+
+#include <simd/simd.h>
+
+namespace WebCore {
+
+TransformationMatrix::TransformationMatrix(const simd_float4x4& t)
+: TransformationMatrix(t.columns[0][0], t.columns[0][1], t.columns[0][2], t.columns[0][3],
+    t.columns[1][0], t.columns[1][1], t.columns[1][2], t.columns[1][3],
+    t.columns[2][0], t.columns[2][1], t.columns[2][2], t.columns[2][3],
+    t.columns[3][0], t.columns[3][1], t.columns[3][2], t.columns[3][3])
+{
+}
+
+TransformationMatrix::operator simd_float4x4() const
+{
+    return simd_float4x4 {
+        simd_float4 { (float)m11(), (float)m12(), (float)m13(), (float)m14() },
+        simd_float4 { (float)m21(), (float)m22(), (float)m23(), (float)m24() },
+        simd_float4 { (float)m31(), (float)m32(), (float)m33(), (float)m34() },
+        simd_float4 { (float)m41(), (float)m42(), (float)m43(), (float)m44() }
+    };
+}
+
+}
+
+#endif // PLATFORM(COCOA)

Source/WebCore/platform/graphics/transforms/TransformationMatrix.h

@@ -41,6 +41,9 @@ typedef struct CATransform3D CATransform3D;
 #if USE(CG)
 typedef struct CGAffineTransform CGAffineTransform;
 #endif
+#if PLATFORM(COCOA)
+#include <simd/simd.h>
+#endif
 #if USE(SKIA)
 class SkM44;
 #endif
@@ -399,6 +402,10 @@ class TransformationMatrix {
     WEBCORE_EXPORT TransformationMatrix(const CGAffineTransform&);
     WEBCORE_EXPORT operator CGAffineTransform() const;
 #endif
+#if PLATFORM(COCOA)
+    WEBCORE_EXPORT TransformationMatrix(const simd_float4x4&);
+    WEBCORE_EXPORT operator simd_float4x4() const;
+#endif
 #if USE(SKIA)
     TransformationMatrix(const SkM44&);
     operator SkM44() const;

Source/WebKit/ModelProcess/cocoa/ModelProcessModelPlayerProxy.h

@@ -64,6 +64,8 @@ class ModelProcessModelPlayerProxy final
     static Ref<ModelProcessModelPlayerProxy> create(ModelProcessModelPlayerManagerProxy&, WebCore::ModelPlayerIdentifier, Ref<IPC::Connection>&&);
     ~ModelProcessModelPlayerProxy();
 
+    static bool transformSupported(const simd_float4x4& transform);
+
     WebCore::ModelPlayerIdentifier identifier() const { return m_id; }
     void invalidate();
     void didReceiveMessage(IPC::Connection&, IPC::Decoder&) final;
@@ -88,6 +90,7 @@ class ModelProcessModelPlayerProxy final
     PlatformLayer* layer() final;
     std::optional<WebCore::LayerHostingContextIdentifier> layerHostingContextIdentifier() final;
     void setBackgroundColor(WebCore::Color) final;
+    void setEntityTransform(WebCore::TransformationMatrix) final;
     void enterFullscreen() final;
     bool supportsMouseInteraction() final;
     bool supportsDragging() final;
@@ -112,6 +115,8 @@ class ModelProcessModelPlayerProxy final
 private:
     ModelProcessModelPlayerProxy(ModelProcessModelPlayerManagerProxy&, WebCore::ModelPlayerIdentifier, Ref<IPC::Connection>&&);
 
+    void computeTransform();
+
     WebCore::ModelPlayerIdentifier m_id;
     Ref<IPC::Connection> m_webProcessConnection;
     WeakPtr<ModelProcessModelPlayerManagerProxy> m_manager;
@@ -127,6 +132,8 @@ class ModelProcessModelPlayerProxy final
     float m_pitch { 0 };
     float m_yaw { 0 };
     REAnimationPlaybackToken m_animationPlaybackToken { kInvalidAnimationToken };
+
+    RESRT m_transformSRT; // SRT=Scaling/Rotation/Translation. This is stricter than a WebCore::TransformationMatrix.
 };
 
 } // namespace WebKit

Source/WebKit/ModelProcess/cocoa/ModelProcessModelPlayerProxy.messages.in

@@ -30,6 +30,7 @@ messages -> ModelProcessModelPlayerProxy {
     LoadModel(Ref<WebCore::Model> url, WebCore::LayoutSize layoutSize)
     SizeDidChange(WebCore::LayoutSize layoutSize)
     SetBackgroundColor(WebCore::Color color)
+    SetEntityTransform(WebCore::TransformationMatrix transform)
 }
 
 #endif