Bug 1975174 - Fix rotateAxisAngleSelf not to always call Ensure3DMatrix r=smaug

I wonder it's worth keeping the current implementation deviation that we don't have is2D check but instead have mMatrix2D/3D. Seems that only complicates the implementation without a clear win.

Differential Revision: https://phabricator.services.mozilla.com/D256115

Author: saschanaz

dom/base/DOMMatrix.cpp

@@ -949,19 +949,42 @@ DOMMatrix* DOMMatrix::RotateSelf(double aRotX, const Optional<double>& aRotY,
   return this;
 }
 
+// https://drafts.fxtf.org/geometry/#dom-dommatrix-rotateaxisangleself
 DOMMatrix* DOMMatrix::RotateAxisAngleSelf(double aX, double aY, double aZ,
                                           double aAngle) {
+  // (Unspecified but rather obvious optimization)
   if (fmod(aAngle, 360) == 0) {
     return this;
   }
 
   aAngle *= radPerDegree;
 
-  Ensure3DMatrix();
+  // Step 1: Post-multiply a rotation transformation on the current matrix
+  // around the specified vector x, y, z by the specified rotation angle in
+  // degrees.
+  // (But actual multiplication happens below with step 2)
   gfx::Matrix4x4Double m;
   m.SetRotateAxisAngle(aX, aY, aZ, aAngle);
 
-  *mMatrix3D = m * *mMatrix3D;
+  // Step 2: If x or y are not 0 or -0, set is 2D of the current matrix to
+  // false.
+  // (But we don't have "is 2D" flag and need to multiply either 2D or 3D
+  // matrix)
+  if (mMatrix3D || aX != 0 || aY != 0) {
+    Ensure3DMatrix();
+    *mMatrix3D = m * *mMatrix3D;
+    return this;
+  }
+
+  gfx::Matrix4x4Double result = m * gfx::Matrix4x4Double::From2D(*mMatrix2D);
+  mMatrix2D->_11 = result._11;
+  mMatrix2D->_12 = result._12;
+  mMatrix2D->_21 = result._21;
+  mMatrix2D->_22 = result._22;
+
+  // Different field names, see ::From2D implementation
+  mMatrix2D->_31 = result._41;
+  mMatrix2D->_32 = result._42;
 
   return this;
 }

testing/web-platform/tests/css/geometry/DOMMatrix-003.html

@@ -227,6 +227,12 @@
           checkDOMMatrix(result, expected);
         },"test rotateAxisAngle() ");
 
+        test(function() {
+          var result = new DOMMatrix([1, 1, 1, 1, 1, 1]).rotateAxisAngle(0, 0, 3, 90);
+          var expected = new DOMMatrix([1, 1, -1, -1, 1, 1]);
+          checkDOMMatrix(result, expected);
+        },"test rotateAxisAngle() on 2D matrix with 2D rotation");
+
         test(function() {
           var angleDeg = 75;
           var result = initialDOMMatrix().skewX(angleDeg);