Create first "real" runtime test suite (#5794)

## Summary Previous tests were created to test and debug testing environment. <table> <tr><td width="50%">Changes</td><td width="50%">Why</td></tr> <tr><td> Increase max line width for all files in app/src/examples/RuntimeTests </td><td> It's convenient to have one set of test parameters in one line </td></tr> <tr><td> Disable prettier for all snapshot files ``` diff + app/**/*.snapshot.ts ``` </td><td>It allows to have one snapshot per line. The file is better organised and you can immediately see how many individual snapshots you have</td> </tr> <tr><td>Improve snapshot mismatch formatting, create the following function to have equal length for all the printed numbers ```jsx export function appendWhiteSpaceToMatchLength( message: string | number, length: number ) ``` > [!WARNING] > Temporarily the function throws an error when message has less characters than length </td> <td> It allows to quickly see the difference between the recorded and actual snapshot. <img width="443" alt="image" src="https://github.com/software-mansion/react-native-reanimated/assets/56199675/fd0ca951-c330-4f54-8856-da1ec968dc94"> </td> </tr> <tr><td>Change the way we render component in test - don't display anything if no component is provided</td><td>We can render invisible components (white views on white backgrounds) to reduce blinking and flickering </td></tr> <tr><td> Add test.each syntax like in jest <a href="https://jestjs.io/docs/api#testeachtablename-fn-timeout">LINK</a> </td><td> Possibility to create simpler tests </td></tr> </table> ## Recording https://github.com/software-mansion/react-native-reanimated/assets/56199675/eb923b01-226a-42c5-b6a2-d202e90414d1 ## Full test logs ![image](https://github.com/software-mansion/react-native-reanimated/assets/56199675/5516e257-9688-4986-9bc6-2c1f0ffc255d)  ## Test plan  --------- Co-authored-by: Latropos <aleksandracynk@Aleksandras-MacBook-Pro-3.local> Co-authored-by: Latropos <aleksandracynk@aleksandras-macbook-pro-3.home>
software-mansion · Apr 11, 2024 · 52c86f1 · 52c86f1
1 parent 3163572
commit 52c86f1
Show file tree

Hide file tree

Showing 24 changed files with 860 additions and 570 deletions.
diff --git a/.prettierignore b/.prettierignore
@@ -1,2 +1,3 @@
 plugin/**/*.js
 !plugin/**/*.json
+app/**/*.snapshot.ts
diff --git a/app/.eslintrc.js b/app/.eslintrc.js
@@ -1,7 +1,8 @@
 module.exports = {
   root: true,
-  extends: '@react-native',
+  extends: ['@react-native', 'prettier'],
   plugins: ['eslint-plugin-no-inline-styles', 'reanimated'],
+  ignorePatterns: ['**/*.snapshot.ts'],
   rules: {
     '@typescript-eslint/no-use-before-define': 'off',
     '@typescript-eslint/no-shadow': 'off',

diff --git a/app/src/examples/RuntimeTests/.prettierrc.js b/app/src/examples/RuntimeTests/.prettierrc.js
@@ -0,0 +1,7 @@
+module.exports = {
+  arrowParens: 'avoid',
+  bracketSameLine: true,
+  singleQuote: true,
+  trailingComma: 'all',
+  printWidth: 120, // Increase line width to make test cases more compact
+};
diff --git a/app/src/examples/RuntimeTests/ReanimatedRuntimeTestsRunner/Asserts.ts b/app/src/examples/RuntimeTests/ReanimatedRuntimeTestsRunner/Asserts.ts
@@ -1,26 +1,20 @@
-import { RUNTIME_TEST_ERRORS } from './LogMessageUtils';
+import { RUNTIME_TEST_ERRORS } from './stringFormatUtils';
 import { TestCase, TestSuite } from './types';
 
-export function assertMockedAnimationTimestamp(
-  timestamp: number | undefined
-): asserts timestamp is number {
+export function assertMockedAnimationTimestamp(timestamp: number | undefined): asserts timestamp is number {
   'worklet';
   if (timestamp === undefined) {
     throw new Error(RUNTIME_TEST_ERRORS.NO_MOCKED_TIMESTAMP);
   }
 }
 
-export function assertTestSuite(
-  test: TestSuite | null
-): asserts test is TestSuite {
+export function assertTestSuite(test: TestSuite | null): asserts test is TestSuite {
   if (!test) {
     throw new Error(RUNTIME_TEST_ERRORS.UNDEFINED_TEST_SUITE);
   }
 }
 
-export function assertTestCase(
-  test: TestCase | null
-): asserts test is TestCase {
+export function assertTestCase(test: TestCase | null): asserts test is TestCase {
   if (!test) {
     throw new Error(RUNTIME_TEST_ERRORS.UNDEFINED_TEST_CASE);
   }

diff --git a/app/src/examples/RuntimeTests/ReanimatedRuntimeTestsRunner/Comparators.ts b/app/src/examples/RuntimeTests/ReanimatedRuntimeTestsRunner/Comparators.ts
@@ -10,8 +10,7 @@ const COMPARATORS: {
   },
 
   [ComparisonMode.NUMBER]: (expected, value) => {
-    const bothAreNumbers =
-      typeof value === 'number' && typeof expected === 'number';
+    const bothAreNumbers = typeof value === 'number' && typeof expected === 'number';
     const bothAreNaN = bothAreNumbers && isNaN(value) && isNaN(expected);
     return bothAreNaN || value === expected;
   },
@@ -21,21 +20,16 @@ const COMPARATORS: {
       return false;
     }
     const expectedUnified = expected.toLowerCase();
-    const colorRegex = new RegExp('^#?([a-f0-9]{6}|[a-f0-9]{3})$');
+    const colorRegex = new RegExp('^#?([a-f0-9]{6})$');
     if (!colorRegex.test(expectedUnified)) {
-      throw Error(
-        `Invalid color format "${expectedUnified}", please use hex color (like #123abc)`
-      );
+      throw Error(`Invalid color format "${expectedUnified}", please use hex color (like #123abc)`);
     }
     return value === expected;
   },
 
   [ComparisonMode.DISTANCE]: (expected, value) => {
     const valueAsNumber = Number(value);
-    return (
-      !isNaN(valueAsNumber) &&
-      Math.abs(valueAsNumber - Number(expected)) < DISTANCE_TOLERANCE
-    );
+    return !isNaN(valueAsNumber) && Math.abs(valueAsNumber - Number(expected)) < DISTANCE_TOLERANCE;
   },
 
   [ComparisonMode.ARRAY]: (expected, value) => {
@@ -46,12 +40,7 @@ const COMPARATORS: {
       return false;
     }
     for (let i = 0; i < expected.length; i++) {
-      if (
-        !COMPARATORS[ComparisonMode.AUTO](
-          expected[i] as TestValue,
-          value[i] as TestValue
-        )
-      ) {
+      if (!COMPARATORS[ComparisonMode.AUTO](expected[i] as TestValue, value[i] as TestValue)) {
         return false;
       }
     }
@@ -68,12 +57,7 @@ const COMPARATORS: {
       return false;
     }
     for (const key of expectedKeys) {
-      if (
-        !COMPARATORS[ComparisonMode.AUTO](
-          expected[key as keyof typeof expected],
-          value[key as keyof typeof value]
-        )
-      ) {
+      if (!COMPARATORS[ComparisonMode.AUTO](expected[key as keyof typeof expected], value[key as keyof typeof value])) {
         return false;
       }
     }

diff --git a/app/src/examples/RuntimeTests/ReanimatedRuntimeTestsRunner/LogMessageUtils.ts b/app/src/examples/RuntimeTests/ReanimatedRuntimeTestsRunner/LogMessageUtils.ts
diff --git a/app/src/examples/RuntimeTests/ReanimatedRuntimeTestsRunner/Matchers.ts b/app/src/examples/RuntimeTests/ReanimatedRuntimeTestsRunner/Matchers.ts
@@ -1,12 +1,6 @@
 import { getComparator } from './Comparators';
-import { color } from './LogMessageUtils';
-import {
-  ComparisonMode,
-  OperationUpdate,
-  TestCase,
-  TestValue,
-  TrackerCallCount,
-} from './types';
+import { appendWhiteSpaceToMatchLength, color } from './stringFormatUtils';
+import { ComparisonMode, OperationUpdate, TestCase, TestValue, TrackerCallCount } from './types';
 
 type MatcherFunction = (
   currentValue: TestValue,
@@ -21,25 +15,19 @@ export class Matchers {
   private _negation = false;
   constructor(private _currentValue: TestValue, private _testCase: TestCase) {}
 
-  private static _assertValueIsCallTracker(
-    value: TrackerCallCount | TestValue
-  ): asserts value is TrackerCallCount {
-    if (
-      typeof value !== 'object' ||
-      !('name' in value && 'onJS' in value && 'onUI' in value)
-    ) {
+  private static _assertValueIsCallTracker(value: TrackerCallCount | TestValue): asserts value is TrackerCallCount {
+    if (typeof value !== 'object' || !('name' in value && 'onJS' in value && 'onUI' in value)) {
       throw Error('Invalid value');
     }
   }
 
   private _toBeMatcher: MatcherFunction = (
     currentValue: TestValue,
     expectedValue: TestValue,
-    comparisonModeUnknown: unknown
+    comparisonModeUnknown: unknown,
   ) => {
     const comparisonMode: ComparisonMode =
-      typeof comparisonModeUnknown === 'string' &&
-      comparisonModeUnknown in ComparisonMode
+      typeof comparisonModeUnknown === 'string' && comparisonModeUnknown in ComparisonMode
         ? (comparisonModeUnknown as ComparisonMode)
         : ComparisonMode.AUTO;
 
@@ -57,10 +45,7 @@ export class Matchers {
     };
   };
 
-  private _toBeCalledMatcher: MatcherFunction = (
-    currentValue: TestValue,
-    times = 1
-  ) => {
+  private _toBeCalledMatcher: MatcherFunction = (currentValue: TestValue, times = 1) => {
     Matchers._assertValueIsCallTracker(currentValue);
     const callsCount = currentValue.onUI + currentValue.onJS;
     const name = color(currentValue.name, 'green');
@@ -74,10 +59,7 @@ export class Matchers {
     };
   };
 
-  private _toBeCalledUIMatcher: MatcherFunction = (
-    currentValue: TestValue,
-    times = 1
-  ) => {
+  private _toBeCalledUIMatcher: MatcherFunction = (currentValue: TestValue, times = 1) => {
     Matchers._assertValueIsCallTracker(currentValue);
     const callsCount = currentValue.onUI;
     const name = color(currentValue.name, 'green');
@@ -93,10 +75,7 @@ export class Matchers {
     };
   };
 
-  private _toBeCalledJSMatcher: MatcherFunction = (
-    currentValue: TestValue,
-    times = 1
-  ) => {
+  private _toBeCalledJSMatcher: MatcherFunction = (currentValue: TestValue, times = 1) => {
     Matchers._assertValueIsCallTracker(currentValue);
     const callsCount = currentValue.onJS;
     const name = color(currentValue.name, 'green');
@@ -114,11 +93,7 @@ export class Matchers {
 
   private decorateMatcher(matcher: MatcherFunction) {
     return (expectedValue: TestValue, ...args: Array<unknown>) => {
-      const { pass, message } = matcher(
-        this._currentValue,
-        expectedValue,
-        ...args
-      );
+      const { pass, message } = matcher(this._currentValue, expectedValue, ...args);
       if ((!pass && !this._negation) || (pass && this._negation)) {
         this._testCase.errors.push(message);
       }
@@ -136,56 +111,46 @@ export class Matchers {
   }
 
   public toMatchSnapshots(expectedSnapshots: Array<Record<string, unknown>>) {
-    const capturedSnapshots = this._currentValue as Array<
-      Record<string, unknown>
-    >;
+    const capturedSnapshots = this._currentValue as Array<Record<string, unknown>>;
     if (capturedSnapshots.length !== expectedSnapshots.length) {
-      const errorMessage = this.formatMismatchLengthErrorMessage(
-        expectedSnapshots.length,
-        capturedSnapshots.length
-      );
+      const errorMessage = this.formatMismatchLengthErrorMessage(expectedSnapshots.length, capturedSnapshots.length);
       this._testCase.errors.push(errorMessage);
     }
     let errorString = '';
-    expectedSnapshots.forEach(
-      (expectedSnapshots: Record<string, unknown>, index: number) => {
-        const capturedSnapshot = capturedSnapshots[index];
-        const isEquals = getComparator(ComparisonMode.AUTO);
-        if (!isEquals(expectedSnapshots, capturedSnapshot)) {
-          const expected = color(
-            `${JSON.stringify(expectedSnapshots)}`,
-            'green'
-          );
-          const received = color(`${JSON.stringify(capturedSnapshot)}`, 'red');
-          errorString += `\tAt index ${index}:\n\t\texpected: ${expected}\n\t\treceived: ${received}\n`;
-        }
+    expectedSnapshots.forEach((expectedSnapshots: Record<string, unknown>, index: number) => {
+      const capturedSnapshot = capturedSnapshots[index];
+      const isEquals = getComparator(ComparisonMode.AUTO);
+      if (!isEquals(expectedSnapshots, capturedSnapshot)) {
+        const expected = color(`${JSON.stringify(expectedSnapshots)}`, 'green');
+        const received = color(`${JSON.stringify(capturedSnapshot)}`, 'red');
+        errorString += `\tAt index ${index}:\n\t\texpected: ${expected}\n\t\treceived: ${received}\n`;
       }
-    );
+    });
     if (errorString !== '') {
       this._testCase.errors.push('Snapshot mismatch: \n' + errorString);
     }
   }
 
-  public toMatchNativeSnapshots(nativeSnapshots: Array<OperationUpdate>) {
-    /*
+  /**
       The TestRunner can collect two types of snapshots:
-      - JS snapshots: animation updates sent via `_updateProps`
-      - Native snapshots: snapshots obtained from the native side via `getViewProp`
-      Updates applied through `_updateProps` are not synchronously applied to the native side. Instead, they are batched and applied at the end of each frame. Therefore, it is not allowed to take a native snapshot immediately after the `_updateProps` call. To address this issue, we need to wait for the next frame before capturing the native snapshot. That's why native snapshots are one frame behind JS snapshots. To account for this delay, one additional native snapshot is taken during the execution of the `getNativeSnapshots` function.
-    */
+      - **JS snapshots:** animation updates sent via `_updateProps`
+      - **Native snapshots:** snapshots obtained from the native side via `getViewProp`
+      The purpose of this function is to compare this two suits of snapshots.
+
+      @param expectNegativeMismatch - Some props expose unexpected behavior, when negative.
+      For example negative `width` may render a full-width component.
+      It means that JS snapshot is negative and the native one is positive, which is a valid behavior.
+      Set this property to true to expect all comparisons with negative value of JS snapshot **NOT** to match.
+   */
+  public toMatchNativeSnapshots(nativeSnapshots: Array<OperationUpdate>, expectNegativeMismatch = false) {
     let errorString = '';
     const jsUpdates = this._currentValue as Array<OperationUpdate>;
     for (let i = 0; i < jsUpdates.length; i++) {
-      errorString += this.compareJsAndNativeSnapshot(
-        jsUpdates,
-        nativeSnapshots,
-        i
-      );
+      errorString += this.compareJsAndNativeSnapshot(jsUpdates, nativeSnapshots, i, expectNegativeMismatch);
     }
+
     if (jsUpdates.length !== nativeSnapshots.length - 1) {
-      errorString += `Expected ${jsUpdates.length} snapshots, but received ${
-        nativeSnapshots.length - 1
-      } snapshots\n`;
+      errorString += `Expected ${jsUpdates.length} snapshots, but received ${nativeSnapshots.length - 1} snapshots\n`;
     }
     if (errorString !== '') {
       this._testCase.errors.push('Native snapshot mismatch: \n' + errorString);
@@ -195,8 +160,22 @@ export class Matchers {
   private compareJsAndNativeSnapshot(
     jsSnapshots: Array<OperationUpdate>,
     nativeSnapshots: Array<OperationUpdate>,
-    i: number
+    i: number,
+    expectNegativeMismatch: Boolean,
   ) {
+    /**
+      The TestRunner can collect two types of snapshots:
+      - JS snapshots: animation updates sent via `_updateProps`
+      - Native snapshots: snapshots obtained from the native side via `getViewProp`
+
+      Updates applied through `_updateProps` are not synchronously applied to the native side.
+      Instead, they are batched and applied at the end of each frame. Therefore, it is not allowed
+      to take a native snapshot immediately after the `_updateProps` call. To address this issue,
+      we need to wait for the next frame before capturing the native snapshot.
+      That's why native snapshots are one frame behind JS snapshots. To account for this delay,
+      one additional native snapshot is taken during the execution of the `getNativeSnapshots` function.
+   */
+
     let errorString = '';
     const jsSnapshot = jsSnapshots[i];
     const nativeSnapshot = nativeSnapshots[i + 1];
@@ -206,33 +185,26 @@ export class Matchers {
       const jsValue = jsSnapshot[typedKey];
       const nativeValue = nativeSnapshot[typedKey];
       const isEqual = getComparator(ComparisonMode.AUTO);
-      if (!isEqual(jsValue, nativeValue)) {
-        errorString += this.formatSnapshotErrorMessage(
-          jsValue,
-          nativeValue,
-          key,
-          i
-        );
+
+      const expectMismatch = jsValue < 0 && expectNegativeMismatch;
+      const valuesAreMatching = isEqual(jsValue, nativeValue);
+      if ((!valuesAreMatching && !expectMismatch) || (valuesAreMatching && expectMismatch)) {
+        errorString += this.formatSnapshotErrorMessage(jsValue, nativeValue, key, i);
       }
     }
     return errorString;
   }
 
-  private formatSnapshotErrorMessage(
-    jsValue: TestValue,
-    nativeValue: TestValue,
-    propName: string,
-    index: number
-  ) {
+  private formatSnapshotErrorMessage(jsValue: TestValue, nativeValue: TestValue, propName: string, index: number) {
     const expected = color(jsValue, 'green');
     const received = color(nativeValue, 'red');
-    return `\tAt index ${index}, value of prop ${propName}:\n\t\texpected: ${expected}\n\t\treceived: ${received}\n`;
+    return `\tIndex ${index} ${propName}\t expected: ${appendWhiteSpaceToMatchLength(
+      expected,
+      30,
+    )} received: ${appendWhiteSpaceToMatchLength(received, 30)}\n`;
   }
 
-  private formatMismatchLengthErrorMessage(
-    expectedLength: number,
-    receivedLength: number
-  ) {
+  private formatMismatchLengthErrorMessage(expectedLength: number, receivedLength: number) {
     const expected = color(expectedLength, 'green');
     const received = color(receivedLength, 'red');
     return `Expected ${expected} snapshots, but received ${received} snapshots\n`;