ci(): upload failed visuals artifact + perf + cleanup

.github/workflows/build.yml

@@ -7,27 +7,16 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    outputs:
+      stats: ${{ steps.b.outputs.result }}
     steps:
       - uses: actions/checkout@v3
-      - name: Uglified build
-        uses: actions/setup-node@v1
-        with:
-          node-version: 16.x
-      - run: npm ci
-      - run: npm run build
-  stats:
-    name: Build stats
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
       - uses: actions/setup-node@v3
         with:
           node-version: 16.x
+          cache: npm
       - run: npm ci
-      - name: build stats
-        run: npm run build -- -s
+      - run: npm run build -- -s
       - name: persist
         id: b
         uses: actions/github-script@v6
@@ -39,8 +28,21 @@ jobs:
               size: JSON.parse(fs.readFileSync('cli_output/build_size.json')),
               stats: JSON.parse(fs.readFileSync('cli_output/build_stats.json')),
             });
+  stats:
+    name: Build stats
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+          cache: npm
+      - run: npm ci
       - name: checkout src files
-        run: git checkout origin/master -- src
+        run: git checkout origin/master -- src index.js HEADER.js
       - name: upstream build stats
         run: npm run build -- -s
       - name: persist
@@ -65,46 +67,30 @@ jobs:
               context,
               core,
               a: JSON.parse('${{ steps.a.outputs.result }}'),
-              b: JSON.parse('${{ steps.b.outputs.result }}'),
+              b: JSON.parse('${{ needs.build.outputs.stats }}'),
             });
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Linting
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: 16.x
+          cache: npm
       - run: npm ci
       - run: npm run lint
   prettier:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Prettier check
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: 16.x
+          cache: npm
       - run: npm ci
       - run: npm run prettier:check
-  coverage:
-    if: ${{ always() }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Code Coverage Report Updater
-        uses: actions/setup-node@v1
-        with:
-          node-version: 16.x
-      - run: npm ci
-      - run: npm run build -- -f
-      - run: npm run test:coverage && npm run test:visual:coverage
-      - run: npm run coverage:report
-      - uses: ShaMan123/lcov-reporter-action@v1.1.1
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          delete-old-comments: true
-          update-comment: true
   changelog:
     if: ${{ always() }}
     runs-on: ubuntu-latest

.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/npmpublish-next.yml

@@ -11,9 +11,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
           node-version: 16
+          cache: npm
           registry-url: https://registry.npmjs.org/
       - run: npm ci
       - run: node publish-next.js

.github/workflows/npmpublish.yml

@@ -12,9 +12,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
           node-version: 16
+          cache: npm
           registry-url: https://registry.npmjs.org/
       - run: npm install
       - run: npm run build

.github/workflows/tests.yml

@@ -7,47 +7,76 @@ on:
 
 jobs:
   browser:
-    name: ${{ matrix.target }} ${{ matrix.suite }} tests
+    name: ${{ matrix.target }} tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [16.x]
         target: [chrome, firefox]
-        suite: [unit, visual]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
-      - name: Unit tests on Chrome
-        uses: actions/setup-node@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
+          cache: npm
       - run: npm ci
       - run: npm run build -- -f
-      - name: Run ${{ matrix.target }} ${{ matrix.suite }} headless test
+      - name: Run browser headless test
         uses: GabrielBB/xvfb-action@v1
         with:
-          run: npm run test -- -c ${{ matrix.target }} -s ${{ matrix.suite }}
+          run: npm run test -- -c ${{ matrix.target }} -s unit visual
+      - name: Archive test results
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: cli_output/test_results
   node:
     runs-on: ubuntu-latest
-    name: Node ${{ matrix.node-version }} ${{ matrix.suite }} tests
+    name: Node ${{ matrix.node-version }} tests
     strategy:
       fail-fast: false
       matrix:
         # For more information see:
         # https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
         # supported Node.js release schedule: https://nodejs.org/en/about/releases/
-        node-version: [14.x, 16.x, 18.x]
-        suite: [unit, visual]
+        node-version: [14, 16, 18]
+    env:
+      flag: ${{ (matrix.node-version == 16 && '--coverage') || '' }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install node-canvas
         run: sudo apt-get install libgif-dev libpng-dev libpango1.0-dev libjpeg8-dev librsvg2-dev libcairo2-dev
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: ${{ matrix.node-version }}.x
+          cache: npm
       - run: npm ci
-      - name: Build
-        run: npm run build -- -f
-      - name: Run ${{ matrix.suite }} tests
-        run: npm run test -- -c node -s ${{ matrix.suite }}
+      - run: npm run build -- -f
+      - name: Run tests
+        run: npm run test -- -c node -a ${{ env.flag }}
+      - name: Prepare Coverage Report
+        if: matrix.node-version == 16 && always()
+        run: npm run coverage:report
+      - name: Post Coverage Report
+        if: matrix.node-version == 16 && always()
+        uses: ShaMan123/lcov-reporter-action@v1.1.1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          delete-old-comments: true
+          update-comment: true
+      - name: Rename artifact (*node > *node${{ matrix.node-version }})
+        if: failure()
+        run: |
+          mv cli_output/test_results/visual/node.txt cli_output/test_results/visual/node${{ matrix.node-version }}.txt
+          mv cli_output/test_results/unit/node.txt cli_output/test_results/unit/node${{ matrix.node-version }}.txt
+          src=cli_output/test_results/visual/node
+          [ ! -d $src ] || mv $src ${src}${{ matrix.node-version }}
+      - name: Archive visual test results
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: cli_output/test_results

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [next]
 
+- ci(): fixed visual tests not to create a missing golden when running from CI and to fail instead + improve visual test suite [#8311](https://github.com/fabricjs/fabric.js/pull/8311)
 - chore(TS): migrate Rect [#8411](https://github.com/fabricjs/fabric.js/pull/8411)
 - chore(TS): migrate Ellipse [#8408](https://github.com/fabricjs/fabric.js/pull/8408)
 - chore(TS): migrate Triangle to TS [#8410](https://github.com/fabricjs/fabric.js/pull/8410)

package.json

@@ -62,8 +62,6 @@
     "test:unit-browser": "npm run test -- -s unit -p 8080 -l -c ",
     "test:visual-browser": "npm run test -- -s visual -p 8080 -l -c ",
     "test": "npm run cli -- test",
-    "test:coverage": "nyc --silent qunit test/node_test_setup.js test/lib test/unit",
-    "test:visual:coverage": "nyc --silent --no-clean qunit test/node_test_setup.js test/lib test/visual",
     "coverage:report": "nyc report --reporter=lcov --reporter=text",
     "lint": "eslint --config .eslintrc.js src/**/*.ts",
     "lint_tests": "eslint test/unit --config .eslintrc_tests && eslint test/visual --config .eslintrc_tests",

scripts/dirname.mjs

@@ -8,7 +8,6 @@ export const __dirname = path.dirname(__filename);
 export const wd = path.resolve(__dirname, '..');
 export const dumpsPath = path.resolve(wd, 'cli_output');
 export const CLI_CACHE = path.resolve(dumpsPath, 'cli_cache.json');
+export const testResultsPath = path.resolve(dumpsPath, 'test_results');
 
-if (!fs.existsSync(dumpsPath)) {
-  fs.mkdirSync(dumpsPath);
-}
+fs.ensureDir(dumpsPath);