test() When a test fails in node, dump the result in cli_output

.github/workflows/visual-node.yml

@@ -27,3 +27,11 @@ jobs:
     - run: npm ci
     - run: npm run build -- -f -x gestures,accessors
     - run: npm run test -- -s visual -p 8080 -c node
+      env:
+        RUNNER: node
+    - name: Archive artifacts
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: node-failing-visual-tests
+        path: cli_output/failed_visual_tests/node/*

.github/workflows/visual-test-chrome.yml

@@ -23,3 +23,9 @@ jobs:
       uses: GabrielBB/xvfb-action@v1
       with:
         run: npm run test -- -s visual -p 8080 -c chrome
+    - name: Archive artifacts
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: chrome-failing-visual-tests
+        path: cli_output/failed_visual_tests/chrome/*

.github/workflows/visual-test-firefox.yml

@@ -23,3 +23,9 @@ jobs:
       uses: GabrielBB/xvfb-action@v1
       with:
         run: npm run test -- -s visual -p 8080 -c firefox
+    - name: Archive artifacts
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: firefox-failing-visual-tests
+        path: cli_output/failed_visual_tests/firefox/*

scripts/index.js

@@ -277,7 +277,8 @@ async function test(suite, tests, options = {}) {
             QUNIT_DEBUG_VISUAL_TESTS: Number(options.debug),
             QUNIT_RECREATE_VISUAL_REFS: Number(options.recreate),
             QUNIT_FILTER: options.filter,
-            REPORT_FILE: options.out
+            REPORT_FILE: options.out,
+            RUNNER: options.context[0],
         },
         shell: true,
         stdio: 'inherit',

test/GoldensServer.js

@@ -7,9 +7,9 @@ const busboy = require('busboy');
 const wd = path.resolve(__dirname, '..');
 
 /**
- * 
- * @param {http.IncomingMessage} req 
- * @returns 
+ *
+ * @param {http.IncomingMessage} req
+ * @returns
  */
 function parseRequest(req) {
     const bb = busboy({ headers: req.headers });
@@ -48,12 +48,18 @@ function startGoldensServer() {
                 const goldenPath = path.resolve(wd, 'test', 'visual', 'golden', filename);
                 res.end(JSON.stringify({ exists: fs.existsSync(goldenPath) }));
             }
-            else if (req.method.toUpperCase() === 'POST') {
+            else if (req.method.toUpperCase() === 'POST' && req.url === '/goldens') {
                 const { files: [{ rawData, filename }] } = await parseRequest(req);
                 const goldenPath = path.resolve(wd, 'test', 'visual', 'golden', filename);
                 console.log(chalk.gray('[info]'), `creating golden ${path.relative(wd, goldenPath)}`);
                 fs.writeFileSync(goldenPath, rawData, { encoding: 'binary' });
                 res.end();
+            } else if (req.method.toUpperCase() === 'POST' && req.url === '/failed') {
+                const { files: [{ rawData, filename }] } = await parseRequest(req);
+                console.log('here', filename);
+                const filepath = path.resolve(wd, 'cli_output', 'failed_visual_tests', 'chrome', filename);
+                fs.writeFileSync(filepath, rawData, { encoding: 'binary' });
+                res.status(200).end();
             }
         }).listen();
     const port = server.address().port;

test/lib/visualTestLoop.js

@@ -106,6 +106,38 @@
     img.src = filename;
   }
 
+  function dumpFailedTest(testName, original, golden, difference, runner) {
+    if (original && difference && golden) {
+      var largeCanvas = fabric.util.createCanvasElement();
+      largeCanvas.width = original.width + golden.width + difference.width;
+      largeCanvas.height = Math.max(original.height, golden.height, difference.height);
+      var ctx = largeCanvas.getContext('2d');
+      ctx.drawImage(original, 0, 0);
+      ctx.putImageData(difference, original.width, 0);
+      ctx.drawImage(golden, original.width + difference.width, 0);
+      var path = `../../cli_output/failed_visual_tests/${runner}`;
+      const fileName = localPath(path, `${testName.replaceAll(' ', '_')}.png`);
+      console.log('dumping failed test', testName, runner);
+      if (fabric.isLikelyNode) {
+        var dataUrl = largeCanvas.toDataURL().split(',')[1];
+          if (!fs.existsSync(`${__dirname}/${path}`)) {
+          fs.mkdirSync(`${__dirname}/${path}`, { recursive: true });
+        }
+        fs.writeFileSync(fileName.replace('file://', ''), dataUrl, { encoding: 'base64' });
+      } else {
+        console.log('running here', fileName)
+        largeCanvas.toBlob(blob => {
+          const formData = new FormData();
+          formData.append('file', blob, fileName);
+          const request = new XMLHttpRequest();
+          request.open('POST', '/failed', true);
+          request.send(formData);
+        }, 'image/png');
+      }
+    }
+
+  }
+
   exports.visualTestLoop = function(QUnit) {
     var _pixelMatch;
     var visualCallback;
@@ -175,8 +207,13 @@
               testName + ' [' + golden + '] has too many different pixels ' + differentPixels + '(' + okDiff + ') representing ' + percDiff + '% (>' + (percentage * 100) + '%)'
             );
             if (!isOK) {
-              var stringa = imageDataToChalk(output);
-              console.log(stringa);
+              // var stringa = imageDataToChalk(output);
+              // console.log(stringa);
+              try {
+                dumpFailedTest(testName, renderedCanvas, canvas, output, QUnit.runner);
+              } catch(e) {
+                console.log(e)
+              }
             }
             if ((!isOK && QUnit.debugVisual) || QUnit.recreateVisualRefs) {
               generateGolden(getGoldeName(golden), renderedCanvas);

test/node_test_setup.js

@@ -29,7 +29,6 @@ global.getAssetName = require('./lib/visualTestLoop').getAssetName;
 global.simulateEvent = require('./lib/event.simulate').simulateEvent;
 global.imageDataToChalk = function(imageData) {
   // actually this does not work on travis-ci, so commenting it out
-  return '';
   var block = String.fromCharCode(9608);
   var data = imageData.data;
   var width = imageData.width;
@@ -126,4 +125,4 @@ QUnit.assert.strictEqual = function (actual, expected, message) {
     expected: getLoggingRepresentation(expected),
     message
   });
-};
+};

test/testem.visual.js

@@ -38,5 +38,6 @@ module.exports = {
     ...config.qunit,
     recreate: Number(process.env.QUNIT_RECREATE_VISUAL_REFS) || false,
     debug: Number(process.env.QUNIT_DEBUG_VISUAL_TESTS) || false,
+    runner: process.env.RUNNER,
   },
 }

test/tests.mustache

@@ -36,8 +36,9 @@ var vendorName = winNav.vendor;
 var isOpera = typeof window.opr !== "undefined";
 var isIEedge = winNav.userAgent.indexOf("Edg") > -1;
 var isIOSChrome = winNav.userAgent.match("CriOS");
+QUnit.runner = {{qunit.runner}};
 if(
-  isIOSChrome || 
+  isIOSChrome ||
   (isChromium !== null &&
   //typeof isChromium !== "undefined" &&    //  fails on headless chrome
   vendorName === "Google Inc." &&
@@ -48,6 +49,7 @@ if(
     QUnit.recreateVisualRefs = {{qunit.recreate}};
     QUnit.debugVisual = {{qunit.debug}};
     {{#qunit.filter}}QUnit.config.filter = {{qunit.filter}};{{/qunit.filter}}
+    QUnit.runner = 'chrome';
 }
 </script>
 {{/visual}}