test() When a test fails in node, dump the result in cli_output

.github/workflows/tests.yml

@@ -17,8 +17,7 @@ jobs:
       fail-fast: false
     steps:
     - uses: actions/checkout@v2
-    - name: Unit tests on Chrome
-      uses: actions/setup-node@v1
+    - uses: actions/setup-node@v1
       with:
         node-version: ${{ matrix.node-version }}
     - run: npm ci
@@ -36,7 +35,7 @@ jobs:
         # For more information see:
         # https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
         # supported Node.js release schedule: https://nodejs.org/en/about/releases/
-        node-version: [14.x, 16.x] 
+        node-version: [14.x, 16.x]
         suite: [unit, visual]
     steps:
     - uses: actions/checkout@v2
@@ -51,3 +50,9 @@ jobs:
       run: npm run build -- -f
     - name: Run ${{ matrix.suite }} tests
       run: npm run test -- -c node -s ${{ matrix.suite }}
+    - name: Archive artifacts
+      if: always() && matrix.suite == 'visual' && matrix.node-version == '16.x'
+      uses: actions/upload-artifact@v3
+      with:
+        name: node-failing-visual-tests
+        path: cli_output/failed_visual_tests/node/*

scripts/index.js

@@ -3,7 +3,7 @@
 /**
  * Dearest fabric maintainer 💗,
  * This file contains the cli logic, which governs most of the available commands fabric has to offer.
- * 
+ *
  * 📢 **IMPORTANT**
  * CI uses these commands.
  * In order for CI to correctly report the result of the command, the process must receive a correct exit code
@@ -166,7 +166,7 @@ function build(options = {}) {
             process.exit(1);
         }
     }
-    
+
 }
 
 function startWebsite() {
@@ -273,7 +273,7 @@ function exportToWebsite(options) {
 }
 
 /**
- * 
+ *
  * @returns {Promise<boolean | undefined>} true if some tests failed
  */
 async function runTestem({ suite, port, launch, dev, processOptions, context } = {}) {

test/GoldensServer.js

@@ -7,9 +7,9 @@ const busboy = require('busboy');
 const wd = path.resolve(__dirname, '..');
 
 /**
- * 
- * @param {http.IncomingMessage} req 
- * @returns 
+ *
+ * @param {http.IncomingMessage} req
+ * @returns
  */
 function parseRequest(req) {
     const bb = busboy({ headers: req.headers });
@@ -48,12 +48,17 @@ function startGoldensServer() {
                 const goldenPath = path.resolve(wd, 'test', 'visual', 'golden', filename);
                 res.end(JSON.stringify({ exists: fs.existsSync(goldenPath) }));
             }
-            else if (req.method.toUpperCase() === 'POST') {
+            else if (req.method.toUpperCase() === 'POST' && req.url === '/goldens') {
                 const { files: [{ rawData, filename }] } = await parseRequest(req);
                 const goldenPath = path.resolve(wd, 'test', 'visual', 'golden', filename);
                 console.log(chalk.gray('[info]'), `creating golden ${path.relative(wd, goldenPath)}`);
                 fs.writeFileSync(goldenPath, rawData, { encoding: 'binary' });
                 res.end();
+            } else if (req.method.toUpperCase() === 'POST' && req.url === '/failed') {
+                const { files: [{ rawData, filename }] } = await parseRequest(req);
+                const filepath = path.resolve(wd, 'cli_output', 'failed_visual_tests', 'chrome', filename);
+                fs.writeFileSync(filepath, rawData, { encoding: 'binary' });
+                res.status(200).end();
             }
         }).listen();
     const port = server.address().port;

test/lib/visualTestLoop.js

@@ -106,6 +106,37 @@
     img.src = filename;
   }
 
+  function dumpFailedTest(testName, original, golden, difference, runner) {
+    if (original && difference && golden) {
+      var largeCanvas = fabric.util.createCanvasElement();
+      largeCanvas.width = original.width + golden.width + difference.width;
+      largeCanvas.height = Math.max(original.height, golden.height, difference.height);
+      var ctx = largeCanvas.getContext('2d');
+      ctx.drawImage(original, 0, 0);
+      ctx.putImageData(difference, original.width, 0);
+      ctx.drawImage(golden, original.width + difference.width, 0);
+      var path = `../../cli_output/failed_visual_tests/${runner}`;
+      const fileName = localPath(path, `${testName.replaceAll(' ', '_')}.png`);
+      console.log('dumping failed test', testName, runner);
+      if (fabric.isLikelyNode) {
+        var dataUrl = largeCanvas.toDataURL().split(',')[1];
+          if (!fs.existsSync(`${__dirname}/${path}`)) {
+          fs.mkdirSync(`${__dirname}/${path}`, { recursive: true });
+        }
+        fs.writeFileSync(fileName.replace('file://', ''), dataUrl, { encoding: 'base64' });
+      } else {
+        largeCanvas.toBlob(blob => {
+          const formData = new FormData();
+          formData.append('file', blob, fileName);
+          const request = new XMLHttpRequest();
+          request.open('POST', '/failed', true);
+          request.send(formData);
+        }, 'image/png');
+      }
+    }
+
+  }
+
   exports.visualTestLoop = function(QUnit) {
     var _pixelMatch;
     var visualCallback;
@@ -175,8 +206,13 @@
               testName + ' [' + golden + '] has too many different pixels ' + differentPixels + '(' + okDiff + ') representing ' + percDiff + '% (>' + (percentage * 100) + '%)'
             );
             if (!isOK) {
-              var stringa = imageDataToChalk(output);
-              console.log(stringa);
+              // var stringa = imageDataToChalk(output);
+              // console.log(stringa);
+              try {
+                dumpFailedTest(testName, renderedCanvas, canvas, output, QUnit.runner);
+              } catch(e) {
+                console.log(e)
+              }
             }
             if ((!isOK && QUnit.debugVisual) || QUnit.recreateVisualRefs) {
               generateGolden(getGoldeName(golden), renderedCanvas);

test/node_test_setup.js

@@ -36,24 +36,6 @@ global.getFixture = require('./lib/visualTestLoop').getFixture;
 global.getAsset = require('./lib/visualTestLoop').getAsset;
 global.getAssetName = require('./lib/visualTestLoop').getAssetName;
 global.simulateEvent = require('./lib/event.simulate').simulateEvent;
-global.imageDataToChalk = function(imageData) {
-  // actually this does not work on travis-ci, so commenting it out
-  return '';
-  var block = String.fromCharCode(9608);
-  var data = imageData.data;
-  var width = imageData.width;
-  var height = imageData.height;
-  var outputString = '';
-  var cp = 0;
-  for (var i = 0; i < height; i++) {
-    outputString += '\n';
-    for (var j = 0; j < width; j++) {
-      cp = (i * width + j) * 4;
-      outputString += chalk.rgb(data[cp], data[cp + 1], data[cp + 2])(block);
-    }
-  }
-  return outputString;
-};
 QUnit.config.testTimeout = 15000;
 QUnit.config.noglobals = true;
 QUnit.config.hidepassed = true;
@@ -135,4 +117,4 @@ QUnit.assert.strictEqual = function (actual, expected, message) {
     expected: getLoggingRepresentation(expected),
     message
   });
-};
+};

test/testem.visual.js

@@ -38,5 +38,6 @@ module.exports = {
     ...config.qunit,
     recreate: Number(process.env.QUNIT_RECREATE_VISUAL_REFS) || false,
     debug: Number(process.env.QUNIT_DEBUG_VISUAL_TESTS) || false,
+    runner: process.env.RUNNER,
   },
 }

test/tests.mustache

@@ -40,8 +40,9 @@ var vendorName = winNav.vendor;
 var isOpera = typeof window.opr !== "undefined";
 var isIEedge = winNav.userAgent.indexOf("Edg") > -1;
 var isIOSChrome = winNav.userAgent.match("CriOS");
+QUnit.runner = "{{qunit.runner}}";
 if(
-  isIOSChrome || 
+  isIOSChrome ||
   (isChromium !== null &&
   //typeof isChromium !== "undefined" &&    //  fails on headless chrome
   vendorName === "Google Inc." &&
@@ -52,6 +53,7 @@ if(
     QUnit.recreateVisualRefs = {{qunit.recreate}};
     QUnit.debugVisual = {{qunit.debug}};
     {{#qunit.filter}}QUnit.config.filter = {{qunit.filter}};{{/qunit.filter}}
+    QUnit.runner = 'chrome';
 }
 </script>
 {{/visual}}