added option to generate junit report from test results (#473)

package.json

@@ -65,6 +65,7 @@
     "meow": "3.7.0",
     "mustache": "4.2.0",
     "x-img-diff-js": "0.3.5",
+    "xmlbuilder2": "^3.1.1",
     "yargs-parser": "^21.0.1"
   },
   "devDependencies": {

src/cli.js

@@ -26,6 +26,7 @@ const cli = meow(
     -I, --ignoreChange If true, error will not be thrown when image change detected.
     -E, --extendedErrors If true, also added/deleted images will throw an error. If omitted false.
     -R, --report Output html report to specified directory.
+    --junit Output junit report to specified file.
     -P, --urlPrefix Add prefix to all image src.
     -M, --matchingThreshold Matching threshold, ranges from 0 to 1. Smaller values make the comparison more sensitive. 0 by default.
     -T, --thresholdRate Rate threshold for detecting change. When the difference ratio of the image is larger than the set rate detects the change.
@@ -70,7 +71,7 @@ const json = cli.flags.json ? cli.flags.json.toString() : './reg.json'; // defau
 const urlPrefix = typeof cli.flags.urlPrefix === 'string' ? cli.flags.urlPrefix : './';
 
 const report = typeof cli.flags.report === 'string' ? cli.flags.report : !!cli.flags.report ? './report.html' : '';
-
+const junitReport = typeof cli.flags.junit === 'string' ? cli.flags.junit : !!cli.flags.junit ? './junit.xml' : '';
 const actualDir = process.argv[2];
 const expectedDir = process.argv[3];
 const diffDir = process.argv[4];
@@ -98,6 +99,8 @@ if (from) {
       ...params,
       json: json || './reg.json',
       report: report || './report.html',
+      junitReport: junitReport || '',
+      extendedErrors,
       urlPrefix: urlPrefix || '',
       enableClientAdditionalDetection,
       fromJSON: true,
@@ -116,6 +119,8 @@ const observer = compare({
   diffDir,
   update,
   report,
+  junitReport,
+  extendedErrors,
   json,
   urlPrefix,
   matchingThreshold: Number(cli.flags.matchingThreshold || 0),

src/index.js

@@ -152,6 +152,8 @@ module.exports = (params: RegParams) => {
     concurrency = 4,
     update,
     report,
+    junitReport,
+    extendedErrors,
     urlPrefix,
     threshold,
     matchingThreshold = 0,
@@ -194,6 +196,8 @@ module.exports = (params: RegParams) => {
         expectedDir,
         diffDir,
         report: report || '',
+        junitReport: junitReport || '',
+        extendedErrors,
         urlPrefix: urlPrefix || '',
         enableClientAdditionalDetection: !!enableClientAdditionalDetection,
       });

src/report.js

@@ -6,6 +6,7 @@ import * as detectDiff from 'x-img-diff-js';
 import fs from 'fs';
 import mkdirp from 'mkdirp';
 import path from 'path';
+import * as xmlBuilder from 'xmlbuilder2';
 
 export type ReportParams = {
   passedItems: string[],
@@ -20,6 +21,8 @@ export type ReportParams = {
   expectedDir: string,
   diffDir: string,
   report: string,
+  junitReport: string,
+  extendedErrors: boolean,
   urlPrefix: string,
   enableClientAdditionalDetection: boolean,
   fromJSON?: boolean,
@@ -90,6 +93,43 @@ const createHTMLReport = params => {
   return Mustache.render(template.toString(), view);
 };
 
+const createJunitReport = params => {
+  const failedTests = params.failedItems.length + params.newItems.length + params.deletedItems.length;
+  const numberOfTests = failedTests + params.passedItems.length;
+  const doc = xmlBuilder.create({ version: '1.0' });
+  const testsuitesElement = doc.ele('testsuites', { name: 'reg-cli tests', tests: numberOfTests, failures: failedTests });
+  const testsuiteElement = testsuitesElement.ele('testsuite', { name: 'reg-cli', tests: numberOfTests, failures: failedTests });
+  params.failedItems.forEach(item => {
+    addFailedJunitTestElement(testsuiteElement, item, 'failed');
+  });
+  params.newItems.forEach(item => {
+    if (params.extendedErrors) {
+      addFailedJunitTestElement(testsuiteElement, item, 'newItem');
+    } else {
+      addPassedJunitTestElement(testsuiteElement, item);
+    }
+  });
+  params.deletedItems.forEach(item => {
+    if (params.extendedErrors) {
+      addFailedJunitTestElement(testsuiteElement, item, 'deletedItem');
+    } else {
+      addPassedJunitTestElement(testsuiteElement, item);
+    }
+  });
+  params.passedItems.forEach(item => {
+    addPassedJunitTestElement(testsuiteElement, item);
+  });
+  return doc.end({ prettyPrint: true });
+};
+
+function addPassedJunitTestElement(testsuiteElement, item: string) {
+  testsuiteElement.ele('testcase', { name: item });
+}
+
+function addFailedJunitTestElement(testsuiteElement, item: string, reason: string) {
+  testsuiteElement.ele('testcase', { name: item }).ele('failure', { message: reason });
+}
+
 function createXimdiffWorker(params: ReportParams) {
   const file = path.join(__dirname, '../template/worker_pre.js');
   const moduleJs = fs.readFileSync(path.join(__dirname, '../report/ui/dist/worker.js'), 'utf8');
@@ -111,6 +151,11 @@ export default (params: ReportParams) => {
       fs.writeFileSync(path.resolve(path.dirname(params.report), 'detector.wasm'), wasmBuf);
     }
   }
+  if (!!params.junitReport) {
+    const junitXml = createJunitReport(params);
+    mkdirp.sync(path.dirname(params.junitReport));
+    fs.writeFileSync(params.junitReport, junitXml);
+  }
 
   const json = createJSONReport(params);
   if (!params.fromJSON) {

test/cli.test.js

@@ -216,6 +216,37 @@ test.serial('should generate report html to `${WORKSPACE}/dist/report.html` when
   }
 });
 
+test.serial('should generate junit report xml to `${WORKSPACE}/dist/report.xml` when `--junit` option enabled', async t => {
+  await new Promise(resolve => {
+    const p = spawn('./dist/cli.js', [
+      `${WORKSPACE}/resource/actual`,
+      `${WORKSPACE}/resource/expected`,
+      `${WORKSPACE}/diff`,
+      `--junit`,
+      `${WORKSPACE}/dist/report.xml`,
+    ]);
+    p.on('close', code => resolve(code));
+    p.stderr.on('data', data => console.error(data));
+  });
+
+  try {
+    let report = fs.readFileSync(`${WORKSPACE}/dist/report.xml`);
+    t.is(report.toString(),
+      `<?xml version="1.0"?>
+<testsuites name="reg-cli tests" tests="1" failures="1">
+  <testsuite name="reg-cli" tests="1" failures="1">
+    <testcase name="sample(cal).png">
+      <failure message="failed"/>
+    </testcase>
+  </testsuite>
+</testsuites>`);
+    t.pass();
+  } catch (e) {
+    console.error(e);
+    t.fail();
+  }
+});
+
 test.serial(
   'should generate worker js and wasm files under `${WORKSPACE}/dist` when `-R -X` option enabled',
   async t => {
@@ -581,6 +612,32 @@ test.serial('should generate report from json', async t => {
   }
 });
 
+
+test.serial('should generate Junit report from json', async t => {
+  await new Promise((resolve, reject) => {
+    const p = spawn('./dist/cli.js', [`-F`, `${WORKSPACE}/resource/reg.json`, '--junit','./from-json.xml']);
+    p.on('close', code => resolve(code));
+    p.stderr.on('data', data => reject(data));
+  });
+
+  try {
+    const report = fs.readFileSync(`./from-json.xml`);
+    t.true(!!report);
+    t.is(report.toString(),
+      `<?xml version="1.0"?>
+<testsuites name="reg-cli tests" tests="1" failures="1">
+  <testsuite name="reg-cli" tests="1" failures="1">
+    <testcase name="sample.png">
+      <failure message="failed"/>
+    </testcase>
+  </testsuite>
+</testsuites>`);
+  } catch (e) {
+    console.log(e);
+    t.fail();
+  }
+});
+
 test.serial('perf', async t => {
   const copy = (s, d, done) => {
     const r = fs.createReadStream(s);

yarn.lock

@@ -209,6 +209,35 @@
   dependencies:
     "@octokit/openapi-types" "^11.2.0"
 
+"@oozcitak/dom@1.15.10":
+  version "1.15.10"
+  resolved "https://registry.yarnpkg.com/@oozcitak/dom/-/dom-1.15.10.tgz#dca7289f2b292cff2a901ea4fbbcc0a1ab0b05c2"
+  integrity sha512-0JT29/LaxVgRcGKvHmSrUTEvZ8BXvZhGl2LASRUgHqDTC1M5g1pLmVv56IYNyt3bG2CUjDkc67wnyZC14pbQrQ==
+  dependencies:
+    "@oozcitak/infra" "1.0.8"
+    "@oozcitak/url" "1.0.4"
+    "@oozcitak/util" "8.3.8"
+
+"@oozcitak/infra@1.0.8":
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/@oozcitak/infra/-/infra-1.0.8.tgz#b0b089421f7d0f6878687608301fbaba837a7d17"
+  integrity sha512-JRAUc9VR6IGHOL7OGF+yrvs0LO8SlqGnPAMqyzOuFZPSZSXI7Xf2O9+awQPSMXgIWGtgUf/dA6Hs6X6ySEaWTg==
+  dependencies:
+    "@oozcitak/util" "8.3.8"
+
+"@oozcitak/url@1.0.4":
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/@oozcitak/url/-/url-1.0.4.tgz#ca8b1c876319cf5a648dfa1123600a6aa5cda6ba"
+  integrity sha512-kDcD8y+y3FCSOvnBI6HJgl00viO/nGbQoCINmQ0h98OhnGITrWR3bOGfwYCthgcrV8AnTJz8MzslTQbC3SOAmw==
+  dependencies:
+    "@oozcitak/infra" "1.0.8"
+    "@oozcitak/util" "8.3.8"
+
+"@oozcitak/util@8.3.8":
+  version "8.3.8"
+  resolved "https://registry.yarnpkg.com/@oozcitak/util/-/util-8.3.8.tgz#10f65fe1891fd8cde4957360835e78fd1936bfdd"
+  integrity sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==
+
 "@sindresorhus/is@^2.0.0":
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-2.1.1.tgz#ceff6a28a5b4867c2dd4a1ba513de278ccbe8bb1"
@@ -3297,7 +3326,7 @@ js-tokens@^4.0.0:
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
   integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==
 
-js-yaml@^3.10.0:
+js-yaml@3.14.1, js-yaml@^3.10.0:
   version "3.14.1"
   resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537"
   integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==
@@ -3680,14 +3709,7 @@ mimic-response@^2.0.0, mimic-response@^2.1.0:
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43"
   integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==
 
-minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
-  integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
-  dependencies:
-    brace-expansion "^1.1.7"
-
-minimatch@^3.1.1:
+minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.1.1:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
   integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
@@ -3696,6 +3718,8 @@ minimatch@^3.1.1:
 
 minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5:
   version "1.2.8"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
+  integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
 
 minipass@^3.0.0:
   version "3.1.6"
@@ -5586,6 +5610,16 @@ xdg-basedir@^3.0.0:
   resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4"
   integrity sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=
 
+xmlbuilder2@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/xmlbuilder2/-/xmlbuilder2-3.1.1.tgz#b977ef8a6fb27a1ea7ffa7d850d2c007ff343bc0"
+  integrity sha512-WCSfbfZnQDdLQLiMdGUQpMxxckeQ4oZNMNhLVkcekTu7xhD4tuUDyAPoY8CwXvBYE6LwBHd6QW2WZXlOWr1vCw==
+  dependencies:
+    "@oozcitak/dom" "1.15.10"
+    "@oozcitak/infra" "1.0.8"
+    "@oozcitak/util" "8.3.8"
+    js-yaml "3.14.1"
+
 xtend@^4.0.0:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"