Add generate report feature. Refs NimaSoroush#10

Supports HTML and JSON report types.

.gitignore

@@ -33,4 +33,6 @@ dist/
 docs/
 .vscode/
 differencify_report/
-screenshots/
+screenshots/
+# OSX hidden files
+.DS_Store

API.md

@@ -5,6 +5,7 @@
 |`update`|[TestOptions](https://github.com/NimaSoroush/differencify#testoptions)|Creates reference screenshots|
 |`test`|[TestOptions](https://github.com/NimaSoroush/differencify#testoptions)|Validate your changes by testing against reference screenshots|
 |`cleanup`|no argument|Closes all leftover browser instances|
+|`generateReport`|Config object, for example: `{ html: 'index.html' }`|Generates a report file. Supports `html` and `json` report types|
 
 ### Steps Methods
 

examples/example-test.js

@@ -7,4 +7,8 @@ const differencify = new Differencify(globalConfig);
 differencify.test(testConfig).then((result) => {
   console.log(result); // true or false
   differencify.cleanup();
+  differencify.generateReport({
+    html: 'index.html',
+    json: 'report.json',
+  });
 });

examples/jest-example.js

@@ -7,6 +7,10 @@ const differencify = new Differencify(globalConfig.default);
 describe('My website', () => {
   afterAll(() => {
     differencify.cleanup();
+    differencify.generateReport({
+      html: 'index.html',
+      json: 'report.json',
+    });
   });
   it('validate visual regression test', async () => {
     const result = await differencify.test(testConfig.default);

package.json

@@ -33,6 +33,7 @@
     "babel-preset-env": "^1.5.2",
     "babel-preset-es2015": "^6.24.1",
     "babel-preset-jest": "^20.0.3",
+    "cheerio": "^1.0.0-rc.2",
     "eslint": "^3.19.0",
     "eslint-config-airbnb": "^13.0.0",
     "eslint-plugin-import": "^2.2.0",

src/Reporter.js

@@ -0,0 +1,57 @@
+import fs from 'fs';
+import path from 'path';
+import logger from './logger';
+import getHtmlReport from './reportTypes/htmlReport';
+import getJsonReport from './reportTypes/jsonReport';
+
+const saveReport = (filepath, contents) => {
+  fs.writeFileSync(filepath, contents);
+};
+
+const getReport = (key, results) => {
+  switch (key) {
+    case 'json':
+      return getJsonReport(results);
+    case 'html':
+    default:
+      return getHtmlReport(results);
+  }
+};
+
+class Reporter {
+
+  constructor() {
+    this.results = [];
+  }
+
+  addResult(outcome, fileName, message, diff) {
+    this.results.push({
+      outcome,
+      fileName: path.basename(fileName),
+      message,
+      diff: diff ? path.basename(diff) : null,
+    });
+  }
+
+  getResults() {
+    return this.results;
+  }
+
+  generate(types, testReportPath) {
+    Object.keys(types).forEach((type) => {
+      const filepath = path.join(testReportPath, types[type]);
+      try {
+        const template = getReport(type, this.getResults());
+        saveReport(filepath, template);
+        logger.log(`Generated ${type} report at ${filepath}`);
+      } catch (err) {
+        logger.error(`Unable to generate ${type} report at ${filepath}: ${err}`);
+      }
+    });
+    return true;
+  }
+}
+
+export { getHtmlReport, getJsonReport };
+
+export default Reporter;

src/Reporter.test.js

@@ -0,0 +1,95 @@
+import fs from 'fs';
+import Reporter, { getHtmlReport, getJsonReport } from './Reporter';
+import logger from './logger';
+
+jest.mock('fs', () => ({
+  writeFileSync: jest.fn(),
+}));
+
+jest.mock('./logger', () => ({
+  log: jest.fn(),
+}));
+
+const results = [
+  {
+    outcome: true,
+    fileName: 'image1.png',
+    message: 'no mismatch found',
+    diff: null,
+  },
+  {
+    outcome: true,
+    fileName: 'image2.png',
+    message: 'no mismatch found',
+    diff: null,
+  },
+  {
+    outcome: false,
+    fileName: 'image2.png',
+    message: 'mismatch found!',
+    diff: 'image2_diff.png',
+  },
+];
+
+describe('Generate report index', () => {
+  let reporter;
+
+  beforeEach(() => {
+    reporter = new Reporter();
+    results.forEach(result =>
+      reporter.addResult(
+        result.outcome,
+        result.fileName,
+        result.message,
+        result.diff,
+      ),
+    );
+  });
+
+  afterEach(() => {
+    fs.writeFileSync.mockClear();
+    logger.log.mockClear();
+  });
+
+  it('generates a HTML report', () => {
+    reporter.generate(
+      {
+        html: 'index.html',
+      },
+      './example/path',
+    );
+    expect(fs.writeFileSync).toHaveBeenCalledWith(
+      'example/path/index.html',
+      getHtmlReport(results),
+    );
+  });
+
+  it('generates a JSON report', () => {
+    reporter.generate(
+      {
+        json: 'report.json',
+      },
+      './example/path',
+    );
+    expect(fs.writeFileSync).toHaveBeenCalledWith(
+      'example/path/report.json',
+      getJsonReport(results),
+    );
+  });
+
+  it('logs the report paths', () => {
+    reporter.generate(
+      {
+        html: 'index.html',
+        json: 'report.json',
+      },
+      './example/path',
+    );
+    expect(logger.log).toHaveBeenCalledWith(
+      'Generated json report at example/path/report.json',
+    );
+    expect(logger.log).toHaveBeenCalledWith(
+      'Generated html report at example/path/index.html',
+    );
+  });
+});

src/chromyRunner.js

@@ -14,7 +14,7 @@ const saveImage = (testName, image, testType, screenshotsPath, testReportPath) =
   return fs.writeFileSync(filePath, image);
 };
 
-const run = async (chromy, options, test) => {
+const run = async (chromy, options, test, reporter) => {
   const prefixedLogger = logger.prefix(test.name);
   // eslint-disable-next-line no-restricted-syntax
   for (const action of test.steps) {
@@ -64,7 +64,7 @@ const run = async (chromy, options, test) => {
         break;
       case actions.test:
         try {
-          const result = await compareImage(options, test.name);
+          const result = await compareImage(options, test.name, reporter);
           prefixedLogger.log(result);
         } catch (error) {
           prefixedLogger.error(error);

src/compareImage.js

@@ -1,7 +1,7 @@
 import Jimp from 'jimp';
 import logger from './logger';
 
-const compareImage = async (options, testName) => {
+const compareImage = async (options, testName, reporter) => {
   const prefixedLogger = logger.prefix(testName);
   const referenceFile = `${options.screenshots}/${testName}.png`;
   const testFile = `${options.testReportPath}/${testName}.png`;
@@ -25,25 +25,32 @@ const compareImage = async (options, testName) => {
   const distance = Jimp.distance(referenceImage, testImage);
   const diff = Jimp.diff(referenceImage, testImage, options.mismatchThreshold);
   if (distance < options.mismatchThreshold && diff.percent < options.mismatchThreshold) {
-    return 'no mismatch found ✅';
+    const result = 'no mismatch found ✅';
+    reporter.addResult(true, testFile, result);
+    return result;
   }
 
+  const result = `mismatch found❗
+    Result:
+      distance: ${distance}
+      diff: ${diff.percent}
+      misMatchThreshold: ${options.mismatchThreshold}
+  `;
+
   if (options.saveDifferencifiedImage) {
     try {
       const diffPath = `${options.testReportPath}/${testName}_differencified.png`;
       diff.image.write(diffPath);
       prefixedLogger.log(`saved the diff image to disk at ${diffPath}`);
+      reporter.addResult(false, testFile, result, diffPath);
     } catch (err) {
       throw new Error(`failed to save the diff image ${err}`);
     }
+  } else {
+    reporter.addResult(false, testFile, result);
   }
 
-  throw new Error(`mismatch found❗
-    Result:
-      distance: ${distance}
-      diff: ${diff.percent}
-      misMatchThreshold: ${options.mismatchThreshold}
-  `);
+  throw new Error(result);
 };
 
 export default compareImage;

src/compareImage.test.js

@@ -1,5 +1,6 @@
 import Jimp from 'jimp';
 import compareImage from './compareImage';
+import Reporter from './Reporter';
 
 const mockConfig = {
   screenshots: './screenshots',
@@ -21,15 +22,22 @@ jest.mock('./logger', () => ({
   }),
 }));
 
+jest.mock('./Reporter', () => () => ({
+  addResult: jest.fn(),
+}));
+
 describe('Compare Image', () => {
+  let mockReporter;
+
   beforeEach(() => {
+    mockReporter = new Reporter();
     Jimp.distance.mockReturnValue(0);
     Jimp.diff.mockReturnValue({ percent: 0 });
   });
 
   it('calls Jimp with correct image names', async () => {
     expect.assertions(2);
-    await compareImage(mockConfig, 'test');
+    await compareImage(mockConfig, 'test', mockReporter);
     expect(Jimp.read).toHaveBeenCalledWith('./differencify_report/test.png');
     expect(Jimp.read).toHaveBeenCalledWith('./screenshots/test.png');
   });
@@ -42,20 +50,20 @@ describe('Compare Image', () => {
       .mockReturnValueOnce(Promise.reject('error2'));
 
     try {
-      await compareImage(mockConfig, 'test');
+      await compareImage(mockConfig, 'test', mockReporter);
     } catch (err) {
       expect(err.message).toEqual('failed to read reference image error1');
     }
 
     try {
-      expect(await compareImage(mockConfig, 'test')).toThrow();
+      expect(await compareImage(mockConfig, 'test', mockReporter)).toThrow();
     } catch (err) {
       expect(err.message).toEqual('failed to read test image error2');
     }
   });
 
   it('returns correct value if difference below threshold', async () => {
-    const result = await compareImage(mockConfig, 'test');
+    const result = await compareImage(mockConfig, 'test', mockReporter);
     expect(result).toEqual('no mismatch found ✅');
   });
 
@@ -64,7 +72,7 @@ describe('Compare Image', () => {
     Jimp.diff.mockReturnValue({ percent: 0.02 });
 
     try {
-      await compareImage(mockConfig, 'test');
+      await compareImage(mockConfig, 'test', mockReporter);
     } catch (err) {
       expect(err.message).toEqual(`mismatch found❗
     Result:
@@ -80,7 +88,7 @@ describe('Compare Image', () => {
     Jimp.distance.mockReturnValue(0.02);
 
     try {
-      await compareImage(mockConfig, 'test');
+      await compareImage(mockConfig, 'test', mockReporter);
     } catch (err) {
       expect(err.message).toEqual(`mismatch found❗
     Result:
@@ -97,7 +105,7 @@ describe('Compare Image', () => {
     Jimp.diff.mockReturnValue({ percent: 0.02 });
 
     try {
-      await compareImage(mockConfig, 'test');
+      await compareImage(mockConfig, 'test', mockReporter);
     } catch (err) {
       expect(err.message).toEqual(`mismatch found❗
     Result:
@@ -120,7 +128,7 @@ describe('Compare Image', () => {
     });
     try {
       // eslint-disable-next-line prefer-object-spread/prefer-object-spread
-      await compareImage(Object.assign({}, mockConfig, { saveDifferencifiedImage: true }), 'test');
+      await compareImage(Object.assign({}, mockConfig, { saveDifferencifiedImage: true }), 'test', mockReporter);
     } catch (err) {
       expect(mockWrite).toHaveBeenCalledWith('./differencify_report/test_differencified.png');
     }

src/index.js

@@ -7,6 +7,7 @@ import run from './chromyRunner';
 import logger from './logger';
 import { configTypes } from './defaultConfig';
 import actions from './actions';
+import Reporter from './Reporter';
 
 const CHROME_WIDTH = 800;
 const CHROME_HEIGHT = 600;
@@ -28,9 +29,10 @@ const getFreePort = async () => {
 };
 
 export default class Differencify {
-  constructor(conf) {
+  constructor(conf, reporter = new Reporter()) {
     this.configuration = sanitiseGlobalConfiguration(conf);
     this.chromeInstances = {};
+    this.reporter = reporter;
     if (this.configuration.debug === true) {
       logger.enable();
     }
@@ -77,7 +79,7 @@ export default class Differencify {
     }
     this._updateChromeInstances(chromy);
     testConfig.type = type;
-    const result = await run(chromy, this.configuration, testConfig);
+    const result = await run(chromy, this.configuration, testConfig, this.reporter);
     await this._closeChrome(chromy, testConfig.name);
     return result;
   }
@@ -91,6 +93,10 @@ export default class Differencify {
     return await this._run(config, configTypes.test);
   }
 
+  async generateReport(config) {
+    return await this.reporter.generate(config, this.configuration.testReportPath);
+  }
+
   async cleanup() {
     await Promise.all(
       Object.values(this.chromeInstances).map(chromeInstance => chromeInstance.close()),