reporters: built-in report map object and results method refactor

README.md

@@ -21,11 +21,15 @@ CI runs accessibility tests against multiple URLs and reports on any issues. Thi
   - [Default configuration](#default-configuration)
   - [URL configuration](#url-configuration)
   - [Sitemaps](#sitemaps)
+  - [Reporters](#reporters)
+    - [Use Multiple reporters](#use-multiple-reporters)
+    - [Write a custom reporter](#write-a-custom-reporter)
 - [Tutorials and articles](#tutorials-and-articles)
 - [Contributing](#contributing)
 - [License](#license)
 
 
+
 ## Requirements
 
 This command line tool requires [Node.js] 8+. You can install through npm:
@@ -186,40 +190,43 @@ You can use multiple reporters by setting them on the `defaults.reporters` array
 
 Pa11y CI reporters use the same interface as [pa11y reporters] with some additions:
 
-- A `beforeAll(urls)` and `afterAll(urls)` optional methods called respectively at the beginning of a test run and after every URL has been checked. `urls` is the URLs array defined in your config.
-- The `results()` method is called with the following arguments:
-  - `results`: the results of a test run
-  - `config`: the current [URL configuration object](#url-configuration)
-  - `url`: the current URL string
+- Every reporter method receives an additional `report` argument. This object is an instance of a [JavaScript Map](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map) that can be used to initialize and collect data across each tested URL.
+- You can define a `beforeAll(urls, report)` and `afterAll(urls, report)` optional methods called respectively at the beginning and at the very end of the process with the following arguments:
   - `urls`: the URLs array defined in your config
+  - `report`: the report object
+- The `results()` method receives a third `option` argument with the following properties:
+    - `config`: the current [URL configuration object](#url-configuration)
+    - `url`: the current URL under test
+    - `urls`: the URLs array defined in your config
 
 **Note**: to prevent a reporter from logging to stdout, ensure its methods return a falsy value or a Promise resolving to a falsy value.
 
 Here is an example of a custom reporter logging to a file
 
 ```js
-let report;
 const fs = require('fs');
 
-// initialize an empty report
-function beforeAll() {
-  report = {
-    results: {},
-    violations: 0,
-  };
+// initialize an empty report data
+// "report" is a JavaScript Map instance 
+function beforeAll(_, report) {
+    report.set('data', {
+        results: {},
+      violations: 0,
+    });
 }
 
 // add test results to the report
-function results(results, config, url) {
-  report.results[url] = results;
-  report.violations += results.issues.length;
+function results(results, report, { url }) {
+    const data = report.get('data');
+    data.results[url] = results;
+    data.violations += results.issues.length;
 }
 
 // write to a file
-function afterAll() {
-  fs.writeFileSync('./report.json', JSON.stringify(report), 'utf8');
-  // or Node 10+ you can use:
-  // return fs.promises.writeFile('./report.json', JSON.stringify(report), 'utf8');`
+function afterAll(_, report) {
+    fs.writeFileSync('./report.json', JSON.stringify(report.get('data')), 'utf8');
+    // or Node 10+ you can use:
+    // return fs.promises.writeFile('./report.json', JSON.stringify(report.get('data')), 'utf8');`
 }
 
 module.exports = {

lib/helpers/reporter.js

@@ -0,0 +1,47 @@
+'use strict';
+
+/**
+ * Build a Pa11y reporter.
+ *
+ * Same as 'pa11y/lib/reporter' but reporter methods accept multiple arguments
+ * @private
+ * @param {Object} methods - The reporter methods.
+ * @returns {Promise} Returns a promise which resolves with the new reporter.
+ */
+module.exports = function buildReporter(methods) {
+	const reporter = {
+		report: new Map(),
+		supports: methods.supports,
+		beforeAll: buildReporterMethod(methods.beforeAll),
+		afterAll: buildReporterMethod(methods.afterAll),
+		begin: buildReporterMethod(methods.begin),
+		results: buildReporterMethod(methods.results),
+		log: {
+			debug: buildReporterMethod(methods.debug),
+			error: buildReporterMethod(methods.error, 'error'),
+			info: buildReporterMethod(methods.info)
+		}
+	};
+
+	return reporter;
+};
+
+/**
+ * Build a Pa11y reporter method, making it async and only outputting when
+ * actual output is returned.
+ * @private
+ * @param {Function} method - The reporter method to build.
+ * @param {String} [consoleMethod='log'] - The console method to use in reporting.
+ * @returns {Function} Returns a built async reporter method.
+ */
+function buildReporterMethod(method, consoleMethod = 'log') {
+	if (typeof method !== 'function') {
+		return () => Promise.resolve();
+	}
+	return async (...args) => {
+		const output = await method(...args);
+		if (output) {
+			console[consoleMethod](output);
+		}
+	};
+}

lib/helpers/resolver.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const buildReporter = require('./reporter');
+
+module.exports = function resolveReporters(reporters) {
+	return [].concat(reporters).map(reporter => {
+		if (typeof reporter !== 'string') {
+			return undefined;
+		}
+		try {
+			return require(reporter);
+		} catch (_) {
+			const localModule = path.isAbsolute(reporter) ?
+				reporter : path.resolve(process.cwd(), reporter);
+			if (fs.existsSync(localModule)) {
+				return require(localModule);
+			}
+			console.error(`Unable to locale reporter "${reporter}"`);
+			return undefined;
+		}
+	}).filter(Boolean).map(reporterModule => {
+		return buildReporter(reporterModule);
+	});
+};

lib/pa11y-ci.js

@@ -11,7 +11,7 @@ const pa11y = require('pa11y');
 const queue = require('async/queue');
 const wordwrap = require('wordwrap');
 const puppeteer = require('puppeteer');
-const {resolveReporters} = require('./helpers');
+const resolveReporters = require('./helpers/resolver');
 const {Console} = require('console');
 
 // Just an empty function to use as default
@@ -30,28 +30,28 @@ const defaultLog = {
 	info: noop
 };
 
-// The default configuration object. This is extended with
-// whatever configurations the user passes in from the
-// command line
-module.exports.defaults = {
-	concurrency: 2,
-	log: defaultLog,
-	wrapWidth: 80,
-	useIncognitoBrowserContext: false
-};
-
-function cycleReporters(reporters, method, ...args) {
+function cycleReporters(reporters, method, payload, options = {}) {
 	if (!reporters.length) {
 		return Promise.resolve();
 	}
 	return Promise.all(reporters.map(reporter => {
 		if (typeof reporter[method] === 'function') {
-			return reporter[method](...args);
+			return reporter[method](payload, reporter.report, options);
 		}
 		return Promise.resolve();
 	}));
 }
 
+// The default configuration object. This is extended with
+// whatever configurations the user passes in from the
+// command line
+module.exports.defaults = {
+	concurrency: 2,
+	log: defaultLog,
+	wrapWidth: 80,
+	useIncognitoBrowserContext: false
+};
+
 
 // This function does all the setup and actually runs Pa11y
 // against the passed in URLs. It accepts options in the form
@@ -148,7 +148,9 @@ function pa11yCi(urls, options) {
 			// results to the report object
 			try {
 				const results = await pa11y(url, config);
-				await cycleReporters(reporters, 'results', results, config, url, urls);
+				await cycleReporters(reporters, 'results', results, {config,
+					url,
+					urls});
 				processResults(results, config, url);
 			} catch (error) {
 				log.error(` ${chalk.cyan('>')} ${url} - ${chalk.red('Failed to run')}`);

test/unit/lib/pa11y-ci.test.js

@@ -445,28 +445,34 @@ describe('lib/pa11y-ci', () => {
 
 
 		it('calls the beforeAll method once', () => {
-			assert.calledWith(reporter.beforeAll, userUrls);
+			assert.calledWithMatch(reporter.beforeAll, userUrls, sinon.match.map, sinon.match.object);
 			assert.callCount(reporter.beforeAll, 1);
 		});
 		it('calls the afterAll method once', () => {
-			assert.calledWith(reporter.afterAll, userUrls);
+			assert.calledWithMatch(reporter.afterAll, userUrls, sinon.match.map, sinon.match.object);
 			assert.callCount(reporter.afterAll, 1);
 		});
 
 		it('calls the begin method for each URL', () => {
 			userUrls.forEach((url, i) => {
-				reporter.begin.getCall(i).calledWith(url);
+				reporter.begin.getCall(i).calledWith(url, sinon.match.map);
 			});
 			assert.callCount(reporter.begin, userUrls.length);
 		});
 
 		it('calls the results method for each URL', () => {
 			userUrls.forEach((url, i) => {
 				const spyCall = reporter.results.getCall(i);
-				assert.deepEqual(spyCall.args[0].issues, report.results[url]);
-				assert.isObject(spyCall.args[1]);
-				assert.equal(spyCall.args[2], url);
-				assert.equal(spyCall.args[3], userUrls);
+				assert.calledWithMatch(
+					spyCall,
+					sinon.match({issues: report.results[url]}),
+					sinon.match.map,
+					sinon.match({
+						config: sinon.match.object,
+						urls: userUrls,
+						url
+					})
+				);
 			});
 			assert.callCount(reporter.results, userUrls.length);
 		});

test/unit/lib/reporter.test.js

@@ -0,0 +1,90 @@
+/* eslint max-len: 'off' */
+'use strict';
+
+const assert = require('proclaim');
+// Const mockery = require('mockery');
+const sinon = require('sinon');
+
+describe('lib/helpers/reporter', () => {
+	describe('buildReporter', () => {
+		let methods;
+		const buildReporter = require('../../../lib/helpers/reporter');
+
+
+		before(() => {
+			sinon.stub(console, 'log');
+		});
+
+		after(() => {
+			console.log.restore();
+		});
+
+
+		beforeEach(() => {
+			methods = {
+				supports: '__MOCK__',
+				beforeAll: sinon.stub(),
+				afterAll: sinon.stub(),
+				begin: sinon.stub(),
+				results: sinon.stub(),
+				debug: sinon.stub(),
+				error: sinon.stub(),
+				info: sinon.stub()
+			};
+		});
+		it('adds a "report" property', () => {
+			const reporter = buildReporter(methods);
+			assert.isInstanceOf(reporter.report, Map);
+		});
+
+		it('wraps function methods in async functions', () => {
+			const reporter = buildReporter(methods);
+			assert.isFunction(reporter.beforeAll);
+			assert.isFunction(reporter.afterAll);
+			assert.isFunction(reporter.begin);
+			assert.isFunction(reporter.results);
+			assert.isFunction(reporter.log.debug);
+			assert.isFunction(reporter.log.error);
+			assert.isFunction(reporter.log.info);
+			assert.isNotFunction(reporter.support);
+
+			assert.isInstanceOf(reporter.beforeAll(), Promise);
+			assert.isInstanceOf(reporter.afterAll(), Promise);
+			assert.isInstanceOf(reporter.begin(), Promise);
+			assert.isInstanceOf(reporter.results(), Promise);
+			assert.isInstanceOf(reporter.log.debug(), Promise);
+			assert.isInstanceOf(reporter.log.error(), Promise);
+			assert.isInstanceOf(reporter.log.info(), Promise);
+		});
+
+		it('wrapped function passes its argument to inner function', () => {
+			const reporter = buildReporter(methods);
+			const args = ['payload', {}];
+			for (const method of ['beforeAll', 'afterAll', 'begin', 'results']) {
+				reporter[method](...args);
+				assert.calledWith(methods[method], ...args);
+			}
+
+			for (const method of ['debug', 'error', 'info']) {
+				reporter.log[method](...args);
+				assert.calledWith(methods[method], ...args);
+			}
+		});
+
+		it('methods returning truthy values will be logged to console', async () => {
+			const reporter = buildReporter(methods);
+			methods.begin.returns('__MOCK__');
+			methods.results.returns(false);
+
+			await reporter.begin();
+			assert.calledWith(console.log, '__MOCK__');
+
+			console.log.resetHistory();
+
+			await reporter.results();
+			assert.notCalled(console.log);
+
+		});
+	});
+
+});