feat: add ability to redirect output of reporter to the file

BREAKING CHANGE: reporters specified as function and used through programmatic API must have a static create method for initialization

README.md

@@ -1370,10 +1370,20 @@ hermione --config ./config.js --reporter flat --browser firefox --grep name
 **Note.** All CLI options override config values.
 
 ### Reporters
+
 You can choose `flat` or `plain` reporter by option `-r, --reporter`. Default is `flat`.
+Information about test results is displayed to the command line by default. But there is an ability to redirect the output to a file, for example:
+```
+hermione --reporter '{"type": "flat", "path": "./some-path/result.txt"}'
+```
 
-* `flat` – all information about failed and retried tests would be grouped by browsers at the end of the report.
+In that example specified file path and all directories will be created automatically. Moreover you can use few reporters:
+```
+hermione --reporter '{"type": "flat", "path": "./some-path/result.txt"}' --reporter flat
+```
 
+Information about each report type:
+* `flat` – all information about failed and retried tests would be grouped by browsers at the end of the report;
 * `plain` – information about fails and retries would be placed after each test.
 
 ### Require modules

lib/cli/index.js

@@ -30,9 +30,9 @@ exports.run = () => {
 
     program
         .on('--help', () => logger.log(info.configOverriding))
-        .option('-r, --reporter <reporter>', 'test reporters', collect)
         .option('-b, --browser <browser>', 'run tests only in specified browser', collect)
         .option('-s, --set <set>', 'run tests only in the specified set', collect)
+        .option('-r, --reporter <reporter>', 'test reporters', collect)
         .option('--require <module>', 'require module', collect)
         .option('--grep <grep>', 'run only tests matching the pattern')
         .option('--update-refs', 'update screenshot references or gather them if they do not exist ("assertView" command)')

lib/constants/test-statuses.js

@@ -0,0 +1,6 @@
+module.exports = {
+    SUCCESS: 'success',
+    FAIL: 'fail',
+    RETRY: 'retry',
+    SKIPPED: 'skipped'
+};

lib/hermione.js

@@ -12,6 +12,7 @@ const signalHandler = require('./signal-handler');
 const TestReader = require('./test-reader');
 const TestCollection = require('./test-collection');
 const validateUnknownBrowsers = require('./validators').validateUnknownBrowsers;
+const {initReporters} = require('./reporters');
 const logger = require('./utils/logger');
 
 module.exports = class Hermione extends BaseHermione {
@@ -25,7 +26,7 @@ module.exports = class Hermione extends BaseHermione {
         this.emit(RunnerEvents.CLI, parser);
     }
 
-    async run(testPaths, {browsers, sets, grep, updateRefs, requireModules, inspectMode, reporters} = {}) {
+    async run(testPaths, {browsers, sets, grep, updateRefs, requireModules, inspectMode, reporters = []} = {}) {
         validateUnknownBrowsers(browsers, _.keys(this._config.browsers));
 
         RuntimeConfig.getInstance().extend({updateRefs, requireModules, inspectMode});
@@ -36,7 +37,7 @@ module.exports = class Hermione extends BaseHermione {
             .on(RunnerEvents.TEST_FAIL, () => this._fail())
             .on(RunnerEvents.ERROR, (err) => this.halt(err));
 
-        _.forEach(reporters, (reporter) => applyReporter(this, reporter));
+        await initReporters(reporters, this);
 
         eventsUtils.passthroughEvent(this._runner, this, _.values(RunnerEvents.getSync()));
         eventsUtils.passthroughEventAsync(this._runner, this, _.values(RunnerEvents.getAsync()));
@@ -113,23 +114,3 @@ module.exports = class Hermione extends BaseHermione {
         }, timeout).unref();
     }
 };
-
-function applyReporter(runner, reporter) {
-    if (typeof reporter === 'string') {
-        try {
-            reporter = require('./reporters/' + reporter);
-        } catch (e) {
-            if (e.code === 'MODULE_NOT_FOUND') {
-                throw new Error('No such reporter: ' + reporter);
-            }
-            throw e;
-        }
-    }
-    if (typeof reporter !== 'function') {
-        throw new TypeError('Reporter must be a string or a function');
-    }
-
-    var Reporter = reporter;
-
-    new Reporter().attachRunner(runner);
-}

lib/reporters/base.js

@@ -1,12 +1,22 @@
 'use strict';
 
 const chalk = require('chalk');
-const logger = require('../utils/logger');
 const RunnerEvents = require('../constants/runner-events');
 const icons = require('./utils/icons');
 const helpers = require('./utils/helpers');
+const {initInformer} = require('./informers');
 
 module.exports = class BaseReporter {
+    static async create(opts = {}) {
+        const informer = await initInformer(opts);
+
+        return new this(informer, opts);
+    }
+
+    constructor(informer) {
+        this.informer = informer;
+    }
+
     attachRunner(runner) {
         runner.on(RunnerEvents.TEST_PASS, (test) => this._onTestPass(test));
         runner.on(RunnerEvents.TEST_FAIL, (test) => this._onTestFail(test));
@@ -29,7 +39,7 @@ module.exports = class BaseReporter {
 
     _onRetry(test) {
         this._logTestInfo(test, icons.RETRY);
-        logger.log('Will be retried. Retries left: %s', chalk.yellow(test.retriesLeft));
+        this.informer.log(`Will be retried. Retries left: ${chalk.yellow(test.retriesLeft)}`);
     }
 
     _onTestPending(test) {
@@ -45,22 +55,23 @@ module.exports = class BaseReporter {
             `Retries: ${chalk.yellow(stats.retries)}`
         ];
 
-        logger.log(message.join(' '));
+        const method = this.__proto__.hasOwnProperty('_onRunnerEnd') ? 'log' : 'end';
+        this.informer[method](message.join(' '));
     }
 
     _onWarning(info) {
-        logger.warn(info);
+        this.informer.warn(info);
     }
 
     _onError(error) {
-        logger.error(chalk.red(error));
+        this.informer.error(chalk.red(error));
     }
 
     _onInfo(info) {
-        logger.log(info);
+        this.informer.log(info);
     }
 
     _logTestInfo(test, icon) {
-        logger.log(`${icon}${helpers.formatTestInfo(test)}`);
+        this.informer.log(`${icon}${helpers.formatTestInfo(test)}`);
     }
 };

lib/reporters/flat.js

@@ -1,11 +1,13 @@
 'use strict';
 
+const _ = require('lodash');
 const BaseReporter = require('./base');
 const helpers = require('./utils/helpers');
+const icons = require('./utils/icons');
 
 module.exports = class FlatReporter extends BaseReporter {
-    constructor() {
-        super();
+    constructor(...args) {
+        super(...args);
 
         this._tests = [];
     }
@@ -25,6 +27,20 @@ module.exports = class FlatReporter extends BaseReporter {
     _onRunnerEnd(stats) {
         super._onRunnerEnd(stats);
 
-        helpers.logFailedTestsInfo(this._tests);
+        const failedTests = helpers.formatFailedTests(this._tests);
+
+        failedTests.forEach((test, index) => {
+            this.informer.log(`\n${index + 1}) ${test.fullTitle}`);
+            this.informer.log(`   in file ${test.file}\n`);
+
+            _.forEach(test.browsers, (testCase) => {
+                const icon = testCase.isFailed ? icons.FAIL : icons.RETRY;
+
+                this.informer.log(`   ${testCase.browserId}`);
+                this.informer.log(`     ${icon} ${testCase.error}`);
+            });
+        });
+
+        this.informer.end();
     }
 };

lib/reporters/index.js

@@ -0,0 +1,100 @@
+exports.initReporters = async (rawReporters, runner) => {
+    await Promise.all([].concat(rawReporters).map((rawReporter) => applyReporter(rawReporter, runner)));
+};
+
+const reporterHandlers = [
+    {
+        isMatched: (rawReporter) => typeof rawReporter === 'string' && isJSON(rawReporter),
+        initReporter: (rawReporter) => initReporter(getReporterDefinition(rawReporter, JSON.parse))
+    },
+    {
+        isMatched: (rawReporter) => typeof rawReporter === 'string',
+        initReporter: (rawReporter) => initReporter({...getReporterDefinition(rawReporter), type: rawReporter})
+    },
+    {
+        isMatched: (rawReporter) => typeof rawReporter === 'object',
+        initReporter: (rawReporter) => initReporter(getReporterDefinition(rawReporter, (v) => v))
+    },
+    {
+        isMatched: (rawReporter) => typeof rawReporter === 'function',
+        initReporter: (rawReporter) => {
+            validateReporter(rawReporter);
+            return rawReporter.create(getReporterDefinition(rawReporter));
+        }
+    },
+    {
+        isMatched: () => true,
+        initReporter: (rawReporter) => {
+            throw new TypeError(`Specified reporter must be a string, object or function, but got: "${typeof rawReporter}"`);
+        }
+    }
+];
+
+async function applyReporter(rawReporter, runner) {
+    for (const handler of reporterHandlers) {
+        if (!handler.isMatched(rawReporter)) {
+            continue;
+        }
+
+        const reporter = await handler.initReporter(rawReporter);
+
+        if (typeof reporter.attachRunner !== 'function') {
+            throw new TypeError(
+                'Initialized reporter must have an "attachRunner" function for subscribe on test result events'
+            );
+        }
+
+        return reporter.attachRunner(runner);
+    }
+}
+
+function initReporter(reporter) {
+    let Reporter;
+
+    try {
+        Reporter = require(`./${reporter.type}`);
+    } catch (e) {
+        if (e.code === 'MODULE_NOT_FOUND') {
+            throw new Error(`No such reporter: "${reporter.type}"`);
+        }
+        throw e;
+    }
+
+    validateReporter(Reporter);
+
+    return Reporter.create(reporter);
+}
+
+function getReporterDefinition(rawReporter, parser) {
+    if (!parser) {
+        return {type: null, path: null};
+    }
+
+    const {type, path} = parser(rawReporter);
+
+    if (!type) {
+        const strRawReporter = typeof rawReporter !== 'string' ? JSON.stringify(rawReporter) : rawReporter;
+        throw new Error(`Failed to find required "type" field in reporter definition: "${strRawReporter}"`);
+    }
+
+    return {type, path};
+}
+
+function validateReporter(Reporter) {
+    if (typeof Reporter !== 'function') {
+        throw new TypeError(`Imported reporter must be a function, but got: "${typeof Reporter}"`);
+    }
+
+    if (typeof Reporter.create !== 'function') {
+        throw new TypeError('Imported reporter must have a "create" function for initialization');
+    }
+}
+
+function isJSON(str) {
+    try {
+        JSON.parse(str);
+    } catch (e) {
+        return false;
+    }
+    return true;
+}

lib/reporters/informers/base.js

@@ -0,0 +1,21 @@
+module.exports = class BaseInformer {
+    static create(...args) {
+        return new this(...args);
+    }
+
+    log() {
+        throw new Error('Method must be implemented in child classes');
+    }
+
+    warn() {
+        throw new Error('Method must be implemented in child classes');
+    }
+
+    error() {
+        throw new Error('Method must be implemented in child classes');
+    }
+
+    end() {
+        throw new Error('Method must be implemented in child classes');
+    }
+};

lib/reporters/informers/console.js

@@ -0,0 +1,22 @@
+const BaseInformer = require('./base');
+const logger = require('../../utils/logger');
+
+module.exports = class ConsoleInformer extends BaseInformer {
+    log(message) {
+        logger.log(message);
+    }
+
+    warn(message) {
+        logger.warn(message);
+    }
+
+    error(message) {
+        logger.error(message);
+    }
+
+    end(message) {
+        if (message) {
+            logger.log(message);
+        }
+    }
+};

lib/reporters/informers/file.js

@@ -0,0 +1,41 @@
+const fs = require('fs');
+const chalk = require('chalk');
+const BaseInformer = require('./base');
+const logger = require('../../utils/logger');
+
+module.exports = class FileInformer extends BaseInformer {
+    constructor(opts) {
+        super(opts);
+
+        this._fileStream = fs.createWriteStream(opts.path);
+        this._reporterType = opts.type;
+
+        logger.log(`Information with test results for report: "${opts.type}" will be saved to a file: "${opts.path}"`);
+    }
+
+    log(message) {
+        this._fileStream.write(`${this._prepareMsg(message)}\n`);
+    }
+
+    warn(message) {
+        this.log(message);
+    }
+
+    error(message) {
+        this.log(message);
+    }
+
+    end(message) {
+        if (message) {
+            this._fileStream.end(`${this._prepareMsg(message)}\n`);
+        } else {
+            this._fileStream.end();
+        }
+    }
+
+    _prepareMsg(msg) {
+        return typeof msg === 'object'
+            ? JSON.stringify(msg)
+            : chalk.stripColor(msg);
+    }
+};

lib/reporters/informers/index.js

@@ -0,0 +1,12 @@
+const path = require('path');
+const fs = require('fs-extra');
+
+exports.initInformer = async (opts) => {
+    if (opts.path) {
+        await fs.ensureDir(path.dirname(opts.path));
+    }
+
+    const informerType = opts.path ? 'file' : 'console';
+
+    return require(`./${informerType}`).create(opts);
+};