Improved using of reporters (#193)

* improved xml/html/cli reports

* moved header output to cli reporter to be compatible with xunit

* moved version and root output to cli reporter

* added reporters documentation

* parse options

* fixed tests

* tests are fixed

Author: DavertMik

docs/reports.md

@@ -0,0 +1,140 @@
+# Reporters
+
+## Cli (default)
+
+By default CodeceptJS provides cli reporter with console output.
+Test names and failures will be printed to screen.
+
+```
+GitHub --
+ ✓ search in 2577ms
+ ✓ signin in 2170ms
+ ✖ register in 1306ms
+
+-- FAILURES:
+
+  1) GitHub: register:
+      Field q not found by name|text|CSS|XPath
+
+  Scenario Steps:
+
+  - Within .js-signup-form: I.fillField("q", "aaa") at examples/github_test.js:29:7
+  - Within .js-signup-form: I.fillField("user[password]", "user@user.com") at examples/github_test.js:28:7
+  - Within .js-signup-form: I.fillField("user[email]", "user@user.com") at examples/github_test.js:27:7
+  - Within .js-signup-form: I.fillField("user[login]", "User") at examples/github_test.js:26:7
+
+
+
+  Run with --verbose flag to see NodeJS stacktrace
+```
+
+For dynamic step-by-step output add `--steps` option to `run` command:
+
+```
+GitHub --
+ search
+ • I am on page "https://github.com"
+ • I am on page "https://github.com/search"
+ • I fill field "Search GitHub", "CodeceptJS"
+ • I press key "Enter"
+ • I see "Codeception/CodeceptJS", "a"
+ ✓ OK in 2681ms
+
+ signin
+ • I am on page "https://github.com"
+ • I click "Sign in"
+ • I see "Sign in to GitHub"
+ • I fill field "Username or email address", "something@totest.com"
+ • I fill field "Password", "123456"
+ • I click "Sign in"
+ • I see "Incorrect username or password.", ".flash-error"
+ ✓ OK in 2252ms
+
+ register
+ • I am on page "https://github.com"
+   Within .js-signup-form:
+   • I fill field "user[login]", "User"
+   • I fill field "user[email]", "user@user.com"
+   • I fill field "user[password]", "user@user.com"
+   • I fill field "q", "aaa"
+ ✖ FAILED in 1260ms
+```
+To get additional information about test execution use `--debug` option. This will show execution steps
+as well as notices from test runner. To get even more information with more technical details like error stacktraces,
+and global promises, or events use `--verbose` mode.
+
+```
+GitHub --
+ register
+   [1] Starting recording promises
+   Emitted | test.before
+ > WebDriverIO._before
+   [1] Queued | hook WebDriverIO._before()
+   [1] Queued | amOnPage: https://github.com
+   Emitted | step.before (I am on page "https://github.com")
+ • I am on page "https://github.com"
+   Emitted | step.after (I am on page "https://github.com")
+   Emitted | test.start ([object Object])
+...
+```
+
+Please use verbose output when reporting issues to GitHub.
+
+## XML
+
+Use default xunit reporter of Mocha to print xml reports. Provide `--reporter xunit` to get the report to screen.
+It is recommended to use more powerful [`mocha-junit-reporter`](https://www.npmjs.com/package/mocha-junit-reporter) package
+ot get better support for Jenkins CI.
+
+Install it via NPM (locally or globally, depending on CodeceptJS installation type):
+
+```
+npm i mocha-junit-reporter
+```
+
+Additional configuration should be added to `codecept.json` to print xml report to `output` directory:
+
+```json
+  "mocha": {
+    "reporterOptions": {
+        "mochaFile": "output/result.xml"
+    }
+  },
+```
+
+Execute CodeceptJS with JUnit reporter:
+
+```
+codeceptjs run --reporter mocha-junit-reporter
+```
+
+Result will be located at `output/result.xml` file.
+
+
+## Html
+
+Best HTML reports could be prodused with [mochawesome](https://www.npmjs.com/package/mochawesome) reporter.
+Install it via NPM:
+
+```
+npm i mochawesome
+```
+
+Configure it to use `output` directory to print HTML reports:
+
+```json
+  "mocha": {
+    "reporterOptions": {
+        "reportDir": "output"
+    }
+  },
+```
+
+Execute CodeceptJS with HTML reporter:
+
+```
+codeceptjs run --reporter mochawesome.
+```
+
+Result will be located at `output/index.html` file.
+

examples/codecept.json

@@ -6,10 +6,14 @@
     "WebDriverIO": {
       "url": "http://localhost",
       "browser": "firefox",
-      "restart": false
+      "restart": true
+    }
+  },
+  "mocha": {
+    "reporterOptions": {
+        "mochaFile": "./output/result.xml"
     }
   },
-  "mocha": {},
   "name": "tests",
   "bootstrap": "./bootstrap.js",
   "include": {

lib/assert/empty.js

@@ -20,12 +20,12 @@ class EmptinessAssertion extends Assertion {
     if (Array.isArray(this.params.value)) {
       this.params.value = '[' + this.params.value.join(', ') + ']';
     }
-    
+
     let err = new AssertionFailedError(this.params, "{{customMessage}}expected {{subject}} '{{value}}' {{type}}");
 
     err.cliMessage = () => {
-      let msg = err.message
-        .replace('{{value}}', output.colors.yellow.bold('{{value}}'))
+      let msg = err.template
+        .replace('{{value}}', output.colors.bold('{{value}}'))
         .replace('{{subject}}', output.colors.bold('{{subject}}'));
       return template(msg, this.params);
     };

lib/assert/equal.js

@@ -20,7 +20,7 @@ class EqualityAssertion extends Assertion {
     let err = new AssertionFailedError(params, "{{customMessage}}expected {{jar}} '{{expected}}' {{type}} '{{actual}}'");
     err.showDiff = false;
     err.cliMessage = () => {
-      let msg = err.message
+      let msg = err.template
         .replace('{{jar}}', output.colors.bold('{{jar}}'));
       return template(msg, this.params);
     };

lib/assert/error.js

@@ -25,7 +25,7 @@ function AssertionFailedError(params, template) {
   this.inspect = () => {
     let params = this.params || {};
     let msg = params.customMessage || '';
-		                            return msg + subs(this.template, params);
+    return msg + subs(this.template, params);
   };
 
   this.cliMessage = () => {

lib/assert/include.js

@@ -24,16 +24,16 @@ class InclusionAssertion extends Assertion {
   getException() {
     let params = this.params;
     params.jar = template(params.jar, params);
-    let err = new AssertionFailedError(params, "{{customMessage}}expected {{jar}} {{type}} {{needle}}");
+    let err = new AssertionFailedError(params, '{{customMessage}}expected {{jar}} {{type}} "{{needle}}"');
     err.expected = params.needle;
     err.actual = params.haystack;
     if (Array.isArray(this.params.haystack)) {
       this.params.haystack = this.params.haystack.join("\n___(next element)___\n");
     }
-    err.cliMessage = () => {
-      let msg = err.message
-        .replace('{{needle}}', output.colors.yellow.bold('{{needle}}'))
-        .replace('{{jar}}', output.colors.bold('{{jar}}'));
+    err.cliMessage = function() {
+      let msg = this.template
+        .replace('{{jar}}', output.colors.bold('{{jar}}'))
+        .replace('{{needle}}', output.colors.bold('{{needle}}'));
       return template(msg, this.params);
     };
     return err;

lib/codecept.js

@@ -21,7 +21,7 @@ class Codecept {
     this.mocha.ui(scenarioUi);
     if (opts.reporter) {
       // custom reporters
-      this.mocha.reporter(opts.reporter, opts.reporterOptions);
+      this.mocha.reporter(opts.reporter, this.reporterOptions());
     } else {
       // standard reporter. TODO: make them run side by side
       this.mocha.reporter(reporter, opts);
@@ -43,6 +43,7 @@ class Codecept {
     require('./listener/steps');
     require('./listener/helpers');
     require('./listener/exit');
+    require('./listener/trace');
 
     // loading bootstrap
     if (this.config.bootstrap && fileExists(fsPath.join(dir, this.config.bootstrap))) {
@@ -64,6 +65,24 @@ class Codecept {
     });
   }
 
+  reporterOptions() {
+    var reporterOptions = Object.assign(this.config.mocha.reporterOptions || {});
+    if (this.opts.reporterOptions !== undefined) {
+        this.opts.reporterOptions.split(",").forEach(function(opt) {
+            var L = opt.split("=");
+            if (L.length > 2 || L.length === 0) {
+                throw new Error("invalid reporter option '" + opt + "'");
+            } else if (L.length === 2) {
+                reporterOptions[L[0]] = L[1];
+            } else {
+                reporterOptions[L[0]] = true;
+            }
+        });
+    }
+    return reporterOptions;
+
+  }
+
   run(test) {
     this.mocha.files = this.testFiles;
     if (test) {

lib/command/run.js

@@ -1,19 +1,18 @@
 'use strict';
 let getConfig = require('./utils').getConfig;
 let getTestRoot = require('./utils').getTestRoot;
-var Codecept = require('../codecept');
+let Codecept = require('../codecept');
+let output = require('../output');
 
 module.exports = function (suite, test, options) {
-  console.log('CodeceptJS v' + Codecept.version());
   let testRoot = getTestRoot(suite);
   let config = getConfig(testRoot);
-  try {    
+  try {
     let codecept = new Codecept(config, options);
     codecept.init(testRoot);
-    codecept.loadTests();  
+    codecept.loadTests();
     codecept.run(test);
   } catch (err) {
-    let output = require('../output');
     output.print('');
     output.error(err.message);
     output.print('');

lib/command/utils.js

@@ -7,12 +7,6 @@ let colors = require('colors');
 
 module.exports.getTestRoot = function (currentPath) {
   let testsPath = path.resolve(currentPath || '.');
-
-  if (!currentPath) {
-    output.print(`Test root is assumed to be ${colors.yellow.bold(testsPath)}`);
-  } else {
-    output.print(`Using test root ${colors.bold(testsPath)}`);
-  }
   return testsPath;
 };
 

lib/event.js

@@ -30,7 +30,13 @@ module.exports = {
       msg += ` (${param.toString()})`;
     }
     log(msg);
-    this.dispatcher.emit.apply(this.dispatcher, arguments);
+    try {
+      this.dispatcher.emit.apply(this.dispatcher, arguments);
+    } catch (err) {
+      log(`Error processing ${event} event:`);
+      log(err.stack);
+      throw err;
+    }
   },
 
   // for testing only!