feat(serenity-mocha): Tests that timed out or failed with an exceptio…

…n are recognised as "Compromised"

spec/integration/mocha/recognises_scenario_results.ts

@@ -61,10 +61,10 @@ describe('When working with Mocha', function () {
             });
         });
 
-        it('reports failing scenarios', () => {
+        it('reports sync scenarios failing due to an AssertionError', () => {
 
             let spawned = protractor('protractor.conf.js',
-                '--mochaOpts.grep', 'A sample test that fails',
+                '--mochaOpts.grep', 'A sample test that fails due to an AssertionError',
             );
 
             return expect(spawned.result).to.be.eventually.rejected.then(() => {
@@ -77,6 +77,80 @@ describe('When working with Mocha', function () {
             });
         });
 
+        it('reports async scenarios failing due to an AssertionError', () => {
+
+            let spawned = protractor('protractor.conf.js',
+                '--mochaOpts.grep', 'ails due to an async AssertionError',
+            );
+
+            return expect(spawned.result).to.be.eventually.rejected.then(() => {
+                expect(spawned.messages).to.have.lengthOf(2);
+
+                let lastMessage = spawned.messages.pop();
+
+                expect(lastMessage).to.be.instanceOf(SceneFinished);
+                expect(Result[lastMessage.value.result]).to.equal(Result[Result.FAILURE]);
+                expect(lastMessage.value.error).to.deep.equal({
+                    actual: 'async pass',
+                    expected: 'async fail',
+                    message: 'expected \'async pass\' to equal \'async fail\'',
+                    name: 'AssertionError',
+                    showDiff: true,
+                });
+            });
+        });
+
+        it('reports scenarios that timed out', function () {
+
+            let spawned = protractor('protractor.conf.js',
+                '--mochaOpts.grep', 'A sample test that times out',
+            );
+
+            return expect(spawned.result).to.be.eventually.rejected.then(() => {
+                expect(spawned.messages).to.have.lengthOf(2);
+
+                let lastMessage = spawned.messages.pop();
+
+                expect(lastMessage).to.be.instanceOf(SceneFinished);
+                expect(Result[lastMessage.value.result]).to.equal(Result[Result.COMPROMISED]);
+                expect(lastMessage.value.error.message).to.match(/Timeout of 5ms exceeded./);
+            });
+        });
+
+        it('reports scenarios that failed with an error', function () {
+
+            let spawned = protractor('protractor.conf.js',
+                '--mochaOpts.grep', 'A sample test that fails with an error',
+            );
+
+            return expect(spawned.result).to.be.eventually.rejected.then(() => {
+                expect(spawned.messages).to.have.lengthOf(2);
+
+                let lastMessage = spawned.messages.pop();
+
+                expect(lastMessage).to.be.instanceOf(SceneFinished);
+                expect(Result[lastMessage.value.result]).to.equal(Result[Result.COMPROMISED]);
+                expect(lastMessage.value.error.message).to.match(/Expected problem/);
+            });
+        });
+
+        it('reports scenarios that failed with an error asynchronously', function () {
+
+            let spawned = protractor('protractor.conf.js',
+                '--mochaOpts.grep', 'A sample test that asynchronously fails with an error',
+            );
+
+            return expect(spawned.result).to.be.eventually.rejected.then(() => {
+                expect(spawned.messages).to.have.lengthOf(2);
+
+                let lastMessage = spawned.messages.pop();
+
+                expect(lastMessage).to.be.instanceOf(SceneFinished);
+                expect(Result[lastMessage.value.result]).to.equal(Result[Result.COMPROMISED]);
+                expect(lastMessage.value.error.message).to.match(/Expected async problem/);
+            });
+        });
+
         it ('recognises the name of the feature under test (the outer-most `describe`)', () => {
             let spawned = protractor('protractor.conf.js',
                 '--mochaOpts.grep', 'A sample test that passes',

spec/integration/mocha/spec/results.spec.ts

@@ -8,15 +8,30 @@ describe ('Integration with Mocha', () => {
             expect('pass').to.equal('pass');
         });
 
-        it ('fails', () => {
+        it ('fails due to an AssertionError', () => {
             expect('pass').to.equal('fail');
         });
 
+        it ('fails due to an async AssertionError', () => {
+            return expect(Promise.resolve('async pass')).to.eventually.equal('async fail');
+        });
+
         it ('is pending');
 
         xit ('is skipped', () => {
             expect('pass').to.equal('fail');
         });
-    });
 
+        it ('times out', function (done) {
+            this.timeout(5);
+
+            setTimeout(done, 100);
+        });
+
+        it ('fails with an error', () => {
+            throw new Error('Expected problem');
+        });
+
+        it ('asynchronously fails with an error', () => Promise.reject(new Error('Expected async problem')));
+    });
 });

spec/support/child_process_reporter.ts

@@ -2,13 +2,35 @@ import { DomainEvent } from '../../src/serenity/domain/events';
 import { StageCrewMember } from '../../src/serenity/stage/stage_manager';
 
 export class ChildProcessReporter implements StageCrewMember {
+    constructor() {
+        this.enableSerialisationOfErrors();
+    }
+
     assignTo(stage) {
         stage.manager.registerInterestIn([ DomainEvent ], this);
     }
 
     notifyOf(event) {
         process.send(event);
     }
+
+    private enableSerialisationOfErrors() {
+        if (! ('toJSON' in Error.prototype)) {
+            Object.defineProperty(Error.prototype, 'toJSON', {
+                value: function () {    // tslint:disable-line:object-literal-shorthand needed for `this` to work
+                    let alt = {};
+
+                    Object.getOwnPropertyNames(this).forEach(function (key) {
+                        alt[key] = this[key];
+                    }, this);
+
+                    return alt;
+                },
+                configurable: true,
+                writable: true,
+            });
+        }
+    }
 }
 
 export function childProcessReporter() {

src/serenity-mocha/controlflow.ts

@@ -77,32 +77,31 @@ function seal(fn) {
  * @return {!Function}
  */
 function makeAsyncTestFn(fn) {
-    var isAsync = fn.length > 0;
-    var isGenerator = promise.isGenerator(fn);
+    const isAsync = fn.length > 0;
+    const isGenerator = promise.isGenerator(fn);
     if (isAsync && isGenerator) {
-        throw new TypeError(
+        throw TypeError(
             'generator-based tests must not take a callback; for async testing,'
             + ' return a promise (or yield on a promise)');
     }
 
-    var ret = /** @type {function(this: mocha.Context)}*/ function(done) {
-        var self = this;
-        var runTest = function(resolve, reject) {
+    var ret = /** @type {function(this: mocha.Context)}*/ (function(done) {
+        const runTest = (resolve, reject) => {
             try {
-                if (self.isAsync) {
-                    fn.call(self, function(err) { err ? reject(err) : resolve(); });
-                } else if (self.isGenerator) {
-                    resolve(promise.consume(fn, self));
+                if (isAsync) {
+                    fn.call(this, err => err ? reject(err) : resolve());
+                } else if (isGenerator) {
+                    resolve(promise.consume(fn, this));
                 } else {
-                    resolve(fn.call(self));
+                    resolve(fn.call(this));
                 }
             } catch (ex) {
                 reject(ex);
             }
         };
 
         if (!promise.USE_PROMISE_MANAGER) {
-            new promise.Promise(runTest).then(seal(done), done);
+            new Promise(runTest).then(seal(done), done);
             return;
         }
 
@@ -118,7 +117,7 @@ function makeAsyncTestFn(fn) {
                 return runTest(fulfill, reject);
             }, flow);
         }, runnable.fullTitle()).then(seal(done), done);
-    };
+    });
 
     ret.toString = function() {
         return fn.toString();

src/serenity-mocha/model.ts

@@ -58,17 +58,27 @@ function finalStateOf(scenario: ExecutedScenario): Result {
         return Result.SUCCESS;
     }
 
-    if (scenario.state === 'failed') {
-        return Result.FAILURE;
+    if (wasCompromised(scenario)) {
+        return Result.COMPROMISED;
     }
 
-    if (scenario.timedOut) {
-        return Result.COMPROMISED;
+    if (scenario.state === 'failed') {
+        return Result.FAILURE;
     }
 
     return Result.COMPROMISED;
 }
 
+function wasCompromised(scenario: ExecutedScenario) {
+    // Mocha sets the `timedOut` flag *after* notifying the reporter of a test failure, hence the regex check.
+    const timedOut = s => s.timedOut || (s.err && /^Timeout.*exceeded/.test(s.err.message));
+
+    // anything other than an assertion error
+    const blewUp = s => s.err && s.err.constructor && ! /AssertionError/.test(s.err.constructor.name);
+
+    return timedOut(scenario) || blewUp(scenario);
+}
+
 type ScenarioOrSuite = { title: string, parent: ScenarioOrSuite };
 
 function fullNameOf(scenario: ScenarioOrSuite) {

src/serenity-protractor/framework/test_framework_detector.ts

@@ -1,5 +1,5 @@
 import { CucumberTestFramework } from '../../serenity-cucumber/cucumber_test_framework';
-import { MochaTestFramework } from '../../serenity-mocha/mocha_test_framework';
+import { MochaTestFramework } from '../../serenity-mocha';
 import { SerenityFrameworkConfig } from './serenity_framework_config';
 import { TestFramework } from './test_framework';