Permalink
Browse files

feat(serenity-mocha): Tests that timed out or failed with an exceptio…

…n are recognised as "Compromised"
  • Loading branch information...
jan-molak committed Jan 29, 2017
1 parent c50a4bf commit 532fb86eac2a54691549801ff93c2db495a73c45
@@ -61,10 +61,10 @@ describe('When working with Mocha', function () {
});
});
it('reports failing scenarios', () => {
it('reports sync scenarios failing due to an AssertionError', () => {
let spawned = protractor('protractor.conf.js',
'--mochaOpts.grep', 'A sample test that fails',
'--mochaOpts.grep', 'A sample test that fails due to an AssertionError',
);
return expect(spawned.result).to.be.eventually.rejected.then(() => {
@@ -77,6 +77,80 @@ describe('When working with Mocha', function () {
});
});
it('reports async scenarios failing due to an AssertionError', () => {
let spawned = protractor('protractor.conf.js',
'--mochaOpts.grep', 'ails due to an async AssertionError',
);
return expect(spawned.result).to.be.eventually.rejected.then(() => {
expect(spawned.messages).to.have.lengthOf(2);
let lastMessage = spawned.messages.pop();
expect(lastMessage).to.be.instanceOf(SceneFinished);
expect(Result[lastMessage.value.result]).to.equal(Result[Result.FAILURE]);
expect(lastMessage.value.error).to.deep.equal({
actual: 'async pass',
expected: 'async fail',
message: 'expected \'async pass\' to equal \'async fail\'',
name: 'AssertionError',
showDiff: true,
});
});
});
it('reports scenarios that timed out', function () {
let spawned = protractor('protractor.conf.js',
'--mochaOpts.grep', 'A sample test that times out',
);
return expect(spawned.result).to.be.eventually.rejected.then(() => {
expect(spawned.messages).to.have.lengthOf(2);
let lastMessage = spawned.messages.pop();
expect(lastMessage).to.be.instanceOf(SceneFinished);
expect(Result[lastMessage.value.result]).to.equal(Result[Result.COMPROMISED]);
expect(lastMessage.value.error.message).to.match(/Timeout of 5ms exceeded./);
});
});
it('reports scenarios that failed with an error', function () {
let spawned = protractor('protractor.conf.js',
'--mochaOpts.grep', 'A sample test that fails with an error',
);
return expect(spawned.result).to.be.eventually.rejected.then(() => {
expect(spawned.messages).to.have.lengthOf(2);
let lastMessage = spawned.messages.pop();
expect(lastMessage).to.be.instanceOf(SceneFinished);
expect(Result[lastMessage.value.result]).to.equal(Result[Result.COMPROMISED]);
expect(lastMessage.value.error.message).to.match(/Expected problem/);
});
});
it('reports scenarios that failed with an error asynchronously', function () {
let spawned = protractor('protractor.conf.js',
'--mochaOpts.grep', 'A sample test that asynchronously fails with an error',
);
return expect(spawned.result).to.be.eventually.rejected.then(() => {
expect(spawned.messages).to.have.lengthOf(2);
let lastMessage = spawned.messages.pop();
expect(lastMessage).to.be.instanceOf(SceneFinished);
expect(Result[lastMessage.value.result]).to.equal(Result[Result.COMPROMISED]);
expect(lastMessage.value.error.message).to.match(/Expected async problem/);
});
});
it ('recognises the name of the feature under test (the outer-most `describe`)', () => {
let spawned = protractor('protractor.conf.js',
'--mochaOpts.grep', 'A sample test that passes',
@@ -8,15 +8,30 @@ describe ('Integration with Mocha', () => {
expect('pass').to.equal('pass');
});
it ('fails', () => {
it ('fails due to an AssertionError', () => {
expect('pass').to.equal('fail');
});
it ('fails due to an async AssertionError', () => {
return expect(Promise.resolve('async pass')).to.eventually.equal('async fail');
});
it ('is pending');
xit ('is skipped', () => {
expect('pass').to.equal('fail');
});
});
it ('times out', function (done) {
this.timeout(5);
setTimeout(done, 100);
});
it ('fails with an error', () => {
throw new Error('Expected problem');
});
it ('asynchronously fails with an error', () => Promise.reject(new Error('Expected async problem')));
});
});
@@ -2,13 +2,35 @@ import { DomainEvent } from '../../src/serenity/domain/events';
import { StageCrewMember } from '../../src/serenity/stage/stage_manager';
export class ChildProcessReporter implements StageCrewMember {
constructor() {
this.enableSerialisationOfErrors();
}
assignTo(stage) {
stage.manager.registerInterestIn([ DomainEvent ], this);
}
notifyOf(event) {
process.send(event);
}
private enableSerialisationOfErrors() {
if (! ('toJSON' in Error.prototype)) {
Object.defineProperty(Error.prototype, 'toJSON', {
value: function () { // tslint:disable-line:object-literal-shorthand needed for `this` to work
let alt = {};
Object.getOwnPropertyNames(this).forEach(function (key) {
alt[key] = this[key];
}, this);
return alt;
},
configurable: true,
writable: true,
});
}
}
}
export function childProcessReporter() {
@@ -77,32 +77,31 @@ function seal(fn) {
* @return {!Function}
*/
function makeAsyncTestFn(fn) {
var isAsync = fn.length > 0;
var isGenerator = promise.isGenerator(fn);
const isAsync = fn.length > 0;
const isGenerator = promise.isGenerator(fn);
if (isAsync && isGenerator) {
throw new TypeError(
throw TypeError(
'generator-based tests must not take a callback; for async testing,'
+ ' return a promise (or yield on a promise)');
}
var ret = /** @type {function(this: mocha.Context)}*/ function(done) {
var self = this;
var runTest = function(resolve, reject) {
var ret = /** @type {function(this: mocha.Context)}*/ (function(done) {
const runTest = (resolve, reject) => {
try {
if (self.isAsync) {
fn.call(self, function(err) { err ? reject(err) : resolve(); });
} else if (self.isGenerator) {
resolve(promise.consume(fn, self));
if (isAsync) {
fn.call(this, err => err ? reject(err) : resolve());
} else if (isGenerator) {
resolve(promise.consume(fn, this));
} else {
resolve(fn.call(self));
resolve(fn.call(this));
}
} catch (ex) {
reject(ex);
}
};
if (!promise.USE_PROMISE_MANAGER) {
new promise.Promise(runTest).then(seal(done), done);
new Promise(runTest).then(seal(done), done);
return;
}
@@ -118,7 +117,7 @@ function makeAsyncTestFn(fn) {
return runTest(fulfill, reject);
}, flow);
}, runnable.fullTitle()).then(seal(done), done);
};
});
ret.toString = function() {
return fn.toString();
@@ -58,17 +58,27 @@ function finalStateOf(scenario: ExecutedScenario): Result {
return Result.SUCCESS;
}
if (scenario.state === 'failed') {
return Result.FAILURE;
if (wasCompromised(scenario)) {
return Result.COMPROMISED;
}
if (scenario.timedOut) {
return Result.COMPROMISED;
if (scenario.state === 'failed') {
return Result.FAILURE;
}
return Result.COMPROMISED;
}
function wasCompromised(scenario: ExecutedScenario) {
// Mocha sets the `timedOut` flag *after* notifying the reporter of a test failure, hence the regex check.
const timedOut = s => s.timedOut || (s.err && /^Timeout.*exceeded/.test(s.err.message));
// anything other than an assertion error
const blewUp = s => s.err && s.err.constructor && ! /AssertionError/.test(s.err.constructor.name);
return timedOut(scenario) || blewUp(scenario);
}
type ScenarioOrSuite = { title: string, parent: ScenarioOrSuite };
function fullNameOf(scenario: ScenarioOrSuite) {
@@ -1,5 +1,5 @@
import { CucumberTestFramework } from '../../serenity-cucumber/cucumber_test_framework';
import { MochaTestFramework } from '../../serenity-mocha/mocha_test_framework';
import { MochaTestFramework } from '../../serenity-mocha';
import { SerenityFrameworkConfig } from './serenity_framework_config';
import { TestFramework } from './test_framework';

0 comments on commit 532fb86

Please sign in to comment.