diff --git a/integration/jasmine/examples/screenplay/ability-discard-error-should-not-affect-stage-cue.spec.js b/integration/jasmine/examples/screenplay/ability-discard-error-should-not-affect-stage-cue.spec.js new file mode 100644 index 00000000000..c519dbcc566 --- /dev/null +++ b/integration/jasmine/examples/screenplay/ability-discard-error-should-not-affect-stage-cue.spec.js @@ -0,0 +1,61 @@ +const { actorCalled, engage, Interaction } = require('@serenity-js/core'); + +describe('Mocha reporting', () => { + + beforeAll(() => engage(new Actors())); + + describe('A screenplay scenario', () => { + + // even if an ability is not discarded successfully, the subsequent tests should still be executed + it(`fails when discarding an ability fails`, () => + actorCalled('Donald') + .attemptsTo( + NotDoTooMuch(), + )); + + it(`succeeds when ability is discarded successfully`, () => + actorCalled('Alice') + .attemptsTo( + NotDoTooMuch(), + )); + + it(`fails if the ability fails to discard again`, () => + actorCalled('Donald') + .attemptsTo( + NotDoTooMuch(), + )); + }); +}); + +class Actors { + prepare(actor) { + switch (actor.name) { + case 'Donald': + return actor.whoCan(new CauseErrorWhenAbilityDiscarded()); + default: + return actor.whoCan(new SucceedWhenAbilityDiscarded()); + } + } +} + +const NotDoTooMuch = () => Interaction.where(`#actor doesn't do much`, () => void 0); + +class CauseErrorWhenAbilityDiscarded { + static as(actor) { + return actor.abilityTo(CauseErrorWhenAbilityDiscarded); + } + + discard() { + return Promise.reject(new TypeError(`Some internal error in ability`)); + } +} + +class SucceedWhenAbilityDiscarded { + static as(actor) { + return actor.abilityTo(CauseErrorWhenAbilityDiscarded); + } + + discard() { + return Promise.resolve(); + } +} diff --git a/integration/jasmine/spec/screenplay.spec.ts b/integration/jasmine/spec/screenplay.spec.ts index 7e621f5ba34..33c70950d8a 100644 --- a/integration/jasmine/spec/screenplay.spec.ts +++ b/integration/jasmine/spec/screenplay.spec.ts @@ -2,7 +2,7 @@ import 'mocha'; import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools'; import { SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events'; -import { ExecutionFailedWithAssertionError, ExecutionFailedWithError, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model'; +import { ExecutionFailedWithAssertionError, ExecutionFailedWithError, ExecutionSuccessful, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model'; import { jasmine } from '../src/jasmine'; describe('@serenity-js/Jasmine', function () { @@ -79,5 +79,43 @@ describe('@serenity-js/Jasmine', function () { }) ; })); + + it(`executes all the scenarios in the test suite even when some of them fail because of an error when discarding an ability`, () => + jasmine('examples/screenplay/ability-discard-error-should-not-affect-stage-cue.spec.js') + .then(ifExitCodeIsOtherThan(1, logOutput)) + .then(res => { + expect(res.exitCode).to.equal(1); + + PickEvent.from(res.events) + .next(SceneStarts, event => expect(event.details.name).to.equal(new Name('A screenplay scenario fails when discarding an ability fails'))) + .next(SceneFinished, event => { + const outcome: ProblemIndication = event.outcome as ProblemIndication; + + expect(outcome).to.be.instanceOf(ExecutionFailedWithError); + expect(outcome.error.name).to.equal('Error'); + + const message = outcome.error.message.split('\n'); + + expect(message[0]).to.equal('1 async operation has failed to complete:'); + expect(message[1]).to.equal('[Stage] Dismissing Donald... - TypeError: Some internal error in ability'); + }) + .next(SceneStarts, event => expect(event.details.name).to.equal(new Name('A screenplay scenario succeeds when ability is discarded successfully'))) + .next(SceneFinished, event => { + expect(event.outcome).to.be.instanceOf(ExecutionSuccessful); + }) + .next(SceneStarts, event => expect(event.details.name).to.equal(new Name('A screenplay scenario fails if the ability fails to discard again'))) + .next(SceneFinished, event => { + const outcome: ProblemIndication = event.outcome as ProblemIndication; + + expect(outcome).to.be.instanceOf(ExecutionFailedWithError); + expect(outcome.error.name).to.equal('Error'); + + const message = outcome.error.message.split('\n'); + + expect(message[0]).to.equal('1 async operation has failed to complete:'); + expect(message[1]).to.equal('[Stage] Dismissing Donald... - TypeError: Some internal error in ability'); + }) + ; + })); }); }); diff --git a/integration/jasmine/src/jasmine.ts b/integration/jasmine/src/jasmine.ts index 5f35c55f806..6c0b2f82433 100644 --- a/integration/jasmine/src/jasmine.ts +++ b/integration/jasmine/src/jasmine.ts @@ -22,6 +22,8 @@ export function jasmine(...params: string[]): Promise { ...params, + '--random=false', + // The path to the reporter needs to be relative to the Jasmine module. // Normally this will be simply "@serenity-js/jasmine" as the Serenity/JS adapter for Jasmine // will be installed next to it. diff --git a/integration/mocha/examples/screenplay/ability-discard-error-should-not-affect-stage-cue.spec.js b/integration/mocha/examples/screenplay/ability-discard-error-should-not-affect-stage-cue.spec.js new file mode 100644 index 00000000000..15c7c8d63d8 --- /dev/null +++ b/integration/mocha/examples/screenplay/ability-discard-error-should-not-affect-stage-cue.spec.js @@ -0,0 +1,61 @@ +const { actorCalled, engage, Interaction } = require('@serenity-js/core'); + +describe('Mocha reporting', () => { + + before(() => engage(new Actors())); + + describe('A screenplay scenario', () => { + + // even if an ability is not discarded successfully, the subsequent tests should still be executed + it(`fails when discarding an ability fails`, () => + actorCalled('Donald') + .attemptsTo( + NotDoTooMuch(), + )); + + it(`succeeds when ability is discarded successfully`, () => + actorCalled('Alice') + .attemptsTo( + NotDoTooMuch(), + )); + + it(`fails if the ability fails to discard again`, () => + actorCalled('Donald') + .attemptsTo( + NotDoTooMuch(), + )); + }); +}); + +class Actors { + prepare(actor) { + switch (actor.name) { + case 'Donald': + return actor.whoCan(new CauseErrorWhenAbilityDiscarded()); + default: + return actor.whoCan(new SucceedWhenAbilityDiscarded()); + } + } +} + +const NotDoTooMuch = () => Interaction.where(`#actor doesn't do much`, () => void 0); + +class CauseErrorWhenAbilityDiscarded { + static as(actor) { + return actor.abilityTo(CauseErrorWhenAbilityDiscarded); + } + + discard() { + return Promise.reject(new TypeError(`Some internal error in ability`)); + } +} + +class SucceedWhenAbilityDiscarded { + static as(actor) { + return actor.abilityTo(CauseErrorWhenAbilityDiscarded); + } + + discard() { + return Promise.resolve(); + } +} diff --git a/integration/mocha/spec/screenplay.spec.ts b/integration/mocha/spec/screenplay.spec.ts index c13a7558237..6a46628152c 100644 --- a/integration/mocha/spec/screenplay.spec.ts +++ b/integration/mocha/spec/screenplay.spec.ts @@ -2,7 +2,7 @@ import 'mocha'; import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools'; import { SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events'; -import { ExecutionFailedWithAssertionError, ExecutionFailedWithError, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model'; +import { ExecutionFailedWithAssertionError, ExecutionFailedWithError, ExecutionSuccessful, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model'; import { mocha } from '../src/mocha'; describe('@serenity-js/mocha', function () { @@ -79,5 +79,43 @@ describe('@serenity-js/mocha', function () { }) ; })); + + it(`executes all the scenarios in the test suite even when some of them fail because of an error when discarding an ability`, () => + mocha('examples/screenplay/ability-discard-error-should-not-affect-stage-cue.spec.js') + .then(ifExitCodeIsOtherThan(2, logOutput)) + .then(res => { + expect(res.exitCode).to.equal(2); // 2 failures, so Mocha returns an exit code of 2 + + PickEvent.from(res.events) + .next(SceneStarts, event => expect(event.details.name).to.equal(new Name('A screenplay scenario fails when discarding an ability fails'))) + .next(SceneFinished, event => { + const outcome: ProblemIndication = event.outcome as ProblemIndication; + + expect(outcome).to.be.instanceOf(ExecutionFailedWithError); + expect(outcome.error.name).to.equal('Error'); + + const message = outcome.error.message.split('\n'); + + expect(message[0]).to.equal('1 async operation has failed to complete:'); + expect(message[1]).to.equal('[Stage] Dismissing Donald... - TypeError: Some internal error in ability'); + }) + .next(SceneStarts, event => expect(event.details.name).to.equal(new Name('A screenplay scenario succeeds when ability is discarded successfully'))) + .next(SceneFinished, event => { + expect(event.outcome).to.be.instanceOf(ExecutionSuccessful); + }) + .next(SceneStarts, event => expect(event.details.name).to.equal(new Name('A screenplay scenario fails if the ability fails to discard again'))) + .next(SceneFinished, event => { + const outcome: ProblemIndication = event.outcome as ProblemIndication; + + expect(outcome).to.be.instanceOf(ExecutionFailedWithError); + expect(outcome.error.name).to.equal('Error'); + + const message = outcome.error.message.split('\n'); + + expect(message[0]).to.equal('1 async operation has failed to complete:'); + expect(message[1]).to.equal('[Stage] Dismissing Donald... - TypeError: Some internal error in ability'); + }) + ; + })); }); }); diff --git a/packages/core/src/stage/Stage.ts b/packages/core/src/stage/Stage.ts index de0d60feea9..8a7feaa41e2 100644 --- a/packages/core/src/stage/Stage.ts +++ b/packages/core/src/stage/Stage.ts @@ -74,6 +74,9 @@ export class Stage { const newActor = new Actor(name, this); actor = this.cast.prepare(newActor); + + // todo this.manager.notifyOf(ActorStarts) + // todo: map this in Serenity BDD Reporter so that the "cast" is recorded } catch (error) { throw new ConfigurationError(`${ this.typeOf(this.cast) } encountered a problem when preparing actor "${ name }" for stage`, error); @@ -215,8 +218,10 @@ export class Stage { new Description(`[${ this.constructor.name }] Dismissed ${ actor.name } successfully`), id, ))) + // todo: ActorFinished .catch(error => - this.announce(new AsyncOperationFailed(error, id)), + this.announce(new AsyncOperationFailed(error, id)), // todo: serialise the error! + // todo: ActorFinished (error ?) ); })) diff --git a/packages/core/src/stage/StageManager.ts b/packages/core/src/stage/StageManager.ts index 04a18471ff7..bd30e68ae0e 100644 --- a/packages/core/src/stage/StageManager.ts +++ b/packages/core/src/stage/StageManager.ts @@ -5,25 +5,12 @@ import { CorrelationId, Description, Duration, Timestamp } from '../model'; import { ListensToDomainEvents } from '../screenplay'; import { Clock } from './Clock'; -interface AsyncOperationDetails { - taskDescription: Description; - startedAt: Timestamp; -} - -interface FailedAsyncOperationDetails { - taskDescription: Description; - startedAt: Timestamp; - duration: Duration; - error: Error; -} - export class StageManager { private readonly subscribers: ListensToDomainEvents[] = []; - private readonly wip = new WIP(); - private readonly failedOperations: FailedAsyncOperationDetails[] = []; + private readonly wip: WIP; - constructor(private readonly cueTimeout: Duration, - private readonly clock: Clock) { + constructor(private readonly cueTimeout: Duration, private readonly clock: Clock) { + this.wip = new WIP(cueTimeout, clock); } register(...subscribers: ListensToDomainEvents[]) { @@ -35,18 +22,12 @@ export class StageManager { } notifyOf(event: DomainEvent): void { - this.handleAsyncOperation(event); + this.wip.recordIfAsync(event); this.subscribers.forEach(crewMember => crewMember.notifyOf(event)); } waitForNextCue(): Promise { - function header(numberOfFailures: number) { - return numberOfFailures === 1 - ? `1 async operation has failed to complete` - : `${ numberOfFailures } async operations have failed to complete`; - } - return new Promise((resolve, reject) => { let interval: NodeJS.Timer, @@ -55,28 +36,18 @@ export class StageManager { timeout = setTimeout(() => { clearInterval(interval); - const now = this.clock.now(); + if (this.wip.hasFailedOperations()) { + const error = new Error(this.wip.descriptionOfFailedOperations()); - if (this.wip.size() > 0) { - const timedOutOperations = this.wip.filter(op => now.diff(op.startedAt).isGreaterThanOrEqualTo(this.cueTimeout)); + this.wip.resetFailedOperations(); - const message = timedOutOperations.reduce( - (acc, op) => - acc.concat(`${ now.diff(op.startedAt) } - ${ op.taskDescription.value }`), - [ `${ header(timedOutOperations.length) } within a ${ this.cueTimeout } cue timeout:` ], - ).join('\n'); - - return reject(new Error(message)); + return reject(error); } - if (this.failedOperations.length > 0) { - let message = `${ header(this.failedOperations.length) }:\n`; - - this.failedOperations.forEach((op: FailedAsyncOperationDetails) => { - message += `${ op.taskDescription.value } - ${ op.error.stack }\n---\n`; - }); + if (this.wip.hasActiveOperations()) { + const error = new Error(this.wip.descriptionOfTimedOutOperations()); - return reject(new Error(message)); + return reject(error); } // "else" can't happen because this case is covered by the interval check below @@ -84,10 +55,19 @@ export class StageManager { }, this.cueTimeout.inMilliseconds()); interval = setInterval(() => { - if (this.wip.size() === 0 && this.failedOperations.length === 0) { + if (this.wip.hasAllOperationsCompleted()) { clearTimeout(timeout); clearInterval(interval); + if (this.wip.hasFailedOperations()) { + + const error = new Error(this.wip.descriptionOfFailedOperations()); + + this.wip.resetFailedOperations(); + + return reject(error); + } + return resolve(); } }, 10); @@ -97,65 +77,104 @@ export class StageManager { currentTime(): Timestamp { return this.clock.now(); } +} + +/** + * @package + */ +class WIP { + private readonly wip = new Map(); + private readonly failedOperations: FailedAsyncOperationDetails[] = []; + + constructor( + private readonly cueTimeout: Duration, + private readonly clock: Clock, + ) { + } - private handleAsyncOperation(event: DomainEvent): void { + recordIfAsync(event: DomainEvent): void { if (event instanceof AsyncOperationAttempted) { - this.wip.set(event.correlationId, { - taskDescription: event.taskDescription, - startedAt: event.timestamp, + this.set(event.correlationId, { + taskDescription: event.taskDescription, + startedAt: event.timestamp, }); } - else if (event instanceof AsyncOperationCompleted) { - this.wip.delete(event.correlationId); + + if (event instanceof AsyncOperationCompleted) { + this.delete(event.correlationId); } - else if (event instanceof AsyncOperationFailed) { - const original = this.wip.get(event.correlationId); + + if (event instanceof AsyncOperationFailed) { + const original = this.get(event.correlationId); + this.failedOperations.push({ taskDescription: original.taskDescription, startedAt: original.startedAt, duration: event.timestamp.diff(original.startedAt), error: event.error, }); - this.wip.delete(event.correlationId); + + this.delete(event.correlationId) } } -} -/** - * @package - */ -class WIP { - private wip = new Map(); + hasAllOperationsCompleted(): boolean { + return this.wip.size === 0; + } + + hasActiveOperations(): boolean { + return this.wip.size > 0; + } + + hasFailedOperations(): boolean { + return this.failedOperations.length > 0; + } + + descriptionOfTimedOutOperations(): string { + const now = this.clock.now(); - set(key: Key, value: Value) { - this.wip.set(key, value); + const timedOutOperations = Array.from(this.wip.values()) + .filter(op => now.diff(op.startedAt).isGreaterThanOrEqualTo(this.cueTimeout)); + + return timedOutOperations.reduce( + (acc, op) => acc.concat(`${ now.diff(op.startedAt) } - ${ op.taskDescription.value }`), + [`${ this.header(this.wip.size) } within a ${ this.cueTimeout } cue timeout:`], + ).join('\n'); } - get(key: Key): Value { - return this.wip.get(this.asReference(key)); + descriptionOfFailedOperations() { + let message = `${ this.header(this.failedOperations.length) }:\n`; + + this.failedOperations.forEach((op: FailedAsyncOperationDetails) => { + message += `${ op.taskDescription.value } - ${ op.error.stack }\n---\n`; + }); + + return message; } - delete(key: Key): boolean { - return this.wip.delete(this.asReference(key)); + resetFailedOperations() { + this.failedOperations.length = 0; } - has(key): boolean { - return !! this.asReference(key); + private header(numberOfFailures): string { + return numberOfFailures === 1 + ? `1 async operation has failed to complete` + : `${ numberOfFailures } async operations have failed to complete`; } - size(): number { - return this.wip.size; + private set(correlationId: CorrelationId, details: AsyncOperationDetails) { + return this.wip.set(correlationId, details); } - forEach(callback: (value: Value, key: Key, map: Map) => void, thisArg?: any) { - return this.wip.forEach(callback); + private get(correlationId: CorrelationId) { + return this.wip.get(this.asReference(correlationId)); } - filter(callback: (value: Value, index: number, array: Value[]) => boolean): Value[] { - return Array.from(this.wip.values()).filter(callback); + private delete(correlationId: CorrelationId) { + this.wip.delete(this.asReference(correlationId)) } - private asReference(key: Key) { + private asReference(key: CorrelationId) { for (const [ k, v ] of this.wip.entries()) { if (k.equals(key)) { return k; @@ -165,3 +184,23 @@ class WIP { return undefined; } } + +/** + * @package + */ +interface AsyncOperationDetails { + taskDescription: Description; + startedAt: Timestamp; + duration?: Duration; + error?: Error; +} + +/** + * @package + */ +interface FailedAsyncOperationDetails { + taskDescription: Description; + startedAt: Timestamp; + duration: Duration; + error: Error; +} diff --git a/packages/mocha/src/SerenityReporterForMocha.ts b/packages/mocha/src/SerenityReporterForMocha.ts index 7348cad141f..f5cce685de1 100644 --- a/packages/mocha/src/SerenityReporterForMocha.ts +++ b/packages/mocha/src/SerenityReporterForMocha.ts @@ -3,7 +3,7 @@ import { Serenity } from '@serenity-js/core'; import { DomainEvent, SceneFinished, SceneFinishes, SceneStarts, SceneTagged, TestRunFinished, TestRunFinishes, TestRunnerDetected } from '@serenity-js/core/lib/events'; import { ArbitraryTag, CorrelationId, ExecutionFailedWithError, ExecutionRetriedTag, FeatureTag, Name } from '@serenity-js/core/lib/model'; -import { MochaOptions, reporters, Runner, Test } from 'mocha'; +import { MochaOptions, reporters, Runnable, Runner, Test } from 'mocha'; import { MochaOutcomeMapper, MochaTestMapper } from './mappers'; import { OutcomeRecorder } from './OutcomeRecorder'; @@ -68,12 +68,12 @@ export class SerenityReporterForMocha extends reporters.Base { const announceSceneFinishedFor = this.announceSceneFinishedFor.bind(this); runner.suite.afterEach('Serenity/JS', function () { - return announceSceneFinishedFor(this.currentTest); + return announceSceneFinishedFor(this.currentTest, this.test); }); // https://github.com/cypress-io/cypress/issues/7562 runner.on('test:after:run', (test: Test) => { - return announceSceneFinishedFor(test); + return announceSceneFinishedFor(test, test); }); // Tests without body don't trigger the above custom afterEach hook @@ -111,7 +111,7 @@ export class SerenityReporterForMocha extends reporters.Base { ); } - private announceSceneFinishedFor(test: Test): Promise { + private announceSceneFinishedFor(test: Test, runnable: Runnable): Promise { const scenario = this.testMapper.detailsOf(test); this.emit( @@ -142,7 +142,9 @@ export class SerenityReporterForMocha extends reporters.Base { this.recorder.erase(test); - throw error; + // re-throwing an error here would cause Mocha to halt test suite, which we don't want to do + // https://github.com/mochajs/mocha/issues/1635 + (runnable as any).error(error); }); }