Skip to content
Permalink
Browse files
fix(cucumber): Compromised tests are now correctly reported
  • Loading branch information
jan-molak committed Feb 14, 2019
1 parent abbac18 commit cf49a752969fa37ec600dcf7b374ba85797fe1e7
Showing 7 changed files with 89 additions and 25 deletions.
@@ -0,0 +1,13 @@
Feature: Reports compromised scenarios

In order to see how Serenity/JS reports a compromised Cucumber scenario
As a curious developer
I'd like to see an example implementation

Scenario: A compromised scenario

Here's an example of a compromised scenario.
A compromised scenario is one that has failed due to issues external to the system.
For example, if the firewall is blocked we can't test a REST API.

Given a step that fails with an error compromising the test
@@ -0,0 +1,13 @@
Feature: Reports failing scenarios

In order to see how Serenity/JS reports a failing Cucumber scenario
As a curious developer
I'd like to see an example implementation

Scenario: A broken scenario

Here's an example of a scenario that fails with a generic error.
Throwing any error will mark the scenario as failed, but generic errors are different
from say AssertionErrors as they might indicate an issue with the test itself.

Given a step that fails with a generic error
@@ -6,8 +6,8 @@ Feature: Reports failing scenarios

Scenario: A failing scenario

Here's an example of a failing scenario.
Here's an example of a scenario failing due to an assertion error.
The rule here is simple: provided that one of the steps fail the scenario fails as well.

Given a step that passes
And a step that fails
And a step that fails with an assertion error
@@ -1,13 +1,22 @@
import { AssertionError, TestCompromisedError } from '@serenity-js/core';
import { Given, TableDefinition, Then, When } from 'cucumber';

Given(/^.*step.*passes$/, function() {
return Promise.resolve();
});

Given(/^.*step.*fails$/, function() {
Given(/^.*step.*fails with an assertion error$/, function() {
return Promise.reject(new AssertionError('expected true to equal false', false, true));
});

Given(/^.*step.*fails with a generic error$/, function() {
return Promise.reject(new Error(`Something's wrong`));
});

Given(/^.*step.*fails with an error compromising the test$/, function() {
return Promise.reject(new TestCompromisedError(`Something's wrong`));
});

Given(/^.*step.*marked as pending/, function() {
return Promise.resolve('pending');
});
@@ -2,7 +2,7 @@ import 'mocha';

import * as sinon from 'sinon';

import { TestCompromisedError } from '../../../../src/errors';
import { AssertionError, TestCompromisedError } from '../../../../src/errors';
import {
ArtifactGenerated,
SceneFinished,
@@ -276,22 +276,45 @@ describe('SerenityBDDReporter', () => {
*/
it('has failed with an assertion error', () => {

try {
expect(true).to.equal(false);
} catch (assertionError) {
const assertionError = new AssertionError('expected true to equal false', false, true);

given(reporter).isNotifiedOfFollowingEvents(
new SceneFinished(defaultCardScenario, new ExecutionFailedWithAssertionError(assertionError)),
new TestRunFinished(),
);
given(reporter).isNotifiedOfFollowingEvents(
new SceneFinished(defaultCardScenario, new ExecutionFailedWithAssertionError(assertionError)),
new TestRunFinished(),
);

report = stageManager.notifyOf.firstCall.lastArg.artifact.map(_ => _);
report = stageManager.notifyOf.firstCall.lastArg.artifact.map(_ => _);

expect(report.result).to.equal('FAILURE');
expect(report.testFailureCause.errorType).to.equal('AssertionError');
expect(report.testFailureCause.message).to.equal('expected true to equal false');
expect(report.testFailureCause.stackTrace).to.be.an('array');
});

expect(report.result).to.equal('FAILURE');
expect(report.testFailureCause.errorType).to.equal('AssertionError');
expect(report.testFailureCause.message).to.equal('expected true to equal false');
expect(report.testFailureCause.stackTrace).to.be.an('array');
}
/**
* @test {SerenityBDDReporter}
* @test {SceneStarts}
* @test {SceneFinished}
* @test {TestRunFinished}
* @test {TestCompromisedError}
* @test {ExecutionCompromised}
*/
it('has been compromised', () => {

// const assertionError = new AssertionError('expected true to equal false', false, true);
const assertionError = new TestCompromisedError('expected true to equal false');

given(reporter).isNotifiedOfFollowingEvents(
new SceneFinished(defaultCardScenario, new ExecutionCompromised(assertionError)),
new TestRunFinished(),
);

report = stageManager.notifyOf.firstCall.lastArg.artifact.map(_ => _);

expect(report.result).to.equal('COMPROMISED');
expect(report.testFailureCause.errorType).to.equal('TestCompromisedError');
expect(report.testFailureCause.message).to.equal('expected true to equal false');
expect(report.testFailureCause.stackTrace).to.be.an('array');
});

/**
@@ -1,6 +1,7 @@
import { AssertionError, UnknownError } from '@serenity-js/core/lib/errors';
import { AssertionError, TestCompromisedError, UnknownError } from '@serenity-js/core/lib/errors';
import { ErrorSerialiser, Path } from '@serenity-js/core/lib/io';
import {
ExecutionCompromised,
ExecutionFailedWithAssertionError,
ExecutionFailedWithError,
ExecutionSkipped,
@@ -103,9 +104,11 @@ export function cucumberEventProtocolAdapter({ notifier, mapper, cache }: Depend

case 'ambiguous':
case 'failed':
return error instanceof AssertionError
? new ExecutionFailedWithAssertionError(error)
: new ExecutionFailedWithError(error);
switch (true) {
case error instanceof AssertionError: return new ExecutionFailedWithAssertionError(error as AssertionError);
case error instanceof TestCompromisedError: return new ExecutionCompromised(error as TestCompromisedError);
default: return new ExecutionFailedWithError(error);
}

case 'pending':
return new ImplementationPending();
@@ -1,6 +1,7 @@
import { AssertionError, serenity } from '@serenity-js/core';
import { AssertionError, serenity, TestCompromisedError } from '@serenity-js/core';
import { Path } from '@serenity-js/core/lib/io';
import {
ExecutionCompromised,
ExecutionFailedWithAssertionError,
ExecutionFailedWithError,
ExecutionSkipped,
@@ -182,9 +183,11 @@ function outcomeFrom(status: string, error?: Error) {
return new ExecutionFailedWithError(error);

case status === 'failed':
return error instanceof AssertionError
? new ExecutionFailedWithAssertionError(error)
: new ExecutionFailedWithError(error);
switch (true) {
case error instanceof AssertionError: return new ExecutionFailedWithAssertionError(error as AssertionError);
case error instanceof TestCompromisedError: return new ExecutionCompromised(error as TestCompromisedError);
default: return new ExecutionFailedWithError(error);
}

case status === 'pending':
return new ImplementationPending();

0 comments on commit cf49a75

Please sign in to comment.