feat(core): better assertion errors reporting in Visual Studio Code

The message of a Serenity/JS AssertionError now includes the diff of expected and actual values, which allows for better error reporting in Visual Studio Code. I've also rewritten the diff mechanism itself to make it easier for developers to read, and to allow support for colour-coding in the future. Related tickets: Closes #1486
serenity-js · Jan 21, 2023 · 3b94b7d · 3b94b7d
1 parent fae2523
commit 3b94b7d
Show file tree

Hide file tree

Showing 93 changed files with 1,260 additions and 548 deletions.
diff --git a/examples/cucumber-reporting/features/reporting_results/reports_failing_scenarios.feature b/examples/cucumber-reporting/features/reporting_results/reports_failing_scenarios.feature
@@ -12,6 +12,22 @@ Feature: Reports failing scenarios
     Given a step that passes
       And a step that fails with a Serenity/JS AssertionError
 
+  Scenario: A scenario failing with a Serenity/JS AssertionError in an async step
+
+    Here's an example of a scenario failing due to an assertion error
+    in an async step.
+
+    Given a step that passes
+      And an async step that fails with a Serenity/JS AssertionError
+
+  Scenario: A scenario failing with a Serenity/JS Screenplay AssertionError
+
+    Here's an example of a scenario failing due to an assertion error
+    thrown by the interaction to `Ensure`
+
+    Given a step that passes
+      And a step that fails with a Serenity/JS Screenplay AssertionError
+
   Scenario: A scenario failing with a Node.js AssertionError
 
     Did you know thatSerenity/JS picks up generic AssertionErrors too?

diff --git a/examples/cucumber-reporting/features/step_definitions/sample.steps.ts b/examples/cucumber-reporting/features/step_definitions/sample.steps.ts
@@ -1,13 +1,24 @@
 import { DataTable, Given, Then, When } from '@cucumber/cucumber';
-import { AssertionError, TestCompromisedError } from '@serenity-js/core';
+import { Ensure, equals } from '@serenity-js/assertions';
+import { actorCalled, AssertionError, TestCompromisedError } from '@serenity-js/core';
 import { strictEqual } from 'assert';
 
 Given(/^.*step.*passes$/, function () {
     return Promise.resolve();
 });
 
-Given(/^.*step.*fails with a Serenity\/JS AssertionError$/, function () {
-    throw new AssertionError('expected true to equal false', false, true);
+Given(/^.*a step.*fails with a Serenity\/JS AssertionError$/, function () {
+    throw new AssertionError('expected true to equal false');
+});
+
+Given(/^.* async step.*fails with a Serenity\/JS AssertionError$/, async function () {
+    throw new AssertionError('expected true to equal false');
+});
+
+Given(/^.*step.*fails with a Serenity\/JS Screenplay AssertionError$/, async function () {
+    await actorCalled('Alice').attemptsTo(
+        Ensure.that(true, equals(false)),
+    );
 });
 
 Given(/^.*step.*fails with a Node.js AssertionError$/, function () {

diff --git a/integration/cucumber-1/src/step_definitions/common.steps.ts b/integration/cucumber-1/src/step_definitions/common.steps.ts
@@ -13,7 +13,7 @@ export = function () {
     });
 
     this.Given(/^.*step .* fails with an assertion error$/, function () {
-        return Promise.reject(new AssertionError(`Expected false to equal true`, false, true));
+        return Promise.reject(new AssertionError(`Expected false to equal true`));
     });
 
     this.Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/cucumber-2/src/step_definitions/common.steps.ts b/integration/cucumber-2/src/step_definitions/common.steps.ts
@@ -12,7 +12,7 @@ defineSupportCode(({ Given }) => {
     });
 
     Given(/^.*step .* fails with an assertion error$/, function () {
-        return Promise.reject(new AssertionError(`Expected false to equal true`, false, true));
+        return Promise.reject(new AssertionError(`Expected false to equal true`));
     });
 
     Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/cucumber-3/src/step_definitions/common.steps.ts b/integration/cucumber-3/src/step_definitions/common.steps.ts
@@ -12,7 +12,7 @@ defineSupportCode(({ Given }) => {
     });
 
     Given(/^.*step .* fails with an assertion error$/, function () {
-        return Promise.reject(new AssertionError(`Expected false to equal true`, false, true));
+        return Promise.reject(new AssertionError(`Expected false to equal true`));
     });
 
     Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/cucumber-4/src/step_definitions/common.steps.ts b/integration/cucumber-4/src/step_definitions/common.steps.ts
@@ -11,7 +11,7 @@ Given(/^.*step .* fails with a generic error$/, function () {
 });
 
 Given(/^.*step .* fails with an assertion error$/, function () {
-    return Promise.reject(new AssertionError(`Expected false to equal true`, false, true));
+    return Promise.reject(new AssertionError(`Expected false to equal true`));
 });
 
 Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/cucumber-5/src/step_definitions/common.steps.ts b/integration/cucumber-5/src/step_definitions/common.steps.ts
@@ -11,7 +11,7 @@ Given(/^.*step .* fails with a generic error$/, function () {
 });
 
 Given(/^.*step .* fails with an assertion error$/, function () {
-    return Promise.reject(new AssertionError(`Expected false to equal true`, false, true));
+    return Promise.reject(new AssertionError(`Expected false to equal true`));
 });
 
 Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/cucumber-6/src/step_definitions/common.steps.ts b/integration/cucumber-6/src/step_definitions/common.steps.ts
@@ -11,7 +11,7 @@ Given(/^.*step .* fails with a generic error$/, function () {
 });
 
 Given(/^.*step .* fails with an assertion error$/, function () {
-    return Promise.reject(new AssertionError(`Expected false to equal true`, false, true));
+    return Promise.reject(new AssertionError(`Expected false to equal true`));
 });
 
 Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/cucumber-7/src/step_definitions/common.steps.ts b/integration/cucumber-7/src/step_definitions/common.steps.ts
@@ -11,7 +11,7 @@ Given(/^.*step .* fails with a generic error$/, function () {
 });
 
 Given(/^.*step .* fails with an assertion error$/, function () {
-    throw new AssertionError(`Expected false to equal true`, false, true);
+    throw new AssertionError(`Expected false to equal true`);
 });
 
 Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/cucumber-8-javascript-api/src/step_definitions/common.steps.ts b/integration/cucumber-8-javascript-api/src/step_definitions/common.steps.ts
@@ -11,7 +11,7 @@ Given(/^.*step .* fails with a generic error$/, function () {
 });
 
 Given(/^.*step .* fails with an assertion error$/, function () {
-    throw new AssertionError(`Expected false to equal true`, false, true);
+    throw new AssertionError(`Expected false to equal true`);
 });
 
 Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/cucumber-8-legacy-api/src/step_definitions/common.steps.ts b/integration/cucumber-8-legacy-api/src/step_definitions/common.steps.ts
@@ -11,7 +11,7 @@ Given(/^.*step .* fails with a generic error$/, function () {
 });
 
 Given(/^.*step .* fails with an assertion error$/, function () {
-    throw new AssertionError(`Expected false to equal true`, false, true);
+    throw new AssertionError(`Expected false to equal true`);
 });
 
 Given(/^.*step .* fails with a non-Serenity assertion error$/, function () {

diff --git a/integration/jasmine/spec/failing_scenarios.spec.ts b/integration/jasmine/spec/failing_scenarios.spec.ts
@@ -1,6 +1,7 @@
 import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools';
 import { AssertionError } from '@serenity-js/core';
 import { ActivityFinished, ActivityStarts, SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events';
+import { trimmed } from '@serenity-js/core/lib/io';
 import { ExecutionFailedWithAssertionError, ExecutionFailedWithError, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model';
 import { describe, it } from 'mocha';
 
@@ -65,9 +66,12 @@ describe('@serenity-js/jasmine', function () {
                         const error = outcome.error as AssertionError;
 
                         expect(error).to.be.instanceof(AssertionError);
-                        expect(error.message).to.equal('Expected false to equal true.');
-                        expect(error.expected).to.equal(true);
-                        expect(error.actual).to.equal(false);
+                        expect(error.message).to.equal(trimmed`
+                            | Expected false to equal true.
+                            | 
+                            | Expected boolean: true
+                            | Actual boolean:   false
+                            |`);
                         expect(error.cause.message).to.equal(`Expected false to equal true.`);
                     })
                 ;
@@ -103,9 +107,12 @@ describe('@serenity-js/jasmine', function () {
                         const error = outcome.error as AssertionError;
 
                         expect(error).to.be.instanceof(AssertionError);
-                        expect(error.message).to.equal('Expected false to equal true.');
-                        expect(error.expected).to.equal(true);
-                        expect(error.actual).to.equal(false);
+                        expect(error.message).to.equal(trimmed`
+                            | Expected false to equal true.
+                            | 
+                            | Expected boolean: true
+                            | Actual boolean:   false
+                            |`);
                         expect(error.cause.message).to.equal(`Expected false to equal true.`);
                     })
                     .next(SceneFinished,       event => {

diff --git a/integration/jasmine/spec/screenplay.spec.ts b/integration/jasmine/spec/screenplay.spec.ts
@@ -1,5 +1,6 @@
 import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools';
 import { SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events';
+import { trimmed } from '@serenity-js/core/lib/io';
 import { ExecutionFailedWithAssertionError, ExecutionFailedWithError, ExecutionSuccessful, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model';
 import { describe, it } from 'mocha';
 
@@ -27,7 +28,11 @@ describe('@serenity-js/Jasmine', function () {
 
                             expect(outcome).to.be.instanceOf(ExecutionFailedWithAssertionError);
                             expect(outcome.error.name).to.equal('AssertionError');
-                            expect(outcome.error.message).to.equal('Expected false to equal true');
+                            expect(outcome.error.message).to.equal(trimmed`
+                                | Expected false to equal true
+                                | 
+                                | Expected boolean: true
+                                | Actual boolean:   false`);
                         })
                     ;
                 }));

diff --git a/integration/mocha/spec/failing_scenarios.spec.ts b/integration/mocha/spec/failing_scenarios.spec.ts
@@ -1,6 +1,7 @@
 import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools';
 import { AssertionError, TestCompromisedError } from '@serenity-js/core';
 import { SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events';
+import { trimmed } from '@serenity-js/core/lib/io';
 import { ExecutionCompromised, ExecutionFailedWithAssertionError, ExecutionFailedWithError, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model';
 import { describe, it } from 'mocha';
 
@@ -85,8 +86,12 @@ describe('@serenity-js/mocha', function () {
 
                         const error = outcome.error as AssertionError;
 
-                        expect(error.expected).to.equal('true');
-                        expect(error.actual).to.equal('false');
+                        expect(error.message).to.equal(trimmed`
+                            | Expected values to be strictly equal:
+                            |
+                            | false !== true
+                            |
+                        `)
                     })
                 ;
             }));

diff --git a/integration/mocha/spec/screenplay.spec.ts b/integration/mocha/spec/screenplay.spec.ts
@@ -1,5 +1,6 @@
 import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools';
 import { SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events';
+import { trimmed } from '@serenity-js/core/lib/io';
 import { ExecutionFailedWithAssertionError, ExecutionFailedWithError, ExecutionSuccessful, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model';
 import { describe, it } from 'mocha';
 
@@ -27,7 +28,12 @@ describe('@serenity-js/mocha', function () {
 
                             expect(outcome).to.be.instanceOf(ExecutionFailedWithAssertionError);
                             expect(outcome.error.name).to.equal('AssertionError');
-                            expect(outcome.error.message).to.equal('Expected false to equal true');
+                            expect(outcome.error.message).to.equal(trimmed`
+                                | Expected false to equal true
+                                | 
+                                | Expected boolean: true
+                                | Actual boolean:   false
+                                |`);
                         })
                     ;
                 }));

diff --git a/integration/playwright-test/examples/failing/failing-serenity-js-assertion.spec.ts b/integration/playwright-test/examples/failing/failing-serenity-js-assertion.spec.ts
@@ -6,7 +6,7 @@ describe('Playwright Test reporting', () => {
     describe('A scenario', () => {
 
         it('fails when the assertion fails', () => {
-            throw new AssertionError('Expected true to equal false', true, false);
+            throw new AssertionError('Expected true to equal false');
         });
     });
 });
diff --git a/integration/playwright-test/spec/failing_scenarios.spec.ts b/integration/playwright-test/spec/failing_scenarios.spec.ts
@@ -1,6 +1,7 @@
 import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools';
 import { AssertionError, LogicError, TestCompromisedError } from '@serenity-js/core';
 import { SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events';
+import { trimmed } from '@serenity-js/core/lib/io';
 import { ExecutionCompromised, ExecutionFailedWithAssertionError, ExecutionFailedWithError, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model';
 import { describe, it } from 'mocha';
 
@@ -47,7 +48,6 @@ describe('@serenity-js/playwright-test', function () {
 
                         expect(error.name).to.equal('Error');
 
-                        // TestError provided by Playwright is already serialised so we can't know the original expected and actual values
                         expect(error.message.split('\n')).to.deep.equal([
                             'expect(received).toEqual(expected) // deep equality',
                             '',
@@ -98,9 +98,12 @@ describe('@serenity-js/playwright-test', function () {
                         const error = outcome.error as AssertionError;
 
                         expect(error.name).to.equal('AssertionError');
-                        expect(error.message).to.equal(`Expected 'Hello' to equal 'Hola'`);
-                        expect(error.expected).to.equal('Hola');
-                        expect(error.actual).to.equal('Hello');
+                        expect(error.message).to.equal(trimmed`
+                            | Expected 'Hello' to equal 'Hola'
+                            | 
+                            | Expected string: Hola
+                            | Actual string:   Hello
+                            |`);
                     })
                 ;
             }));

diff --git a/integration/playwright-test/spec/screenplay.spec.ts b/integration/playwright-test/spec/screenplay.spec.ts
@@ -1,5 +1,6 @@
 import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools';
 import { AsyncOperationAttempted, AsyncOperationCompleted, InteractionFinished, InteractionStarts, SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events';
+import { trimmed } from '@serenity-js/core/lib/io';
 import { CorrelationId, Description, ExecutionFailedWithAssertionError, ExecutionFailedWithError, ExecutionSuccessful, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model';
 import { describe, it } from 'mocha';
 
@@ -120,7 +121,12 @@ describe('@serenity-js/playwright-test', function () {
 
                             expect(outcome).to.be.instanceOf(ExecutionFailedWithAssertionError);
                             expect(outcome.error.name).to.equal('AssertionError');
-                            expect(outcome.error.message).to.equal('Expected false to equal true');
+                            expect(outcome.error.message).to.equal(trimmed`
+                                | Expected false to equal true
+                                |
+                                | Expected boolean: true
+                                | Actual boolean:   false
+                                |`);
                         })
                     ;
                 }));

diff --git a/integration/protractor-jasmine/spec/failing_scenario.spec.ts b/integration/protractor-jasmine/spec/failing_scenario.spec.ts
@@ -1,6 +1,7 @@
 import { expect, ifExitCodeIsOtherThan, logOutput, PickEvent } from '@integration/testing-tools';
 import { AssertionError } from '@serenity-js/core';
 import { SceneFinished, SceneStarts, SceneTagged, TestRunnerDetected } from '@serenity-js/core/lib/events';
+import { trimmed } from '@serenity-js/core/lib/io';
 import { ExecutionFailedWithAssertionError, FeatureTag, Name, ProblemIndication } from '@serenity-js/core/lib/model';
 import { describe, it } from 'mocha';
 
@@ -31,9 +32,12 @@ describe('@serenity-js/jasmine', function () {
                     const error = outcome.error as AssertionError;
 
                     expect(error).to.be.instanceof(AssertionError);
-                    expect(error.message).to.equal('Expected false to be true.');
-                    expect(error.expected).to.equal(true);
-                    expect(error.actual).to.equal(false);
+                    expect(error.message).to.equal(trimmed`
+                        | Expected false to be true.
+                        | 
+                        | Expected boolean: true
+                        | Actual boolean:   false
+                        |`);
                     expect(error.cause.message).to.equal(`Expected false to be true.`);
                 })
             ;

diff --git a/integration/protractor-mocha/spec/failing_scenario.spec.ts b/integration/protractor-mocha/spec/failing_scenario.spec.ts
@@ -31,9 +31,7 @@ describe('@serenity-js/mocha', function () {
 
                     const error = outcome.error as AssertionError;
 
-                    expect(error.message).to.equal('Expected false to be true.');
-                    expect(error.expected).to.equal('true');
-                    expect(error.actual).to.equal('false');
+                    expect(error.message).to.equal(`Expected false to be true.`);
                 })
             ;
         }));

diff --git a/integration/testing-tools/src/expect/index.ts b/integration/testing-tools/src/expect/index.ts
@@ -8,4 +8,6 @@ chai.use(chaiAsPromised);
 chai.use(sinonChai);
 chai.use(assertions);
 
+import 'chai-as-promised';
+
 export const expect = chai.expect;