From 85e2bf1ad06b00aa7d917e762ce0c2e0cb964533 Mon Sep 17 00:00:00 2001 From: Mateusz Kruk Date: Sat, 24 Feb 2024 16:03:48 +0100 Subject: [PATCH] feat(console-reporter): test with reporter developement --- .../crew/console-reporter/ConsoleReporter.ts | 6 + .../src/events/GlobalExceptionEncountered.ts | 28 +++ packages/core/src/events/index.ts | 1 + .../SerenityReporterForPlaywrightTest.ts | 204 ++++++++++++------ 4 files changed, 170 insertions(+), 69 deletions(-) create mode 100644 packages/core/src/events/GlobalExceptionEncountered.ts diff --git a/packages/console-reporter/src/stage/crew/console-reporter/ConsoleReporter.ts b/packages/console-reporter/src/stage/crew/console-reporter/ConsoleReporter.ts index fd8263e8190..18e35559dda 100644 --- a/packages/console-reporter/src/stage/crew/console-reporter/ConsoleReporter.ts +++ b/packages/console-reporter/src/stage/crew/console-reporter/ConsoleReporter.ts @@ -5,6 +5,7 @@ import type { DomainEvent} from '@serenity-js/core/lib/events'; import { ActivityRelatedArtifactGenerated, + GlobalExceptionEncountered, InteractionFinished, InteractionStarts, SceneFinished, @@ -267,6 +268,11 @@ export class ConsoleReporter implements ListensToDomainEvents { this.printScene(event.sceneId); } + if (event instanceof GlobalExceptionEncountered && event.outcome instanceof ProblemIndication) { + // this.globalError.recordIfNeeded(event.outcome.error); + this.printTestRunErrorOutcome(event.outcome); + } + if (event instanceof TestRunFinished) { this.summary.recordTestRunFinishedAt(event.timestamp); diff --git a/packages/core/src/events/GlobalExceptionEncountered.ts b/packages/core/src/events/GlobalExceptionEncountered.ts new file mode 100644 index 00000000000..3a5cdc2fb8b --- /dev/null +++ b/packages/core/src/events/GlobalExceptionEncountered.ts @@ -0,0 +1,28 @@ +import type { JSONObject } from 'tiny-types'; + +import type { SerialisedOutcome } from '../model'; +import { Outcome } from '../model'; +import { Timestamp } from '../screenplay'; +import { DomainEvent } from './DomainEvent'; + +/** + * Emitted when the last test in the test suite has finished running + * and it's time for any last-minute reporting activities to take place. + * + * @group Events + */ +export class GlobalExceptionEncountered extends DomainEvent { + static fromJSON(o: JSONObject): GlobalExceptionEncountered { + return new GlobalExceptionEncountered( + Outcome.fromJSON(o.error as SerialisedOutcome), + Timestamp.fromJSON(o.timestamp as string), + ); + } + + constructor( + public readonly error: Outcome, + timestamp?: Timestamp + ) { + super(timestamp); + } +} \ No newline at end of file diff --git a/packages/core/src/events/index.ts b/packages/core/src/events/index.ts index 222befd69c2..107f409c2e1 100644 --- a/packages/core/src/events/index.ts +++ b/packages/core/src/events/index.ts @@ -12,6 +12,7 @@ export * from './BusinessRuleDetected'; export * from './DomainEvent'; export * from './EmitsDomainEvents'; export * from './FeatureNarrativeDetected'; +export * from './GlobalExceptionEncountered'; export * from './InteractionFinished'; export * from './InteractionStarts'; export * from './RetryableSceneDetected'; diff --git a/packages/playwright-test/src/reporter/SerenityReporterForPlaywrightTest.ts b/packages/playwright-test/src/reporter/SerenityReporterForPlaywrightTest.ts index 7195f3a8ca3..a7904a917b4 100644 --- a/packages/playwright-test/src/reporter/SerenityReporterForPlaywrightTest.ts +++ b/packages/playwright-test/src/reporter/SerenityReporterForPlaywrightTest.ts @@ -1,11 +1,27 @@ import type { FullConfig } from '@playwright/test'; -import type { Reporter, Suite, TestCase, TestError, TestResult } from '@playwright/test/reporter'; -import type { ClassDescription, Serenity, StageCrewMember, StageCrewMemberBuilder, Timestamp } from '@serenity-js/core'; -import { LogicError, serenity as reporterSerenityInstance } from '@serenity-js/core'; +import type { + Reporter, + Suite, + TestCase, + TestError, + TestResult, +} from '@playwright/test/reporter'; +import type { + ClassDescription, + Serenity, + StageCrewMember, + StageCrewMemberBuilder, + Timestamp, +} from '@serenity-js/core'; +import { + LogicError, + serenity as reporterSerenityInstance, +} from '@serenity-js/core'; import type { OutputStream } from '@serenity-js/core/lib/adapter'; import type { DomainEvent } from '@serenity-js/core/lib/events'; import * as events from '@serenity-js/core/lib/events'; import { + GlobalExceptionEncountered, InteractionFinished, RetryableSceneDetected, SceneFinished, @@ -16,7 +32,12 @@ import { TestRunnerDetected, TestRunStarts } from '@serenity-js/core/lib/events'; -import { FileSystem, FileSystemLocation, Path, RequirementsHierarchy } from '@serenity-js/core/lib/io'; +import { + FileSystem, + FileSystemLocation, + Path, + RequirementsHierarchy, +} from '@serenity-js/core/lib/io'; import type { CorrelationId, Outcome, Tag } from '@serenity-js/core/lib/model'; import { ArbitraryTag, @@ -40,26 +61,25 @@ import { SERENITY_JS_DOMAIN_EVENTS_ATTACHMENT_CONTENT_TYPE } from './PlaywrightA * See {@apilink SerenityOptions} for usage examples. */ export interface SerenityReporterForPlaywrightTestConfig { - /** - * A list of {@apilink StageCrewMemberBuilder|StageCrewMemberBuilders} or {@apilink StageCrewMember|StageCrewMembers} - * to be instantiated in Playwright Test reporter process and notified of {@apilink DomainEvent|DomainEvents} that occur during the scenario execution. - * Note that the `crew` can also be configured using {@apilink ClassDescription|ClassDescriptions}. - * - * #### Learn more - * - {@apilink SerenityOptions} - * - {@apilink SerenityConfig.crew} - */ + * A list of {@apilink StageCrewMemberBuilder|StageCrewMemberBuilders} or {@apilink StageCrewMember|StageCrewMembers} + * to be instantiated in Playwright Test reporter process and notified of {@apilink DomainEvent|DomainEvents} that occur during the scenario execution. + * Note that the `crew` can also be configured using {@apilink ClassDescription|ClassDescriptions}. + * + * #### Learn more + * - {@apilink SerenityOptions} + * - {@apilink SerenityConfig.crew} + */ crew?: Array; /** - * An output stream to be injected into {@apilink StageCrewMemberBuilder|StageCrewMemberBuilders} - * - * Defaults to [`process.stdout`](https://nodejs.org/api/process.html#process_process_stdout). - * - * #### Learn more - * - {@apilink SerenityConfig.outputStream} - */ + * An output stream to be injected into {@apilink StageCrewMemberBuilder|StageCrewMemberBuilders} + * + * Defaults to [`process.stdout`](https://nodejs.org/api/process.html#process_process_stdout). + * + * #### Learn more + * - {@apilink SerenityConfig.outputStream} + */ outputStream?: OutputStream; } @@ -69,49 +89,64 @@ export interface SerenityReporterForPlaywrightTestConfig { * Serenity/JS {@apilink StageCrewMember|stage crew members}. */ export class SerenityReporterForPlaywrightTest implements Reporter { - private errorParser = new PlaywrightErrorParser(); private sceneIds: Map = new Map(); /** - * @param config - * @param serenity - * Instance of {@apilink Serenity}, specific to the Node process running this Serenity reporter. - * Note that Playwright runs test workers and reporters in separate processes. - * @param requirementsHierarchy - * Root directory of the requirements hierarchy, used to determine capabilities and themes. - */ + * @param config + * @param serenity + * Instance of {@apilink Serenity}, specific to the Node process running this Serenity reporter. + * Note that Playwright runs test workers and reporters in separate processes. + * @param requirementsHierarchy + * Root directory of the requirements hierarchy, used to determine capabilities and themes. + */ constructor( config: SerenityReporterForPlaywrightTestConfig, private readonly serenity: Serenity = reporterSerenityInstance, - private requirementsHierarchy: RequirementsHierarchy = new RequirementsHierarchy(new FileSystem(Path.from(process.cwd()))), + private requirementsHierarchy: RequirementsHierarchy = new RequirementsHierarchy( + new FileSystem(Path.from(process.cwd())), + ), ) { this.serenity.configure(config); } onBegin(config: FullConfig, suite: Suite): void { - this.requirementsHierarchy = new RequirementsHierarchy(new FileSystem(Path.from(config.rootDir))); + this.requirementsHierarchy = new RequirementsHierarchy( + new FileSystem(Path.from(config.rootDir)), + ); this.serenity.announce(new TestRunStarts(this.now())); } onTestBegin(test: TestCase): void { - const currentSceneId = this.serenity.assignNewSceneId(); this.sceneIds.set(test.id, currentSceneId); const scenario = this.scenarioDetailsFrom(test); - - const tags: Tag[] = Tags.from(`${scenario.category.toString()} ${scenario.name.toString().replace(')', '')}`); + + const tags: Tag[] = Tags.from( + `${scenario.category.toString()} ${scenario.name.toString().replace(')', '')}`, + ); this.emit( new SceneStarts(currentSceneId, scenario, this.serenity.currentTime()), - ...this.requirementsHierarchy.requirementTagsFor(scenario.location.path, scenario.category.value) - .map(tag => new SceneTagged(currentSceneId, tag, this.serenity.currentTime())), - new TestRunnerDetected(currentSceneId, new Name('Playwright'), this.serenity.currentTime()), - ...tags.map(tag => new SceneTagged(currentSceneId, tag, this.serenity.currentTime())), + ...this.requirementsHierarchy + .requirementTagsFor(scenario.location.path, scenario.category.value) + .map( + (tag) => + new SceneTagged(currentSceneId, tag, this.serenity.currentTime()), + ), + new TestRunnerDetected( + currentSceneId, + new Name('Playwright'), + this.serenity.currentTime(), + ), + ...tags.map( + (tag) => + new SceneTagged(currentSceneId, tag, this.serenity.currentTime()), + ), ); } @@ -125,7 +160,6 @@ export class SerenityReporterForPlaywrightTest implements Reporter { // } onTestEnd(test: TestCase, result: TestResult): void { - this.announceRetryIfNeeded(test, result); const currentSceneId = this.sceneIds.get(test.id); @@ -133,7 +167,12 @@ export class SerenityReporterForPlaywrightTest implements Reporter { let worstInteractionOutcome: Outcome = new ExecutionSuccessful(); for (const attachment of result.attachments) { - if (! (attachment.contentType === SERENITY_JS_DOMAIN_EVENTS_ATTACHMENT_CONTENT_TYPE && attachment.body)) { + if ( + !( + attachment.contentType === + SERENITY_JS_DOMAIN_EVENTS_ATTACHMENT_CONTENT_TYPE && attachment.body + ) + ) { continue; } @@ -148,7 +187,10 @@ export class SerenityReporterForPlaywrightTest implements Reporter { this.serenity.announce(event); - if (event instanceof InteractionFinished && event.outcome.isWorseThan(worstInteractionOutcome)) { + if ( + event instanceof InteractionFinished && + event.outcome.isWorseThan(worstInteractionOutcome) + ) { worstInteractionOutcome = event.outcome; } } @@ -162,11 +204,25 @@ export class SerenityReporterForPlaywrightTest implements Reporter { this.scenarioDetailsFrom(test), this.determineScenarioOutcome(worstInteractionOutcome, scenarioOutcome), this.now(), - ) + ), ); } - private determineScenarioOutcome(worstInteractionOutcome: Outcome, scenarioOutcome: Outcome): Outcome { + onError(error: TestError): void { + const parsedError = this.errorParser.errorFrom(error); + + this.serenity.announce( + new GlobalExceptionEncountered( + new ExecutionFailedWithError(parsedError), + this.serenity.currentTime() + ), + ); + } + + private determineScenarioOutcome( + worstInteractionOutcome: Outcome, + scenarioOutcome: Outcome, + ): Outcome { if (worstInteractionOutcome instanceof ExecutionFailedWithAssertionError) { return worstInteractionOutcome; } @@ -177,7 +233,6 @@ export class SerenityReporterForPlaywrightTest implements Reporter { } private outcomeFrom(test: TestCase, result: TestResult): Outcome { - const outcome = test.outcome(); if (outcome === 'skipped') { @@ -185,39 +240,42 @@ export class SerenityReporterForPlaywrightTest implements Reporter { } if (outcome === 'unexpected' && result.status === 'passed') { - return new ExecutionFailedWithError(new LogicError(`Scenario expected to fail, but ${ result.status }`)); + return new ExecutionFailedWithError( + new LogicError(`Scenario expected to fail, but ${result.status}`), + ); } if (['failed', 'interrupted', 'timedOut'].includes(result.status)) { - if (test.retries > result.retry) { return new ExecutionIgnored(this.errorParser.errorFrom(result.error)); } - return new ExecutionFailedWithError(this.errorParser.errorFrom(result.error)); + return new ExecutionFailedWithError( + this.errorParser.errorFrom(result.error), + ); } return new ExecutionSuccessful(); } private scenarioDetailsFrom(test: TestCase) { - const [ root_, browserName_, fileName, describeOrItBlockTitle, ...nestedTitles] = test.titlePath(); + const [ + root_, + browserName_, + fileName, + describeOrItBlockTitle, + ...nestedTitles + ] = test.titlePath(); const path = new Path(test.location.file); const scenarioName = nestedTitles.join(' ').trim(); - const featureName = scenarioName - ? describeOrItBlockTitle - : fileName; + const featureName = scenarioName ? describeOrItBlockTitle : fileName; return new ScenarioDetails( new Name(scenarioName || describeOrItBlockTitle), new Category(featureName), - new FileSystemLocation( - path, - test.location.line, - test.location.column, - ), + new FileSystemLocation(path, test.location.line, test.location.column), ); } @@ -226,10 +284,19 @@ export class SerenityReporterForPlaywrightTest implements Reporter { try { await this.serenity.waitForNextCue(); - this.serenity.announce(new TestRunFinished(new ExecutionSuccessful(), this.serenity.currentTime())); - } - catch (error) { - this.serenity.announce(new TestRunFinished(new ExecutionFailedWithError(error), this.serenity.currentTime())); + this.serenity.announce( + new TestRunFinished( + new ExecutionSuccessful(), + this.serenity.currentTime(), + ), + ); + } catch (error) { + this.serenity.announce( + new TestRunFinished( + new ExecutionFailedWithError(error), + this.serenity.currentTime(), + ), + ); throw error; } } @@ -239,8 +306,8 @@ export class SerenityReporterForPlaywrightTest implements Reporter { // reporter.onStdOut(chunk, test, result) private emit(...events: DomainEvent[]): void { - events.forEach(event => { - this.serenity.announce(event) + events.forEach((event) => { + this.serenity.announce(event); }); } @@ -252,13 +319,10 @@ export class SerenityReporterForPlaywrightTest implements Reporter { const currentSceneId = this.sceneIds.get(test.id); this.emit( - new RetryableSceneDetected( - currentSceneId, - this.now(), - ), + new RetryableSceneDetected(currentSceneId, this.now()), new SceneTagged( currentSceneId, - new ArbitraryTag('retried'), // todo: replace with a dedicated tag + new ArbitraryTag('retried'), // todo: replace with a dedicated tag this.now(), ), ); @@ -290,9 +354,11 @@ class PlaywrightErrorParser { ); public errorFrom(testError: TestError): Error { - - const message = testError.message && PlaywrightErrorParser.stripAsciiFrom(testError.message); - let stack = testError.stack && PlaywrightErrorParser.stripAsciiFrom(testError.stack); + const message = + testError.message && + PlaywrightErrorParser.stripAsciiFrom(testError.message); + let stack = + testError.stack && PlaywrightErrorParser.stripAsciiFrom(testError.stack); // TODO: Do I need to process it? // const value = testError.value;