From 0cfcfa1c5fb9eeb04514c368b3256eda7f8650f0 Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Mon, 3 May 2021 20:39:18 +0200 Subject: [PATCH 1/6] feat(report): report test states in the html reporter --- e2e/package-lock.json | 14 +++++++------- e2e/package.json | 2 +- e2e/test/karma-webpack-with-ts/package-lock.json | 14 +++++++------- e2e/test/karma-webpack-with-ts/package.json | 4 ++-- packages/api/package.json | 4 ++-- packages/core/package.json | 4 ++-- packages/test-helpers/package.json | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/e2e/package-lock.json b/e2e/package-lock.json index 81a708c68f..e35cbe6450 100644 --- a/e2e/package-lock.json +++ b/e2e/package-lock.json @@ -10560,18 +10560,18 @@ } }, "mutation-testing-metrics": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/mutation-testing-metrics/-/mutation-testing-metrics-1.6.2.tgz", - "integrity": "sha512-Rhmp/Mvs27RKTJO2cIzViiPN76ERR3CGVNlBZsM0sbdnbIaTuvISlEHFuIZHEQl/+zySX9Z+h/k/b47JxOskiA==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/mutation-testing-metrics/-/mutation-testing-metrics-1.7.2.tgz", + "integrity": "sha512-QFIf/dYdE7MnVkOfotr+97JaMXpG1fRF1T+m0MgbmAdTe+cNf+tY2R030vkC9C5Bdf88VO/rQabHFtL6Cr9v1w==", "dev": true, "requires": { - "mutation-testing-report-schema": "^1.6.0" + "mutation-testing-report-schema": "^1.7.1" } }, "mutation-testing-report-schema": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mutation-testing-report-schema/-/mutation-testing-report-schema-1.6.0.tgz", - "integrity": "sha512-aQlE9Tx7X1MoW2592miQSxPk3oll2GjVCE+0bYWIoSfe9UAKp3TSULd8IDHDXhDgQ4QNSGUARhCObMz3XhNXEw==", + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/mutation-testing-report-schema/-/mutation-testing-report-schema-1.7.1.tgz", + "integrity": "sha512-yElCLI/NOz6QWG6HwRvDMtzND5EkvgC/3KvmgO6rSz8+rlK0EO4OfvnFp8a7r5m/2gEN67JABMnlx76tWBWS1Q==", "dev": true }, "mz": { diff --git a/e2e/package.json b/e2e/package.json index 4a60b2bce4..e9246cadc3 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -31,7 +31,7 @@ "link-parent-bin": "^2.0.0", "load-grunt-tasks": "~5.1.0", "mocha": "~8.2.0", - "mutation-testing-metrics": "~1.6.2", + "mutation-testing-metrics": "1.7.2", "rxjs": "~6.5.3", "semver": "~6.3.0", "ts-jest": "~26.3.0", diff --git a/e2e/test/karma-webpack-with-ts/package-lock.json b/e2e/test/karma-webpack-with-ts/package-lock.json index 9530e8f9b0..ab6b48ad66 100644 --- a/e2e/test/karma-webpack-with-ts/package-lock.json +++ b/e2e/test/karma-webpack-with-ts/package-lock.json @@ -23,17 +23,17 @@ "integrity": "sha512-GSJHHXMGLZDzTRq59IUfL9FCdAlGfqNp/dEa7k7aBaaWD+JKaCjsAk9KYm2V12ItonVaYx2dprN66Zdm1AuBTQ==" }, "mutation-testing-metrics": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/mutation-testing-metrics/-/mutation-testing-metrics-1.6.2.tgz", - "integrity": "sha512-Rhmp/Mvs27RKTJO2cIzViiPN76ERR3CGVNlBZsM0sbdnbIaTuvISlEHFuIZHEQl/+zySX9Z+h/k/b47JxOskiA==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/mutation-testing-metrics/-/mutation-testing-metrics-1.7.2.tgz", + "integrity": "sha512-QFIf/dYdE7MnVkOfotr+97JaMXpG1fRF1T+m0MgbmAdTe+cNf+tY2R030vkC9C5Bdf88VO/rQabHFtL6Cr9v1w==", "requires": { - "mutation-testing-report-schema": "^1.6.0" + "mutation-testing-report-schema": "^1.7.1" } }, "mutation-testing-report-schema": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mutation-testing-report-schema/-/mutation-testing-report-schema-1.6.0.tgz", - "integrity": "sha512-aQlE9Tx7X1MoW2592miQSxPk3oll2GjVCE+0bYWIoSfe9UAKp3TSULd8IDHDXhDgQ4QNSGUARhCObMz3XhNXEw==" + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/mutation-testing-report-schema/-/mutation-testing-report-schema-1.7.1.tgz", + "integrity": "sha512-yElCLI/NOz6QWG6HwRvDMtzND5EkvgC/3KvmgO6rSz8+rlK0EO4OfvnFp8a7r5m/2gEN67JABMnlx76tWBWS1Q==" } } } diff --git a/e2e/test/karma-webpack-with-ts/package.json b/e2e/test/karma-webpack-with-ts/package.json index 1f022191c6..6ebdd57017 100644 --- a/e2e/test/karma-webpack-with-ts/package.json +++ b/e2e/test/karma-webpack-with-ts/package.json @@ -8,8 +8,8 @@ }, "dependencies": { "lit-element": "~2.3.1", - "mutation-testing-metrics": "~1.6.2", - "mutation-testing-report-schema": "~1.6.0" + "mutation-testing-metrics": "1.7.2", + "mutation-testing-report-schema": "1.7.1" }, "devDependencies": { "@types/webpack-env": "~1.15.2" diff --git a/packages/api/package.json b/packages/api/package.json index 56257a8185..985fe5dedb 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -39,8 +39,8 @@ "node": ">=10" }, "dependencies": { - "mutation-testing-report-schema": "~1.6.0", - "mutation-testing-metrics": "~1.6.2", + "mutation-testing-report-schema": "1.7.1", + "mutation-testing-metrics": "1.7.2", "surrial": "~2.0.2", "tslib": "~2.2.0" }, diff --git a/packages/core/package.json b/packages/core/package.json index 29244e79a3..32226d3958 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -72,8 +72,8 @@ "log4js": "~6.2.1", "minimatch": "~3.0.4", "mkdirp": "~1.0.3", - "mutation-testing-elements": "~1.6.2", - "mutation-testing-metrics": "~1.6.2", + "mutation-testing-elements": "1.7.2", + "mutation-testing-metrics": "1.7.2", "npm-run-path": "~4.0.1", "progress": "~2.0.0", "rimraf": "~3.0.0", diff --git a/packages/test-helpers/package.json b/packages/test-helpers/package.json index 961d389dca..36f460b695 100644 --- a/packages/test-helpers/package.json +++ b/packages/test-helpers/package.json @@ -19,7 +19,7 @@ "license": "ISC", "dependencies": { "ajv": "~8.1.0", - "mutation-testing-metrics": "~1.6.2" + "mutation-testing-metrics": "1.7.2" }, "devDependencies": { "@stryker-mutator/api": "4.6.0", From 56cb01e369f832f9863f63c177f29065a418ca14 Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Tue, 4 May 2021 22:10:15 +0200 Subject: [PATCH 2/6] Add test logging for console reporting --- .../core/src/reporters/clear-text-reporter.ts | 61 ++++++++++++++++--- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/packages/core/src/reporters/clear-text-reporter.ts b/packages/core/src/reporters/clear-text-reporter.ts index 5e1406d2eb..0fa19fe42e 100644 --- a/packages/core/src/reporters/clear-text-reporter.ts +++ b/packages/core/src/reporters/clear-text-reporter.ts @@ -1,11 +1,11 @@ import os from 'os'; -import chalk from 'chalk'; +import chalk, { Color } from 'chalk'; import { schema, Position, StrykerOptions } from '@stryker-mutator/api/core'; import { Logger } from '@stryker-mutator/api/logging'; import { commonTokens } from '@stryker-mutator/api/plugin'; import { Reporter } from '@stryker-mutator/api/report'; -import { MetricsResult, MutantModel, TestModel, MutationTestMetricsResult } from 'mutation-testing-metrics'; +import { MetricsResult, MutantModel, TestModel, MutationTestMetricsResult, TestFileModel, TestMetrics, TestStatus } from 'mutation-testing-metrics'; import { tokens } from 'typed-inject'; import { plural } from '../utils/string-utils'; @@ -22,9 +22,13 @@ export class ClearTextReporter implements Reporter { private readonly out: NodeJS.WritableStream = process.stdout; - private writeLine(output?: string) { + private readonly writeLine = (output?: string) => { this.out.write(`${output ?? ''}${os.EOL}`); - } + }; + + private readonly writeDebugLine = (input: string) => { + this.log.debug(input); + }; private configConsoleColor() { if (!this.options.allowConsoleColors) { @@ -33,18 +37,48 @@ export class ClearTextReporter implements Reporter { } public onMutationTestReportReady(_report: schema.MutationTestResult, metrics: MutationTestMetricsResult): void { + this.reportAllTests(metrics); this.reportAllMutants(metrics); this.writeLine(new ClearTextScoreTable(metrics.systemUnderTestMetrics, this.options.thresholds).draw()); } + private reportAllTests(metrics: MutationTestMetricsResult) { + function indent(depth: number) { + return new Array(depth).fill(' ').join(''); + } + + if (metrics.testMetrics) { + const reportTests = ({ file, childResults }: MetricsResult, depth = 0) => { + if (file) { + this.writeLine(`${indent(depth)}${file.name || 'All tests'}`); + file?.tests.forEach((test) => { + switch (test.status) { + case TestStatus.Killing: + this.writeLine( + `${indent(depth + 1)}${this.color('greenBright', '✓')} ${this.color('grey', test.name)} (killed ${test.killedMutants?.length})` + ); + break; + case TestStatus.Covering: + this.writeLine( + `${indent(depth + 1)}${this.color('blueBright', '~')} ${this.color('grey', test.name)} (covered ${test.coveredMutants?.length})` + ); + break; + case TestStatus.NotCovering: + this.writeLine(`${indent(depth + 1)}${this.color('redBright', '✘')} ${this.color('grey', test.name)} (covered 0)`); + break; + } + }); + } + childResults.forEach((childResult) => reportTests(childResult, depth + 1)); + }; + reportTests(metrics.testMetrics); + } + } + private reportAllMutants({ systemUnderTestMetrics }: MutationTestMetricsResult): void { this.writeLine(); let totalTests = 0; - // use these functions in order to preserve the 'this` pointer - const logDebugFn = (input: string) => this.log.debug(input); - const writeLineFn = (input: string) => this.writeLine(input); - const reportMutants = (metrics: MetricsResult[]) => { metrics.forEach((child) => { child.file?.mutants.forEach((result) => { @@ -54,11 +88,11 @@ export class ClearTextReporter implements Reporter { case MutantStatus.Timeout: case MutantStatus.RuntimeError: case MutantStatus.CompileError: - this.reportMutantResult(result, logDebugFn); + this.reportMutantResult(result, this.writeDebugLine); break; case MutantStatus.Survived: case MutantStatus.NoCoverage: - this.reportMutantResult(result, writeLineFn); + this.reportMutantResult(result, this.writeLine); break; default: } @@ -110,6 +144,13 @@ export class ClearTextReporter implements Reporter { return [chalk.cyan(fileName), chalk.yellow(`${position.line}`), chalk.yellow(`${position.column}`)].join(':'); } + private color(color: typeof Color, ...text: string[]) { + if (this.options.clearTextReporter.allowColor) { + return chalk[color](...text); + } + return text.join(''); + } + private logExecutedTests(tests: TestModel[], logImplementation: (input: string) => void) { if (!this.options.clearTextReporter.logTests) { return; From 53ce76f4f380a56f77c4943b20cfcaec56f2ac16 Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Wed, 5 May 2021 14:19:47 +0200 Subject: [PATCH 3/6] Merge deep directories with only 1 entry --- .../core/src/reporters/clear-text-reporter.ts | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/packages/core/src/reporters/clear-text-reporter.ts b/packages/core/src/reporters/clear-text-reporter.ts index 0fa19fe42e..57b2a35647 100644 --- a/packages/core/src/reporters/clear-text-reporter.ts +++ b/packages/core/src/reporters/clear-text-reporter.ts @@ -48,28 +48,31 @@ export class ClearTextReporter implements Reporter { } if (metrics.testMetrics) { - const reportTests = ({ file, childResults }: MetricsResult, depth = 0) => { - if (file) { - this.writeLine(`${indent(depth)}${file.name || 'All tests'}`); - file?.tests.forEach((test) => { - switch (test.status) { - case TestStatus.Killing: - this.writeLine( - `${indent(depth + 1)}${this.color('greenBright', '✓')} ${this.color('grey', test.name)} (killed ${test.killedMutants?.length})` - ); - break; - case TestStatus.Covering: - this.writeLine( - `${indent(depth + 1)}${this.color('blueBright', '~')} ${this.color('grey', test.name)} (covered ${test.coveredMutants?.length})` - ); - break; - case TestStatus.NotCovering: - this.writeLine(`${indent(depth + 1)}${this.color('redBright', '✘')} ${this.color('grey', test.name)} (covered 0)`); - break; - } - }); + const reportTests = (currentResult: MetricsResult, depth = 0) => { + const nameParts: string[] = [currentResult.name]; + while (!currentResult.file && currentResult.childResults.length === 1) { + currentResult = currentResult.childResults[0]; + nameParts.push(currentResult.name); } - childResults.forEach((childResult) => reportTests(childResult, depth + 1)); + this.writeLine(`${indent(depth)}${nameParts.join('/')}`); + currentResult.file?.tests.forEach((test) => { + switch (test.status) { + case TestStatus.Killing: + this.writeLine( + `${indent(depth + 1)}${this.color('greenBright', '✓')} ${this.color('grey', test.name)} (killed ${test.killedMutants?.length})` + ); + break; + case TestStatus.Covering: + this.writeLine( + `${indent(depth + 1)}${this.color('blueBright', '~')} ${this.color('grey', test.name)} (covered ${test.coveredMutants?.length})` + ); + break; + case TestStatus.NotCovering: + this.writeLine(`${indent(depth + 1)}${this.color('redBright', '✘')} ${this.color('grey', test.name)} (covered 0)`); + break; + } + }); + currentResult.childResults.forEach((childResult) => reportTests(childResult, depth + 1)); }; reportTests(metrics.testMetrics); } @@ -137,14 +140,10 @@ export class ClearTextReporter implements Reporter { } private colorSourceFileAndLocation(fileName: string, position: Position): string { - if (!this.options.clearTextReporter.allowColor) { - return `${fileName}:${position.line}:${position.column}`; - } - - return [chalk.cyan(fileName), chalk.yellow(`${position.line}`), chalk.yellow(`${position.column}`)].join(':'); + return [this.color('cyan', fileName), this.color('yellow', position.line), this.color('yellow', position.column)].join(':'); } - private color(color: typeof Color, ...text: string[]) { + private color(color: typeof Color, ...text: unknown[]) { if (this.options.clearTextReporter.allowColor) { return chalk[color](...text); } From 83ced19d87ef35ea1fcb18443d17e7bb53917aa8 Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Wed, 5 May 2021 14:19:55 +0200 Subject: [PATCH 4/6] Unit test test output --- .../reporters/clear-text-reporter.spec.ts | 370 ++++++++++++------ 1 file changed, 248 insertions(+), 122 deletions(-) diff --git a/packages/core/test/unit/reporters/clear-text-reporter.spec.ts b/packages/core/test/unit/reporters/clear-text-reporter.spec.ts index 10be9d4949..7a9b0aef9d 100644 --- a/packages/core/test/unit/reporters/clear-text-reporter.spec.ts +++ b/packages/core/test/unit/reporters/clear-text-reporter.spec.ts @@ -21,37 +21,6 @@ describe(ClearTextReporter.name, () => { }); describe(ClearTextReporter.prototype.onMutationTestReportReady.name, () => { - let report: schema.MutationTestResult; - let mutant: schema.MutantResult; - - beforeEach(() => { - mutant = factory.mutationTestReportSchemaMutantResult({ - id: '1', - location: { start: { line: 2, column: 1 }, end: { line: 2, column: 4 } }, - replacement: 'bar', - mutatorName: 'Math', - }); - report = factory.mutationTestReportSchemaMutationTestResult({ - files: { - 'foo.js': factory.mutationTestReportSchemaFileResult({ - source: '\nfoo\n', - mutants: [mutant], - }), - }, - testFiles: { - 'foo.spec.js': factory.mutationTestReportSchemaTestFile({ - tests: [ - factory.mutationTestReportSchemaTestDefinition({ id: '1', name: 'foo should be bar' }), - factory.mutationTestReportSchemaTestDefinition({ id: '2', name: 'bar should be baz' }), - factory.mutationTestReportSchemaTestDefinition({ id: '3', name: 'baz should be qux' }), - factory.mutationTestReportSchemaTestDefinition({ id: '4', name: 'qux should be quux' }), - factory.mutationTestReportSchemaTestDefinition({ id: '5', name: 'quux should be corge' }), - ], - }), - }, - }); - }); - it('should report the clear text table with correct values', () => { testInjector.options.coverageAnalysis = 'all'; @@ -102,113 +71,270 @@ describe(ClearTextReporter.name, () => { expect(chalk.level).to.eq(0); }); - it('should report a killed mutant to debug', async () => { - mutant.status = MutantStatus.Killed; - mutant.killedBy = ['1']; - act(report); - expect(testInjector.logger.debug).calledWithMatch(sinon.match('1. [Killed] Math')); - expect(testInjector.logger.debug).calledWith(`${chalk.red('- foo')}`); - expect(testInjector.logger.debug).calledWith(`${chalk.green('+ bar')}`); - expect(testInjector.logger.debug).calledWith('Killed by: foo should be bar'); - }); + describe('mutants', () => { + let report: schema.MutationTestResult; + let mutant: schema.MutantResult; + beforeEach(() => { + mutant = factory.mutationTestReportSchemaMutantResult({ + id: '1', + location: { start: { line: 2, column: 1 }, end: { line: 2, column: 4 } }, + replacement: 'bar', + mutatorName: 'Math', + }); + report = factory.mutationTestReportSchemaMutationTestResult({ + files: { + 'foo.js': factory.mutationTestReportSchemaFileResult({ + source: '\nfoo\n', + mutants: [mutant], + }), + }, + testFiles: { + 'foo.spec.js': factory.mutationTestReportSchemaTestFile({ + tests: [ + factory.mutationTestReportSchemaTestDefinition({ id: '1', name: 'foo should be bar' }), + factory.mutationTestReportSchemaTestDefinition({ id: '2', name: 'bar should be baz' }), + factory.mutationTestReportSchemaTestDefinition({ id: '3', name: 'baz should be qux' }), + factory.mutationTestReportSchemaTestDefinition({ id: '4', name: 'qux should be quux' }), + factory.mutationTestReportSchemaTestDefinition({ id: '5', name: 'quux should be corge' }), + ], + }), + }, + }); + }); + it('should report a killed mutant to debug', async () => { + mutant.status = MutantStatus.Killed; + mutant.killedBy = ['1']; + act(report); + expect(testInjector.logger.debug).calledWithMatch(sinon.match('1. [Killed] Math')); + expect(testInjector.logger.debug).calledWith(`${chalk.red('- foo')}`); + expect(testInjector.logger.debug).calledWith(`${chalk.green('+ bar')}`); + expect(testInjector.logger.debug).calledWith('Killed by: foo should be bar'); + }); - it('should report a CompileError mutant to debug', async () => { - mutant.status = MutantStatus.CompileError; - mutant.statusReason = 'could not call bar of undefined'; - act(report); - expect(testInjector.logger.debug).calledWithMatch(sinon.match('1. [CompileError] Math')); - expect(testInjector.logger.debug).calledWith(`${chalk.red('- foo')}`); - expect(testInjector.logger.debug).calledWith(`${chalk.green('+ bar')}`); - expect(testInjector.logger.debug).calledWith('Error message: could not call bar of undefined'); - }); + it('should report a CompileError mutant to debug', async () => { + mutant.status = MutantStatus.CompileError; + mutant.statusReason = 'could not call bar of undefined'; + act(report); + expect(testInjector.logger.debug).calledWithMatch(sinon.match('1. [CompileError] Math')); + expect(testInjector.logger.debug).calledWith(`${chalk.red('- foo')}`); + expect(testInjector.logger.debug).calledWith(`${chalk.green('+ bar')}`); + expect(testInjector.logger.debug).calledWith('Error message: could not call bar of undefined'); + }); - it('should report a NoCoverage mutant to stdout', async () => { - mutant.status = MutantStatus.NoCoverage; - act(report); - expect(stdoutStub).calledWithMatch(sinon.match('1. [NoCoverage] Math')); - expect(stdoutStub).calledWith(`${chalk.red('- foo')}${os.EOL}`); - expect(stdoutStub).calledWith(`${chalk.green('+ bar')}${os.EOL}`); - }); + it('should report a NoCoverage mutant to stdout', async () => { + mutant.status = MutantStatus.NoCoverage; + act(report); + expect(stdoutStub).calledWithMatch(sinon.match('1. [NoCoverage] Math')); + expect(stdoutStub).calledWith(`${chalk.red('- foo')}${os.EOL}`); + expect(stdoutStub).calledWith(`${chalk.green('+ bar')}${os.EOL}`); + }); - it('should report a Survived mutant to stdout', async () => { - mutant.status = MutantStatus.Survived; - act(report); - expect(stdoutStub).calledWithMatch(sinon.match('1. [Survived] Math')); - }); + it('should report a Survived mutant to stdout', async () => { + mutant.status = MutantStatus.Survived; + act(report); + expect(stdoutStub).calledWithMatch(sinon.match('1. [Survived] Math')); + }); - it('should report a Timeout mutant to stdout', async () => { - mutant.status = MutantStatus.Timeout; - act(report); - expect(testInjector.logger.debug).calledWithMatch(sinon.match('1. [Timeout] Math')); - }); + it('should report a Timeout mutant to stdout', async () => { + mutant.status = MutantStatus.Timeout; + act(report); + expect(testInjector.logger.debug).calledWithMatch(sinon.match('1. [Timeout] Math')); + }); - it('should report the tests ran for a Survived mutant to stdout for "perTest" coverage analysis', async () => { - mutant.coveredBy = ['1', '2', '3']; - mutant.status = MutantStatus.Survived; - act(report); - expect(stdoutStub).calledWithExactly(`Tests ran:${os.EOL}`); - expect(stdoutStub).calledWithExactly(` foo should be bar${os.EOL}`); - expect(stdoutStub).calledWithExactly(` bar should be baz${os.EOL}`); - expect(stdoutStub).calledWithExactly(` baz should be qux${os.EOL}`); - }); + it('should report the tests ran for a Survived mutant to stdout for "perTest" coverage analysis', async () => { + mutant.coveredBy = ['1', '2', '3']; + mutant.status = MutantStatus.Survived; + act(report); + expect(stdoutStub).calledWithExactly(`Tests ran:${os.EOL}`); + expect(stdoutStub).calledWithExactly(` foo should be bar${os.EOL}`); + expect(stdoutStub).calledWithExactly(` bar should be baz${os.EOL}`); + expect(stdoutStub).calledWithExactly(` baz should be qux${os.EOL}`); + }); - it('should report the max tests to log and however many more tests', async () => { - testInjector.options.clearTextReporter.maxTestsToLog = 2; - mutant.coveredBy = ['1', '2', '3']; - mutant.status = MutantStatus.Survived; - act(report); - expect(stdoutStub).calledWithExactly(`Tests ran:${os.EOL}`); - expect(stdoutStub).calledWithExactly(` foo should be bar${os.EOL}`); - expect(stdoutStub).calledWithExactly(` bar should be baz${os.EOL}`); - expect(stdoutStub).not.calledWithMatch(sinon.match('baz should be qux')); - expect(stdoutStub).calledWithExactly(` and 1 more test!${os.EOL}`); - }); + it('should report the max tests to log and however many more tests', async () => { + testInjector.options.clearTextReporter.maxTestsToLog = 2; + mutant.coveredBy = ['1', '2', '3']; + mutant.status = MutantStatus.Survived; + act(report); + expect(stdoutStub).calledWithExactly(`Tests ran:${os.EOL}`); + expect(stdoutStub).calledWithExactly(` foo should be bar${os.EOL}`); + expect(stdoutStub).calledWithExactly(` bar should be baz${os.EOL}`); + const allCalls = stdoutStub.getCalls().map((call) => call.args.join('')); + expect(allCalls.filter((call) => call.includes('baz should be qux'))).lengthOf(1, 'Test "baz should be qux" was written more than once'); + expect(stdoutStub).calledWithExactly(` and 1 more test!${os.EOL}`); + }); - it('should report that all tests have ran for a surviving mutant that is static', async () => { - testInjector.options.clearTextReporter.maxTestsToLog = 2; - mutant.static = true; - mutant.status = MutantStatus.Survived; - act(report); - expect(stdoutStub).calledWithExactly(`Ran all tests for this mutant.${os.EOL}`); - }); + it('should report that all tests have ran for a surviving mutant that is static', async () => { + testInjector.options.clearTextReporter.maxTestsToLog = 2; + mutant.static = true; + mutant.status = MutantStatus.Survived; + act(report); + expect(stdoutStub).calledWithExactly(`Ran all tests for this mutant.${os.EOL}`); + }); - it('should not log individual ran tests when logTests is not true', () => { - testInjector.options.clearTextReporter.logTests = false; - mutant.coveredBy = ['1', '2', '3']; - mutant.status = MutantStatus.Survived; - act(report); + it('should not log individual ran tests when logTests is not true', () => { + testInjector.options.clearTextReporter.logTests = false; + mutant.coveredBy = ['1', '2', '3']; + mutant.status = MutantStatus.Survived; + act(report); - expect(process.stdout.write).not.calledWithMatch(sinon.match('Tests ran: ')); - expect(process.stdout.write).not.calledWithMatch(sinon.match('foo should be bar')); - expect(process.stdout.write).not.calledWithMatch(sinon.match('Ran all tests for this mutant.')); - }); + const allCalls = stdoutStub.getCalls().map((call) => call.args.join('')); + expect(process.stdout.write).not.calledWithMatch(sinon.match('Tests ran: ')); + expect(allCalls.filter((call) => call.includes('foo should be bar'))).lengthOf(1, 'Test "foo should be bar" was written more than once'); + expect(process.stdout.write).not.calledWithMatch(sinon.match('Ran all tests for this mutant.')); + }); - it('should correctly report tests run per mutant on avg', () => { - mutant.testsCompleted = 4; - report.files['foo.js'].mutants.push(factory.mutationTestReportSchemaMutantResult({ testsCompleted: 5 })); - report.files['foo.js'].mutants.push(factory.mutationTestReportSchemaMutantResult({ testsCompleted: 1 })); - act(report); + it('should correctly report tests run per mutant on avg', () => { + mutant.testsCompleted = 4; + report.files['foo.js'].mutants.push(factory.mutationTestReportSchemaMutantResult({ testsCompleted: 5 })); + report.files['foo.js'].mutants.push(factory.mutationTestReportSchemaMutantResult({ testsCompleted: 1 })); + act(report); - expect(stdoutStub).calledWithExactly(`Ran 3.33 tests per mutant on average.${os.EOL}`); - }); + expect(stdoutStub).calledWithExactly(`Ran 3.33 tests per mutant on average.${os.EOL}`); + }); - it('should log source file location', () => { - mutant.status = MutantStatus.Survived; - mutant.location.start = { line: 4, column: 6 }; - act(report); + it('should log source file location', () => { + mutant.status = MutantStatus.Survived; + mutant.location.start = { line: 4, column: 6 }; + act(report); - expect(stdoutStub).to.have.been.calledWithMatch(sinon.match(`${chalk.cyan('foo.js')}:${chalk.yellow('4')}:${chalk.yellow('6')}`)); + expect(stdoutStub).to.have.been.calledWithMatch(sinon.match(`${chalk.cyan('foo.js')}:${chalk.yellow('4')}:${chalk.yellow('6')}`)); + }); + + it('should log source file names without colored text when clearTextReporter is not false and allowConsoleColors is false', () => { + testInjector.options.allowConsoleColors = false; + mutant.status = MutantStatus.Survived; + mutant.location.start = { line: 4, column: 6 }; + // Recreate, color setting is set in constructor + sut = testInjector.injector.injectClass(ClearTextReporter); + act(report); + + expect(stdoutStub).calledWithMatch(sinon.match('foo.js:4:6')); + }); }); - it('should log source file names without colored text when clearTextReporter is not false and allowConsoleColors is false', () => { - testInjector.options.allowConsoleColors = false; - mutant.status = MutantStatus.Survived; - mutant.location.start = { line: 4, column: 6 }; - // Recreate, color setting is set in constructor - sut = testInjector.injector.injectClass(ClearTextReporter); - act(report); + describe('tests', () => { + it('should report a big list of tests if file names are unknown', () => { + testInjector.options.clearTextReporter.allowColor = false; + const report = factory.mutationTestReportSchemaMutationTestResult({ + files: { + 'foo.js': factory.mutationTestReportSchemaFileResult({ + mutants: [ + factory.mutationTestReportSchemaMutantResult({ killedBy: ['0'] }), + factory.mutationTestReportSchemaMutantResult({ coveredBy: ['1'] }), + ], + }), + }, + testFiles: { + '': factory.mutationTestReportSchemaTestFile({ + tests: [ + factory.mutationTestReportSchemaTestDefinition({ id: '0', name: 'foo should bar' }), + factory.mutationTestReportSchemaTestDefinition({ id: '1', name: 'baz should qux' }), + factory.mutationTestReportSchemaTestDefinition({ id: '2', name: 'quux should corge' }), + ], + }), + }, + }); + act(report); + expect(stdoutStub).calledWithMatch(sinon.match('All tests')); + expect(stdoutStub).calledWithMatch(sinon.match(' ✓ foo should bar (killed 1)')); + expect(stdoutStub).calledWithMatch(sinon.match(' ~ baz should qux (covered 1)')); + expect(stdoutStub).calledWithMatch(sinon.match(' ✘ quux should corge (covered 0)')); + }); + + it('should report in the correct colors', () => { + testInjector.options.clearTextReporter.allowColor = true; + const report = factory.mutationTestReportSchemaMutationTestResult({ + files: { + 'foo.js': factory.mutationTestReportSchemaFileResult({ + mutants: [ + factory.mutationTestReportSchemaMutantResult({ killedBy: ['0'] }), + factory.mutationTestReportSchemaMutantResult({ coveredBy: ['1'] }), + ], + }), + }, + testFiles: { + '': factory.mutationTestReportSchemaTestFile({ + tests: [ + factory.mutationTestReportSchemaTestDefinition({ id: '0', name: 'foo should bar' }), + factory.mutationTestReportSchemaTestDefinition({ id: '1', name: 'baz should qux' }), + factory.mutationTestReportSchemaTestDefinition({ id: '2', name: 'quux should corge' }), + ], + }), + }, + }); + act(report); + expect(stdoutStub).calledWithMatch(sinon.match('All tests')); + expect(stdoutStub).calledWithMatch(sinon.match(`${chalk.greenBright('✓')} ${chalk.grey('foo should bar')} (killed 1)`)); + expect(stdoutStub).calledWithMatch(sinon.match(`${chalk.blueBright('~')} ${chalk.grey('baz should qux')} (covered 1)`)); + expect(stdoutStub).calledWithMatch(sinon.match(`${chalk.redBright('✘')} ${chalk.grey('quux should corge')} (covered 0)`)); + }); + + it('should report tests per file if file names are unknown', () => { + testInjector.options.clearTextReporter.allowColor = false; + const report = factory.mutationTestReportSchemaMutationTestResult({ + files: { + 'foo.js': factory.mutationTestReportSchemaFileResult({ + mutants: [ + factory.mutationTestReportSchemaMutantResult({ killedBy: ['0'] }), + factory.mutationTestReportSchemaMutantResult({ coveredBy: ['1'] }), + ], + }), + }, + testFiles: { + 'foo.spec.js': factory.mutationTestReportSchemaTestFile({ + tests: [ + factory.mutationTestReportSchemaTestDefinition({ id: '0', name: 'foo should bar' }), + factory.mutationTestReportSchemaTestDefinition({ id: '1', name: 'baz should qux' }), + ], + }), + 'foo.test.js': factory.mutationTestReportSchemaTestFile({ + tests: [factory.mutationTestReportSchemaTestDefinition({ id: '2', name: 'quux should corge' })], + }), + }, + }); + act(report); + expect(stdoutStub).calledWithMatch(sinon.match('All tests')); + expect(stdoutStub).calledWithMatch(sinon.match(' foo.spec.js')); + expect(stdoutStub).calledWithMatch(sinon.match(' ✓ foo should bar (killed 1)')); + expect(stdoutStub).calledWithMatch(sinon.match(' ~ baz should qux (covered 1)')); + expect(stdoutStub).calledWithMatch(sinon.match(' foo.test.js')); + expect(stdoutStub).calledWithMatch(sinon.match(' ✘ quux should corge (covered 0)')); + }); - expect(stdoutStub).calledWithMatch(sinon.match('foo.js:4:6')); + it('should merge deep directories with only one entry', () => { + testInjector.options.clearTextReporter.allowColor = false; + const report = factory.mutationTestReportSchemaMutationTestResult({ + files: { + 'components/foo.js': factory.mutationTestReportSchemaFileResult({ + mutants: [ + factory.mutationTestReportSchemaMutantResult({ killedBy: ['0'] }), + factory.mutationTestReportSchemaMutantResult({ coveredBy: ['1'] }), + ], + }), + }, + testFiles: { + 'test/unit/components/foo.spec.js': factory.mutationTestReportSchemaTestFile({ + tests: [factory.mutationTestReportSchemaTestDefinition({ id: '0', name: 'foo should bar' })], + }), + 'test/unit/components/foo.test.js': factory.mutationTestReportSchemaTestFile({ + tests: [factory.mutationTestReportSchemaTestDefinition({ id: '1', name: 'baz should qux' })], + }), + 'test/integration/components/foo.it.test.js': factory.mutationTestReportSchemaTestFile({ + tests: [factory.mutationTestReportSchemaTestDefinition({ id: '2', name: 'quux should corge' })], + }), + }, + }); + act(report); + expect(stdoutStub).calledWithMatch(sinon.match('All tests')); + expect(stdoutStub).calledWithMatch(sinon.match(' unit/components')); + expect(stdoutStub).calledWithMatch(sinon.match(' foo.spec.js')); + expect(stdoutStub).calledWithMatch(sinon.match(' ✓ foo should bar (killed 1)')); + expect(stdoutStub).calledWithMatch(sinon.match(' foo.test.js')); + expect(stdoutStub).calledWithMatch(sinon.match(' ~ baz should qux (covered 1)')); + expect(stdoutStub).calledWithMatch(sinon.match(' integration/components/foo.it.test.js')); + expect(stdoutStub).calledWithMatch(sinon.match(' ✘ quux should corge (covered 0)')); + }); }); }); From 24d183eefa470cfe87932e9a7cb3427c187dfcad Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Wed, 5 May 2021 16:48:36 +0200 Subject: [PATCH 5/6] Add tests to e2e test --- e2e/test/reporters-e2e/verify/verify.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/e2e/test/reporters-e2e/verify/verify.ts b/e2e/test/reporters-e2e/verify/verify.ts index a1055e6202..94ca94189b 100644 --- a/e2e/test/reporters-e2e/verify/verify.ts +++ b/e2e/test/reporters-e2e/verify/verify.ts @@ -27,6 +27,10 @@ describe('Verify stryker has ran correctly', () => { stdout = await fs.promises.readFile('reports/stdout.txt', 'utf8'); }) + it('should report all tests', () => { + expect(stdout).matches(createTestsRegex()); + }); + it('should report NoCoverage mutants', () => { expect(stdout).matches(createNoCoverageMutantRegex()); }); @@ -51,11 +55,11 @@ describe('Verify stryker has ran correctly', () => { const indexOfClearTextTable = clearTextTableRegex.exec(stdout).index; expect(indexOfSurvivedMutant).lessThan(indexOfClearTextTable); }); - }) - - + }); }); +const createTestsRegex = () => /All tests\s*✓ Add should be able to add two numbers \(killed 2\)/; + const createNoCoverageMutantRegex = () => /#6\.\s*\[NoCoverage\]/; const createSurvivedMutantRegex = () => /#20\.\s*\[Survived\]/; From 909dc4bff9d0f53077d055ec8a07c8859eb81e4f Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Wed, 5 May 2021 19:39:47 +0200 Subject: [PATCH 6/6] Add line numbers --- .../core/src/reporters/clear-text-reporter.ts | 12 +++++++----- .../unit/reporters/clear-text-reporter.spec.ts | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/packages/core/src/reporters/clear-text-reporter.ts b/packages/core/src/reporters/clear-text-reporter.ts index 57b2a35647..e463502ccf 100644 --- a/packages/core/src/reporters/clear-text-reporter.ts +++ b/packages/core/src/reporters/clear-text-reporter.ts @@ -37,6 +37,7 @@ export class ClearTextReporter implements Reporter { } public onMutationTestReportReady(_report: schema.MutationTestResult, metrics: MutationTestMetricsResult): void { + this.writeLine(); this.reportAllTests(metrics); this.reportAllMutants(metrics); this.writeLine(new ClearTextScoreTable(metrics.systemUnderTestMetrics, this.options.thresholds).draw()); @@ -46,6 +47,9 @@ export class ClearTextReporter implements Reporter { function indent(depth: number) { return new Array(depth).fill(' ').join(''); } + const formatTestLine = (test: TestModel, state: string): string => { + return `${this.color('grey', `${test.name}${test.location ? ` [line ${test.location.start.line}]` : ''}`)} (${state})`; + }; if (metrics.testMetrics) { const reportTests = (currentResult: MetricsResult, depth = 0) => { @@ -58,17 +62,15 @@ export class ClearTextReporter implements Reporter { currentResult.file?.tests.forEach((test) => { switch (test.status) { case TestStatus.Killing: - this.writeLine( - `${indent(depth + 1)}${this.color('greenBright', '✓')} ${this.color('grey', test.name)} (killed ${test.killedMutants?.length})` - ); + this.writeLine(`${indent(depth + 1)}${this.color('greenBright', '✓')} ${formatTestLine(test, `killed ${test.killedMutants?.length}`)}`); break; case TestStatus.Covering: this.writeLine( - `${indent(depth + 1)}${this.color('blueBright', '~')} ${this.color('grey', test.name)} (covered ${test.coveredMutants?.length})` + `${indent(depth + 1)}${this.color('blueBright', '~')} ${formatTestLine(test, `covered ${test.coveredMutants?.length}`)}` ); break; case TestStatus.NotCovering: - this.writeLine(`${indent(depth + 1)}${this.color('redBright', '✘')} ${this.color('grey', test.name)} (covered 0)`); + this.writeLine(`${indent(depth + 1)}${this.color('redBright', '✘')} ${formatTestLine(test, 'covered 0')}`); break; } }); diff --git a/packages/core/test/unit/reporters/clear-text-reporter.spec.ts b/packages/core/test/unit/reporters/clear-text-reporter.spec.ts index 7a9b0aef9d..dac07191bc 100644 --- a/packages/core/test/unit/reporters/clear-text-reporter.spec.ts +++ b/packages/core/test/unit/reporters/clear-text-reporter.spec.ts @@ -302,6 +302,21 @@ describe(ClearTextReporter.name, () => { expect(stdoutStub).calledWithMatch(sinon.match(' ✘ quux should corge (covered 0)')); }); + it('should report the line number if they are known', () => { + testInjector.options.clearTextReporter.allowColor = false; + const report = factory.mutationTestReportSchemaMutationTestResult({ + testFiles: { + 'foo.spec.js': factory.mutationTestReportSchemaTestFile({ + tests: [ + factory.mutationTestReportSchemaTestDefinition({ id: '0', name: 'foo should bar', location: { start: { line: 7, column: 1 } } }), + ], + }), + }, + }); + act(report); + expect(stdoutStub).calledWithMatch(sinon.match('✘ foo should bar [line 7] (covered 0)')); + }); + it('should merge deep directories with only one entry', () => { testInjector.options.clearTextReporter.allowColor = false; const report = factory.mutationTestReportSchemaMutationTestResult({