From c49d9a5e5e71d67972747cf07edd27fae110b86d Mon Sep 17 00:00:00 2001 From: Xander Vedder <34238405+xandervedder@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:24:53 +0200 Subject: [PATCH] feat(schema): add `Pending` status to JSON schema (#2425) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(schema): add `Pending` status to JSON schema See #2424 for the reasoning behind this. * fix: add `MutantStatus.Pending` where necessary Fixes build errors in places where `MutantStatus.Pending` was missing. * feat(metrics-scala): add `Pending` to `metrics-scala` * fix(scala-metrics): fix compile errors * fix: change `⏳` to `⏰` in `getEmojiForStatus` * fix: change the correct emoji * fix: update integration test * feat: add pending to totalMutants metric * fix: run prettier * fix: compile errors * feat(metrics-scala): add pending to totalMutants metric * feat(elements): add pending to tooltip * feat: make pending mutants visible by default The user wants to see mutants if they are present in the file. To have a filter for this would be weird, since the number in the filter would count down as time passes. For this reason `Pending` is not filterable. * fix: reword tooltip --- packages/elements/src/components/file/file.component.ts | 3 ++- packages/elements/src/components/file/file.scss | 1 + .../elements/src/components/mutant-view/mutant-view.ts | 2 +- packages/elements/src/lib/html-helpers.ts | 3 +++ packages/elements/test/helpers/factory.ts | 1 + .../test/unit/components/mutant-view.component.spec.ts | 2 +- .../test/unit/components/state-filter.component.spec.ts | 2 +- .../testResources/csharp-example/mutation-report.json | 2 +- .../circe/src/main/scala/mutationtesting/circe.scala | 1 + .../src/main/scala/mutationtesting/MetricsResult.scala | 7 ++++++- .../src/main/scala/mutationtesting/MutantStatus.scala | 1 + .../src/test/scala/mutationtesting/MetricsResultTest.scala | 5 ++++- packages/metrics/src/calculateMetrics.ts | 4 +++- packages/metrics/src/model/metrics.ts | 4 ++++ packages/metrics/test/unit/calculateMetrics.spec.ts | 4 +++- .../report-schema/src/mutation-testing-report-schema.json | 2 +- 16 files changed, 34 insertions(+), 10 deletions(-) diff --git a/packages/elements/src/components/file/file.component.ts b/packages/elements/src/components/file/file.component.ts index 13992f504..9b2d33382 100644 --- a/packages/elements/src/components/file/file.component.ts +++ b/packages/elements/src/components/file/file.component.ts @@ -38,7 +38,8 @@ export class FileComponent extends LitElement { private codeRef = createRef(); private readonly filtersChanged = (event: MteCustomEvent<'filters-changed'>) => { - this.selectedMutantStates = event.detail as MutantStatus[]; + // Pending is not filterable, but they should still be shown to the user. + this.selectedMutantStates = (event.detail as MutantStatus[]).concat([MutantStatus.Pending]); }; private codeClicked = (ev: MouseEvent) => { diff --git a/packages/elements/src/components/file/file.scss b/packages/elements/src/components/file/file.scss index 246a16973..8f1abc919 100644 --- a/packages/elements/src/components/file/file.scss +++ b/packages/elements/src/components/file/file.scss @@ -3,6 +3,7 @@ @import '../../style/code.scss'; $mutant-themes: ( + 'Pending': theme('colors.neutral.400'), 'Killed': theme('colors.green.600'), 'NoCoverage': theme('colors.orange.500'), 'Survived': theme('colors.red.500'), diff --git a/packages/elements/src/components/mutant-view/mutant-view.ts b/packages/elements/src/components/mutant-view/mutant-view.ts index 7bd8be4b4..25aa2549f 100644 --- a/packages/elements/src/components/mutant-view/mutant-view.ts +++ b/packages/elements/src/components/mutant-view/mutant-view.ts @@ -118,7 +118,7 @@ const COLUMNS: Column[] = [ { key: 'totalMutants', label: 'Total', - tooltip: 'All mutants (valid + invalid + ignored)', + tooltip: 'All mutants (except runtimeErrors + compileErrors)', category: 'number', width: 'large', isBold: true, diff --git a/packages/elements/src/lib/html-helpers.ts b/packages/elements/src/lib/html-helpers.ts index 344897d18..ad649730a 100644 --- a/packages/elements/src/lib/html-helpers.ts +++ b/packages/elements/src/lib/html-helpers.ts @@ -43,6 +43,7 @@ export function getContextClassForStatus(status: MutantStatus) { return 'warning'; case MutantStatus.Ignored: case MutantStatus.RuntimeError: + case MutantStatus.Pending: case MutantStatus.CompileError: return 'secondary'; } @@ -81,6 +82,8 @@ export function getEmojiForStatus(status: MutantStatus) { case MutantStatus.Survived: return renderEmoji('👽', status); case MutantStatus.Timeout: + return renderEmoji('⏰', status); + case MutantStatus.Pending: return renderEmoji('⌛', status); case MutantStatus.RuntimeError: case MutantStatus.CompileError: diff --git a/packages/elements/test/helpers/factory.ts b/packages/elements/test/helpers/factory.ts index 8fa239ca1..7b3e4757c 100644 --- a/packages/elements/test/helpers/factory.ts +++ b/packages/elements/test/helpers/factory.ts @@ -81,6 +81,7 @@ export function createTestMetrics(overrides?: TestMetrics): TestMetrics { export function createMetrics(overrides?: Metrics): Metrics { const defaults: Metrics = { + pending: 0, killed: 0, survived: 0, timeout: 0, diff --git a/packages/elements/test/unit/components/mutant-view.component.spec.ts b/packages/elements/test/unit/components/mutant-view.component.spec.ts index d7d646b5e..3cb0b1c43 100644 --- a/packages/elements/test/unit/components/mutant-view.component.spec.ts +++ b/packages/elements/test/unit/components/mutant-view.component.spec.ts @@ -90,7 +90,7 @@ describe(MutationTestReportMutantViewComponent.name, () => { { key: 'totalMutants', label: 'Total', - tooltip: 'All mutants (valid + invalid + ignored)', + tooltip: 'All mutants (except runtimeErrors + compileErrors)', category: 'number', width: 'large', isBold: true, diff --git a/packages/elements/test/unit/components/state-filter.component.spec.ts b/packages/elements/test/unit/components/state-filter.component.spec.ts index b315adad7..ca7754160 100644 --- a/packages/elements/test/unit/components/state-filter.component.spec.ts +++ b/packages/elements/test/unit/components/state-filter.component.spec.ts @@ -57,7 +57,7 @@ describe(FileStateFilterComponent.name, () => { '👽 Survived (1)', '🙈 NoCoverage (1)', '🤥 Ignored (1)', - '⌛ Timeout (1)', + '⏰ Timeout (1)', '💥 CompileError (1)', '💥 RuntimeError (1)', ]); diff --git a/packages/elements/testResources/csharp-example/mutation-report.json b/packages/elements/testResources/csharp-example/mutation-report.json index a33ed4a93..56400800d 100644 --- a/packages/elements/testResources/csharp-example/mutation-report.json +++ b/packages/elements/testResources/csharp-example/mutation-report.json @@ -41,7 +41,7 @@ "column": 20 } }, - "status": "Survived" + "status": "Pending" }, { "id": 2, diff --git a/packages/metrics-scala/circe/src/main/scala/mutationtesting/circe.scala b/packages/metrics-scala/circe/src/main/scala/mutationtesting/circe.scala index 274f553c8..756404a04 100644 --- a/packages/metrics-scala/circe/src/main/scala/mutationtesting/circe.scala +++ b/packages/metrics-scala/circe/src/main/scala/mutationtesting/circe.scala @@ -127,6 +127,7 @@ object circe { case MutantStatus.CompileError => "CompileError" case MutantStatus.RuntimeError => "RuntimeError" case MutantStatus.Ignored => "Ignored" + case MutantStatus.Pending => "Pending" }) implicit lazy val testDefinitionCodec: Codec[TestDefinition] = diff --git a/packages/metrics-scala/metrics/src/main/scala/mutationtesting/MetricsResult.scala b/packages/metrics-scala/metrics/src/main/scala/mutationtesting/MetricsResult.scala index 839b1131b..55ba31ccd 100644 --- a/packages/metrics-scala/metrics/src/main/scala/mutationtesting/MetricsResult.scala +++ b/packages/metrics-scala/metrics/src/main/scala/mutationtesting/MetricsResult.scala @@ -1,6 +1,9 @@ package mutationtesting sealed trait MetricsResult { + /** The total number of mutants that are pending, meaning that they have been generated but not yet run. + */ + def pending: Int /** At least one test failed while this mutant was active. The mutant is killed. This is what you want, good job! */ @@ -59,7 +62,7 @@ sealed trait MetricsResult { /** All mutants. */ - lazy val totalMutants: Int = totalValid + totalInvalid + ignored + lazy val totalMutants: Int = totalValid + totalInvalid + ignored + pending /** The total percentage of mutants that were killed. Or a {{Double.NaN}} if there are no mutants. */ @@ -75,6 +78,7 @@ sealed trait MetricsResult { sealed trait DirOps extends MetricsResult { val files: Iterable[MetricsResult] + override lazy val pending: Int = sumOfChildrenWith(_.pending) override lazy val killed: Int = sumOfChildrenWith(_.killed) override lazy val timeout: Int = sumOfChildrenWith(_.timeout) override lazy val survived: Int = sumOfChildrenWith(_.survived) @@ -91,6 +95,7 @@ sealed trait DirOps extends MetricsResult { sealed trait FileOps extends MetricsResult { val mutants: Iterable[MetricMutant] + override lazy val pending: Int = countWhere(MutantStatus.Pending) override lazy val killed: Int = countWhere(MutantStatus.Killed) override lazy val timeout: Int = countWhere(MutantStatus.Timeout) override lazy val survived: Int = countWhere(MutantStatus.Survived) diff --git a/packages/metrics-scala/metrics/src/main/scala/mutationtesting/MutantStatus.scala b/packages/metrics-scala/metrics/src/main/scala/mutationtesting/MutantStatus.scala index 7594058c6..03fcfa818 100644 --- a/packages/metrics-scala/metrics/src/main/scala/mutationtesting/MutantStatus.scala +++ b/packages/metrics-scala/metrics/src/main/scala/mutationtesting/MutantStatus.scala @@ -12,5 +12,6 @@ object MutantStatus { case object CompileError extends MutantStatus case object RuntimeError extends MutantStatus case object Ignored extends MutantStatus + case object Pending extends MutantStatus } diff --git a/packages/metrics-scala/metrics/src/test/scala/mutationtesting/MetricsResultTest.scala b/packages/metrics-scala/metrics/src/test/scala/mutationtesting/MetricsResultTest.scala index f5a3bc36e..f3f569ad1 100644 --- a/packages/metrics-scala/metrics/src/test/scala/mutationtesting/MetricsResultTest.scala +++ b/packages/metrics-scala/metrics/src/test/scala/mutationtesting/MetricsResultTest.scala @@ -4,6 +4,7 @@ class MetricsResultTest extends munit.FunSuite { test("all should be 0 on empty root") { val sut = MetricsResultRoot(Nil) + assertEquals(sut.pending, 0) assertEquals(sut.killed, 0) assertEquals(sut.survived, 0) assertEquals(sut.timeout, 0) @@ -31,6 +32,7 @@ class MetricsResultTest extends munit.FunSuite { } private lazy val expectedSet: List[(String, MetricsResult => Number, Number)] = List( + ("pending", _.pending, 1), ("killed", _.killed, 2), ("survived", _.survived, 2), ("timeout", _.timeout, 2), @@ -43,7 +45,7 @@ class MetricsResultTest extends munit.FunSuite { ("totalCovered", _.totalCovered, 6), ("totalValid", _.totalValid, 9), ("totalInvalid", _.totalInvalid, 2), - ("totalMutants", _.totalMutants, 12), + ("totalMutants", _.totalMutants, 13), ("mutationScore", _.mutationScore, (4d / 9d) * 100), ( "mutationScoreBasedOnCoveredCode", @@ -60,6 +62,7 @@ class MetricsResultTest extends munit.FunSuite { MetricsFile( "bar.scala", List( + MetricMutant(MutantStatus.Pending), MetricMutant(MutantStatus.Killed), MetricMutant(MutantStatus.Killed), MetricMutant(MutantStatus.Survived), diff --git a/packages/metrics/src/calculateMetrics.ts b/packages/metrics/src/calculateMetrics.ts index 5fa8c989c..52d1cce19 100644 --- a/packages/metrics/src/calculateMetrics.ts +++ b/packages/metrics/src/calculateMetrics.ts @@ -142,6 +142,7 @@ function countTestFileMetrics(testFile: TestFileModel[]): TestMetrics { function countFileMetrics(fileResult: FileUnderTestModel[]): Metrics { const mutants = fileResult.flatMap((_) => _.mutants); const count = (status: MutantStatus) => mutants.filter((_) => _.status === status).length; + const pending = count(MutantStatus.Pending); const killed = count(MutantStatus.Killed); const timeout = count(MutantStatus.Timeout); const survived = count(MutantStatus.Survived); @@ -155,6 +156,7 @@ function countFileMetrics(fileResult: FileUnderTestModel[]): Metrics { const totalValid = totalUndetected + totalDetected; const totalInvalid = runtimeErrors + compileErrors; return { + pending, killed, timeout, survived, @@ -168,7 +170,7 @@ function countFileMetrics(fileResult: FileUnderTestModel[]): Metrics { totalValid, totalInvalid, mutationScore: totalValid > 0 ? (totalDetected / totalValid) * 100 : DEFAULT_SCORE, - totalMutants: totalValid + totalInvalid + ignored, + totalMutants: totalValid + totalInvalid + ignored + pending, mutationScoreBasedOnCoveredCode: totalValid > 0 ? (totalDetected / totalCovered) * 100 || 0 : DEFAULT_SCORE, }; } diff --git a/packages/metrics/src/model/metrics.ts b/packages/metrics/src/model/metrics.ts index dad3f2106..2ce5e87b8 100644 --- a/packages/metrics/src/model/metrics.ts +++ b/packages/metrics/src/model/metrics.ts @@ -2,6 +2,10 @@ * Container for the metrics of mutation testing */ export interface Metrics { + /** + * The total number of mutants that are pending, meaning that they have been generated but not yet run + */ + pending: number; /** * The total number of mutants that were killed */ diff --git a/packages/metrics/test/unit/calculateMetrics.spec.ts b/packages/metrics/test/unit/calculateMetrics.spec.ts index c91a90d26..226d438d3 100644 --- a/packages/metrics/test/unit/calculateMetrics.spec.ts +++ b/packages/metrics/test/unit/calculateMetrics.spec.ts @@ -128,6 +128,7 @@ describe(calculateMetrics.name, () => { const input: FileResultDictionary = { 'foo.js': createFileResult({ mutants: [ + createMutantResult({ status: MutantStatus.Pending }), createMutantResult({ status: MutantStatus.RuntimeError }), createMutantResult({ status: MutantStatus.Killed }), createMutantResult({ status: MutantStatus.CompileError }), @@ -144,6 +145,7 @@ describe(calculateMetrics.name, () => { const actual = calculateMetrics(input); // Assert + expect(actual.metrics.pending, 'pending').to.eq(1); expect(actual.metrics.killed, 'killed').to.eq(2); expect(actual.metrics.compileErrors, 'compileErrors').eq(1); expect(actual.metrics.runtimeErrors, 'runtimeErrors').eq(1); @@ -153,7 +155,7 @@ describe(calculateMetrics.name, () => { expect(actual.metrics.noCoverage, 'ignored').to.eq(1); expect(actual.metrics.totalCovered, 'totalCovered').to.eq(4); expect(actual.metrics.totalDetected, 'detected').to.eq(3); - expect(actual.metrics.totalMutants, 'mutants').to.eq(8); + expect(actual.metrics.totalMutants, 'mutants').to.eq(9); expect(actual.metrics.totalValid, 'mutants').to.eq(5); expect(actual.metrics.totalInvalid, 'mutants').to.eq(2); expect(actual.metrics.totalUndetected, 'undetected').to.eq(2); diff --git a/packages/report-schema/src/mutation-testing-report-schema.json b/packages/report-schema/src/mutation-testing-report-schema.json index 588a3e7df..9da7a1e03 100644 --- a/packages/report-schema/src/mutation-testing-report-schema.json +++ b/packages/report-schema/src/mutation-testing-report-schema.json @@ -89,7 +89,7 @@ "type": "string", "title": "MutantStatus", "description": "Result of the mutation.", - "enum": ["Killed", "Survived", "NoCoverage", "CompileError", "RuntimeError", "Timeout", "Ignored"] + "enum": ["Killed", "Survived", "NoCoverage", "CompileError", "RuntimeError", "Timeout", "Ignored", "Pending"] }, "statusReason": { "type": "string",