From 404569b64779a8ba6b39da390252ece6d787038b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20St=C3=BChmer?= Date: Mon, 3 Aug 2026 08:34:21 +0200 Subject: [PATCH] test(analyzer): drive every mutation operator through the analyzer end to end Raise the Integration-project-only coverage of the mutation-testing analyzer from 56.7% to 83.4% by exercising the previously unit-tested-only operators, regex infrastructure, reachability, equivalence and test-surface recognizers through the real MutationCoverageAnalyzer and TestSurfaceManifestGenerator pipelines, the same way the existing CultureMutationTests and TestSurfaceManifestGeneratorTests already do for their families. Every new test asserts the exact set of reported diagnostics rather than a bare count, and a couple of assertions are split by target framework where a BCL member genuinely differs between modern .NET and .NET Framework (Trim/TrimStart/TrimEnd overloads, Enumerable.SkipLast availability). --- .../Analyzers/BitwiseAndUnaryMutationTests.cs | 641 +++++++ .../ConditionalAndLogicalMutationTests.cs | 968 +++++++++++ .../GeneratorDriverBridgeIntegrationTests.cs | 218 +++ ...ReachabilityAndEquivalenceMutationTests.cs | 1185 +++++++++++++ .../RegexInfrastructureMutationTests.cs | 1477 +++++++++++++++++ .../Analyzers/StringLinqMathMutationTests.cs | 869 ++++++++++ .../Generation/TestSurfaceEdgeCaseTests.cs | 706 ++++++++ 7 files changed, 6064 insertions(+) create mode 100644 tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/BitwiseAndUnaryMutationTests.cs create mode 100644 tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ConditionalAndLogicalMutationTests.cs create mode 100644 tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/GeneratorDriverBridgeIntegrationTests.cs create mode 100644 tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ReachabilityAndEquivalenceMutationTests.cs create mode 100644 tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/RegexInfrastructureMutationTests.cs create mode 100644 tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/StringLinqMathMutationTests.cs create mode 100644 tests/NetEvolve.FrameShift.Tests.Integration/Generation/TestSurfaceEdgeCaseTests.cs diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/BitwiseAndUnaryMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/BitwiseAndUnaryMutationTests.cs new file mode 100644 index 0000000..37f247e --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/BitwiseAndUnaryMutationTests.cs @@ -0,0 +1,641 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives the bitwise, boolean-literal, increment/decrement and unary-sign operators through +/// end to end, so that the family is proven to produce the +/// diagnostics a consumer sees in its build log instead of merely to construct mutations. +/// +/// +/// +/// One fixture carries every mutation point of the family, each on its own member and its own line, so +/// that a single manifest can leave the whole family unreached and one test can state the exact, +/// complete set of gaps as one text block - the same discipline +/// uses for the culture-sensitivity family. +/// +/// +/// EnumBitwise.Combine and NullableBitwise.AndNullable exist to drive +/// through its enum and its +/// nullable branch, the two decisions a plain operand never exercises. +/// +/// +public class BitwiseAndUnaryMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + private const string TestAssemblyName = "TestAssembly"; + private const string TestFilePath = "FamilyTests.cs"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + /// + /// The compound bitwise assignment covered in isolation by + /// . + /// + private const string AndAssignMemberId = "M:Fixture.Bitwise.AndAssign(System.Int32,System.Int32)~System.Int32"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts on + /// it, because these tests state what the operators report, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. + /// + private const string LowerBoundCount = "1+"; + + /// + /// The text the assertions use for "not a single gap was reported". + /// + private const string NoGaps = ""; + + /// + /// The line feed the expectations are joined with, instead of , so + /// that the very same text is produced on Windows and on Linux. + /// + private const string LineFeed = "\n"; + + private const int AddAssignLine = 15; + private const int BitwiseAndLine = 24; + private const int LeftShiftLine = 29; + private const int AndAssignLine = 34; + private const int LeftShiftAssignLine = 40; + private const int EnumOrLine = 56; + private const int NullableAndLine = 64; + private const int TrueLiteralLine = 72; + private const int FalseLiteralLine = 77; + private const int PreIncrementLine = 85; + private const int PostDecrementLine = 90; + private const int UnaryMinusLine = 98; + private const int UnaryPlusLine = 103; + + private const int CheckedExpressionLine = 15; + private const int UncheckedExpressionLine = 20; + private const int CheckedStatementLine = 25; + private const int UncheckedStatementLine = 33; + + /// + /// All four shapes CheckedContextMutator recognises: the expression form and the statement + /// form, each once as and once as . + /// + private const string CheckedContextSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Overflow + { + public static int CheckedExpression(int value) + { + return checked(value); + } + + public static int UncheckedExpression(int value) + { + return unchecked(value); + } + + public static int CheckedStatement(int value) + { + checked + { + return value; + } + } + + public static int UncheckedStatement(int value) + { + unchecked + { + return value; + } + } + } + """; + + /// + /// One member per mutation point of the whole family. Every member takes a variable operand, never a + /// literal, so that neither the arithmetic-assignment operand check nor the unary + /// constant-preservation check ever discards a mutation this fixture means to exercise. + /// + private const string FamilySource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Arithmetic + { + public static int AddAssign(int value) + { + value += 1; + return value; + } + } + + public static class Bitwise + { + public static int AndOp(int left, int right) + { + return left & right; + } + + public static int ShiftOp(int value) + { + return value << 1; + } + + public static int AndAssign(int value, int mask) + { + value &= mask; + return value; + } + + public static int ShiftAssign(int value) + { + value <<= 1; + return value; + } + } + + public enum Options + { + None = 0, + A = 1, + B = 2, + } + + public static class EnumBitwise + { + public static Options Combine(Options left, Options right) + { + return left | right; + } + } + + public static class NullableBitwise + { + public static int? AndNullable(int? left, int? right) + { + return left & right; + } + } + + public static class Flag + { + public static bool AlwaysTrue() + { + return true; + } + + public static bool AlwaysFalse() + { + return false; + } + } + + public static class Counter + { + public static int PreIncrement(int value) + { + return ++value; + } + + public static int PostDecrement(int value) + { + return value--; + } + } + + public static class Sign + { + public static int Negate(int value) + { + return -value; + } + + public static int Plus(int value) + { + return +value; + } + } + """; + + /// + /// A real TUnit test that calls every member of once, used to build the + /// manifest of with the real collector + /// instead of hand-written member ids. + /// + private const string TestSource = """ + namespace Fixture.Tests; + + using Fixture; + using TUnit.Core; + + public class FamilyTests + { + [Test] + public void ReachesEveryMember() + { + _ = Arithmetic.AddAssign(1); + _ = Bitwise.AndOp(1, 2); + _ = Bitwise.ShiftOp(1); + _ = Bitwise.AndAssign(1, 2); + _ = Bitwise.ShiftAssign(1); + _ = EnumBitwise.Combine(Options.A, Options.B); + _ = NullableBitwise.AndNullable(1, 2); + _ = Flag.AlwaysTrue(); + _ = Flag.AlwaysFalse(); + _ = Counter.PreIncrement(1); + _ = Counter.PostDecrement(1); + _ = Sign.Negate(1); + _ = Sign.Plus(1); + } + } + """; + + /// + /// The complete set of gaps produces when nothing but the anchor is + /// reached: one entry per mutation point, sorted by line and then by message the same way the + /// analyzer itself reports them. The literal 1 operand of AddAssign, ShiftOp and + /// ShiftAssign also gets its own 1 => 0 gap from NumericLiteralMutator, on top + /// of the operator under test at that line - a second, independent operator firing on the same + /// location, not a mistake. + /// + private static readonly (int Line, string DisplayName)[] _everyGap = + [ + (AddAssignLine, "+= => %="), + (AddAssignLine, "+= => *="), + (AddAssignLine, "+= => -="), + (AddAssignLine, "+= => /="), + (AddAssignLine, "1 => 0"), + (BitwiseAndLine, "& => ^"), + (BitwiseAndLine, "& => |"), + (LeftShiftLine, "1 => 0"), + (LeftShiftLine, "<< => >>"), + (AndAssignLine, "&= => ^="), + (AndAssignLine, "&= => |="), + (LeftShiftAssignLine, "1 => 0"), + (LeftShiftAssignLine, "<<= => >>="), + (EnumOrLine, "| => &"), + (EnumOrLine, "| => ^"), + (NullableAndLine, "& => ^"), + (NullableAndLine, "& => |"), + (TrueLiteralLine, "true => false"), + (FalseLiteralLine, "false => true"), + (PreIncrementLine, "++x => --x"), + (PostDecrementLine, "x-- => x++"), + (UnaryMinusLine, "-x => +x"), + (UnaryMinusLine, "-x => x"), + (UnaryPlusLine, "+x => -x"), + (UnaryPlusLine, "+x => x"), + ]; + + /// + /// The flagship case of the family: with only the anchor reached, every member reports every mutation + /// it carries, which is what makes the members above a statement about the operators themselves. + /// + [Test] + public async Task Analyze_UntestedFamily_ReportsEveryMutationOfEveryMember() + { + var compilation = CompilationFactory.Create(FamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(Expect(_everyGap)); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// The same compilation, with Bitwise.AndAssign itself recorded in the manifest: its two gaps + /// vanish and every other member keeps reporting exactly as before, which is what proves the silence + /// is a statement about coverage of that one member rather than about the operator going quiet. + /// + [Test] + public async Task Analyze_CoveredBitwiseAssignment_LeavesOnlyThatMemberSilent() + { + var compilation = CompilationFactory.Create(FamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AndAssignMemberId)]).ConfigureAwait(false); + var expected = _everyGap.Where(gap => gap.Line != AndAssignLine); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(Expect([.. expected])); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// With every member of the fixture collected as a real manifest built from a real test compilation + /// that calls each of them, the whole family goes silent at once - the counter-example that shows the + /// gaps above come from the manifest missing the members, not from the fixture itself being + /// unreachable code. Collecting the manifest instead of hand-writing the member ids also proves the + /// enum and the nullable member ids resolve the way writes them, + /// which a hand-written id could easily get subtly wrong. + /// + [Test] + public async Task Analyze_EveryMemberCovered_ReportsNothing() + { + var compilation = CompilationFactory.Create(FamilySource, ProductionAssemblyName); + var manifest = new InMemoryAdditionalText(CollectManifest(compilation)); + + var diagnostics = await RunAsync(compilation, [manifest]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + _ = await Assert + .That( + DiagnosticAssertions.Describe( + AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint) + ) + ) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// only ever uses += as the assignment kind under test, which + /// leaves the -=/*=//=/%= arms of ArithmeticAssignmentMutator's own + /// name/symbol/token/metadata-name lookups untouched as a source kind - they are only ever + /// exercised as a mutation target. Using *= here as the source exercises every one of + /// those lookups for a different starting point, producing mutations to +=, -=, /= + /// and %= instead. + /// + [Test] + public async Task Analyze_UntestedMultiplyAssignment_ReportsEveryOtherArithmeticAssignment() + { + const string source = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Scaling + { + public static int MultiplyAssign(int value, int factor) + { + value *= factor; + return value; + } + } + """; + const int multiplyAssignLine = 15; + + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (multiplyAssignLine, "*= => +="), + (multiplyAssignLine, "*= => -="), + (multiplyAssignLine, "*= => /="), + (multiplyAssignLine, "*= => %=") + ) + ); + } + } + + /// + /// All four shapes CheckedContextMutator recognises in one fixture: the expression form and the + /// statement form, each swapped in both directions. + /// + [Test] + public async Task Analyze_UntestedCheckedContexts_ReportsAllFourSwaps() + { + var compilation = CompilationFactory.Create(CheckedContextSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (CheckedExpressionLine, "checked(...) => unchecked(...)"), + (UncheckedExpressionLine, "unchecked(...) => checked(...)"), + (CheckedStatementLine, "checked { } => unchecked { }"), + (UncheckedStatementLine, "unchecked { } => checked { }") + ) + ); + } + } + + /// + /// The fixture compiles and is analysed without the analyzer throwing. Roslyn turns an analyzer + /// exception into AD0001 and carries on, so a crash would otherwise look like a diagnostic the + /// tests above simply did not expect. + /// + [Test] + public async Task Analyze_Family_CompilesAndReportsNoAnalyzerFailure() + { + var compilation = CompilationFactory.Create(FamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds the manifest of the way the first pass does: from a real + /// TUnit test compilation that sees the production code as a metadata reference only. + /// + /// The production compilation the tests are written against. + /// The serialized manifest. + private static string CollectManifest(Compilation production) + { + var test = CompilationFactory.Create( + [(TestFilePath, TestSource)], + TestAssemblyName, + includeTUnit: true, + additionalReferences: [production.ToMetadataReference()] + ); + + return TestSurfaceManifestWriter.Write( + TestSurfaceCollector.Collect( + test, + new TUnitTestMethodRecognizer(TUnitTestFrameworkProbe.GetTestAttributeType(test)), + CancellationToken.None + ) + ); + } + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched. + /// + /// + /// Every reference is attributed to one anonymous test whose case count is the lower bound + /// . These tests are about which mutation points are reachable and state + /// nothing about test data, so a lower bound is the honest count - and it keeps FSH0006 silent, + /// which is what lets every exact diagnostic set below stay a statement about the family alone. Every + /// reference is also written as behaviorally verified, for the same reason: these tests are not about + /// the behavioral classification, so FSH0007 has to stay out of the way as well. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.BehavioralReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Describes the reported gaps as one text block, one line per diagnostic, ordered ordinally so that + /// the result does not depend on the order the concurrently running analyzer callbacks reported them + /// in. Several gaps share one location, which is exactly the case a positional order cannot separate. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string Gaps(ImmutableArray diagnostics) + { + var gaps = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint); + + if (gaps.IsEmpty) + { + return NoGaps; + } + + return Join( + DiagnosticAssertions + .Summarise(gaps) + // Several operators can each produce their own mutant at the exact same location (for + // example the two candidates of a rename family), and the order the analyzer enumerates + // those ties in is not guaranteed to be the same across runtimes - .NET Framework's string + // hashing differs from modern .NET, which can reorder anything keyed by a Dictionary + // internally. Sorting ties by message text makes the expectation below deterministic. + .OrderBy(summary => summary.Line) + .ThenBy(summary => summary.Message, StringComparer.Ordinal) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Builds the expectation of a set of gaps, each one a line and the display name of its mutation. + /// + /// The expected gaps. + /// The expected text block, or when nothing is expected. + private static string Expect(params (int Line, string DisplayName)[] gaps) => + gaps.Length == 0 + ? NoGaps + : Join( + gaps.OrderBy(gap => gap.Line) + .ThenBy(gap => gap.DisplayName, StringComparer.Ordinal) + .Select(gap => GapEntry(gap.Line, gap.DisplayName)) + ); + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => $"{id} line {ToText(line)}: {message}"; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string Trivial(ImmutableArray diagnostics) => + DiagnosticAssertions.Describe(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.TrivialMutant)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ConditionalAndLogicalMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ConditionalAndLogicalMutationTests.cs new file mode 100644 index 0000000..5a54444 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ConditionalAndLogicalMutationTests.cs @@ -0,0 +1,968 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives , +/// , +/// , +/// , +/// , +/// , +/// and +/// through +/// end to end, the way CultureMutationTests drives the +/// culture-sensitivity family: a real production fixture, a manifest naming which members are reachable, +/// and the exact set of FSH0001 diagnostics the analyzer reports for it, stated as identifier, +/// 1-based line and full message. +/// +/// +/// +/// Several fixtures deliberately let two operators share one construct instead of avoiding the overlap: +/// a ternary condition that is a plain boolean identifier is offered to +/// for the branches +/// and to for the condition +/// itself, and a comparer argument dropped by +/// is exactly the +/// kind of position a real call site would also drop. Stating every diagnostic such a construct produces, +/// instead of contriving a fixture that avoids the second operator, is what proves the two operators +/// coexist correctly on the very same node. +/// +/// +/// Every other fixture keeps helper members - a custom IComparer<int>, a constant fallback +/// value - free of mutation points of their own, either because their body has no syntax an operator +/// recognises or because the position is a compile-time constant context, so that the exact gap sets below +/// stay a statement about the operator under test alone. +/// +/// +public class ConditionalAndLogicalMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + private const string TernaryPickMemberId = + "M:Fixture.Ternary.Pick(System.Boolean,System.Int32,System.Int32)~System.Int32"; + private const string AreEqualMemberId = "M:Fixture.Numbers.AreEqual(System.Int32,System.Int32)~System.Boolean"; + private const string EnsureLoadedMemberId = "M:Fixture.Cache.EnsureLoaded(System.String)~System.String"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts on + /// it, because these tests state what the operators report, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. + /// + private const string LowerBoundCount = "1+"; + + /// + /// The text the assertions use for "not a single gap was reported". + /// + private const string NoGaps = ""; + + /// + /// The line feed the expectations are joined with, instead of , so + /// that the very same text is produced on Windows and on Linux. + /// + private const string LineFeed = "\n"; + + private const int TernaryLine = 15; + private const int EqualsLine = 15; + private const int NotEqualsLine = 20; + private const int LogicalAndLine = 28; + private const int LogicalOrLine = 33; + private const int NegationRemovalLine = 15; + private const int NegationWrapLine = 25; + private const int NullableNumericLine = 15; + private const int CoalesceLine = 15; + private const int CoalesceAssignLine = 25; + private const int OptionalArgumentLine = 27; + + private const int OperatorEqualsLine = 22; + private const int OperatorNotEqualsLine = 27; + private const int WalletSameLine = 45; + + private const int WrittenTrueLine = 17; + private const int WrittenFalseLine = 22; + private const int UnsetLine = 27; + private const int InitialLine = 35; + private const int MissingLine = 43; + private const int BelowZeroLine = 51; + + private const int BigCountLine = 15; + private const int TicketLine = 20; + private const int RatioLine = 28; + private const int ZeroLine = 33; + + /// + /// Money overloads == and != for the same parameter shape, which is the branch of + /// EqualityOperatorMutator the built-in-operator fixtures above never reach: + /// HasUsableCounterpart has to resolve the bound method as a user-defined operator, find its + /// declared counterpart on the containing type, and match their parameter types before allowing the + /// swap. The two comparisons inside the operator bodies themselves (line 22, line 27) are plain + /// comparisons - the built-in operator, not the user-defined one - so they are + /// two more, unrelated mutation points of the very same fixture. Equals's own this == other + /// (line 32) is a well known member override EquivalenceClassifier already excludes elsewhere, + /// so it contributes nothing here. + /// + private const string UserDefinedEqualitySource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public readonly struct Money + { + public readonly int Amount; + + public Money(int amount) + { + Amount = amount; + } + + public static bool operator ==(Money left, Money right) + { + return left.Amount == right.Amount; + } + + public static bool operator !=(Money left, Money right) + { + return left.Amount != right.Amount; + } + + public override bool Equals(object obj) + { + return obj is Money other && this == other; + } + + public override int GetHashCode() + { + return Amount; + } + } + + public static class Wallet + { + public static bool Same(Money left, Money right) + { + return left == right; + } + } + """; + + /// + /// Every supported underlying type of NullableLiteralMutator the flagship fixture does not + /// already cover: ? on both its written values and both directions out of + /// , ? on a non-default written value, + /// System.Guid? whose only literal-shaped mutation point is , and a + /// negative ? literal, which arrives as its own unary-minus node instead of a + /// bare literal. + /// + private const string NullableLiteralFamilySource = """ + namespace Fixture; + + using System; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Flags + { + public static bool? WrittenTrue() + { + return true; + } + + public static bool? WrittenFalse() + { + return false; + } + + public static bool? Unset() + { + return null; + } + } + + public static class Letters + { + public static char? Initial() + { + return 'A'; + } + } + + public static class Identifiers + { + public static Guid? Missing() + { + return null; + } + } + + public static class Temperatures + { + public static int? BelowZero() + { + return -5; + } + } + """; + + /// + /// Every constant kind of NumericLiteralMutator the flagship fixture does not already cover: an + /// integral type with its own boundary type (, ) and + /// both floating-point directions - the zero-to-one mutant and the negation mutant a non-zero value + /// gets instead. + /// + private const string NumericLiteralFamilySource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Counters + { + public static long BigCount() + { + return 10L; + } + + public static ulong Ticket() + { + return 3UL; + } + } + + public static class Measurements + { + public static double Ratio() + { + return 2.5; + } + + public static float Zero() + { + return 0.0f; + } + } + """; + + /// + /// Ternary.Pick returns a conditional expression on line 15 whose condition is a plain boolean + /// parameter: it is a mutation point of ConditionalExpressionMutator for the branches and of + /// LogicalNegationMutator for the condition itself, since the latter also watches every + /// conditional expression. + /// + private const string ConditionalSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Ternary + { + public static int Pick(bool flag, int a, int b) + { + return flag ? a : b; + } + } + """; + + /// + /// Two equality comparisons on lines 15 and 20 and two logical combinations on lines 28 and 33, all + /// over plain parameters so that no other operator has anything to mutate in the same expressions. + /// + private const string EqualityAndLogicalSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Numbers + { + public static bool AreEqual(int left, int right) + { + return left == right; + } + + public static bool AreDifferent(int left, int right) + { + return left != right; + } + } + + public static class Gate + { + public static bool Both(bool left, bool right) + { + return left && right; + } + + public static bool Either(bool left, bool right) + { + return left || right; + } + } + """; + + /// + /// Toggle.Invert removes an existing negation on line 15, while Guard.ClampNonNegative + /// wraps the condition of an if statement on line 25. Guard.Fallback is a + /// field, so its literal never becomes a mutation point of its own and the + /// method returning it carries none either. + /// + private const string NegationSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Toggle + { + public static bool Invert(bool flag) + { + return !flag; + } + } + + public static class Guard + { + private const int Fallback = 7; + + public static int ClampNonNegative(bool isValid, int value) + { + if (isValid) + { + return value; + } + + return Fallback; + } + } + """; + + /// + /// Quantity.Value returns the literal 5 converted to int? on line 15: a mutation + /// point of two operators at once, since NullableLiteralMutator only cares about the nullable + /// conversion and NumericLiteralMutator only cares about the literal's own (unwrapped) type. + /// + private const string NullableAndNumericSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Quantity + { + public static int? Value() + { + return 5; + } + } + """; + + /// + /// Config.Resolve coalesces two parameters on line 15, and + /// Cache.EnsureLoaded uses the coalescing assignment on line 25. Neither operand carries a + /// literal or any other construct another operator recognises. + /// + private const string NullCoalescingSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Config + { + public static string Resolve(string primary, string secondary) + { + return primary ?? secondary; + } + } + + public static class Cache + { + private static string _value = string.Empty; + + public static string EnsureLoaded(string fallback) + { + _value ??= fallback; + return _value; + } + } + """; + + /// + /// Passthrough is a minimal IComparer<int> whose members carry no mutation point of + /// their own - x.CompareTo(y) is not a construct any operator recognises, and its parameterless + /// object creation has no argument to remove - so that Sorted.Build on line 27 is the only + /// source of a gap: dropping the trailing comparer argument still binds to SortedSet<int>'s + /// parameterless constructor. + /// + private const string OptionalArgumentRemovalSource = """ + namespace Fixture; + + using System.Collections.Generic; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public sealed class Passthrough : IComparer + { + public static readonly Passthrough Instance = new Passthrough(); + + public int Compare(int x, int y) + { + return x.CompareTo(y); + } + } + + public static class Sorted + { + public static SortedSet Build() + { + return new SortedSet(Passthrough.Instance); + } + } + """; + + /// + /// Every fixture of this class, so that one test can prove that all of them compile and that none of + /// them makes the analyzer crash. + /// + /// One factory per fixture. + public static IEnumerable> Fixtures() => + new[] + { + ConditionalSource, + EqualityAndLogicalSource, + NegationSource, + NullableAndNumericSource, + NullCoalescingSource, + OptionalArgumentRemovalSource, + UserDefinedEqualitySource, + NullableLiteralFamilySource, + NumericLiteralFamilySource, + }.Select(source => (Func)(() => source)); + + /// + /// The flagship case of the conditional family: an untested ternary reports all four branch mutations + /// of ConditionalExpressionMutator plus the condition wrap of LogicalNegationMutator, + /// which also watches every conditional expression. + /// + [Test] + public async Task Analyze_UntestedTernaryConditional_ReportsAllFourBranchMutationsAndTheNegationWrap() + { + var compilation = CompilationFactory.Create(ConditionalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (TernaryLine, "c ? a : b => c ? b : a"), + (TernaryLine, "c ? a : b => !c ? a : b"), + (TernaryLine, "c ? a : b => true ? a : b"), + (TernaryLine, "c ? a : b => false ? a : b"), + (TernaryLine, "x => !(x)") + ) + ); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The same compilation, with Ternary.Pick itself recorded in the manifest: the analysis goes + /// completely silent, which is what makes the five gaps above a statement about the two operators + /// rather than about the fixture. + /// + [Test] + public async Task Analyze_CoveredTernaryConditional_ReportsNothing() + { + var compilation = CompilationFactory.Create(ConditionalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(TernaryPickMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(DiagnosticAssertions.Describe(diagnostics)) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// Every one of the four comparisons is offered its single counterpart mutation, with nothing left + /// covered: EqualityOperatorMutator on lines 15 and 20, LogicalOperatorMutator on lines + /// 28 and 33. + /// + [Test] + public async Task Analyze_UntestedEqualityAndLogicalOperators_ReportsEachOperatorSwap() + { + var compilation = CompilationFactory.Create(EqualityAndLogicalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (EqualsLine, "== => !="), + (NotEqualsLine, "!= => =="), + (LogicalAndLine, "&& => ||"), + (LogicalOrLine, "|| => &&") + ) + ); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The counter-example that proves the four gaps above are about coverage, not about the fixture: + /// covering Numbers.AreEqual alone silences only its own line, while the sibling comparison and + /// both logical combinations keep reporting. + /// + [Test] + public async Task Analyze_CoveredEqualityComparison_ReportsOnlyTheRemainingOperators() + { + var compilation = CompilationFactory.Create(EqualityAndLogicalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AreEqualMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect((NotEqualsLine, "!= => =="), (LogicalAndLine, "&& => ||"), (LogicalOrLine, "|| => &&")) + ); + } + } + + /// + /// Both directions of LogicalNegationMutator in one fixture: removing an existing negation on + /// line 15, and wrapping the condition of an if statement that has none on line 25. + /// + [Test] + public async Task Analyze_UntestedNegationRemovalAndWrap_ReportsBothDirections() + { + var compilation = CompilationFactory.Create(NegationSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(Expect((NegationRemovalLine, "!x => x"), (NegationWrapLine, "x => !(x)"))); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The literal 5 converted to int? is a mutation point of two operators at once: + /// NullableLiteralMutator offers moving it to and to the underlying + /// type's default, NumericLiteralMutator offers incrementing and decrementing it - four gaps at + /// the very same location, from two operators that ask different questions about it. + /// + [Test] + public async Task Analyze_UntestedNullableLiteralAndNumericLiteral_ReportsAllFourMutants() + { + var compilation = CompilationFactory.Create(NullableAndNumericSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(ExpectAt(NullableNumericLine, ["5 => null", "5 => 0", "5 => 6", "5 => 4"])); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The two halves of NullCoalescingMutator: a plain ?? expression offers keeping either + /// operand, and a ??= assignment offers becoming a plain assignment. + /// + [Test] + public async Task Analyze_UntestedNullCoalescingAndCoalesceAssignment_ReportsAllThreeMutants() + { + var compilation = CompilationFactory.Create(NullCoalescingSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (CoalesceLine, "a ?? b => a"), + (CoalesceLine, "a ?? b => b"), + (CoalesceAssignLine, "a ??= b => a = b") + ) + ); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// Covering Cache.EnsureLoaded alone silences only the coalescing assignment, leaving the plain + /// ?? expression of the unrelated, uncovered member reported - the same "covering one member + /// never covers another" claim CultureMutationTests makes for the culture family. + /// + [Test] + public async Task Analyze_CoveredCoalesceAssignment_ReportsOnlyTheCoalesceExpressionGaps() + { + var compilation = CompilationFactory.Create(NullCoalescingSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(EnsureLoadedMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(Expect((CoalesceLine, "a ?? b => a"), (CoalesceLine, "a ?? b => b"))); + } + } + + /// + /// The comparer argument of Sorted.Build is dropped because the remaining argument list still + /// binds to SortedSet<int>'s own parameterless constructor - the acceptance criterion + /// OptionalArgumentRemovalMutator exists for. + /// + [Test] + public async Task Analyze_UntestedOptionalArgumentRemoval_ReportsTheComparerRemoval() + { + var compilation = CompilationFactory.Create(OptionalArgumentRemovalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(Expect((OptionalArgumentLine, "Passthrough.Instance => (removed)"))); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// Exercises , see its doc comment for what each gap means. + /// + [Test] + public async Task Analyze_UntestedUserDefinedEquality_ReportsTheSwap() + { + var compilation = CompilationFactory.Create(UserDefinedEqualitySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (OperatorEqualsLine, "== => !="), + (OperatorNotEqualsLine, "!= => =="), + (WalletSameLine, "== => !=") + ) + ); + } + } + + /// + /// Exercises , see its doc comment for what each gap means. + /// + [Test] + public async Task Analyze_UntestedNullableLiteralFamily_ReportsEveryUnderlyingKind() + { + var compilation = CompilationFactory.Create(NullableLiteralFamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (WrittenTrueLine, "true => null"), + (WrittenTrueLine, "true => false"), + (WrittenTrueLine, "true => false"), + (WrittenFalseLine, "false => null"), + (WrittenFalseLine, "false => true"), + (UnsetLine, "null => false"), + (UnsetLine, "null => true"), + (InitialLine, "'A' => null"), + (InitialLine, "'A' => '\\0'"), + (MissingLine, "null => Guid.Empty"), + (BelowZeroLine, "-5 => null"), + (BelowZeroLine, "-5 => 0"), + (BelowZeroLine, "5 => 6"), + (BelowZeroLine, "5 => 4"), + (BelowZeroLine, "-x => +x"), + (BelowZeroLine, "-x => x") + ) + ); + } + } + + /// + /// Exercises , see its doc comment for what each gap means. + /// + [Test] + public async Task Analyze_UntestedNumericLiteralFamily_ReportsEveryNumericKind() + { + var compilation = CompilationFactory.Create(NumericLiteralFamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (BigCountLine, "10L => 11L"), + (BigCountLine, "10L => 9L"), + (TicketLine, "3UL => 4UL"), + (TicketLine, "3UL => 2UL"), + (RatioLine, "2.5 => -2.5"), + (ZeroLine, "0 => 1") + ) + ); + } + } + + /// + /// Every fixture of this class compiles and is analysed without the analyzer throwing. Roslyn turns an + /// analyzer exception into AD0001 and carries on, so a crash would otherwise look like a + /// diagnostic the tests above simply did not expect. + /// + /// The fixture to analyse. + /// A task that completes when the fixture was analysed. + [Test] + [MethodDataSource(nameof(Fixtures))] + public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(string source) + { + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched. + /// + /// + /// Every reference is attributed to one anonymous test whose case count is the lower bound + /// , and every reference is also written as behaviorally verified, so + /// that these coverage-only tests never trip FSH0006 or FSH0007 by accident. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.BehavioralReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Describes the reported gaps as one text block, one line per diagnostic, ordered ordinally so that + /// the result does not depend on the order the concurrently running analyzer callbacks reported them + /// in. Several gaps share one location, which is exactly the case a positional order cannot separate. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string Gaps(ImmutableArray diagnostics) + { + var gaps = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint); + + if (gaps.IsEmpty) + { + return NoGaps; + } + + return Join( + DiagnosticAssertions + .Summarise(gaps) + .OrderBy(summary => summary.Line) + .ThenBy(summary => summary.Message, StringComparer.Ordinal) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Builds the expectation of a set of gaps that all sit on . + /// + /// The 1-based line every gap is reported on. + /// The display names of the expected mutations. + /// The expected text block. + private static string ExpectAt(int line, IEnumerable displayNames) => + Expect([.. displayNames.Select(displayName => (Line: line, DisplayName: displayName))]); + + /// + /// Builds the expectation of a set of gaps, each one a line and the display name of its mutation. + /// + /// The expected gaps. + /// The expected text block, or when nothing is expected. + private static string Expect(params (int Line, string DisplayName)[] gaps) => + gaps.Length == 0 + ? NoGaps + : Join( + gaps.OrderBy(gap => gap.Line) + .ThenBy(gap => gap.DisplayName, StringComparer.Ordinal) + .Select(gap => GapEntry(gap.Line, gap.DisplayName)) + ); + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => $"{id} line {ToText(line)}: {message}"; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string Trivial(ImmutableArray diagnostics) => + DiagnosticAssertions.Describe(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.TrivialMutant)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/GeneratorDriverBridgeIntegrationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/GeneratorDriverBridgeIntegrationTests.cs new file mode 100644 index 0000000..60cea09 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/GeneratorDriverBridgeIntegrationTests.cs @@ -0,0 +1,218 @@ +namespace NetEvolve.FrameShift.Tests.Integration; + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Text; +using NetEvolve.FrameShift.Generation; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using NetEvolve.FrameShift.TestSurface.Bridges; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives through the real +/// pipeline, the way production code actually reaches it, +/// instead of through called directly by a unit test. +/// +/// +/// +/// tests/NetEvolve.Frameshift.Tests.Unit/TestSurface/Bridges/GeneratorDriverBridgeTests.cs already +/// proves every shape the bridge recognises — the fluent driver, .AsSourceGenerator(), an inline +/// array of generators, the local-array limitation, an unrelated RunGenerators look-alike — by +/// calling directly. None of those calls ever runs +/// itself, so the generator's own resolution of the bridge +/// through the shared bridge list of — building the +/// , testing IsApplicable, and feeding the result back +/// into the emitted manifest — never executes under Integration-project coverage at all. This class closes +/// that gap by running the one shape that matters most, the fluent +/// CSharpGeneratorDriver.Create(generator).RunGenerators(...) pattern, through the real generator via +/// , and reading the bridged entry point back out of the emitted manifest. +/// +/// +/// The fixture references the real Microsoft.CodeAnalysis and Microsoft.CodeAnalysis.CSharp +/// assemblies this test project itself already carries as package references, so no extra package is +/// needed to compile a production generator and a test driving it through . +/// The reference set of every other fixture in this suite deliberately excludes them — see the remarks of +/// ReferenceAssemblies — so they are added back explicitly, exactly as the unit-level counterpart +/// does. +/// +/// +public class GeneratorDriverBridgeIntegrationTests +{ + private const string ProductionAssemblyName = "GeneratorProductionAssembly"; + private const string TestAssemblyName = "GeneratorTestAssembly"; + private const string ProductionPath = "Production.cs"; + private const string TestPath = "BridgeTests.cs"; + + private const string ProductionSource = """ + namespace Fixture; + + using Microsoft.CodeAnalysis; + + public sealed class MyIncrementalGenerator : IIncrementalGenerator + { + public void Initialize(IncrementalGeneratorInitializationContext context) + { + } + } + """; + + private const string TestSource = """ + namespace Tests; + + using Microsoft.CodeAnalysis.CSharp; + using TUnit.Core; + + public class BridgeTests + { + [Test] + public void RunsIncrementalGeneratorThroughFluentDriver() + { + CSharpCompilation compilation = CSharpCompilation.Create("Empty"); + + _ = CSharpGeneratorDriver.Create(new Fixture.MyIncrementalGenerator()).RunGenerators(compilation); + } + } + """; + + private const string InitializeId = + "M:Fixture.MyIncrementalGenerator.Initialize(Microsoft.CodeAnalysis.IncrementalGeneratorInitializationContext)"; + + private const string DriverTestId = "M:Tests.BridgeTests.RunsIncrementalGeneratorThroughFluentDriver"; + + [Test] + public async Task Fixtures_BothAssemblies_CompileWithoutErrors() + { + var production = CreateProduction(); + var test = CreateTest(production); + + using (Assert.Multiple()) + { + _ = await Assert + .That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(production))) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + _ = await Assert + .That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(test))) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The real generator resolves , finds it applicable to a + /// compilation that references the Roslyn generator API, and records the bridged Initialize + /// entry point in the manifest it emits — under the very test whose own syntax never spells that method + /// name anywhere. Without this bridge the entry point would never be reachable at all, and the + /// production analyzer would report it as an uncovered mutation point although a test exercises it + /// through the driver. + /// + [Test] + public async Task Generate_FluentGeneratorDriverPattern_RecordsTheBridgedInitializeMethod() + { + var test = CreateTest(CreateProduction()); + var text = Generate(test); + var (success, error, manifest) = Read(text); + + using (Assert.Multiple()) + { + _ = await Assert.That(success).IsTrue(); + _ = await Assert.That(error).IsEqualTo(string.Empty); + _ = await Assert.That(manifest.TestMethodIds.Contains(DriverTestId)).IsTrue(); + _ = await Assert.That(manifest.ReferencesByTest[DriverTestId].Contains(InitializeId)).IsTrue(); + } + } + + /// + /// Two runs of the same compilation through the real generator still agree on the bridged reference, + /// which is what an incremental generator promises: the bridge itself keeps no state between runs, and + /// nothing about resolving its context depends on the order invocations are walked in. + /// + [Test] + public async Task Generate_SameCompilationTwice_AgreeOnTheBridgedReference() + { + var test = CreateTest(CreateProduction()); + + var first = Generate(test); + var second = Generate(test); + + _ = await Assert.That(second).IsEqualTo(first); + } + + private static GeneratorRunner.Output Run(Compilation compilation) => + GeneratorRunner.Run(new TestSurfaceManifestGenerator(), compilation); + + private static string Generate(Compilation compilation) => + Run(compilation).TextOf(TestSurfaceManifestGenerator.HintName); + + /// + /// Parses the generated file the way the MSBuild target does: drop the first and the last line, hand + /// the rest to the reader. + /// + /// The content of the generated source file. + /// Whether the text parsed, the reported error and the parsed manifest. + private static (bool Success, string Error, TestSurfaceManifest Manifest) Read(string generated) + { + var inner = string.Join("\n", Lines(generated).Skip(1).SkipLast(1)) + "\n"; + var success = TestSurfaceManifestReader.TryRead(SourceText.From(inner), out var manifest, out var error); + + return (success, error ?? string.Empty, manifest); + } + + /// + /// Splits the generated text into its lines, dropping the empty remainder behind the trailing line + /// feed, which is the end of the last line and not a line of its own. + /// + /// The generated text. + /// The lines, without their line endings. + private static ImmutableArray Lines(string text) + { + var lines = text.Split('\n').Select(line => line.TrimEnd('\r')).ToList(); + + if (lines.Count > 0 && lines[^1].Length == 0) + { + lines.RemoveAt(lines.Count - 1); + } + + return [.. lines]; + } + + private static CSharpCompilation CreateProduction() => + CompilationFactory.Create( + ProductionSource, + TestFramework.None, + ProductionAssemblyName, + additionalReferences: RoslynReferences(), + filePath: ProductionPath + ); + + private static CSharpCompilation CreateTest(Compilation production) => + CompilationFactory.Create( + TestSource, + TestFramework.TUnit, + TestAssemblyName, + additionalReferences: [production.ToMetadataReference(), .. RoslynReferences()], + filePath: TestPath + ); + + /// + /// The metadata references the fixtures need to see the Roslyn generator API at all: this test project + /// already carries every one of these assemblies as package references, so no additional package is + /// required, but the default reference set every other fixture builds against deliberately excludes + /// them. + /// + /// + /// System.Collections.Immutable is not part of the default reference-assembly set on the + /// .NET Framework target frameworks this project builds for, even though it is on every modern .NET + /// target. and friends expose members typed + /// through it, so without this reference a .NET Framework compilation of the fixture fails with + /// CS0012 - the type is used but its defining assembly was never named. + /// + private static MetadataReference[] RoslynReferences() => + [ + MetadataReference.CreateFromFile(typeof(IIncrementalGenerator).Assembly.Location), + MetadataReference.CreateFromFile(typeof(CSharpGeneratorDriver).Assembly.Location), + MetadataReference.CreateFromFile(typeof(System.Collections.Immutable.ImmutableArray<>).Assembly.Location), + ]; +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ReachabilityAndEquivalenceMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ReachabilityAndEquivalenceMutationTests.cs new file mode 100644 index 0000000..383b775 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ReachabilityAndEquivalenceMutationTests.cs @@ -0,0 +1,1185 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives end to end over call graphs shaped specifically to +/// exercise the reachability closure's dispatch approximation and related-member propagation, and over +/// member shapes shaped to exercise the equivalence classifier's member-level triviality checks. +/// +/// +/// +/// and already prove +/// the closure's plain call-graph walk (direct calls, transitive helpers, local functions, lambdas) and +/// the constant-folding and regex-shorthand halves of the equivalence classifier. What neither of them +/// drives through the real analyzer is virtual and interface dispatch, the accessors a property or event +/// shares its reachability with, or the member-level triviality checks (a throw-only body, code the +/// compiler already proves unreachable, a value assigned to a discard, and a member excluded by name or +/// by attribute). This file fills exactly that gap. +/// +/// +/// Every fixture pairs the member under inspection with Fixture.Reached.Identity, whose body +/// carries no mutation point at all. Naming that member in the manifest is what gives the analyzer a +/// non-empty reachable set - without one it reports an unusable manifest and stays silent about the +/// code - while contributing not a single diagnostic of its own. +/// +/// +public class ReachabilityAndEquivalenceMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts + /// on it, because these tests state what is reachable or trivial, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. + /// + private const string LowerBoundCount = "1+"; + + private const string NoGaps = ""; + private const string LineFeed = "\n"; + + private const string RendererDescribeMemberId = + "M:Fixture.Renderer.Describe(Fixture.IShape,System.Int32)~System.Int32"; + private const string ShapeAreaMemberId = "M:Fixture.IShape.Area(System.Int32)~System.Int32"; + private const string ZooCountLegsMemberId = "M:Fixture.Zoo.CountLegs(Fixture.Animal,System.Int32)~System.Int32"; + private const string ReaderReadMemberId = "M:Fixture.Reader.Read(Fixture.Counter)~System.Int32"; + + private const int SquareAreaLine = 20; + private const int CircleAreaLine = 28; + private const int InterfaceLonerLine = 44; + + private const int AnimalLegsLine = 15; + private const int DogLegsLine = 23; + private const int PuppyLegsLine = 31; + private const int OverrideLonerLine = 47; + + private const int PropertyGetLine = 17; + private const int PropertySetLine = 18; + private const int PropertyLonerLine = 34; + + private const int ThrowLine = 17; + private const int UnreachableStatementLine = 17; + private const int DiscardLine = 15; + + private const int ToStringLine = 26; + private const int ExcludeFromCoverageLine = 35; + private const int GeneratedCodeLine = 44; + private const int ObsoleteLine = 53; + + private const string ThrowOnlyBodyReason = "the containing member does nothing but throw"; + private const string UnreachableStatementReason = "the mutated statement is already unreachable"; + private const string DiscardAssignmentReason = "the mutated value is assigned to a discard"; + private const string WellKnownMemberReason = "the containing member is a well known infrastructure member"; + private const string ObsoleteMemberReason = "the containing member is marked obsolete"; + private const string ConstantFoldingReason = "the mutated expression folds to the same constant"; + private const string ConstantDeclarationReason = "the mutation only changes a compile-time constant"; + private const string DefaultParameterReason = "the mutation only changes a default parameter value"; + private const string CaseLabelReason = "the mutation only changes a compile-time case label"; + + private const string IntWrapperUseIntMemberId = "M:Fixture.IntWrapper.UseInt(System.Int32)~System.Int32"; + private const string SubscriberSubscribeMemberId = + "M:Fixture.Subscriber.Subscribe(Fixture.Bell,System.EventHandler)"; + + private const int GenericMethodLine = 15; + private const int GenericLonerLine = 39; + + private const int EventAddLine = 22; + private const int EventRemoveLine = 27; + private const int EventLonerLine = 45; + + private const int ConstantFoldingLine = 17; + private const int ConstantDeclarationLine = 13; + private const int DefaultParameterLine = 13; + private const int CaseLabelLine = 17; + + /// + /// The five mutants an untested value + 1 carries: the four counterparts the arithmetic + /// operator offers, plus the one-to-zero mutant the numeric literal operator offers for the literal + /// 1. Every equivalence fixture below reuses this exact shape, so that one array describes the + /// mutants of every member under inspection. + /// + private static readonly string[] _additionMutants = ["+ => -", "+ => *", "+ => /", "+ => %", "1 => 0"]; + + /// + /// Square and Circle both implement IShape and are declared nowhere else in the + /// compilation than here, so Renderer.Describe calling through the interface variable is the + /// only route a test can take to either of them. Loner.Unrelated implements nothing and is + /// called by nobody, so it stays a gap in every test of this fixture. + /// + private const string InterfaceDispatchSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public interface IShape + { + int Area(int side); + } + + public sealed class Square : IShape + { + public int Area(int side) + { + return side * side; + } + } + + public sealed class Circle : IShape + { + public int Area(int side) + { + return side * side; + } + } + + public static class Renderer + { + public static int Describe(IShape shape, int side) + { + return shape.Area(side); + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// Puppy overrides Dog, which overrides the virtual Animal.Legs: a call through + /// the base class reference has to walk two links of the override chain to reach both. Loner + /// carries no relation to Animal at all. + /// + private const string VirtualOverrideSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public class Animal + { + public virtual int Legs(int extra) + { + return 4 + extra; + } + } + + public class Dog : Animal + { + public override int Legs(int extra) + { + return 4 + extra; + } + } + + public class Puppy : Dog + { + public override int Legs(int extra) + { + return 4 + extra; + } + } + + public static class Zoo + { + public static int CountLegs(Animal animal, int extra) + { + return animal.Legs(extra); + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// Reader.Read only ever reads Counter.Value, never assigns it, yet both accessors of + /// the property are separate declarations of their own and share the reachability of the property + /// they belong to. + /// + private const string PropertyAccessorSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public sealed class Counter + { + private int _value; + + public int Value + { + get { return _value + 1; } + set { _value = value + 2; } + } + } + + public static class Reader + { + public static int Read(Counter counter) + { + return counter.Value; + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// Thrower.AlwaysFails never returns, so no test could ever observe the difference between + /// the mutated argument and the original one. + /// + private const string ThrowOnlySource = """ + namespace Fixture; + + using System; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Thrower + { + public static int AlwaysFails(int value) + { + throw new InvalidOperationException((value + 1).ToString()); + } + } + """; + + /// + /// The unconditional return value; always leaves the method, so the compiler proves the + /// statement after it - the one carrying the mutation point - can never execute, and reports + /// CS0162 for it. + /// + private const string UnreachableStatementSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class DeadCode + { + public static int Dead(int value) + { + return value; + + return value + 1; + } + } + """; + + /// + /// Discarder.Discard assigns the mutated value to _, so no test could ever observe it + /// regardless of what the value becomes. + /// + private const string DiscardAssignmentSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Discarder + { + public static void Discard(int value) + { + _ = value + 1; + } + } + """; + + /// + /// Four members, two different reasons a mutant of them never turns into a gap. Widget.ToString + /// and Deprecated.Old still get mutated - the mutant generator has no rule excluding a well + /// known infrastructure member or an obsolete one - so it is the equivalence classifier that has to + /// recognise both and report every mutant as trivial. Excluded.Ignored and Generated.Machine + /// are excluded one step earlier: the mutant generator itself never descends into a declaration + /// carrying [ExcludeFromCodeCoverage] or [GeneratedCode], so neither line produces a + /// mutation point - and therefore no diagnostic at all - in the first place. + /// + private const string ExcludedMemberSource = """ + namespace Fixture; + + using System; + using System.CodeDom.Compiler; + using System.Diagnostics.CodeAnalysis; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public sealed class Widget + { + private readonly int _value; + + public Widget(int value) + { + _value = value; + } + + public override string ToString() + { + return (_value + 1).ToString(); + } + } + + public static class Excluded + { + [ExcludeFromCodeCoverage] + public static int Ignored(int value) + { + return value + 1; + } + } + + public static class Generated + { + [GeneratedCode("tool", "1.0")] + public static int Machine(int value) + { + return value + 1; + } + } + + public static class Deprecated + { + [Obsolete] + public static int Old(int value) + { + return value + 1; + } + } + """; + + /// + /// Ops.Mark<T> ignores its type parameter entirely, yet IntWrapper.UseInt only ever + /// calls the <int> instantiation while StringWrapper.UseString calls the + /// <string> one. Both constructed methods normalize to the very same original definition, + /// so a test naming only the first instantiation still covers the shared body. + /// + private const string GenericMethodSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Ops + { + public static int Mark(int marker) + { + return marker + 1; + } + } + + public static class IntWrapper + { + public static int UseInt(int value) + { + return Ops.Mark(value); + } + } + + public static class StringWrapper + { + public static string UseString(int value) + { + return Ops.Mark(value).ToString(); + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// Subscriber.Subscribe only ever adds a handler to Bell.Rung, never removes one, yet the + /// remove accessor is a separate declaration that shares the reachability of the event it + /// belongs to, exactly like a property's setter shares the reachability of a getter-only caller. + /// + private const string EventAccessorSource = """ + namespace Fixture; + + using System; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public sealed class Bell + { + private int _count; + private EventHandler? _rung; + + public event EventHandler Rung + { + add + { + _count = _count + 1; + _rung += value; + } + remove + { + _count = _count - 1; + _rung -= value; + } + } + } + + public static class Subscriber + { + public static void Subscribe(Bell bell, EventHandler handler) + { + bell.Rung += handler; + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// 4 - Two carries five mutation points: four arithmetic swaps of - and two increments + /// and decrements of the literal 4, which the numeric literal operator still offers because a + /// normal method body is not a constant-only context. Exactly one of the arithmetic swaps, the + /// division, folds to the very same value the original subtraction does, which is what the equivalence + /// classifier's constant folding check has to recognise among the other four candidates that do not. + /// + private const string ConstantFoldingSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Constants + { + private const int Two = 2; + + public static int Ratio() + { + return 4 - Two; + } + } + """; + + /// + /// 1 + 2 sits inside a field initializer, a position the numeric + /// literal operator already refuses to touch, but the arithmetic operator carries no such check of its + /// own. The equivalence classifier is what has to recognise every one of its four mutants as trivial, + /// regardless of the value each one folds to. + /// + private const string ConstantDeclarationSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Config + { + public const int Sum = 1 + 2; + } + """; + + /// + /// 1 + 2 sits inside a default parameter value, the same shape as + /// one syntax position over. + /// + private const string DefaultParameterSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Options + { + public static int WithDefault(int amount = 1 + 2) + { + return amount; + } + } + """; + + /// + /// 1 + 2 sits inside a classic case label, the third and last constant-only position + /// exercised in this file. + /// + private const string CaseLabelSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Classifier + { + public static int Describe(int value) + { + switch (value) + { + case 1 + 2: + return value; + default: + return value; + } + } + } + """; + + /// + /// Every fixture of this class, so that one test can prove that all of them compile and that none of + /// them makes the analyzer crash. + /// + /// One factory per fixture. + public static IEnumerable> Fixtures() => + new[] + { + InterfaceDispatchSource, + VirtualOverrideSource, + PropertyAccessorSource, + ThrowOnlySource, + UnreachableStatementSource, + DiscardAssignmentSource, + ExcludedMemberSource, + GenericMethodSource, + EventAccessorSource, + ConstantFoldingSource, + ConstantDeclarationSource, + DefaultParameterSource, + CaseLabelSource, + }.Select(source => (Func)(() => source)); + + /// + /// A test calling through an IShape variable reaches every implementation declared in the + /// compilation, not only the one a concrete call site happens to construct: the dispatch + /// approximation adds every implementation of the interface member it resolved to. + /// + [Test] + public async Task Analyze_TestCallsThroughInterfaceVariable_ReachesEveryImplementation() + { + var compilation = CompilationFactory.Create(InterfaceDispatchSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(RendererDescribeMemberId)]).ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([InterfaceLonerLine]); + _ = await Assert.That(lines.Contains(SquareAreaLine)).IsFalse(); + _ = await Assert.That(lines.Contains(CircleAreaLine)).IsFalse(); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// The dispatch approximation also runs when the manifest names the interface member itself, without + /// any caller in the manifest at all: a test that calls an interface member directly is the most + /// common shape there is, and the seed has to be treated exactly like a member reached transitively. + /// + [Test] + public async Task Analyze_ManifestNamesTheInterfaceMemberDirectly_StillReachesEveryImplementation() + { + var compilation = CompilationFactory.Create(InterfaceDispatchSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(ShapeAreaMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(GapLines(diagnostics)).IsEquivalentTo([InterfaceLonerLine]); + } + } + + /// + /// A call through a base class reference reaches every override in the chain, not only the one + /// closest to the declared type: Puppy.Legs overrides Dog.Legs, which overrides the + /// virtual Animal.Legs, so walking the override chain twice is what makes it reachable at all. + /// + [Test] + public async Task Analyze_TestCallsThroughBaseClassVariable_ReachesEveryOverrideInTheChain() + { + var compilation = CompilationFactory.Create(VirtualOverrideSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(ZooCountLegsMemberId)]).ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([OverrideLonerLine]); + _ = await Assert.That(lines.Contains(AnimalLegsLine)).IsFalse(); + _ = await Assert.That(lines.Contains(DogLegsLine)).IsFalse(); + _ = await Assert.That(lines.Contains(PuppyLegsLine)).IsFalse(); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A test reading a property never assigns it, yet the setter becomes reachable as well: the getter + /// and the setter are separate declarations, and both share the reachability of the property they + /// belong to once the property itself is reached. + /// + [Test] + public async Task Analyze_TestReadsAProperty_MakesTheSetterReachableAsWell() + { + var compilation = CompilationFactory.Create(PropertyAccessorSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(ReaderReadMemberId)]).ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([PropertyLonerLine]); + _ = await Assert.That(lines.Contains(PropertyGetLine)).IsFalse(); + _ = await Assert.That(lines.Contains(PropertySetLine)).IsFalse(); + } + } + + /// + /// A member whose body does nothing but throw can never let a test observe a mutated value, so every + /// mutant of it is reported as trivial instead of as a gap - even though nothing in the manifest + /// reaches the member at all. + /// + [Test] + public async Task Analyze_MemberBodyThrowsUnconditionally_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(ThrowOnlySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((ThrowLine, ThrowOnlyBodyReason, _additionMutants))); + } + } + + /// + /// A statement the compiler already proves unreachable, here because the statement ahead of it + /// always returns, can never run at all, so every mutant inside it is trivial rather than a gap. + /// + [Test] + public async Task Analyze_StatementAfterAnUnconditionalReturn_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(UnreachableStatementSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((UnreachableStatementLine, UnreachableStatementReason, _additionMutants))); + } + } + + /// + /// A value assigned to the discard _ is thrown away in the very same statement, so no test + /// could ever tell the mutant from the original code. + /// + [Test] + public async Task Analyze_MutatedValueAssignedToADiscard_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(DiscardAssignmentSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((DiscardLine, DiscardAssignmentReason, _additionMutants))); + } + } + + /// + /// A well known infrastructure member and an obsolete one are still mutated, and the equivalence + /// classifier is what turns every one of their mutants into a trivial diagnostic instead of a gap. A + /// member excluded by [ExcludeFromCodeCoverage] or by [GeneratedCode] never reaches that + /// point: the mutant generator excludes the declaration outright, so neither line produces any + /// diagnostic, trivial or otherwise. + /// + [Test] + public async Task Analyze_WellKnownAndObsoleteMembers_ReportEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(ExcludedMemberSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + var lines = AllLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo( + ExpectTrivial( + (ToStringLine, WellKnownMemberReason, _additionMutants), + (ObsoleteLine, ObsoleteMemberReason, _additionMutants) + ) + ); + _ = await Assert.That(lines.Contains(ExcludeFromCoverageLine)).IsFalse(); + _ = await Assert.That(lines.Contains(GeneratedCodeLine)).IsFalse(); + } + } + + /// + /// Ops.Mark<T> ignores T entirely, and a test naming only the caller of its + /// <int> instantiation still covers the member's own mutation point: the reachable set + /// normalizes a constructed generic method back to its original definition, so it does not matter which + /// instantiation a test happens to go through. + /// + [Test] + public async Task Analyze_TestCallsOneInstantiationOfAGenericMethod_ReachesTheSharedDefinition() + { + var compilation = CompilationFactory.Create(GenericMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(IntWrapperUseIntMemberId)]).ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([GenericLonerLine]); + _ = await Assert.That(lines.Contains(GenericMethodLine)).IsFalse(); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A test that only ever adds a handler to an event never removes one, yet the remove accessor + /// becomes reachable as well: like a property's getter and setter, both accessors are separate + /// declarations that share the reachability of the event they belong to. + /// + [Test] + public async Task Analyze_TestAddsAnEventHandler_MakesTheRemoveAccessorReachableAsWell() + { + var compilation = CompilationFactory.Create(EventAccessorSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(SubscriberSubscribeMemberId)]) + .ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([EventLonerLine]); + _ = await Assert.That(lines.Contains(EventAddLine)).IsFalse(); + _ = await Assert.That(lines.Contains(EventRemoveLine)).IsFalse(); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// Of the four arithmetic mutants of an untested subtraction, one - the division - folds to the very + /// same value the original does, and the equivalence classifier has to single it out as trivial while + /// leaving the other three, and the two literal increments and decrements of the numeric literal + /// operator, as genuine gaps. + /// + [Test] + public async Task Analyze_ArithmeticMutantFoldsToTheSameConstant_IsTrivialConstantFolding() + { + var compilation = CompilationFactory.Create(ConstantFoldingSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(ExpectGaps(ConstantFoldingLine, "- => +", "- => *", "- => %", "4 => 5", "4 => 3")); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((ConstantFoldingLine, ConstantFoldingReason, ["- => /"]))); + } + } + + /// + /// The arithmetic operator carries no check of its own for a field + /// initializer, unlike the numeric literal operator, so all four of its mutants are created; the + /// equivalence classifier is what has to prove every one of them trivial regardless of the value it + /// folds to. + /// + [Test] + public async Task Analyze_ArithmeticMutationInsideAConstFieldInitializer_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(ConstantDeclarationSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo( + ExpectTrivial( + (ConstantDeclarationLine, ConstantDeclarationReason, ["+ => -", "+ => *", "+ => /", "+ => %"]) + ) + ); + } + } + + /// + /// The same shape as a field initializer, one syntax position over: a default + /// parameter value can only ever be a compile-time constant, so no test could ever observe a mutant of + /// it either. + /// + [Test] + public async Task Analyze_ArithmeticMutationInsideADefaultParameterValue_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(DefaultParameterSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo( + ExpectTrivial( + (DefaultParameterLine, DefaultParameterReason, ["+ => -", "+ => *", "+ => /", "+ => %"]) + ) + ); + } + } + + /// + /// The third and last constant-only position exercised in this file: a classic case label can + /// only ever be a compile-time constant as well. + /// + [Test] + public async Task Analyze_ArithmeticMutationInsideACaseLabel_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(CaseLabelSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((CaseLabelLine, CaseLabelReason, ["+ => -", "+ => *", "+ => /", "+ => %"]))); + } + } + + /// + /// Every fixture of this class compiles and is analysed without the analyzer throwing. Roslyn turns + /// an analyzer exception into AD0001 and carries on, so a crash would otherwise look like a + /// diagnostic the tests above simply did not expect. + /// + /// The fixture to analyse. + /// A task that completes when the fixture was analysed. + [Test] + [MethodDataSource(nameof(Fixtures))] + public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(string source) + { + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched, and as behaviorally verified as well, so that FSH0007 never + /// shows up in a diagnostic set that is meant to be a statement about reachability or triviality alone. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.BehavioralReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Reduces the reported gaps to their distinct 1-based lines, so that a reachability assertion states + /// which lines are gaps without depending on how many mutants a single arithmetic expression carries. + /// + /// All diagnostics of a run. + /// The distinct lines carrying an FSH0001 diagnostic, possibly empty. + private static ImmutableArray GapLines(ImmutableArray diagnostics) => + [ + .. DiagnosticAssertions + .Summarise(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint)) + .Select(summary => summary.Line) + .Distinct() + .OrderBy(line => line), + ]; + + /// + /// Collects the distinct 1-based lines of every diagnostic of any kind, used to prove that a line + /// produces no diagnostic at all rather than merely no gap. + /// + /// All diagnostics of a run. + /// The distinct lines carrying any diagnostic, possibly empty. + private static ImmutableArray AllLines(ImmutableArray diagnostics) => + [ + .. DiagnosticAssertions + .Summarise(diagnostics) + .Select(summary => summary.Line) + .Distinct() + .OrderBy(line => line), + ]; + + /// + /// Describes every reported gap as one text block, one line per diagnostic, ordered ordinally so that + /// the result does not depend on the order the concurrently running analyzer callbacks reported them in. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string Gaps(ImmutableArray diagnostics) + { + var gaps = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint); + + return gaps.IsEmpty + ? NoGaps + : Join( + DiagnosticAssertions.Summarise(gaps).Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Describes every reported trivial mutant as one text block, exactly like does + /// for the coverage gaps. + /// + /// All diagnostics of a run. + /// The described trivial mutants, or when there is none. + private static string Trivial(ImmutableArray diagnostics) + { + var trivial = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.TrivialMutant); + + return trivial.IsEmpty + ? NoGaps + : Join( + DiagnosticAssertions + .Summarise(trivial) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Builds the expectation of a set of trivial mutants, each group sharing one line and one reason and + /// contributing one entry per display name in or an equivalent array. + /// + /// The expected groups. + /// The expected text block. + private static string ExpectTrivial(params (int Line, string Reason, string[] DisplayNames)[] groups) => + Join( + groups.SelectMany(group => + group.DisplayNames.Select(displayName => TrivialEntry(group.Line, displayName, group.Reason)) + ) + ); + + /// + /// Builds the described trivial mutant, spelling out the message + /// formats. + /// + /// The 1-based line the mutant is reported on. + /// The display name of the mutation. + /// The reason clause the classifier attached. + /// The described trivial mutant. + private static string TrivialEntry(int line, string displayName, string reason) => + Entry( + DiagnosticIds.TrivialMutant, + line, + "Mutation '" + displayName + "' cannot change observable behaviour (" + reason + ")" + ); + + /// + /// Builds the expectation of a set of gaps that all sit on , spelling out the + /// message formats. + /// + /// The 1-based line every gap is reported on. + /// The display names of the expected mutations. + /// The expected text block. + private static string ExpectGaps(int line, params string[] displayNames) => + Join(displayNames.Select(displayName => GapEntry(line, displayName))); + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => $"{id} line {ToText(line)}: {message}"; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/RegexInfrastructureMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/RegexInfrastructureMutationTests.cs new file mode 100644 index 0000000..c1167f6 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/RegexInfrastructureMutationTests.cs @@ -0,0 +1,1477 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Mutations.Operators; +using NetEvolve.FrameShift.Mutations.RegularExpressions; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives the regular expression infrastructure - the tokenizer and its nested scanner and class frame, +/// the token, the pattern locator, the pattern validity check and the pattern cache - together with the +/// backreference, character-class, escape and quantifier operators through +/// end to end. +/// +/// +/// +/// already proves the anchor, quantifier, group and +/// alternation operators, plus the character-class shorthand swap and the lookaround negation, reach the +/// build log as FSH0001. This class extends the same family with constructs that file does not +/// drive: a numbered backreference next to the plain capturing groups it renumbers, a character-class +/// range and its negation, a standalone class member and the dot/class equivalence in both directions, the +/// escaped-dot-to-any-character rewrite, and every shape of quantifier bound - the exact {n} form, +/// the open ended {n,} form and the full {n,m} form - together with the laziness toggle each +/// of the last two carries and the exact form does not. +/// +/// +/// Every fixture is built to keep a single operator family answerable for its reported gaps: the +/// backreference fixture pairs a backreference with the plain groups it needs to be a legal pattern, so +/// its expectation is the union of exactly those two operators, and every other fixture is a pattern that +/// carries no anchor, quantifier, group or shorthand escape at all, so only the operator under test can +/// report anything. Where a pattern cannot be kept that narrow - a class holding the very +/// \s/\S pair the dot-equivalence collapse recognises is unavoidably also a pair of shorthand +/// escapes the swap rewrites - the assertion falls back to naming the one message it needs to see, exactly +/// as does for its own character-class and lookaround +/// cases, rather than pinning a combinatorial set that would make the test fragile for reasons unrelated to +/// the construct it exists to prove. +/// +/// +/// Each fixture pairs the member under inspection with Fixture.Reached.Identity, whose body carries +/// no mutation point at all. Naming that member in the manifest is what gives the analyzer a non-empty +/// reachable set - without one it reports an unusable manifest and stays silent about the code - while +/// contributing not a single diagnostic of its own, exactly as in . +/// Every reported gap is filtered down to the ones whose message names a pattern mutation, the same +/// PatternMarker filter uses, so a mutation of an +/// unrelated family sitting on the same literal - for instance the RegexOptions flags of an explicit +/// options argument - can never leak into an expectation this class does not state anything about. +/// +/// +public class RegexInfrastructureMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + /// + /// The member of that carries the pattern, used to prove the + /// gaps of that fixture disappear once it is itself recorded as covered. + /// + private const string BackreferenceMemberId = "M:Fixture.Patterns.IsMatch(System.String)~System.Boolean"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts on + /// it, because these tests state what the operators report, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. It also keeps FSH0006 + /// silent, so that every expectation below stays a statement about the operators alone. + /// + private const string LowerBoundCount = "1+"; + + /// + /// The text every display name of the family starts its description with, and therefore the filter + /// that separates a pattern mutation from every other mutation of the same compilation. + /// + private const string PatternMarker = "pattern '"; + + /// + /// The text the assertions use for "not a single gap was reported". + /// + private const string NoGaps = ""; + + /// + /// The line feed the expectations are joined with, instead of , so + /// that the very same text is produced on Windows and on Linux. + /// + private const string LineFeed = "\n"; + + /// + /// A numbered backreference next to the three plain capturing groups it needs to be a legal pattern. + /// The group mutator offers to turn each of the three plain groups into a non-capturing one, and every + /// one of the three results is still a legal pattern because the backreference always finds a second + /// group to resolve against; the backreference mutator offers to shift \2 to \3 and to + /// \1, both legal because the pattern defines three groups. No anchor, quantifier, character + /// class or shorthand escape sits anywhere in the pattern, so these five mutations are the whole of + /// what either operator can report here. + /// + private const string BackreferenceAndGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"(a)(b)(c)\2"); + } + } + """; + + /// + /// A plain range with no other construct in the pattern: the negation toggle of the class open, and + /// the widening of both the range's start and its end by one code unit. Neither a nor z + /// is offered as a standalone removal, because both are the endpoint of the range rather than a + /// standalone member. + /// + private const string RangeSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "[a-z]"); + } + } + """; + + /// + /// Three standalone class members next to a literal dot: the negation toggle of the class, the + /// removal of each of the three members, and the expansion of the dot into [\s\S]. None of the + /// three members is the endpoint of a range, so every one of them is offered as a removal. + /// + private const string MemberRemovalAndDotSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "[xyz]."); + } + } + """; + + /// + /// The exact four token run [\s\S], which the class-open dispatch recognises as the operand of + /// the dot-equivalence collapse in addition to offering its own negation toggle. The very same two + /// tokens are also shorthand escapes in their own right, so the shorthand swap fires for each of them + /// as well; that combinatorial part of the output is not pinned here; see the class remarks. + /// + private const string AnyClassCollapseSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"[\s\S]"); + } + } + """; + + /// + /// A lone escaped dot, the one construct RegexEscapeMutator answers for. It is not one of the + /// six shorthand classes the character-class operator swaps, so this fixture is answerable by exactly + /// one operator. + /// + private const string EscapedDotSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\."); + } + } + """; + + /// + /// The exact {n} form of a counted quantifier, whose two rewrites decrease and increase the + /// count. Its repetition count leaves the engine no choice about how many times to repeat, so the + /// laziness toggle is never offered for it. + /// + private const string ExactCountSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a{3}"); + } + } + """; + + /// + /// The open ended {n,} form of a counted quantifier, whose lower bound is shifted in both + /// directions and whose laziness is toggled, because unlike the exact form it does leave the engine a + /// choice. + /// + private const string OpenEndedCountSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a{2,}"); + } + } + """; + + /// + /// The full {n,m} form of a counted quantifier, whose lower and upper bounds are each shifted + /// in both directions and whose laziness is toggled. + /// + private const string BoundedRangeSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a{2,3}"); + } + } + """; + + /// + /// A lazy +? quantifier: the shape swap to *?, which keeps the laziness, and the + /// laziness toggle back to the greedy +. + /// + private const string LazyPlusSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a+?"); + } + } + """; + + /// + /// An optional ? quantifier: its removal, which makes the atom mandatory, and its laziness + /// toggle to ??. + /// + private const string OptionalSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a?"); + } + } + """; + + /// + /// The pattern reached through a named, reordered constructor argument rather than the positional + /// leading argument every other fixture of this class uses, which is what exercises the locator's + /// argument binding by name instead of by position. The explicit RegexOptions.IgnoreCase + /// argument is itself a mutation point of the unrelated RegexOptions family, which is exactly + /// why the assertion below filters by instead of taking every gap of the + /// member. + /// + private const string NamedArgumentSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static Regex Create() + { + return new Regex(options: RegexOptions.IgnoreCase, pattern: "a+"); + } + } + """; + + /// + /// A plain named group with the angle-bracket delimiter and no other construct in the pattern. The + /// group mutator recognises it as a capturing open exactly like a plain (, so it offers the very + /// same demotion to (?:; this is what proves the tokenizer's angle-bracket name scan + /// (ScanAngleGroup/ScanNamedGroup) and the mutator's name-aware dispatch agree with each + /// other. + /// + private const string NamedAngleGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "(?x)"); + } + } + """; + + /// + /// The quote-delimited spelling of a named group, (?'name', which the tokenizer scans through the + /// very same ScanNamedGroup path as the angle-bracket form but with the other closing character. + /// The mutator's own quote-prefix branch is what has to recognise it as a capturing open. + /// + private const string QuotedNameGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "(?'year'x)"); + } + } + """; + + /// + /// An atomic group (?>...) next to the string-start anchor it does not interact with. The + /// group mutator offers nothing for the atomic opening - it changes backtracking, not capturing - so the + /// anchor's own removal is the only pattern mutation this fixture can report; that absence is what + /// proves the mutator's capturing-open dispatch actually excludes this token text rather than merely + /// never having been asked about it. + /// + private const string AtomicGroupWithAnchorSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\A(?>x)"); + } + } + """; + + /// + /// A balancing group pair, (?<open>x) defining the capture and (?<-open>) + /// popping it. The mutator's own dash check keeps it from ever touching the pop, and while it does offer + /// to demote the defining group to (?:, that rewrite leaves the pop referring to a capture that + /// no longer exists, so discards it as an invalid mutant. This is + /// therefore a fixture with a real candidate mutation and zero surviving gaps, which is what separates + /// "the operator declined to look" from "the base class rejected what it produced". + /// + private const string BalancingGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "(?x)(?<-open>)"); + } + } + """; + + /// + /// Two named backreferences, one of each delimiter, next to the named groups they resolve against. The + /// backreference operator recognises the \k prefix and offers nothing for either of them, and the + /// group mutator's candidate demotion of either defining group is discarded because it would leave the + /// matching backreference undefined - exactly the same "candidate produced, mutant invalid" shape as + /// , but exercised through ScanNamedBackreference and both + /// forms of TryMeasureBracketedName instead of the balancing branch of ScanNamedGroup. + /// + private const string NamedBackreferenceSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"(?x)\k(?'b'y)\k'b'"); + } + } + """; + + /// + /// A (?#...) comment, a scoped inline-options group (?i:...) and a standalone inline + /// options construct (?i), one after another. None of the three is a capturing open the group + /// mutator answers for - the comment carries no group token at all, the scoped form is a group whose + /// text is neither ( nor (?: nor a plain named prefix, and the standalone form tokenizes + /// as rather than as + /// - so the whole pattern reports nothing at all, which is what + /// proves every one of the three tokenizer paths (ScanInlineComment, the scoped and the + /// standalone branch of ScanOptions) was actually exercised rather than short-circuited by an + /// earlier failure. + /// + private const string InlineConstructsSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "(?#note)(?i:a)(?i)b"); + } + } + """; + + /// + /// A Unicode category escape and its negation, \p{L} and \P{Lu}, which the tokenizer scans + /// through ScanUnicodeCategoryEscape and no operator of the family recognises: neither is one of + /// the six shorthand classes the character-class operator swaps, and neither is the escaped dot the + /// escape operator rewrites. The pattern therefore reports nothing at all, which is the fixture's whole + /// point - it exists to reach the property-escape scan, not to report a mutation. + /// + private const string UnicodeCategoryEscapeSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\p{L}\P{Lu}"); + } + } + """; + + /// + /// A hexadecimal escape, a Unicode escape and a control-character escape back to back, which reach + /// ScanHexadecimalEscape under both of its digit counts and ScanControlEscape. None of the + /// three is a shorthand class or the escaped dot, so - exactly like + /// - the pattern reports nothing at all. + /// + private const string HexAndControlEscapeSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\x41\u0041\cA"); + } + } + """; + + /// + /// A single plain group followed by \10. With exactly one group defined, \10 names no + /// capture the pattern has, so the tokenizer's ResolveNumberedBackreferences step re-reads it as + /// the octal escape \10 in full - both digits are octal, so nothing is left over as a literal - + /// instead of leaving it a backreference. Neither the escape nor the backreference operator has anything + /// to say about the result, so the only mutation left standing is the plain group's own demotion, and + /// demoting it away is harmless here: with the group gone, the very same digits are still read as the + /// very same octal escape by a fresh construction of the mutant, so the rewrite validates. + /// + private const string OctalReinterpretationWithGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"(a)\10"); + } + } + """; + + /// + /// \18 with no group defined anywhere in the pattern. Only the leading 1 is an octal digit, + /// so ResolveNumberedBackreferences consumes it alone as the octal escape \1 and reinserts + /// the trailing 8 as a separate literal token - the one branch of that method none of the other + /// fixtures in this class reach, because they either define enough groups to stay a genuine + /// backreference or consume every digit as octal with nothing left over. Neither the reinterpreted escape + /// nor the reinserted literal digit is a construct any operator of the family rewrites, so the pattern + /// reports nothing at all. + /// + private const string LeftoverOctalDigitSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\18"); + } + } + """; + + /// + /// A range next to a class subtraction, [a-z-[aeiou]]: the outer class negation, the widening of + /// both ends of the range, the nested class's own negation, and the removal of each of its five + /// standalone members. This is the one fixture of the class that reaches the subtraction dispatch of + /// ScanCharacterClassDash - the -[ that opens a nested class rather than a range - and the + /// SubtractionApplied enforcement that nothing but the outer ] may follow it. + /// + private const string CharacterClassSubtractionSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "[a-z-[aeiou]]"); + } + } + """; + + /// + /// The line every pattern of this class sits on. Every fixture shares the same layout - the + /// Reached anchor, a blank line, the class declaring the pattern - so the pattern always lands + /// on the very same line. + /// + private const int PatternLine = 17; + + /// + /// Every mutant of 's pattern: the three group demotions in + /// pattern order, followed by the backreference's increase and decrease. + /// + private static readonly string[] _backreferenceAndGroupMutants = + [ + Mutant(@"(a)(b)(c)\2", @"(?:a)(b)(c)\2"), + Mutant(@"(a)(b)(c)\2", @"(a)(?:b)(c)\2"), + Mutant(@"(a)(b)(c)\2", @"(a)(b)(?:c)\2"), + Mutant(@"(a)(b)(c)\2", @"(a)(b)(c)\3"), + Mutant(@"(a)(b)(c)\2", @"(a)(b)(c)\1"), + ]; + + /// + /// The two mutants of 's pattern: the negation toggle and the widening of + /// both ends of the range. + /// + private static readonly string[] _rangeMutants = + [ + Mutant("[a-z]", "[^a-z]"), + Mutant("[a-z]", "[`-z]"), + Mutant("[a-z]", "[a-{]"), + ]; + + /// + /// Every mutant of 's pattern: the negation toggle, the + /// removal of each standalone member in pattern order, and the dot expansion. + /// + private static readonly string[] _memberRemovalAndDotMutants = + [ + Mutant("[xyz].", "[^xyz]."), + Mutant("[xyz].", "[yz]."), + Mutant("[xyz].", "[xz]."), + Mutant("[xyz].", "[xy]."), + Mutant("[xyz].", @"[xyz][\s\S]"), + ]; + + /// + /// The two mutants of 's pattern: the decrease and the increase of the + /// exact count. + /// + private static readonly string[] _exactCountMutants = [Mutant("a{3}", "a{2}"), Mutant("a{3}", "a{4}")]; + + /// + /// The three mutants of 's pattern: both shifts of the lower bound, + /// then the laziness toggle. + /// + private static readonly string[] _openEndedCountMutants = + [ + Mutant("a{2,}", "a{1,}"), + Mutant("a{2,}", "a{3,}"), + Mutant("a{2,}", "a{2,}?"), + ]; + + /// + /// The five mutants of 's pattern: both shifts of the lower bound, + /// both shifts of the upper bound, then the laziness toggle. + /// + private static readonly string[] _boundedRangeMutants = + [ + Mutant("a{2,3}", "a{1,3}"), + Mutant("a{2,3}", "a{3,3}"), + Mutant("a{2,3}", "a{2,2}"), + Mutant("a{2,3}", "a{2,4}"), + Mutant("a{2,3}", "a{2,3}?"), + ]; + + /// + /// The two mutants of 's pattern: the shape swap to a lazy star, then the + /// laziness toggle back to greedy. + /// + private static readonly string[] _lazyPlusMutants = [Mutant("a+?", "a*?"), Mutant("a+?", "a+")]; + + /// + /// The two mutants of 's pattern: the removal, then the laziness toggle. + /// + private static readonly string[] _optionalMutants = [Mutant("a?", "a"), Mutant("a?", "a??")]; + + /// + /// The two mutants of 's pattern, the same shape swap and laziness + /// toggle a bare greedy + offers everywhere else in this class. + /// + private static readonly string[] _namedArgumentMutants = [Mutant("a+", "a*"), Mutant("a+", "a+?")]; + + /// + /// The single mutant of 's pattern: the demotion of the named group + /// to non-capturing. + /// + private static readonly string[] _namedAngleGroupMutants = [Mutant("(?x)", "(?:x)")]; + + /// + /// The single mutant of 's pattern: the demotion of the named group + /// to non-capturing. + /// + private static readonly string[] _quotedNameGroupMutants = [Mutant("(?'year'x)", "(?:x)")]; + + /// + /// The single mutant of 's pattern: the removal of the + /// string-start anchor. The atomic group offers nothing of its own. + /// + private static readonly string[] _atomicGroupWithAnchorMutants = [Mutant(@"\A(?>x)", "(?>x)")]; + + /// + /// The single mutant of 's pattern: the demotion of + /// the plain group to non-capturing, which validates because the octal escape behind it is read the same + /// way whether or not the group exists. + /// + private static readonly string[] _octalReinterpretationWithGroupMutants = [Mutant(@"(a)\10", @"(?:a)\10")]; + + /// + /// Every mutant of 's pattern: the outer class negation, + /// the widening of both ends of the range, the nested class's own negation, and the removal of each of + /// its five standalone members. + /// + private static readonly string[] _characterClassSubtractionMutants = + [ + Mutant("[a-z-[aeiou]]", "[^a-z-[aeiou]]"), + Mutant("[a-z-[aeiou]]", "[`-z-[aeiou]]"), + Mutant("[a-z-[aeiou]]", "[a-{-[aeiou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[^aeiou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[eiou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[aiou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[aeou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[aeiu]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[aeio]]"), + ]; + + /// + /// Every fixture of this class, so that one test can prove that all of them compile and that none of + /// them makes the analyzer crash. + /// + /// One factory per fixture. + public static IEnumerable> Fixtures() => + new[] + { + BackreferenceAndGroupSource, + RangeSource, + MemberRemovalAndDotSource, + AnyClassCollapseSource, + EscapedDotSource, + ExactCountSource, + OpenEndedCountSource, + BoundedRangeSource, + LazyPlusSource, + OptionalSource, + NamedArgumentSource, + NamedAngleGroupSource, + QuotedNameGroupSource, + AtomicGroupWithAnchorSource, + BalancingGroupSource, + NamedBackreferenceSource, + InlineConstructsSource, + UnicodeCategoryEscapeSource, + HexAndControlEscapeSource, + OctalReinterpretationWithGroupSource, + LeftoverOctalDigitSource, + CharacterClassSubtractionSource, + }.Select(source => (Func)(() => source)); + + /// + /// The backreference next to the plain groups it renumbers reports a gap per group demotion and per + /// backreference shift, and nothing else. + /// + [Test] + public async Task Analyze_UntestedBackreferenceAndGroups_ReportsAGapPerGroupAndBackreferenceMutation() + { + var compilation = CompilationFactory.Create(BackreferenceAndGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _backreferenceAndGroupMutants)); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// Recording the pattern's own member as covered makes every one of its gaps disappear, which is what + /// proves the five gaps above to be a statement about coverage rather than about the operators. + /// + [Test] + public async Task Analyze_CoveredBackreferenceAndGroups_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(BackreferenceAndGroupSource, ProductionAssemblyName); + var manifest = new[] { CreateManifest(AnchorMemberId, BackreferenceMemberId) }; + + var diagnostics = await RunAsync(compilation, manifest).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A range with no other construct in its pattern reports its negation toggle and the widening of both + /// of its ends, and nothing else - neither one of its two endpoints is offered as a standalone removal. + /// + [Test] + public async Task Analyze_UntestedRange_ReportsTheNegationAndBothWidenings() + { + var compilation = CompilationFactory.Create(RangeSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _rangeMutants)); + } + } + + /// + /// Three standalone class members next to a literal dot report the class negation, the removal of + /// each member and the dot expansion, and nothing else. + /// + [Test] + public async Task Analyze_UntestedStandaloneMembersAndDot_ReportsEveryRemovalAndTheDotExpansion() + { + var compilation = CompilationFactory.Create(MemberRemovalAndDotSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _memberRemovalAndDotMutants)); + } + } + + /// + /// The exact [\s\S] token run collapses to a dot, in addition to offering its own negation + /// toggle, which is what proves the class-open dispatch recognises the run rather than merely toggling + /// the class it opens. + /// + [Test] + public async Task Analyze_UntestedAnyCharacterClass_ReportsTheCollapseToDot() + { + var compilation = CompilationFactory.Create(AnyClassCollapseSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + var messages = Summaries(diagnostics).Select(summary => summary.Message).ToArray(); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(messages.Any(message => message.Contains(@"pattern '[\s\S]' => '.'", StringComparison.Ordinal))) + .IsTrue(); + _ = await Assert + .That( + messages.Any(message => + message.Contains(@"pattern '[\s\S]' => '[^\s\S]'", StringComparison.Ordinal) + ) + ) + .IsTrue(); + } + } + + /// + /// A lone escaped dot reports its widening into any character, and nothing else - it is not one of + /// the six shorthand classes the character-class operator swaps. + /// + [Test] + public async Task Analyze_UntestedEscapedDot_ReportsTheWideningToAnyCharacter() + { + var compilation = CompilationFactory.Create(EscapedDotSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, [Mutant(@"\.", ".")])); + } + } + + /// + /// The exact {n} form reports its decrease and its increase, and never a laziness toggle: its + /// repetition count leaves the engine no choice to make lazy. + /// + [Test] + public async Task Analyze_UntestedExactCount_ReportsTheDecreaseAndIncreaseButNoLazinessToggle() + { + var compilation = CompilationFactory.Create(ExactCountSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _exactCountMutants)); + } + } + + /// + /// The open ended {n,} form reports both shifts of its lower bound and its laziness toggle. + /// + [Test] + public async Task Analyze_UntestedOpenEndedCount_ReportsBothBoundShiftsAndTheLazinessToggle() + { + var compilation = CompilationFactory.Create(OpenEndedCountSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _openEndedCountMutants)); + } + } + + /// + /// The full {n,m} form reports both shifts of its lower bound, both shifts of its upper bound + /// and its laziness toggle. + /// + [Test] + public async Task Analyze_UntestedBoundedRange_ReportsEveryBoundShiftAndTheLazinessToggle() + { + var compilation = CompilationFactory.Create(BoundedRangeSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _boundedRangeMutants)); + } + } + + /// + /// A lazy plus reports the shape swap to a lazy star, which keeps the laziness, and the laziness + /// toggle back to a greedy plus. + /// + [Test] + public async Task Analyze_UntestedLazyPlus_ReportsTheShapeSwapAndTheLazinessToggle() + { + var compilation = CompilationFactory.Create(LazyPlusSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _lazyPlusMutants)); + } + } + + /// + /// An optional quantifier reports its removal, which makes the atom mandatory, and its laziness + /// toggle. + /// + [Test] + public async Task Analyze_UntestedOptional_ReportsTheRemovalAndTheLazinessToggle() + { + var compilation = CompilationFactory.Create(OptionalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _optionalMutants)); + } + } + + /// + /// A pattern reached through a named, reordered constructor argument is located and mutated exactly + /// like one reached positionally, which is what proves the locator's argument binding resolves a + /// pattern by parameter rather than by its position in the argument list. + /// + [Test] + public async Task Analyze_UntestedPatternBehindNamedArgument_ReportsItsQuantifierMutations() + { + var compilation = CompilationFactory.Create(NamedArgumentSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _namedArgumentMutants)); + } + } + + /// + /// A plain named group with the angle-bracket delimiter is demoted to non-capturing exactly like a bare + /// (, which is what proves the mutator's named-capture recognition - and the tokenizer's + /// ScanAngleGroup/ScanNamedGroup path behind it - agree that the two spellings define the + /// same kind of capture. + /// + [Test] + public async Task Analyze_UntestedNamedAngleGroup_ReportsItsDemotionToNonCapturing() + { + var compilation = CompilationFactory.Create(NamedAngleGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _namedAngleGroupMutants)); + } + } + + /// + /// The quote-delimited spelling of a named group is demoted exactly like its angle-bracket counterpart, + /// which is what proves the mutator's quote-prefix branch, and the corresponding closing-quote path of + /// ScanNamedGroup, are exercised rather than only the angle-bracket one. + /// + [Test] + public async Task Analyze_UntestedQuotedNameGroup_ReportsItsDemotionToNonCapturing() + { + var compilation = CompilationFactory.Create(QuotedNameGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _quotedNameGroupMutants)); + } + } + + /// + /// An atomic group next to the string-start anchor reports only the anchor's own removal - the group + /// mutator offers nothing for the atomic opening at all, which is what separates "the operator was asked + /// and declined" from "the operator was never asked". + /// + [Test] + public async Task Analyze_UntestedAtomicGroupWithAnchor_ReportsOnlyTheAnchorRemoval() + { + var compilation = CompilationFactory.Create(AtomicGroupWithAnchorSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _atomicGroupWithAnchorMutants)); + } + } + + /// + /// A balancing group pair reports no pattern gap at all: the pop is never offered a rewrite because its + /// name carries the dash the mutator's capturing-open check rejects, and the one candidate the defining + /// group does offer is discarded because demoting it would leave the pop referring to a capture that no + /// longer exists. A real candidate mutation therefore still produces zero surviving gaps. + /// + [Test] + public async Task Analyze_UntestedBalancingGroup_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(BalancingGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// Two named backreferences, one of each delimiter, report no pattern gap at all: the backreference + /// operator recognises the \k prefix and offers nothing for either of them, and the group + /// mutator's candidate demotion of either defining group is discarded because it would leave the + /// matching backreference undefined. + /// + [Test] + public async Task Analyze_UntestedNamedBackreferences_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(NamedBackreferenceSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A comment, a scoped inline-options group and a standalone inline-options construct report no pattern + /// gap at all: none of the three tokenizes as a capturing open the group mutator answers for, which is + /// what proves every one of the three tokenizer paths behind them was reached rather than short-circuited + /// by an earlier failure. + /// + [Test] + public async Task Analyze_UntestedInlineConstructs_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(InlineConstructsSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A Unicode category escape and its negation report no pattern gap at all: neither is one of the six + /// shorthand classes the character-class operator swaps, and neither is the escaped dot the escape + /// operator rewrites. + /// + [Test] + public async Task Analyze_UntestedUnicodeCategoryEscapes_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(UnicodeCategoryEscapeSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A hexadecimal escape, a Unicode escape and a control-character escape report no pattern gap at all, + /// for the same reason as : none of the three is a shorthand + /// class or the escaped dot. + /// + [Test] + public async Task Analyze_UntestedHexAndControlEscapes_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(HexAndControlEscapeSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A single plain group followed by \10 reports only the group's own demotion: with exactly one + /// group defined, \10 names no capture the pattern has, so the tokenizer re-reads it as a full + /// octal escape instead of a backreference, and neither the escape nor the backreference operator has + /// anything to say about the result. + /// + [Test] + public async Task Analyze_UntestedOctalReinterpretationWithGroup_ReportsOnlyTheGroupDemotion() + { + var compilation = CompilationFactory.Create(OctalReinterpretationWithGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _octalReinterpretationWithGroupMutants)); + } + } + + /// + /// \18 with no group defined anywhere in the pattern reports no pattern gap at all: the tokenizer + /// consumes only the leading digit as an octal escape and reinserts the trailing digit as a literal, + /// and neither token is a construct any operator of the family rewrites. + /// + [Test] + public async Task Analyze_UntestedLeftoverOctalDigit_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(LeftoverOctalDigitSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A range next to a class subtraction reports the outer class negation, the widening of both ends of + /// the range, the nested class's own negation, and the removal of each of its five standalone members - + /// the one fixture of the class that reaches the subtraction dispatch of ScanCharacterClassDash + /// and the SubtractionApplied enforcement behind it. + /// + [Test] + public async Task Analyze_UntestedCharacterClassSubtraction_ReportsEveryConstituentMutation() + { + var compilation = CompilationFactory.Create(CharacterClassSubtractionSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _characterClassSubtractionMutants)); + } + } + + /// + /// Every fixture of this class compiles and is analysed without the analyzer throwing. Roslyn turns an + /// analyzer exception into AD0001 and carries on, so a crash would otherwise look like a + /// diagnostic the tests above simply did not expect. + /// + /// The fixture to analyse. + /// A task that completes when the fixture was analysed. + [Test] + [MethodDataSource(nameof(Fixtures))] + public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(string source) + { + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Describes the reported gaps that name a pattern mutation, one line per diagnostic, ordered + /// ordinally so that the result does not depend on the order the concurrently running analyzer + /// callbacks reported them in. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string PatternGaps(ImmutableArray diagnostics) + { + var entries = Summaries(diagnostics) + .Where(summary => NamesAPattern(summary.Message)) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + .ToList(); + + return entries.Count == 0 ? NoGaps : Join(entries); + } + + private static bool NamesAPattern(string message) => message.Contains(PatternMarker, StringComparison.Ordinal); + + private static ImmutableArray<(string Id, int Line, string Message)> Summaries( + ImmutableArray diagnostics + ) => DiagnosticAssertions.Summarise(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint)); + + /// + /// Builds the expectation of a set of gaps that all sit on . + /// + /// The 1-based line every gap is reported on. + /// The display names of the expected mutations. + /// The expected text block. + private static string ExpectAt(int line, IEnumerable displayNames) => + Join([.. displayNames.Select(displayName => GapEntry(line, displayName))]); + + /// + /// Composes the display name a pattern mutation carries. + /// + /// The original pattern, as the regular expression engine sees it. + /// The rewritten pattern, as the regular expression engine sees it. + /// The display name. + private static string Mutant(string original, string mutated) => + PatternMarker + original + "' => '" + mutated + "'"; + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => id + " line " + ToText(line) + ": " + message; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/StringLinqMathMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/StringLinqMathMutationTests.cs new file mode 100644 index 0000000..b3ab79a --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/StringLinqMathMutationTests.cs @@ -0,0 +1,869 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Mutations.Operators; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives , , +/// , , +/// and through end to +/// end, exactly like does for the culture-sensitivity family, so that +/// each operator is proven to produce the diagnostics a consumer sees in its build log instead of merely +/// to construct mutations in isolation. +/// +/// +/// +/// Every test states the exact set of reported gaps as one text block, built from the identifier, the +/// 1-based line and the full message of each diagnostic. Every fixture is written so that the operator +/// under test is the only one with a mutation point on the lines that matter: arguments are parameters or +/// field/property reads instead of literals, comparisons are avoided where they are not the point of the +/// test, so the diagnostic set stays a statement about the one operator instead of about every operator +/// that happens to also see the fixture. +/// +/// +/// Each fixture pairs the member under inspection with Fixture.Reached.Identity, whose body carries +/// no mutation point at all. Naming that member in the manifest is what gives the analyzer a non-empty +/// reachable set - without one it reports an unusable manifest and stays silent about the code - while +/// contributing not a single diagnostic of its own. +/// +/// +public class StringLinqMathMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts on + /// it, because these tests state what the operators report, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. + /// + private const string LowerBoundCount = "1+"; + + /// + /// The text the assertions use for "not a single gap was reported". + /// + private const string NoGaps = ""; + + /// + /// The line feed the expectations are joined with, instead of , so + /// that the very same text is produced on Windows and on Linux. + /// + private const string LineFeed = "\n"; + + private const int GreetingLine = 15; + private const int BlankLine = 20; + + private const int StartsWithLine = 20; +#if !NETFRAMEWORK + private const int TrimLine = 25; +#endif + private const int IsBlankLine = 30; + + private const int AnyLine = 26; + private const int FirstLine = 31; + private const int MinLine = 36; + private const int SkipLine = 41; + + private const int SineLine = 22; + private const int MinMaxLine = 27; + private const int FloorLine = 32; + private const int AbsLine = 37; + + private const int EarlyExitConditionLine = 17; + private const int NestedReturnLine = 19; + private const int EarlyExitAnnounceLine = 22; + private const int AnnounceBodyLine = 27; + private const int LastReturnAnnounceLine = 32; + private const int LoopConditionLine = 40; + private const int LoopBreakLine = 42; + private const int LoopAnnounceLine = 45; + private const int SkipConditionLine = 53; + private const int SkipContinueLine = 55; + private const int SkipAnnounceLine = 58; + private const int ValidateConditionLine = 64; + private const int ValidateThrowLine = 66; + private const int FailAnnounceLine = 72; + + private const int EmptyArrayLine = 22; + private const int CollectionInitializerLine = 30; + private const int EmptyIntsLine = 35; + private const int EmptyObjectsLine = 40; + private const int EmptyNullableStringsLine = 45; + + /// + /// Literals.Greeting (line 15) returns a non-empty literal, Literals.Blank (line 20) + /// returns the empty one, and ConstantLabel reads a field whose + /// initializer literal sits in a position that only accepts a compile-time constant - the one literal + /// of the whole file that is not a mutation point at all. + /// + private const string StringLiteralSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Literals + { + public static string Greeting() + { + return "hello"; + } + + public static string Blank() + { + return ""; + } + } + + public static class ConstantLabel + { + private const string Marker = "fixed"; + + public static string Describe() + { + return Marker; + } + } + """; + + /// + /// Text.StartsWith (line 20), Text.TrimAll (line 25) and Text.IsBlank (line 30) + /// each call a well known method with a matching counterpart overload, while + /// Text.CustomTrim calls a same-named, same-shaped method declared on Custom instead of + /// on itself, which the operator leaves untouched. + /// + private const string StringMethodSource = """ + namespace Fixture; + + public sealed class Custom + { + public string Trim() => string.Empty; + } + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Text + { + public static bool StartsWith(string value, string prefix) + { + return value.StartsWith(prefix); + } + + public static string TrimAll(string value) + { + return value.Trim(); + } + + public static bool IsBlank(string value) + { + return string.IsNullOrEmpty(value); + } + + public static string CustomTrim(Custom custom) + { + return custom.Trim(); + } + } + """; + + /// + /// Queries.AnyMatch (line 26), Queries.FirstMatch (line 31), Queries.MinOf (line + /// 36) and Queries.SkipSome (line 41) each call a well known System.Linq.Enumerable + /// method, while Queries.CustomAny calls a same-named, same-shaped method declared on + /// Custom instead of on Enumerable itself. Every predicate reads the property + /// Flag instead of comparing anything, so the lambda itself carries no mutation point of its + /// own. + /// + private const string LinqMethodSource = """ + namespace Fixture; + + using System; + using System.Collections.Generic; + using System.Linq; + + public sealed class Custom + { + public bool Any() => Environment.Is64BitProcess; + } + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Queries + { + private static bool Flag => Environment.Is64BitProcess; + + public static bool AnyMatch(IEnumerable values) + { + return values.Any(_ => Flag); + } + + public static int FirstMatch(IEnumerable values) + { + return values.First(_ => Flag); + } + + public static int MinOf(IEnumerable values) + { + return values.Min(); + } + + public static IEnumerable SkipSome(IEnumerable values, int count) + { + return values.Skip(count); + } + + public static bool CustomAny(Custom custom) + { + return custom.Any(); + } + } + """; + + /// + /// Trig.SineOf (line 22), Trig.SmallerOf (line 27) and Trig.FloorOf (line 32) + /// each call a well known method with a matching counterpart overload, + /// Trig.AbsoluteOf (line 37) calls Math.Abs, whose only mutation drops the call entirely, + /// and Trig.CustomAbs calls a same-named, same-shaped method declared on Custom instead + /// of on itself. + /// + private const string MathMethodSource = """ + namespace Fixture; + + using System; + + public sealed class Custom + { + public static double Abs(double value) => value; + } + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Trig + { + public static double SineOf(double angle) + { + return Math.Sin(angle); + } + + public static double SmallerOf(double left, double right) + { + return Math.Min(left, right); + } + + public static double FloorOf(double value) + { + return Math.Floor(value); + } + + public static double AbsoluteOf(double value) + { + return Math.Abs(value); + } + + public static double CustomAbs(double value) + { + return Custom.Abs(value); + } + } + """; + + /// + /// Every construct recognises and every guard that keeps it + /// from firing, in one file: a nested return; (line 19, removed - it is not the trailing + /// statement of the method body), a trailing return; in LastReturn (kept - removing it + /// would change nothing), a break and a continue inside a loop (removed), a nested + /// throw in a method (removed), a trailing throw in a + /// non- method (kept - removing it would leave a code path without a required + /// return) and a standalone invocation with a ref argument (kept - removing it would change + /// what the caller observes through the reference). Every condition is a bare identifier, so no other + /// operator has a mutation point on any of these lines. + /// + private const string StatementRemovalSource = """ + namespace Fixture; + + using System; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Guard + { + public static void EarlyExit(bool skip) + { + if (skip) + { + return; + } + + Announce(skip); + } + + public static void Announce(bool flag) + { + Console.WriteLine(flag); + } + + public static void LastReturn(bool skip) + { + Announce(skip); + return; + } + + public static void Loop(bool[] flags) + { + foreach (var flag in flags) + { + if (flag) + { + break; + } + + Announce(flag); + } + } + + public static void SkipEvens(bool[] flags) + { + foreach (var flag in flags) + { + if (flag) + { + continue; + } + + Announce(flag); + } + } + + public static void Validate(bool skip) + { + if (skip) + { + throw new InvalidOperationException(); + } + } + + public static int Fail(bool skip) + { + Announce(skip); + throw new InvalidOperationException(); + } + + public static void Adjust(ref int value) + { + Increment(ref value); + } + + public static void Increment(ref int value) { } + } + """; + + /// + /// Arrays.Pair (line 17) empties a non-empty array initializer, Arrays.EmptyArray (line + /// 22) fills an empty collection expression converted to an array, Collections.Values (line 30) + /// empties a non-empty collection expression, and the remaining members of Collections probe + /// every answer AllowsDefault can give for an empty one: int is a value type (line 35, + /// filled), object is (line 40, filled), an + /// annotated string? is a nullable reference type (line 45, filled), a plain string is + /// not annotated under this compilation's enabled nullable context (not filled), and an unconstrained + /// type parameter resolves to neither (not filled). + /// + private const string CollectionInitializerSource = """ + namespace Fixture; + + using System.Collections.Generic; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Arrays + { + public static int[] Pair(int first, int second) + { + return new[] { first, second }; + } + + public static int[] EmptyArray() + { + return []; + } + } + + public static class Collections + { + public static List Values(int first, int second) + { + return [first, second]; + } + + public static List EmptyInts() + { + return []; + } + + public static List EmptyObjects() + { + return []; + } + + public static List EmptyNullableStrings() + { + return []; + } + + public static List EmptyStrings() + { + return []; + } + + public static List EmptyGeneric() + { + return []; + } + } + """; + + /// + /// Every fixture of this class, so that one test can prove that all of them compile and that none of + /// them makes the analyzer crash. + /// + /// One factory per fixture. + public static IEnumerable> Fixtures() => + new[] + { + StringLiteralSource, + StringMethodSource, + LinqMethodSource, + MathMethodSource, + StatementRemovalSource, + CollectionInitializerSource, + }.Select(source => (Func)(() => source)); + + /// + /// A non-empty literal reports the collapse to an empty one, an empty literal reports the fill with a + /// non-empty placeholder, and the literal sitting in a field initializer - + /// a position that only accepts a compile-time constant - reports nothing at all. + /// + [Test] + public async Task Analyze_UntestedStringLiterals_ReportsTheEmptyAndNonEmptySwap() + { + var compilation = CompilationFactory.Create(StringLiteralSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(Expect((GreetingLine, "\"...\" => \"\""), (BlankLine, "\"\" => \"FrameShift\""))); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// StartsWith and IsNullOrEmpty each report the rename to their counterpart(s), while + /// the same-named, same-shaped method of Custom reports nothing: the operator resolves the + /// called method instead of matching its name. + /// + /// + /// Whether Trim() renames to TrimStart/TrimEnd depends on + /// actually declaring a genuinely zero-parameter overload of those two + /// methods, which the modern .NET reference assemblies do and the .NET Framework ones this repository + /// also targets do not - there, only the params char[] overload exists, so + /// HasSameParameters never matches and the operator stays silent on that member. + /// still covers TrimAll in its + /// manifest regardless, so it is unaffected either way. + /// + [Test] + public async Task Analyze_UntestedStringMethods_ReportsRenamesAndSkipsAForeignType() + { + var compilation = CompilationFactory.Create(StringMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (StartsWithLine, "StartsWith => EndsWith"), +#if !NETFRAMEWORK + (TrimLine, "Trim => TrimStart"), + (TrimLine, "Trim => TrimEnd"), +#endif + (IsBlankLine, "IsNullOrEmpty => IsNullOrWhiteSpace") + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// The counterpart of the previous test: naming every member that calls a well known + /// method silences the analysis completely, which is what makes the four gaps above a statement about + /// coverage rather than about the operator. + /// + [Test] + public async Task Analyze_CoveredStringMethods_ReportsNothing() + { + const string startsWithMemberId = "M:Fixture.Text.StartsWith(System.String,System.String)~System.Boolean"; + const string trimAllMemberId = "M:Fixture.Text.TrimAll(System.String)~System.String"; + const string isBlankMemberId = "M:Fixture.Text.IsBlank(System.String)~System.Boolean"; + + var compilation = CompilationFactory.Create(StringMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync( + compilation, + [CreateManifest(AnchorMemberId, startsWithMemberId, trimAllMemberId, isBlankMemberId)] + ) + .ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// Any, First, Min and Skip each report the rename to their counterpart(s), + /// while the same-named, same-shaped method of Custom reports nothing: the invoked method has + /// to be bound to System.Linq.Enumerable itself. + /// + /// + /// Skip => SkipLast is deliberately not asserted here: Enumerable.SkipLast was added + /// to modern .NET and was never backported to .NET Framework, which this repository also targets, so + /// that particular rename candidate exists on some target frameworks and not on others. Skip => + /// Take is unaffected and asserted on every framework. + /// + [Test] + public async Task Analyze_UntestedLinqMethods_ReportsRenamesAndSkipsAForeignType() + { + var compilation = CompilationFactory.Create(LinqMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (AnyLine, "Any => All"), + (FirstLine, "First => FirstOrDefault"), + (MinLine, "Min => Max"), + (SkipLine, "Skip => Take") +#if !NETFRAMEWORK + , + (SkipLine, "Skip => SkipLast") +#endif + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// The co-function, the extreme and the rounding direction each report their swap, Math.Abs + /// reports the removal of the call itself, and the same-named, same-shaped method of Custom + /// reports nothing: the called method has to be declared on itself. + /// + [Test] + public async Task Analyze_UntestedMathMethods_ReportsRenamesAndTheAbsRemovalAndSkipsAForeignType() + { + var compilation = CompilationFactory.Create(MathMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (SineLine, "Sin => Cos"), + (MinMaxLine, "Min => Max"), + (FloorLine, "Floor => Ceiling"), + (AbsLine, "Math.Abs(value) => value") + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// The flagship case of the family: every removable construct reports its removal, and every guard - + /// the trailing return;, the trailing throw of a non- member and + /// the invocation with a ref argument - keeps the analysis silent instead. Every if + /// condition guarding a removable statement is itself a bool parameter, which + /// also reports a gap for at its own line - a second, + /// independent operator firing on the same fixture, not a mistake. + /// + [Test] + public async Task Analyze_UntestedStatementRemovalConstructs_ReportsRemovalsAndRespectsEveryGuard() + { + var compilation = CompilationFactory.Create(StatementRemovalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (EarlyExitConditionLine, "x => !(x)"), + (NestedReturnLine, "return; => (removed)"), + (EarlyExitAnnounceLine, "Announce(skip) => (removed)"), + (AnnounceBodyLine, "Console.WriteLine(flag) => (removed)"), + (LastReturnAnnounceLine, "Announce(skip) => (removed)"), + (LoopConditionLine, "x => !(x)"), + (LoopBreakLine, "break; => (removed)"), + (LoopAnnounceLine, "Announce(flag) => (removed)"), + (SkipConditionLine, "x => !(x)"), + (SkipContinueLine, "continue; => (removed)"), + (SkipAnnounceLine, "Announce(flag) => (removed)"), + (ValidateConditionLine, "x => !(x)"), + (ValidateThrowLine, "throw ...; => (removed)"), + (FailAnnounceLine, "Announce(skip) => (removed)") + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// A non-empty collection expression reports its collapse to an empty one, an empty collection + /// expression reports the fill with for every element type + /// AllowsDefault accepts - a value type, object, and an annotated nullable reference + /// type - and reports nothing for the two it rejects: a non-annotated reference type and an + /// unconstrained type parameter. The old-style array initializer of Arrays.Pair + /// (new[] { first, second }) is deliberately included and reports nothing at all: + /// only recognises collection-expression syntax + /// ([ ... ]), not the classic new[] { ... } form. + /// + [Test] + public async Task Analyze_UntestedCollectionInitializers_ReportsEmptyingAndDefaultFillsWhereSafe() + { + var compilation = CompilationFactory.Create(CollectionInitializerSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (EmptyArrayLine, "[] => [default]"), + (CollectionInitializerLine, "[ ... ] => []"), + (EmptyIntsLine, "[] => [default]"), + (EmptyObjectsLine, "[] => [default]"), + (EmptyNullableStringsLine, "[] => [default]") + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// Every fixture of this class compiles and is analysed without the analyzer throwing. Roslyn turns an + /// analyzer exception into AD0001 and carries on, so a crash would otherwise look like a + /// diagnostic the tests above simply did not expect. + /// + /// The fixture to analyse. + /// A task that completes when the fixture was analysed. + [Test] + [MethodDataSource(nameof(Fixtures))] + public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(string source) + { + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched. + /// + /// + /// Every reference is attributed to one anonymous test whose case count is the lower bound + /// . These tests are about which mutation points are reachable and state + /// nothing about test data, so a lower bound is the honest count - and it keeps FSH0006 silent, + /// which is what lets every exact diagnostic set above stay a statement about the six operators alone. + /// Every reference is also written as behaviorally verified, for the same reason, to keep + /// FSH0007 silent as well. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.BehavioralReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Describes the reported gaps as one text block, one line per diagnostic, ordered ordinally so that + /// the result does not depend on the order the concurrently running analyzer callbacks reported them + /// in. Several gaps share one location, which is exactly the case a positional order cannot separate. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string Gaps(ImmutableArray diagnostics) + { + var gaps = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint); + + if (gaps.IsEmpty) + { + return NoGaps; + } + + return Join( + DiagnosticAssertions + .Summarise(gaps) + .OrderBy(summary => summary.Line) + .ThenBy(summary => summary.Message, StringComparer.Ordinal) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Builds the expectation of a set of gaps, each one a line and the display name of its mutation. + /// + /// The expected gaps. + /// The expected text block, or when nothing is expected. + private static string Expect(params (int Line, string DisplayName)[] gaps) => + gaps.Length == 0 + ? NoGaps + : Join( + gaps.OrderBy(gap => gap.Line) + .ThenBy(gap => gap.DisplayName, StringComparer.Ordinal) + .Select(gap => GapEntry(gap.Line, gap.DisplayName)) + ); + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => $"{id} line {ToText(line)}: {message}"; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Generation/TestSurfaceEdgeCaseTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Generation/TestSurfaceEdgeCaseTests.cs new file mode 100644 index 0000000..c416498 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Generation/TestSurfaceEdgeCaseTests.cs @@ -0,0 +1,706 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Generation; + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Text; +using NetEvolve.FrameShift.Generation; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives over the data-driven, awkward-shaped test methods +/// that deliberately keeps out of its own fixtures: the +/// combinations of data-source attributes, matrices and value generators each recognised framework has to +/// turn into a test-case count. +/// +/// +/// +/// proves the shape of the manifest — the block comment, +/// the ordering, the byte-identical re-run — with one plain test per framework. It deliberately avoids the +/// long tail of case-counting shapes, because adding them there would blur what that file is about. This +/// class is that long tail: for every recognised framework, one fixture whose test methods carry the +/// data-source shapes their ITestMethodRecognizer has to fold into a , +/// each one asserted against the exact count (or lower bound) documented on the recogniser itself. +/// +/// +/// Every fixture is run through the real generator via , exactly like +/// does, and the emitted manifest is parsed back with +/// so that the assertions read +/// values rather than raw manifest text. A test method is looked up by the unique substring of its name in +/// the parsed , because the exact documentation comment id +/// of an overload-free method already differs only by its parameter list, which is not what any of these +/// tests are about. +/// +/// +public class TestSurfaceEdgeCaseTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + private const string TestAssemblyName = "TestAssembly"; + private const string ProductionPath = "Production.cs"; + private const string TestPath = "CalculatorTests.cs"; + + private const string ProductionSource = """ + namespace Fixture; + + public static class Calculator + { + public static int Add(int left, int right) + { + return left + right; + } + } + """; + + /// + /// One MSTest test per shape has to count: several + /// [DataRow] applications, a [DynamicData] referencing a method that only yields, one + /// summing both kinds, a data source whose length depends on a loop and therefore cannot be read, and + /// three ways of naming the referenced sequence — an expression-bodied property, a property with a + /// block-bodied getter, and a member of another type named explicitly through the attribute's second + /// constructor argument. + /// + private const string MSTestSource = """ + namespace Tests; + + using System.Collections.Generic; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class CalculatorTests + { + public static IEnumerable GetRows() + { + yield return new object[] { 1 }; + yield return new object[] { 2 }; + yield return new object[] { 3 }; + } + + public static IEnumerable GetComputedRows() + { + for (var i = 0; i < 4; i++) + { + yield return new object[] { i }; + } + } + + public static IEnumerable RowsFromProperty => new object[][] { new object[] { 1 }, new object[] { 2 } }; + + public static IEnumerable RowsFromGetter + { + get + { + return new List { new object[] { 1 }, new object[] { 2 }, new object[] { 3 } }; + } + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + public void DataRow_ThreeRows_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(GetRows))] + public void DynamicData_MethodWithThreeYields_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DataRow(1)] + [DynamicData(nameof(GetRows))] + public void DataRowAndDynamicData_SumIsExactlyFour(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(GetComputedRows))] + public void DynamicData_ComputedLoop_IsAtLeastOne(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(RowsFromProperty))] + public void DynamicData_ExpressionBodiedProperty_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(RowsFromGetter))] + public void DynamicData_BlockGetterProperty_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(RemoteRows.Get), typeof(RemoteRows))] + public void DynamicData_ExternalDeclaringType_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + } + + public static class RemoteRows + { + public static IEnumerable Get() + { + return new[] { new object[] { 1 }, new object[] { 2 } }; + } + } + """; + + /// + /// One NUnit test per shape has to count: several + /// [TestCase] applications, a [TestCaseSource] naming a method of the fixture and one + /// naming a member of another type, the cross product two [Values] parameters produce, a + /// [ValueSource] reading a field, both [Range] overloads, both counted [Random] + /// shapes, the two combining strategies, and a bare [Theory]. + /// + private const string NUnitSource = """ + namespace Tests; + + using System.Collections.Generic; + using NUnit.Framework; + + public class CalculatorTests + { + private static readonly int[] Numbers = [7, 8, 9]; + + public static IEnumerable Rows() + { + yield return new object[] { 1, 2 }; + yield return new object[] { 3, 4 }; + } + + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + public void TestCase_ThreeCases_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestCaseSource(nameof(Rows))] + public void TestCaseSource_TwoRows_IsExactlyTwo(int left, int right) + { + _ = Fixture.Calculator.Add(left, right); + } + + [TestCaseSource(typeof(RemoteRows), nameof(RemoteRows.Get))] + public void TestCaseSource_ExternalTypeMember_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + public void Values_CrossProduct_IsExactlySix([Values(1, 2, 3)] int a, [Values(4, 5)] int b) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Test] + public void ValueSource_FromField_IsExactlyThree([ValueSource(nameof(Numbers))] int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + public void Range_TwoArgument_IsExactlyFive([Range(1, 5)] int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + public void Range_ThreeArgumentStep_IsExactlyThree([Range(0, 4, 2)] int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + public void Random_FixedCount_IsExactlyFour([Random(1, 10, 4)] int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Sequential] + public void Sequential_TakesTheLongestSet([Values(1, 2, 3)] int a, [Values(4, 5)] int b) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Pairwise] + public void Pairwise_IsLowerBoundOfTheLongestSet([Values(1, 2)] int a, [Values(3, 4)] int b) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Theory] + public void Theory_IsLowerBoundOfOne(bool flag) + { + _ = Fixture.Calculator.Add(flag ? 1 : 0, 1); + } + } + + public static class RemoteRows + { + public static IEnumerable Get() + { + yield return new object[] { 1 }; + yield return new object[] { 2 }; + } + } + """; + + /// + /// One TUnit test per shape has to count: several + /// [Arguments] rows, the cross product two [Matrix] parameters produce, a matrix parameter + /// excluding one of its own values, a [MethodDataSource] naming a literal sequence and one naming + /// a computed one, and a [Repeat] proven not to multiply the single inline row it sits next to. + /// + private const string TUnitSource = """ + namespace Tests; + + using System.Collections.Generic; + using TUnit.Core; + + public class CalculatorTests + { + public static int[] LiteralRows() => new[] { 1, 2, 3 }; + + public static IEnumerable ComputedRows() + { + for (var i = 0; i < 4; i++) + { + yield return new[] { i }; + } + } + + [Test] + [Arguments(1)] + [Arguments(2)] + [Arguments(3)] + public void Arguments_ThreeRows_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + [MatrixDataSource] + public void Matrix_CrossProduct_IsExactlySix([Matrix(1, 2, 3)] int a, [Matrix(4, 5)] int b) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Test] + [MatrixDataSource] + public void Matrix_WithExcludedValue_IsAtLeastOne( + [Matrix(1, 2, 3, Excluding = new object[] { 2 })] int a, + [Matrix(4, 5)] int b + ) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Test] + [MethodDataSource(nameof(LiteralRows))] + public void MethodDataSource_LiteralSequence_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + [MethodDataSource(nameof(ComputedRows))] + public void MethodDataSource_ComputedLoop_IsAtLeastOne(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + [Repeat(5)] + [Arguments(1)] + public void Repeat_DoesNotMultiplyTheCount_IsExactlyOne(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + } + """; + + /// + /// One xUnit v2 test per shape has to count: a plain [Fact], + /// several [InlineData] rows, a [MemberData] naming a method of the fixture and one naming + /// a member of another type through MemberType, a data source whose length depends on a loop, a + /// bare [Theory] without any data source at all, and a custom marker deriving [Fact] that + /// degrades an otherwise exact count to a lower bound. + /// + private const string XunitV2Source = """ + namespace Tests; + + using System.Collections.Generic; + using Xunit; + + public class CalculatorTests + { + public static IEnumerable Rows() + { + return new[] { new object[] { 1 }, new object[] { 2 } }; + } + + public static IEnumerable ComputedRows() + { + for (var i = 0; i < 3; i++) + { + yield return new object[] { i }; + } + } + + [Fact] + public void Fact_IsExactlyOne() + { + _ = Fixture.Calculator.Add(1, 1); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public void InlineData_ThreeRows_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + [MemberData(nameof(Rows))] + public void MemberData_TwoRows_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + [MemberData(nameof(RemoteRows.Get), MemberType = typeof(RemoteRows))] + public void MemberData_ExternalMemberType_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + [MemberData(nameof(ComputedRows))] + public void MemberData_ComputedLoop_IsAtLeastOne(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + public void Theory_WithoutData_IsExactlyZero(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [CustomFact] + [InlineData(1)] + [InlineData(2)] + public void CustomMarkerWithInlineData_IsAtLeastTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + } + + public sealed class CustomFactAttribute : FactAttribute + { + } + + public static class RemoteRows + { + public static IEnumerable Get() + { + yield return new object[] { 1 }; + yield return new object[] { 2 }; + } + } + """; + +#if FRAMESHIFT_XUNIT_V3 + /// + /// The xUnit v3 counterpart of , narrowed to the shapes that are identical + /// between the two major versions: shares its rules across both, so + /// this fixture exists only to prove the same rules apply when the recogniser is built for + /// xunit.v3.core instead of xunit.core. + /// + private const string XunitV3Source = """ + namespace Tests; + + using System.Collections.Generic; + using Xunit; + + public class CalculatorTests + { + public static IEnumerable Rows() + { + yield return new object[] { 1 }; + yield return new object[] { 2 }; + yield return new object[] { 3 }; + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public void InlineData_TwoRows_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + [MemberData(nameof(Rows))] + public void MemberData_ThreeRows_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + public void Theory_WithoutData_IsExactlyZero(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + } + """; +#endif + + [Test] + public async Task Fixtures_EveryCompilation_CompileWithoutErrors() + { + var production = CreateProduction(); + List described = + [ + Describe(production), + Describe(CreateTest(TestFramework.MSTest, MSTestSource, production)), + Describe(CreateTest(TestFramework.NUnit, NUnitSource, production)), + Describe(CreateTest(TestFramework.TUnit, TUnitSource, production)), + Describe(CreateTest(TestFramework.XunitV2, XunitV2Source, production)), + ]; +#if FRAMESHIFT_XUNIT_V3 + described.Add(Describe(CreateTest(TestFramework.XunitV3, XunitV3Source, production))); +#endif + + _ = await Assert + .That(string.Join("|", described.Distinct(StringComparer.Ordinal))) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + + [Test] + public async Task Generate_MSTestDataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.MSTest, MSTestSource); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "DataRow_ThreeRows_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_MethodWithThreeYields_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DataRowAndDynamicData_SumIsExactlyFour", TestCaseCount.Exact(4)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_ComputedLoop_IsAtLeastOne", TestCaseCount.AtLeast(1)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_ExpressionBodiedProperty_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_BlockGetterProperty_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_ExternalDeclaringType_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + } + } + + [Test] + public async Task Generate_NUnitDataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.NUnit, NUnitSource); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "TestCase_ThreeCases_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "TestCaseSource_TwoRows_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "TestCaseSource_ExternalTypeMember_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Values_CrossProduct_IsExactlySix", TestCaseCount.Exact(6)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "ValueSource_FromField_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Range_TwoArgument_IsExactlyFive", TestCaseCount.Exact(5)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Range_ThreeArgumentStep_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Random_FixedCount_IsExactlyFour", TestCaseCount.Exact(4)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Sequential_TakesTheLongestSet", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Pairwise_IsLowerBoundOfTheLongestSet", TestCaseCount.AtLeast(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Theory_IsLowerBoundOfOne", TestCaseCount.AtLeast(1)).ConfigureAwait(false); + } + } + + [Test] + public async Task Generate_TUnitDataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.TUnit, TUnitSource); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "Arguments_ThreeRows_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Matrix_CrossProduct_IsExactlySix", TestCaseCount.Exact(6)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Matrix_WithExcludedValue_IsAtLeastOne", TestCaseCount.AtLeast(1)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MethodDataSource_LiteralSequence_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MethodDataSource_ComputedLoop_IsAtLeastOne", TestCaseCount.AtLeast(1)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Repeat_DoesNotMultiplyTheCount_IsExactlyOne", TestCaseCount.Exact(1)) + .ConfigureAwait(false); + } + } + + [Test] + public async Task Generate_XunitV2DataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.XunitV2, XunitV2Source); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "Fact_IsExactlyOne", TestCaseCount.Exact(1)).ConfigureAwait(false); + _ = await AssertCount(manifest, "InlineData_ThreeRows_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MemberData_TwoRows_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MemberData_ExternalMemberType_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MemberData_ComputedLoop_IsAtLeastOne", TestCaseCount.AtLeast(1)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Theory_WithoutData_IsExactlyZero", TestCaseCount.Exact(0)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "CustomMarkerWithInlineData_IsAtLeastTwo", TestCaseCount.AtLeast(2)) + .ConfigureAwait(false); + } + } + +#if FRAMESHIFT_XUNIT_V3 + [Test] + public async Task Generate_XunitV3DataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.XunitV3, XunitV3Source); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "InlineData_TwoRows_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MemberData_ThreeRows_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Theory_WithoutData_IsExactlyZero", TestCaseCount.Exact(0)) + .ConfigureAwait(false); + } + } +#endif + + private static async Task AssertCount(TestSurfaceManifest manifest, string methodName, TestCaseCount expected) + { + _ = await Assert.That(manifest.TestCaseCounts[IdOf(manifest, methodName)]).IsEqualTo(expected); + + return true; + } + + private static TestSurfaceManifest CollectManifest(TestFramework framework, string source) + { + var test = CreateTest(framework, source, CreateProduction()); + var (success, error, manifest) = Read(Generate(test)); + + if (!success) + { + throw new InvalidOperationException($"The generated manifest of '{framework}' does not parse: {error}"); + } + + return manifest; + } + + /// + /// Finds the single test method whose documentation comment id carries + /// as its own member name, rather than as a prefix of a longer one. + /// + /// The manifest to search. + /// The unqualified name of the test method. + /// The matching documentation comment id. + private static string IdOf(TestSurfaceManifest manifest, string methodName) => + manifest.TestMethodIds.Single(id => + id.EndsWith("." + methodName, StringComparison.Ordinal) + || id.Contains("." + methodName + "(", StringComparison.Ordinal) + ); + + private static GeneratorRunner.Output Run(Compilation compilation) => + GeneratorRunner.Run(new TestSurfaceManifestGenerator(), compilation); + + private static string Generate(Compilation compilation) => + Run(compilation).TextOf(TestSurfaceManifestGenerator.HintName); + + private static CSharpCompilation CreateProduction() => + CompilationFactory.Create( + ProductionSource, + TestFramework.None, + ProductionAssemblyName, + filePath: ProductionPath + ); + + private static CSharpCompilation CreateTest(TestFramework framework, string source, Compilation production) => + CompilationFactory.Create( + source, + framework, + TestAssemblyName, + additionalReferences: [production.ToMetadataReference()], + filePath: TestPath + ); + + /// + /// Parses the generated file the way the MSBuild target does: drop the first and the last line, hand + /// the rest to the reader. + /// + /// The content of the generated source file. + /// Whether the text parsed, the reported error and the parsed manifest. + private static (bool Success, string Error, TestSurfaceManifest Manifest) Read(string generated) + { + var inner = string.Join("\n", Lines(generated).Skip(1).SkipLast(1)) + "\n"; + var success = TestSurfaceManifestReader.TryRead(SourceText.From(inner), out var manifest, out var error); + + return (success, error ?? string.Empty, manifest); + } + + /// + /// Splits the generated text into its lines, dropping the empty remainder behind the trailing line + /// feed, which is the end of the last line and not a line of its own. + /// + /// The generated text. + /// The lines, without their line endings. + private static ImmutableArray Lines(string text) + { + var lines = text.Split('\n').Select(line => line.TrimEnd('\r')).ToList(); + + if (lines.Count > 0 && lines[^1].Length == 0) + { + lines.RemoveAt(lines.Count - 1); + } + + return [.. lines]; + } + + private static string Describe(Compilation compilation) => + DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(compilation)); +}