diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/BitwiseAndUnaryMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/BitwiseAndUnaryMutationTests.cs new file mode 100644 index 0000000..37f247e --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/BitwiseAndUnaryMutationTests.cs @@ -0,0 +1,641 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives the bitwise, boolean-literal, increment/decrement and unary-sign operators through +/// end to end, so that the family is proven to produce the +/// diagnostics a consumer sees in its build log instead of merely to construct mutations. +/// +/// +/// +/// One fixture carries every mutation point of the family, each on its own member and its own line, so +/// that a single manifest can leave the whole family unreached and one test can state the exact, +/// complete set of gaps as one text block - the same discipline +/// uses for the culture-sensitivity family. +/// +/// +/// EnumBitwise.Combine and NullableBitwise.AndNullable exist to drive +/// through its enum and its +/// nullable branch, the two decisions a plain operand never exercises. +/// +/// +public class BitwiseAndUnaryMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + private const string TestAssemblyName = "TestAssembly"; + private const string TestFilePath = "FamilyTests.cs"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + /// + /// The compound bitwise assignment covered in isolation by + /// . + /// + private const string AndAssignMemberId = "M:Fixture.Bitwise.AndAssign(System.Int32,System.Int32)~System.Int32"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts on + /// it, because these tests state what the operators report, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. + /// + private const string LowerBoundCount = "1+"; + + /// + /// The text the assertions use for "not a single gap was reported". + /// + private const string NoGaps = ""; + + /// + /// The line feed the expectations are joined with, instead of , so + /// that the very same text is produced on Windows and on Linux. + /// + private const string LineFeed = "\n"; + + private const int AddAssignLine = 15; + private const int BitwiseAndLine = 24; + private const int LeftShiftLine = 29; + private const int AndAssignLine = 34; + private const int LeftShiftAssignLine = 40; + private const int EnumOrLine = 56; + private const int NullableAndLine = 64; + private const int TrueLiteralLine = 72; + private const int FalseLiteralLine = 77; + private const int PreIncrementLine = 85; + private const int PostDecrementLine = 90; + private const int UnaryMinusLine = 98; + private const int UnaryPlusLine = 103; + + private const int CheckedExpressionLine = 15; + private const int UncheckedExpressionLine = 20; + private const int CheckedStatementLine = 25; + private const int UncheckedStatementLine = 33; + + /// + /// All four shapes CheckedContextMutator recognises: the expression form and the statement + /// form, each once as and once as . + /// + private const string CheckedContextSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Overflow + { + public static int CheckedExpression(int value) + { + return checked(value); + } + + public static int UncheckedExpression(int value) + { + return unchecked(value); + } + + public static int CheckedStatement(int value) + { + checked + { + return value; + } + } + + public static int UncheckedStatement(int value) + { + unchecked + { + return value; + } + } + } + """; + + /// + /// One member per mutation point of the whole family. Every member takes a variable operand, never a + /// literal, so that neither the arithmetic-assignment operand check nor the unary + /// constant-preservation check ever discards a mutation this fixture means to exercise. + /// + private const string FamilySource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Arithmetic + { + public static int AddAssign(int value) + { + value += 1; + return value; + } + } + + public static class Bitwise + { + public static int AndOp(int left, int right) + { + return left & right; + } + + public static int ShiftOp(int value) + { + return value << 1; + } + + public static int AndAssign(int value, int mask) + { + value &= mask; + return value; + } + + public static int ShiftAssign(int value) + { + value <<= 1; + return value; + } + } + + public enum Options + { + None = 0, + A = 1, + B = 2, + } + + public static class EnumBitwise + { + public static Options Combine(Options left, Options right) + { + return left | right; + } + } + + public static class NullableBitwise + { + public static int? AndNullable(int? left, int? right) + { + return left & right; + } + } + + public static class Flag + { + public static bool AlwaysTrue() + { + return true; + } + + public static bool AlwaysFalse() + { + return false; + } + } + + public static class Counter + { + public static int PreIncrement(int value) + { + return ++value; + } + + public static int PostDecrement(int value) + { + return value--; + } + } + + public static class Sign + { + public static int Negate(int value) + { + return -value; + } + + public static int Plus(int value) + { + return +value; + } + } + """; + + /// + /// A real TUnit test that calls every member of once, used to build the + /// manifest of with the real collector + /// instead of hand-written member ids. + /// + private const string TestSource = """ + namespace Fixture.Tests; + + using Fixture; + using TUnit.Core; + + public class FamilyTests + { + [Test] + public void ReachesEveryMember() + { + _ = Arithmetic.AddAssign(1); + _ = Bitwise.AndOp(1, 2); + _ = Bitwise.ShiftOp(1); + _ = Bitwise.AndAssign(1, 2); + _ = Bitwise.ShiftAssign(1); + _ = EnumBitwise.Combine(Options.A, Options.B); + _ = NullableBitwise.AndNullable(1, 2); + _ = Flag.AlwaysTrue(); + _ = Flag.AlwaysFalse(); + _ = Counter.PreIncrement(1); + _ = Counter.PostDecrement(1); + _ = Sign.Negate(1); + _ = Sign.Plus(1); + } + } + """; + + /// + /// The complete set of gaps produces when nothing but the anchor is + /// reached: one entry per mutation point, sorted by line and then by message the same way the + /// analyzer itself reports them. The literal 1 operand of AddAssign, ShiftOp and + /// ShiftAssign also gets its own 1 => 0 gap from NumericLiteralMutator, on top + /// of the operator under test at that line - a second, independent operator firing on the same + /// location, not a mistake. + /// + private static readonly (int Line, string DisplayName)[] _everyGap = + [ + (AddAssignLine, "+= => %="), + (AddAssignLine, "+= => *="), + (AddAssignLine, "+= => -="), + (AddAssignLine, "+= => /="), + (AddAssignLine, "1 => 0"), + (BitwiseAndLine, "& => ^"), + (BitwiseAndLine, "& => |"), + (LeftShiftLine, "1 => 0"), + (LeftShiftLine, "<< => >>"), + (AndAssignLine, "&= => ^="), + (AndAssignLine, "&= => |="), + (LeftShiftAssignLine, "1 => 0"), + (LeftShiftAssignLine, "<<= => >>="), + (EnumOrLine, "| => &"), + (EnumOrLine, "| => ^"), + (NullableAndLine, "& => ^"), + (NullableAndLine, "& => |"), + (TrueLiteralLine, "true => false"), + (FalseLiteralLine, "false => true"), + (PreIncrementLine, "++x => --x"), + (PostDecrementLine, "x-- => x++"), + (UnaryMinusLine, "-x => +x"), + (UnaryMinusLine, "-x => x"), + (UnaryPlusLine, "+x => -x"), + (UnaryPlusLine, "+x => x"), + ]; + + /// + /// The flagship case of the family: with only the anchor reached, every member reports every mutation + /// it carries, which is what makes the members above a statement about the operators themselves. + /// + [Test] + public async Task Analyze_UntestedFamily_ReportsEveryMutationOfEveryMember() + { + var compilation = CompilationFactory.Create(FamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(Expect(_everyGap)); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// The same compilation, with Bitwise.AndAssign itself recorded in the manifest: its two gaps + /// vanish and every other member keeps reporting exactly as before, which is what proves the silence + /// is a statement about coverage of that one member rather than about the operator going quiet. + /// + [Test] + public async Task Analyze_CoveredBitwiseAssignment_LeavesOnlyThatMemberSilent() + { + var compilation = CompilationFactory.Create(FamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AndAssignMemberId)]).ConfigureAwait(false); + var expected = _everyGap.Where(gap => gap.Line != AndAssignLine); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(Expect([.. expected])); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// With every member of the fixture collected as a real manifest built from a real test compilation + /// that calls each of them, the whole family goes silent at once - the counter-example that shows the + /// gaps above come from the manifest missing the members, not from the fixture itself being + /// unreachable code. Collecting the manifest instead of hand-writing the member ids also proves the + /// enum and the nullable member ids resolve the way writes them, + /// which a hand-written id could easily get subtly wrong. + /// + [Test] + public async Task Analyze_EveryMemberCovered_ReportsNothing() + { + var compilation = CompilationFactory.Create(FamilySource, ProductionAssemblyName); + var manifest = new InMemoryAdditionalText(CollectManifest(compilation)); + + var diagnostics = await RunAsync(compilation, [manifest]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + _ = await Assert + .That( + DiagnosticAssertions.Describe( + AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint) + ) + ) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// only ever uses += as the assignment kind under test, which + /// leaves the -=/*=//=/%= arms of ArithmeticAssignmentMutator's own + /// name/symbol/token/metadata-name lookups untouched as a source kind - they are only ever + /// exercised as a mutation target. Using *= here as the source exercises every one of + /// those lookups for a different starting point, producing mutations to +=, -=, /= + /// and %= instead. + /// + [Test] + public async Task Analyze_UntestedMultiplyAssignment_ReportsEveryOtherArithmeticAssignment() + { + const string source = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Scaling + { + public static int MultiplyAssign(int value, int factor) + { + value *= factor; + return value; + } + } + """; + const int multiplyAssignLine = 15; + + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (multiplyAssignLine, "*= => +="), + (multiplyAssignLine, "*= => -="), + (multiplyAssignLine, "*= => /="), + (multiplyAssignLine, "*= => %=") + ) + ); + } + } + + /// + /// All four shapes CheckedContextMutator recognises in one fixture: the expression form and the + /// statement form, each swapped in both directions. + /// + [Test] + public async Task Analyze_UntestedCheckedContexts_ReportsAllFourSwaps() + { + var compilation = CompilationFactory.Create(CheckedContextSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (CheckedExpressionLine, "checked(...) => unchecked(...)"), + (UncheckedExpressionLine, "unchecked(...) => checked(...)"), + (CheckedStatementLine, "checked { } => unchecked { }"), + (UncheckedStatementLine, "unchecked { } => checked { }") + ) + ); + } + } + + /// + /// The fixture compiles and is analysed without the analyzer throwing. Roslyn turns an analyzer + /// exception into AD0001 and carries on, so a crash would otherwise look like a diagnostic the + /// tests above simply did not expect. + /// + [Test] + public async Task Analyze_Family_CompilesAndReportsNoAnalyzerFailure() + { + var compilation = CompilationFactory.Create(FamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds the manifest of the way the first pass does: from a real + /// TUnit test compilation that sees the production code as a metadata reference only. + /// + /// The production compilation the tests are written against. + /// The serialized manifest. + private static string CollectManifest(Compilation production) + { + var test = CompilationFactory.Create( + [(TestFilePath, TestSource)], + TestAssemblyName, + includeTUnit: true, + additionalReferences: [production.ToMetadataReference()] + ); + + return TestSurfaceManifestWriter.Write( + TestSurfaceCollector.Collect( + test, + new TUnitTestMethodRecognizer(TUnitTestFrameworkProbe.GetTestAttributeType(test)), + CancellationToken.None + ) + ); + } + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched. + /// + /// + /// Every reference is attributed to one anonymous test whose case count is the lower bound + /// . These tests are about which mutation points are reachable and state + /// nothing about test data, so a lower bound is the honest count - and it keeps FSH0006 silent, + /// which is what lets every exact diagnostic set below stay a statement about the family alone. Every + /// reference is also written as behaviorally verified, for the same reason: these tests are not about + /// the behavioral classification, so FSH0007 has to stay out of the way as well. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.BehavioralReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Describes the reported gaps as one text block, one line per diagnostic, ordered ordinally so that + /// the result does not depend on the order the concurrently running analyzer callbacks reported them + /// in. Several gaps share one location, which is exactly the case a positional order cannot separate. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string Gaps(ImmutableArray diagnostics) + { + var gaps = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint); + + if (gaps.IsEmpty) + { + return NoGaps; + } + + return Join( + DiagnosticAssertions + .Summarise(gaps) + // Several operators can each produce their own mutant at the exact same location (for + // example the two candidates of a rename family), and the order the analyzer enumerates + // those ties in is not guaranteed to be the same across runtimes - .NET Framework's string + // hashing differs from modern .NET, which can reorder anything keyed by a Dictionary + // internally. Sorting ties by message text makes the expectation below deterministic. + .OrderBy(summary => summary.Line) + .ThenBy(summary => summary.Message, StringComparer.Ordinal) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Builds the expectation of a set of gaps, each one a line and the display name of its mutation. + /// + /// The expected gaps. + /// The expected text block, or when nothing is expected. + private static string Expect(params (int Line, string DisplayName)[] gaps) => + gaps.Length == 0 + ? NoGaps + : Join( + gaps.OrderBy(gap => gap.Line) + .ThenBy(gap => gap.DisplayName, StringComparer.Ordinal) + .Select(gap => GapEntry(gap.Line, gap.DisplayName)) + ); + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => $"{id} line {ToText(line)}: {message}"; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string Trivial(ImmutableArray diagnostics) => + DiagnosticAssertions.Describe(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.TrivialMutant)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ConditionalAndLogicalMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ConditionalAndLogicalMutationTests.cs new file mode 100644 index 0000000..5a54444 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ConditionalAndLogicalMutationTests.cs @@ -0,0 +1,968 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives , +/// , +/// , +/// , +/// , +/// , +/// and +/// through +/// end to end, the way CultureMutationTests drives the +/// culture-sensitivity family: a real production fixture, a manifest naming which members are reachable, +/// and the exact set of FSH0001 diagnostics the analyzer reports for it, stated as identifier, +/// 1-based line and full message. +/// +/// +/// +/// Several fixtures deliberately let two operators share one construct instead of avoiding the overlap: +/// a ternary condition that is a plain boolean identifier is offered to +/// for the branches +/// and to for the condition +/// itself, and a comparer argument dropped by +/// is exactly the +/// kind of position a real call site would also drop. Stating every diagnostic such a construct produces, +/// instead of contriving a fixture that avoids the second operator, is what proves the two operators +/// coexist correctly on the very same node. +/// +/// +/// Every other fixture keeps helper members - a custom IComparer<int>, a constant fallback +/// value - free of mutation points of their own, either because their body has no syntax an operator +/// recognises or because the position is a compile-time constant context, so that the exact gap sets below +/// stay a statement about the operator under test alone. +/// +/// +public class ConditionalAndLogicalMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + private const string TernaryPickMemberId = + "M:Fixture.Ternary.Pick(System.Boolean,System.Int32,System.Int32)~System.Int32"; + private const string AreEqualMemberId = "M:Fixture.Numbers.AreEqual(System.Int32,System.Int32)~System.Boolean"; + private const string EnsureLoadedMemberId = "M:Fixture.Cache.EnsureLoaded(System.String)~System.String"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts on + /// it, because these tests state what the operators report, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. + /// + private const string LowerBoundCount = "1+"; + + /// + /// The text the assertions use for "not a single gap was reported". + /// + private const string NoGaps = ""; + + /// + /// The line feed the expectations are joined with, instead of , so + /// that the very same text is produced on Windows and on Linux. + /// + private const string LineFeed = "\n"; + + private const int TernaryLine = 15; + private const int EqualsLine = 15; + private const int NotEqualsLine = 20; + private const int LogicalAndLine = 28; + private const int LogicalOrLine = 33; + private const int NegationRemovalLine = 15; + private const int NegationWrapLine = 25; + private const int NullableNumericLine = 15; + private const int CoalesceLine = 15; + private const int CoalesceAssignLine = 25; + private const int OptionalArgumentLine = 27; + + private const int OperatorEqualsLine = 22; + private const int OperatorNotEqualsLine = 27; + private const int WalletSameLine = 45; + + private const int WrittenTrueLine = 17; + private const int WrittenFalseLine = 22; + private const int UnsetLine = 27; + private const int InitialLine = 35; + private const int MissingLine = 43; + private const int BelowZeroLine = 51; + + private const int BigCountLine = 15; + private const int TicketLine = 20; + private const int RatioLine = 28; + private const int ZeroLine = 33; + + /// + /// Money overloads == and != for the same parameter shape, which is the branch of + /// EqualityOperatorMutator the built-in-operator fixtures above never reach: + /// HasUsableCounterpart has to resolve the bound method as a user-defined operator, find its + /// declared counterpart on the containing type, and match their parameter types before allowing the + /// swap. The two comparisons inside the operator bodies themselves (line 22, line 27) are plain + /// comparisons - the built-in operator, not the user-defined one - so they are + /// two more, unrelated mutation points of the very same fixture. Equals's own this == other + /// (line 32) is a well known member override EquivalenceClassifier already excludes elsewhere, + /// so it contributes nothing here. + /// + private const string UserDefinedEqualitySource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public readonly struct Money + { + public readonly int Amount; + + public Money(int amount) + { + Amount = amount; + } + + public static bool operator ==(Money left, Money right) + { + return left.Amount == right.Amount; + } + + public static bool operator !=(Money left, Money right) + { + return left.Amount != right.Amount; + } + + public override bool Equals(object obj) + { + return obj is Money other && this == other; + } + + public override int GetHashCode() + { + return Amount; + } + } + + public static class Wallet + { + public static bool Same(Money left, Money right) + { + return left == right; + } + } + """; + + /// + /// Every supported underlying type of NullableLiteralMutator the flagship fixture does not + /// already cover: ? on both its written values and both directions out of + /// , ? on a non-default written value, + /// System.Guid? whose only literal-shaped mutation point is , and a + /// negative ? literal, which arrives as its own unary-minus node instead of a + /// bare literal. + /// + private const string NullableLiteralFamilySource = """ + namespace Fixture; + + using System; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Flags + { + public static bool? WrittenTrue() + { + return true; + } + + public static bool? WrittenFalse() + { + return false; + } + + public static bool? Unset() + { + return null; + } + } + + public static class Letters + { + public static char? Initial() + { + return 'A'; + } + } + + public static class Identifiers + { + public static Guid? Missing() + { + return null; + } + } + + public static class Temperatures + { + public static int? BelowZero() + { + return -5; + } + } + """; + + /// + /// Every constant kind of NumericLiteralMutator the flagship fixture does not already cover: an + /// integral type with its own boundary type (, ) and + /// both floating-point directions - the zero-to-one mutant and the negation mutant a non-zero value + /// gets instead. + /// + private const string NumericLiteralFamilySource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Counters + { + public static long BigCount() + { + return 10L; + } + + public static ulong Ticket() + { + return 3UL; + } + } + + public static class Measurements + { + public static double Ratio() + { + return 2.5; + } + + public static float Zero() + { + return 0.0f; + } + } + """; + + /// + /// Ternary.Pick returns a conditional expression on line 15 whose condition is a plain boolean + /// parameter: it is a mutation point of ConditionalExpressionMutator for the branches and of + /// LogicalNegationMutator for the condition itself, since the latter also watches every + /// conditional expression. + /// + private const string ConditionalSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Ternary + { + public static int Pick(bool flag, int a, int b) + { + return flag ? a : b; + } + } + """; + + /// + /// Two equality comparisons on lines 15 and 20 and two logical combinations on lines 28 and 33, all + /// over plain parameters so that no other operator has anything to mutate in the same expressions. + /// + private const string EqualityAndLogicalSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Numbers + { + public static bool AreEqual(int left, int right) + { + return left == right; + } + + public static bool AreDifferent(int left, int right) + { + return left != right; + } + } + + public static class Gate + { + public static bool Both(bool left, bool right) + { + return left && right; + } + + public static bool Either(bool left, bool right) + { + return left || right; + } + } + """; + + /// + /// Toggle.Invert removes an existing negation on line 15, while Guard.ClampNonNegative + /// wraps the condition of an if statement on line 25. Guard.Fallback is a + /// field, so its literal never becomes a mutation point of its own and the + /// method returning it carries none either. + /// + private const string NegationSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Toggle + { + public static bool Invert(bool flag) + { + return !flag; + } + } + + public static class Guard + { + private const int Fallback = 7; + + public static int ClampNonNegative(bool isValid, int value) + { + if (isValid) + { + return value; + } + + return Fallback; + } + } + """; + + /// + /// Quantity.Value returns the literal 5 converted to int? on line 15: a mutation + /// point of two operators at once, since NullableLiteralMutator only cares about the nullable + /// conversion and NumericLiteralMutator only cares about the literal's own (unwrapped) type. + /// + private const string NullableAndNumericSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Quantity + { + public static int? Value() + { + return 5; + } + } + """; + + /// + /// Config.Resolve coalesces two parameters on line 15, and + /// Cache.EnsureLoaded uses the coalescing assignment on line 25. Neither operand carries a + /// literal or any other construct another operator recognises. + /// + private const string NullCoalescingSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Config + { + public static string Resolve(string primary, string secondary) + { + return primary ?? secondary; + } + } + + public static class Cache + { + private static string _value = string.Empty; + + public static string EnsureLoaded(string fallback) + { + _value ??= fallback; + return _value; + } + } + """; + + /// + /// Passthrough is a minimal IComparer<int> whose members carry no mutation point of + /// their own - x.CompareTo(y) is not a construct any operator recognises, and its parameterless + /// object creation has no argument to remove - so that Sorted.Build on line 27 is the only + /// source of a gap: dropping the trailing comparer argument still binds to SortedSet<int>'s + /// parameterless constructor. + /// + private const string OptionalArgumentRemovalSource = """ + namespace Fixture; + + using System.Collections.Generic; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public sealed class Passthrough : IComparer + { + public static readonly Passthrough Instance = new Passthrough(); + + public int Compare(int x, int y) + { + return x.CompareTo(y); + } + } + + public static class Sorted + { + public static SortedSet Build() + { + return new SortedSet(Passthrough.Instance); + } + } + """; + + /// + /// Every fixture of this class, so that one test can prove that all of them compile and that none of + /// them makes the analyzer crash. + /// + /// One factory per fixture. + public static IEnumerable> Fixtures() => + new[] + { + ConditionalSource, + EqualityAndLogicalSource, + NegationSource, + NullableAndNumericSource, + NullCoalescingSource, + OptionalArgumentRemovalSource, + UserDefinedEqualitySource, + NullableLiteralFamilySource, + NumericLiteralFamilySource, + }.Select(source => (Func)(() => source)); + + /// + /// The flagship case of the conditional family: an untested ternary reports all four branch mutations + /// of ConditionalExpressionMutator plus the condition wrap of LogicalNegationMutator, + /// which also watches every conditional expression. + /// + [Test] + public async Task Analyze_UntestedTernaryConditional_ReportsAllFourBranchMutationsAndTheNegationWrap() + { + var compilation = CompilationFactory.Create(ConditionalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (TernaryLine, "c ? a : b => c ? b : a"), + (TernaryLine, "c ? a : b => !c ? a : b"), + (TernaryLine, "c ? a : b => true ? a : b"), + (TernaryLine, "c ? a : b => false ? a : b"), + (TernaryLine, "x => !(x)") + ) + ); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The same compilation, with Ternary.Pick itself recorded in the manifest: the analysis goes + /// completely silent, which is what makes the five gaps above a statement about the two operators + /// rather than about the fixture. + /// + [Test] + public async Task Analyze_CoveredTernaryConditional_ReportsNothing() + { + var compilation = CompilationFactory.Create(ConditionalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(TernaryPickMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(DiagnosticAssertions.Describe(diagnostics)) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// Every one of the four comparisons is offered its single counterpart mutation, with nothing left + /// covered: EqualityOperatorMutator on lines 15 and 20, LogicalOperatorMutator on lines + /// 28 and 33. + /// + [Test] + public async Task Analyze_UntestedEqualityAndLogicalOperators_ReportsEachOperatorSwap() + { + var compilation = CompilationFactory.Create(EqualityAndLogicalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (EqualsLine, "== => !="), + (NotEqualsLine, "!= => =="), + (LogicalAndLine, "&& => ||"), + (LogicalOrLine, "|| => &&") + ) + ); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The counter-example that proves the four gaps above are about coverage, not about the fixture: + /// covering Numbers.AreEqual alone silences only its own line, while the sibling comparison and + /// both logical combinations keep reporting. + /// + [Test] + public async Task Analyze_CoveredEqualityComparison_ReportsOnlyTheRemainingOperators() + { + var compilation = CompilationFactory.Create(EqualityAndLogicalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AreEqualMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect((NotEqualsLine, "!= => =="), (LogicalAndLine, "&& => ||"), (LogicalOrLine, "|| => &&")) + ); + } + } + + /// + /// Both directions of LogicalNegationMutator in one fixture: removing an existing negation on + /// line 15, and wrapping the condition of an if statement that has none on line 25. + /// + [Test] + public async Task Analyze_UntestedNegationRemovalAndWrap_ReportsBothDirections() + { + var compilation = CompilationFactory.Create(NegationSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(Expect((NegationRemovalLine, "!x => x"), (NegationWrapLine, "x => !(x)"))); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The literal 5 converted to int? is a mutation point of two operators at once: + /// NullableLiteralMutator offers moving it to and to the underlying + /// type's default, NumericLiteralMutator offers incrementing and decrementing it - four gaps at + /// the very same location, from two operators that ask different questions about it. + /// + [Test] + public async Task Analyze_UntestedNullableLiteralAndNumericLiteral_ReportsAllFourMutants() + { + var compilation = CompilationFactory.Create(NullableAndNumericSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(ExpectAt(NullableNumericLine, ["5 => null", "5 => 0", "5 => 6", "5 => 4"])); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The two halves of NullCoalescingMutator: a plain ?? expression offers keeping either + /// operand, and a ??= assignment offers becoming a plain assignment. + /// + [Test] + public async Task Analyze_UntestedNullCoalescingAndCoalesceAssignment_ReportsAllThreeMutants() + { + var compilation = CompilationFactory.Create(NullCoalescingSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (CoalesceLine, "a ?? b => a"), + (CoalesceLine, "a ?? b => b"), + (CoalesceAssignLine, "a ??= b => a = b") + ) + ); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// Covering Cache.EnsureLoaded alone silences only the coalescing assignment, leaving the plain + /// ?? expression of the unrelated, uncovered member reported - the same "covering one member + /// never covers another" claim CultureMutationTests makes for the culture family. + /// + [Test] + public async Task Analyze_CoveredCoalesceAssignment_ReportsOnlyTheCoalesceExpressionGaps() + { + var compilation = CompilationFactory.Create(NullCoalescingSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(EnsureLoadedMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(Expect((CoalesceLine, "a ?? b => a"), (CoalesceLine, "a ?? b => b"))); + } + } + + /// + /// The comparer argument of Sorted.Build is dropped because the remaining argument list still + /// binds to SortedSet<int>'s own parameterless constructor - the acceptance criterion + /// OptionalArgumentRemovalMutator exists for. + /// + [Test] + public async Task Analyze_UntestedOptionalArgumentRemoval_ReportsTheComparerRemoval() + { + var compilation = CompilationFactory.Create(OptionalArgumentRemovalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(Expect((OptionalArgumentLine, "Passthrough.Instance => (removed)"))); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// Exercises , see its doc comment for what each gap means. + /// + [Test] + public async Task Analyze_UntestedUserDefinedEquality_ReportsTheSwap() + { + var compilation = CompilationFactory.Create(UserDefinedEqualitySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (OperatorEqualsLine, "== => !="), + (OperatorNotEqualsLine, "!= => =="), + (WalletSameLine, "== => !=") + ) + ); + } + } + + /// + /// Exercises , see its doc comment for what each gap means. + /// + [Test] + public async Task Analyze_UntestedNullableLiteralFamily_ReportsEveryUnderlyingKind() + { + var compilation = CompilationFactory.Create(NullableLiteralFamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (WrittenTrueLine, "true => null"), + (WrittenTrueLine, "true => false"), + (WrittenTrueLine, "true => false"), + (WrittenFalseLine, "false => null"), + (WrittenFalseLine, "false => true"), + (UnsetLine, "null => false"), + (UnsetLine, "null => true"), + (InitialLine, "'A' => null"), + (InitialLine, "'A' => '\\0'"), + (MissingLine, "null => Guid.Empty"), + (BelowZeroLine, "-5 => null"), + (BelowZeroLine, "-5 => 0"), + (BelowZeroLine, "5 => 6"), + (BelowZeroLine, "5 => 4"), + (BelowZeroLine, "-x => +x"), + (BelowZeroLine, "-x => x") + ) + ); + } + } + + /// + /// Exercises , see its doc comment for what each gap means. + /// + [Test] + public async Task Analyze_UntestedNumericLiteralFamily_ReportsEveryNumericKind() + { + var compilation = CompilationFactory.Create(NumericLiteralFamilySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (BigCountLine, "10L => 11L"), + (BigCountLine, "10L => 9L"), + (TicketLine, "3UL => 4UL"), + (TicketLine, "3UL => 2UL"), + (RatioLine, "2.5 => -2.5"), + (ZeroLine, "0 => 1") + ) + ); + } + } + + /// + /// Every fixture of this class compiles and is analysed without the analyzer throwing. Roslyn turns an + /// analyzer exception into AD0001 and carries on, so a crash would otherwise look like a + /// diagnostic the tests above simply did not expect. + /// + /// The fixture to analyse. + /// A task that completes when the fixture was analysed. + [Test] + [MethodDataSource(nameof(Fixtures))] + public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(string source) + { + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched. + /// + /// + /// Every reference is attributed to one anonymous test whose case count is the lower bound + /// , and every reference is also written as behaviorally verified, so + /// that these coverage-only tests never trip FSH0006 or FSH0007 by accident. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.BehavioralReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Describes the reported gaps as one text block, one line per diagnostic, ordered ordinally so that + /// the result does not depend on the order the concurrently running analyzer callbacks reported them + /// in. Several gaps share one location, which is exactly the case a positional order cannot separate. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string Gaps(ImmutableArray diagnostics) + { + var gaps = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint); + + if (gaps.IsEmpty) + { + return NoGaps; + } + + return Join( + DiagnosticAssertions + .Summarise(gaps) + .OrderBy(summary => summary.Line) + .ThenBy(summary => summary.Message, StringComparer.Ordinal) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Builds the expectation of a set of gaps that all sit on . + /// + /// The 1-based line every gap is reported on. + /// The display names of the expected mutations. + /// The expected text block. + private static string ExpectAt(int line, IEnumerable displayNames) => + Expect([.. displayNames.Select(displayName => (Line: line, DisplayName: displayName))]); + + /// + /// Builds the expectation of a set of gaps, each one a line and the display name of its mutation. + /// + /// The expected gaps. + /// The expected text block, or when nothing is expected. + private static string Expect(params (int Line, string DisplayName)[] gaps) => + gaps.Length == 0 + ? NoGaps + : Join( + gaps.OrderBy(gap => gap.Line) + .ThenBy(gap => gap.DisplayName, StringComparer.Ordinal) + .Select(gap => GapEntry(gap.Line, gap.DisplayName)) + ); + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => $"{id} line {ToText(line)}: {message}"; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string Trivial(ImmutableArray diagnostics) => + DiagnosticAssertions.Describe(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.TrivialMutant)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/GeneratorDriverBridgeIntegrationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/GeneratorDriverBridgeIntegrationTests.cs new file mode 100644 index 0000000..60cea09 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/GeneratorDriverBridgeIntegrationTests.cs @@ -0,0 +1,218 @@ +namespace NetEvolve.FrameShift.Tests.Integration; + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Text; +using NetEvolve.FrameShift.Generation; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using NetEvolve.FrameShift.TestSurface.Bridges; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives through the real +/// pipeline, the way production code actually reaches it, +/// instead of through called directly by a unit test. +/// +/// +/// +/// tests/NetEvolve.Frameshift.Tests.Unit/TestSurface/Bridges/GeneratorDriverBridgeTests.cs already +/// proves every shape the bridge recognises — the fluent driver, .AsSourceGenerator(), an inline +/// array of generators, the local-array limitation, an unrelated RunGenerators look-alike — by +/// calling directly. None of those calls ever runs +/// itself, so the generator's own resolution of the bridge +/// through the shared bridge list of — building the +/// , testing IsApplicable, and feeding the result back +/// into the emitted manifest — never executes under Integration-project coverage at all. This class closes +/// that gap by running the one shape that matters most, the fluent +/// CSharpGeneratorDriver.Create(generator).RunGenerators(...) pattern, through the real generator via +/// , and reading the bridged entry point back out of the emitted manifest. +/// +/// +/// The fixture references the real Microsoft.CodeAnalysis and Microsoft.CodeAnalysis.CSharp +/// assemblies this test project itself already carries as package references, so no extra package is +/// needed to compile a production generator and a test driving it through . +/// The reference set of every other fixture in this suite deliberately excludes them — see the remarks of +/// ReferenceAssemblies — so they are added back explicitly, exactly as the unit-level counterpart +/// does. +/// +/// +public class GeneratorDriverBridgeIntegrationTests +{ + private const string ProductionAssemblyName = "GeneratorProductionAssembly"; + private const string TestAssemblyName = "GeneratorTestAssembly"; + private const string ProductionPath = "Production.cs"; + private const string TestPath = "BridgeTests.cs"; + + private const string ProductionSource = """ + namespace Fixture; + + using Microsoft.CodeAnalysis; + + public sealed class MyIncrementalGenerator : IIncrementalGenerator + { + public void Initialize(IncrementalGeneratorInitializationContext context) + { + } + } + """; + + private const string TestSource = """ + namespace Tests; + + using Microsoft.CodeAnalysis.CSharp; + using TUnit.Core; + + public class BridgeTests + { + [Test] + public void RunsIncrementalGeneratorThroughFluentDriver() + { + CSharpCompilation compilation = CSharpCompilation.Create("Empty"); + + _ = CSharpGeneratorDriver.Create(new Fixture.MyIncrementalGenerator()).RunGenerators(compilation); + } + } + """; + + private const string InitializeId = + "M:Fixture.MyIncrementalGenerator.Initialize(Microsoft.CodeAnalysis.IncrementalGeneratorInitializationContext)"; + + private const string DriverTestId = "M:Tests.BridgeTests.RunsIncrementalGeneratorThroughFluentDriver"; + + [Test] + public async Task Fixtures_BothAssemblies_CompileWithoutErrors() + { + var production = CreateProduction(); + var test = CreateTest(production); + + using (Assert.Multiple()) + { + _ = await Assert + .That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(production))) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + _ = await Assert + .That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(test))) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + } + + /// + /// The real generator resolves , finds it applicable to a + /// compilation that references the Roslyn generator API, and records the bridged Initialize + /// entry point in the manifest it emits — under the very test whose own syntax never spells that method + /// name anywhere. Without this bridge the entry point would never be reachable at all, and the + /// production analyzer would report it as an uncovered mutation point although a test exercises it + /// through the driver. + /// + [Test] + public async Task Generate_FluentGeneratorDriverPattern_RecordsTheBridgedInitializeMethod() + { + var test = CreateTest(CreateProduction()); + var text = Generate(test); + var (success, error, manifest) = Read(text); + + using (Assert.Multiple()) + { + _ = await Assert.That(success).IsTrue(); + _ = await Assert.That(error).IsEqualTo(string.Empty); + _ = await Assert.That(manifest.TestMethodIds.Contains(DriverTestId)).IsTrue(); + _ = await Assert.That(manifest.ReferencesByTest[DriverTestId].Contains(InitializeId)).IsTrue(); + } + } + + /// + /// Two runs of the same compilation through the real generator still agree on the bridged reference, + /// which is what an incremental generator promises: the bridge itself keeps no state between runs, and + /// nothing about resolving its context depends on the order invocations are walked in. + /// + [Test] + public async Task Generate_SameCompilationTwice_AgreeOnTheBridgedReference() + { + var test = CreateTest(CreateProduction()); + + var first = Generate(test); + var second = Generate(test); + + _ = await Assert.That(second).IsEqualTo(first); + } + + private static GeneratorRunner.Output Run(Compilation compilation) => + GeneratorRunner.Run(new TestSurfaceManifestGenerator(), compilation); + + private static string Generate(Compilation compilation) => + Run(compilation).TextOf(TestSurfaceManifestGenerator.HintName); + + /// + /// Parses the generated file the way the MSBuild target does: drop the first and the last line, hand + /// the rest to the reader. + /// + /// The content of the generated source file. + /// Whether the text parsed, the reported error and the parsed manifest. + private static (bool Success, string Error, TestSurfaceManifest Manifest) Read(string generated) + { + var inner = string.Join("\n", Lines(generated).Skip(1).SkipLast(1)) + "\n"; + var success = TestSurfaceManifestReader.TryRead(SourceText.From(inner), out var manifest, out var error); + + return (success, error ?? string.Empty, manifest); + } + + /// + /// Splits the generated text into its lines, dropping the empty remainder behind the trailing line + /// feed, which is the end of the last line and not a line of its own. + /// + /// The generated text. + /// The lines, without their line endings. + private static ImmutableArray Lines(string text) + { + var lines = text.Split('\n').Select(line => line.TrimEnd('\r')).ToList(); + + if (lines.Count > 0 && lines[^1].Length == 0) + { + lines.RemoveAt(lines.Count - 1); + } + + return [.. lines]; + } + + private static CSharpCompilation CreateProduction() => + CompilationFactory.Create( + ProductionSource, + TestFramework.None, + ProductionAssemblyName, + additionalReferences: RoslynReferences(), + filePath: ProductionPath + ); + + private static CSharpCompilation CreateTest(Compilation production) => + CompilationFactory.Create( + TestSource, + TestFramework.TUnit, + TestAssemblyName, + additionalReferences: [production.ToMetadataReference(), .. RoslynReferences()], + filePath: TestPath + ); + + /// + /// The metadata references the fixtures need to see the Roslyn generator API at all: this test project + /// already carries every one of these assemblies as package references, so no additional package is + /// required, but the default reference set every other fixture builds against deliberately excludes + /// them. + /// + /// + /// System.Collections.Immutable is not part of the default reference-assembly set on the + /// .NET Framework target frameworks this project builds for, even though it is on every modern .NET + /// target. and friends expose members typed + /// through it, so without this reference a .NET Framework compilation of the fixture fails with + /// CS0012 - the type is used but its defining assembly was never named. + /// + private static MetadataReference[] RoslynReferences() => + [ + MetadataReference.CreateFromFile(typeof(IIncrementalGenerator).Assembly.Location), + MetadataReference.CreateFromFile(typeof(CSharpGeneratorDriver).Assembly.Location), + MetadataReference.CreateFromFile(typeof(System.Collections.Immutable.ImmutableArray<>).Assembly.Location), + ]; +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ReachabilityAndEquivalenceMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ReachabilityAndEquivalenceMutationTests.cs new file mode 100644 index 0000000..383b775 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/ReachabilityAndEquivalenceMutationTests.cs @@ -0,0 +1,1185 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives end to end over call graphs shaped specifically to +/// exercise the reachability closure's dispatch approximation and related-member propagation, and over +/// member shapes shaped to exercise the equivalence classifier's member-level triviality checks. +/// +/// +/// +/// and already prove +/// the closure's plain call-graph walk (direct calls, transitive helpers, local functions, lambdas) and +/// the constant-folding and regex-shorthand halves of the equivalence classifier. What neither of them +/// drives through the real analyzer is virtual and interface dispatch, the accessors a property or event +/// shares its reachability with, or the member-level triviality checks (a throw-only body, code the +/// compiler already proves unreachable, a value assigned to a discard, and a member excluded by name or +/// by attribute). This file fills exactly that gap. +/// +/// +/// Every fixture pairs the member under inspection with Fixture.Reached.Identity, whose body +/// carries no mutation point at all. Naming that member in the manifest is what gives the analyzer a +/// non-empty reachable set - without one it reports an unusable manifest and stays silent about the +/// code - while contributing not a single diagnostic of its own. +/// +/// +public class ReachabilityAndEquivalenceMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts + /// on it, because these tests state what is reachable or trivial, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. + /// + private const string LowerBoundCount = "1+"; + + private const string NoGaps = ""; + private const string LineFeed = "\n"; + + private const string RendererDescribeMemberId = + "M:Fixture.Renderer.Describe(Fixture.IShape,System.Int32)~System.Int32"; + private const string ShapeAreaMemberId = "M:Fixture.IShape.Area(System.Int32)~System.Int32"; + private const string ZooCountLegsMemberId = "M:Fixture.Zoo.CountLegs(Fixture.Animal,System.Int32)~System.Int32"; + private const string ReaderReadMemberId = "M:Fixture.Reader.Read(Fixture.Counter)~System.Int32"; + + private const int SquareAreaLine = 20; + private const int CircleAreaLine = 28; + private const int InterfaceLonerLine = 44; + + private const int AnimalLegsLine = 15; + private const int DogLegsLine = 23; + private const int PuppyLegsLine = 31; + private const int OverrideLonerLine = 47; + + private const int PropertyGetLine = 17; + private const int PropertySetLine = 18; + private const int PropertyLonerLine = 34; + + private const int ThrowLine = 17; + private const int UnreachableStatementLine = 17; + private const int DiscardLine = 15; + + private const int ToStringLine = 26; + private const int ExcludeFromCoverageLine = 35; + private const int GeneratedCodeLine = 44; + private const int ObsoleteLine = 53; + + private const string ThrowOnlyBodyReason = "the containing member does nothing but throw"; + private const string UnreachableStatementReason = "the mutated statement is already unreachable"; + private const string DiscardAssignmentReason = "the mutated value is assigned to a discard"; + private const string WellKnownMemberReason = "the containing member is a well known infrastructure member"; + private const string ObsoleteMemberReason = "the containing member is marked obsolete"; + private const string ConstantFoldingReason = "the mutated expression folds to the same constant"; + private const string ConstantDeclarationReason = "the mutation only changes a compile-time constant"; + private const string DefaultParameterReason = "the mutation only changes a default parameter value"; + private const string CaseLabelReason = "the mutation only changes a compile-time case label"; + + private const string IntWrapperUseIntMemberId = "M:Fixture.IntWrapper.UseInt(System.Int32)~System.Int32"; + private const string SubscriberSubscribeMemberId = + "M:Fixture.Subscriber.Subscribe(Fixture.Bell,System.EventHandler)"; + + private const int GenericMethodLine = 15; + private const int GenericLonerLine = 39; + + private const int EventAddLine = 22; + private const int EventRemoveLine = 27; + private const int EventLonerLine = 45; + + private const int ConstantFoldingLine = 17; + private const int ConstantDeclarationLine = 13; + private const int DefaultParameterLine = 13; + private const int CaseLabelLine = 17; + + /// + /// The five mutants an untested value + 1 carries: the four counterparts the arithmetic + /// operator offers, plus the one-to-zero mutant the numeric literal operator offers for the literal + /// 1. Every equivalence fixture below reuses this exact shape, so that one array describes the + /// mutants of every member under inspection. + /// + private static readonly string[] _additionMutants = ["+ => -", "+ => *", "+ => /", "+ => %", "1 => 0"]; + + /// + /// Square and Circle both implement IShape and are declared nowhere else in the + /// compilation than here, so Renderer.Describe calling through the interface variable is the + /// only route a test can take to either of them. Loner.Unrelated implements nothing and is + /// called by nobody, so it stays a gap in every test of this fixture. + /// + private const string InterfaceDispatchSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public interface IShape + { + int Area(int side); + } + + public sealed class Square : IShape + { + public int Area(int side) + { + return side * side; + } + } + + public sealed class Circle : IShape + { + public int Area(int side) + { + return side * side; + } + } + + public static class Renderer + { + public static int Describe(IShape shape, int side) + { + return shape.Area(side); + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// Puppy overrides Dog, which overrides the virtual Animal.Legs: a call through + /// the base class reference has to walk two links of the override chain to reach both. Loner + /// carries no relation to Animal at all. + /// + private const string VirtualOverrideSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public class Animal + { + public virtual int Legs(int extra) + { + return 4 + extra; + } + } + + public class Dog : Animal + { + public override int Legs(int extra) + { + return 4 + extra; + } + } + + public class Puppy : Dog + { + public override int Legs(int extra) + { + return 4 + extra; + } + } + + public static class Zoo + { + public static int CountLegs(Animal animal, int extra) + { + return animal.Legs(extra); + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// Reader.Read only ever reads Counter.Value, never assigns it, yet both accessors of + /// the property are separate declarations of their own and share the reachability of the property + /// they belong to. + /// + private const string PropertyAccessorSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public sealed class Counter + { + private int _value; + + public int Value + { + get { return _value + 1; } + set { _value = value + 2; } + } + } + + public static class Reader + { + public static int Read(Counter counter) + { + return counter.Value; + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// Thrower.AlwaysFails never returns, so no test could ever observe the difference between + /// the mutated argument and the original one. + /// + private const string ThrowOnlySource = """ + namespace Fixture; + + using System; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Thrower + { + public static int AlwaysFails(int value) + { + throw new InvalidOperationException((value + 1).ToString()); + } + } + """; + + /// + /// The unconditional return value; always leaves the method, so the compiler proves the + /// statement after it - the one carrying the mutation point - can never execute, and reports + /// CS0162 for it. + /// + private const string UnreachableStatementSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class DeadCode + { + public static int Dead(int value) + { + return value; + + return value + 1; + } + } + """; + + /// + /// Discarder.Discard assigns the mutated value to _, so no test could ever observe it + /// regardless of what the value becomes. + /// + private const string DiscardAssignmentSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Discarder + { + public static void Discard(int value) + { + _ = value + 1; + } + } + """; + + /// + /// Four members, two different reasons a mutant of them never turns into a gap. Widget.ToString + /// and Deprecated.Old still get mutated - the mutant generator has no rule excluding a well + /// known infrastructure member or an obsolete one - so it is the equivalence classifier that has to + /// recognise both and report every mutant as trivial. Excluded.Ignored and Generated.Machine + /// are excluded one step earlier: the mutant generator itself never descends into a declaration + /// carrying [ExcludeFromCodeCoverage] or [GeneratedCode], so neither line produces a + /// mutation point - and therefore no diagnostic at all - in the first place. + /// + private const string ExcludedMemberSource = """ + namespace Fixture; + + using System; + using System.CodeDom.Compiler; + using System.Diagnostics.CodeAnalysis; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public sealed class Widget + { + private readonly int _value; + + public Widget(int value) + { + _value = value; + } + + public override string ToString() + { + return (_value + 1).ToString(); + } + } + + public static class Excluded + { + [ExcludeFromCodeCoverage] + public static int Ignored(int value) + { + return value + 1; + } + } + + public static class Generated + { + [GeneratedCode("tool", "1.0")] + public static int Machine(int value) + { + return value + 1; + } + } + + public static class Deprecated + { + [Obsolete] + public static int Old(int value) + { + return value + 1; + } + } + """; + + /// + /// Ops.Mark<T> ignores its type parameter entirely, yet IntWrapper.UseInt only ever + /// calls the <int> instantiation while StringWrapper.UseString calls the + /// <string> one. Both constructed methods normalize to the very same original definition, + /// so a test naming only the first instantiation still covers the shared body. + /// + private const string GenericMethodSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Ops + { + public static int Mark(int marker) + { + return marker + 1; + } + } + + public static class IntWrapper + { + public static int UseInt(int value) + { + return Ops.Mark(value); + } + } + + public static class StringWrapper + { + public static string UseString(int value) + { + return Ops.Mark(value).ToString(); + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// Subscriber.Subscribe only ever adds a handler to Bell.Rung, never removes one, yet the + /// remove accessor is a separate declaration that shares the reachability of the event it + /// belongs to, exactly like a property's setter shares the reachability of a getter-only caller. + /// + private const string EventAccessorSource = """ + namespace Fixture; + + using System; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public sealed class Bell + { + private int _count; + private EventHandler? _rung; + + public event EventHandler Rung + { + add + { + _count = _count + 1; + _rung += value; + } + remove + { + _count = _count - 1; + _rung -= value; + } + } + } + + public static class Subscriber + { + public static void Subscribe(Bell bell, EventHandler handler) + { + bell.Rung += handler; + } + } + + public static class Loner + { + public static int Unrelated(int value) + { + return value + 5; + } + } + """; + + /// + /// 4 - Two carries five mutation points: four arithmetic swaps of - and two increments + /// and decrements of the literal 4, which the numeric literal operator still offers because a + /// normal method body is not a constant-only context. Exactly one of the arithmetic swaps, the + /// division, folds to the very same value the original subtraction does, which is what the equivalence + /// classifier's constant folding check has to recognise among the other four candidates that do not. + /// + private const string ConstantFoldingSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Constants + { + private const int Two = 2; + + public static int Ratio() + { + return 4 - Two; + } + } + """; + + /// + /// 1 + 2 sits inside a field initializer, a position the numeric + /// literal operator already refuses to touch, but the arithmetic operator carries no such check of its + /// own. The equivalence classifier is what has to recognise every one of its four mutants as trivial, + /// regardless of the value each one folds to. + /// + private const string ConstantDeclarationSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Config + { + public const int Sum = 1 + 2; + } + """; + + /// + /// 1 + 2 sits inside a default parameter value, the same shape as + /// one syntax position over. + /// + private const string DefaultParameterSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Options + { + public static int WithDefault(int amount = 1 + 2) + { + return amount; + } + } + """; + + /// + /// 1 + 2 sits inside a classic case label, the third and last constant-only position + /// exercised in this file. + /// + private const string CaseLabelSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Classifier + { + public static int Describe(int value) + { + switch (value) + { + case 1 + 2: + return value; + default: + return value; + } + } + } + """; + + /// + /// Every fixture of this class, so that one test can prove that all of them compile and that none of + /// them makes the analyzer crash. + /// + /// One factory per fixture. + public static IEnumerable> Fixtures() => + new[] + { + InterfaceDispatchSource, + VirtualOverrideSource, + PropertyAccessorSource, + ThrowOnlySource, + UnreachableStatementSource, + DiscardAssignmentSource, + ExcludedMemberSource, + GenericMethodSource, + EventAccessorSource, + ConstantFoldingSource, + ConstantDeclarationSource, + DefaultParameterSource, + CaseLabelSource, + }.Select(source => (Func)(() => source)); + + /// + /// A test calling through an IShape variable reaches every implementation declared in the + /// compilation, not only the one a concrete call site happens to construct: the dispatch + /// approximation adds every implementation of the interface member it resolved to. + /// + [Test] + public async Task Analyze_TestCallsThroughInterfaceVariable_ReachesEveryImplementation() + { + var compilation = CompilationFactory.Create(InterfaceDispatchSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(RendererDescribeMemberId)]).ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([InterfaceLonerLine]); + _ = await Assert.That(lines.Contains(SquareAreaLine)).IsFalse(); + _ = await Assert.That(lines.Contains(CircleAreaLine)).IsFalse(); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// The dispatch approximation also runs when the manifest names the interface member itself, without + /// any caller in the manifest at all: a test that calls an interface member directly is the most + /// common shape there is, and the seed has to be treated exactly like a member reached transitively. + /// + [Test] + public async Task Analyze_ManifestNamesTheInterfaceMemberDirectly_StillReachesEveryImplementation() + { + var compilation = CompilationFactory.Create(InterfaceDispatchSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(ShapeAreaMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(GapLines(diagnostics)).IsEquivalentTo([InterfaceLonerLine]); + } + } + + /// + /// A call through a base class reference reaches every override in the chain, not only the one + /// closest to the declared type: Puppy.Legs overrides Dog.Legs, which overrides the + /// virtual Animal.Legs, so walking the override chain twice is what makes it reachable at all. + /// + [Test] + public async Task Analyze_TestCallsThroughBaseClassVariable_ReachesEveryOverrideInTheChain() + { + var compilation = CompilationFactory.Create(VirtualOverrideSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(ZooCountLegsMemberId)]).ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([OverrideLonerLine]); + _ = await Assert.That(lines.Contains(AnimalLegsLine)).IsFalse(); + _ = await Assert.That(lines.Contains(DogLegsLine)).IsFalse(); + _ = await Assert.That(lines.Contains(PuppyLegsLine)).IsFalse(); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A test reading a property never assigns it, yet the setter becomes reachable as well: the getter + /// and the setter are separate declarations, and both share the reachability of the property they + /// belong to once the property itself is reached. + /// + [Test] + public async Task Analyze_TestReadsAProperty_MakesTheSetterReachableAsWell() + { + var compilation = CompilationFactory.Create(PropertyAccessorSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(ReaderReadMemberId)]).ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([PropertyLonerLine]); + _ = await Assert.That(lines.Contains(PropertyGetLine)).IsFalse(); + _ = await Assert.That(lines.Contains(PropertySetLine)).IsFalse(); + } + } + + /// + /// A member whose body does nothing but throw can never let a test observe a mutated value, so every + /// mutant of it is reported as trivial instead of as a gap - even though nothing in the manifest + /// reaches the member at all. + /// + [Test] + public async Task Analyze_MemberBodyThrowsUnconditionally_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(ThrowOnlySource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((ThrowLine, ThrowOnlyBodyReason, _additionMutants))); + } + } + + /// + /// A statement the compiler already proves unreachable, here because the statement ahead of it + /// always returns, can never run at all, so every mutant inside it is trivial rather than a gap. + /// + [Test] + public async Task Analyze_StatementAfterAnUnconditionalReturn_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(UnreachableStatementSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((UnreachableStatementLine, UnreachableStatementReason, _additionMutants))); + } + } + + /// + /// A value assigned to the discard _ is thrown away in the very same statement, so no test + /// could ever tell the mutant from the original code. + /// + [Test] + public async Task Analyze_MutatedValueAssignedToADiscard_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(DiscardAssignmentSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((DiscardLine, DiscardAssignmentReason, _additionMutants))); + } + } + + /// + /// A well known infrastructure member and an obsolete one are still mutated, and the equivalence + /// classifier is what turns every one of their mutants into a trivial diagnostic instead of a gap. A + /// member excluded by [ExcludeFromCodeCoverage] or by [GeneratedCode] never reaches that + /// point: the mutant generator excludes the declaration outright, so neither line produces any + /// diagnostic, trivial or otherwise. + /// + [Test] + public async Task Analyze_WellKnownAndObsoleteMembers_ReportEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(ExcludedMemberSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + var lines = AllLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo( + ExpectTrivial( + (ToStringLine, WellKnownMemberReason, _additionMutants), + (ObsoleteLine, ObsoleteMemberReason, _additionMutants) + ) + ); + _ = await Assert.That(lines.Contains(ExcludeFromCoverageLine)).IsFalse(); + _ = await Assert.That(lines.Contains(GeneratedCodeLine)).IsFalse(); + } + } + + /// + /// Ops.Mark<T> ignores T entirely, and a test naming only the caller of its + /// <int> instantiation still covers the member's own mutation point: the reachable set + /// normalizes a constructed generic method back to its original definition, so it does not matter which + /// instantiation a test happens to go through. + /// + [Test] + public async Task Analyze_TestCallsOneInstantiationOfAGenericMethod_ReachesTheSharedDefinition() + { + var compilation = CompilationFactory.Create(GenericMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(IntWrapperUseIntMemberId)]).ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([GenericLonerLine]); + _ = await Assert.That(lines.Contains(GenericMethodLine)).IsFalse(); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A test that only ever adds a handler to an event never removes one, yet the remove accessor + /// becomes reachable as well: like a property's getter and setter, both accessors are separate + /// declarations that share the reachability of the event they belong to. + /// + [Test] + public async Task Analyze_TestAddsAnEventHandler_MakesTheRemoveAccessorReachableAsWell() + { + var compilation = CompilationFactory.Create(EventAccessorSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(SubscriberSubscribeMemberId)]) + .ConfigureAwait(false); + var lines = GapLines(diagnostics); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(lines).IsEquivalentTo([EventLonerLine]); + _ = await Assert.That(lines.Contains(EventAddLine)).IsFalse(); + _ = await Assert.That(lines.Contains(EventRemoveLine)).IsFalse(); + _ = await Assert.That(Trivial(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// Of the four arithmetic mutants of an untested subtraction, one - the division - folds to the very + /// same value the original does, and the equivalence classifier has to single it out as trivial while + /// leaving the other three, and the two literal increments and decrements of the numeric literal + /// operator, as genuine gaps. + /// + [Test] + public async Task Analyze_ArithmeticMutantFoldsToTheSameConstant_IsTrivialConstantFolding() + { + var compilation = CompilationFactory.Create(ConstantFoldingSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(ExpectGaps(ConstantFoldingLine, "- => +", "- => *", "- => %", "4 => 5", "4 => 3")); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((ConstantFoldingLine, ConstantFoldingReason, ["- => /"]))); + } + } + + /// + /// The arithmetic operator carries no check of its own for a field + /// initializer, unlike the numeric literal operator, so all four of its mutants are created; the + /// equivalence classifier is what has to prove every one of them trivial regardless of the value it + /// folds to. + /// + [Test] + public async Task Analyze_ArithmeticMutationInsideAConstFieldInitializer_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(ConstantDeclarationSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo( + ExpectTrivial( + (ConstantDeclarationLine, ConstantDeclarationReason, ["+ => -", "+ => *", "+ => /", "+ => %"]) + ) + ); + } + } + + /// + /// The same shape as a field initializer, one syntax position over: a default + /// parameter value can only ever be a compile-time constant, so no test could ever observe a mutant of + /// it either. + /// + [Test] + public async Task Analyze_ArithmeticMutationInsideADefaultParameterValue_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(DefaultParameterSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo( + ExpectTrivial( + (DefaultParameterLine, DefaultParameterReason, ["+ => -", "+ => *", "+ => /", "+ => %"]) + ) + ); + } + } + + /// + /// The third and last constant-only position exercised in this file: a classic case label can + /// only ever be a compile-time constant as well. + /// + [Test] + public async Task Analyze_ArithmeticMutationInsideACaseLabel_ReportsEveryMutantAsTrivial() + { + var compilation = CompilationFactory.Create(CaseLabelSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + _ = await Assert + .That(Trivial(diagnostics)) + .IsEqualTo(ExpectTrivial((CaseLabelLine, CaseLabelReason, ["+ => -", "+ => *", "+ => /", "+ => %"]))); + } + } + + /// + /// Every fixture of this class compiles and is analysed without the analyzer throwing. Roslyn turns + /// an analyzer exception into AD0001 and carries on, so a crash would otherwise look like a + /// diagnostic the tests above simply did not expect. + /// + /// The fixture to analyse. + /// A task that completes when the fixture was analysed. + [Test] + [MethodDataSource(nameof(Fixtures))] + public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(string source) + { + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched, and as behaviorally verified as well, so that FSH0007 never + /// shows up in a diagnostic set that is meant to be a statement about reachability or triviality alone. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.BehavioralReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Reduces the reported gaps to their distinct 1-based lines, so that a reachability assertion states + /// which lines are gaps without depending on how many mutants a single arithmetic expression carries. + /// + /// All diagnostics of a run. + /// The distinct lines carrying an FSH0001 diagnostic, possibly empty. + private static ImmutableArray GapLines(ImmutableArray diagnostics) => + [ + .. DiagnosticAssertions + .Summarise(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint)) + .Select(summary => summary.Line) + .Distinct() + .OrderBy(line => line), + ]; + + /// + /// Collects the distinct 1-based lines of every diagnostic of any kind, used to prove that a line + /// produces no diagnostic at all rather than merely no gap. + /// + /// All diagnostics of a run. + /// The distinct lines carrying any diagnostic, possibly empty. + private static ImmutableArray AllLines(ImmutableArray diagnostics) => + [ + .. DiagnosticAssertions + .Summarise(diagnostics) + .Select(summary => summary.Line) + .Distinct() + .OrderBy(line => line), + ]; + + /// + /// Describes every reported gap as one text block, one line per diagnostic, ordered ordinally so that + /// the result does not depend on the order the concurrently running analyzer callbacks reported them in. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string Gaps(ImmutableArray diagnostics) + { + var gaps = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint); + + return gaps.IsEmpty + ? NoGaps + : Join( + DiagnosticAssertions.Summarise(gaps).Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Describes every reported trivial mutant as one text block, exactly like does + /// for the coverage gaps. + /// + /// All diagnostics of a run. + /// The described trivial mutants, or when there is none. + private static string Trivial(ImmutableArray diagnostics) + { + var trivial = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.TrivialMutant); + + return trivial.IsEmpty + ? NoGaps + : Join( + DiagnosticAssertions + .Summarise(trivial) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Builds the expectation of a set of trivial mutants, each group sharing one line and one reason and + /// contributing one entry per display name in or an equivalent array. + /// + /// The expected groups. + /// The expected text block. + private static string ExpectTrivial(params (int Line, string Reason, string[] DisplayNames)[] groups) => + Join( + groups.SelectMany(group => + group.DisplayNames.Select(displayName => TrivialEntry(group.Line, displayName, group.Reason)) + ) + ); + + /// + /// Builds the described trivial mutant, spelling out the message + /// formats. + /// + /// The 1-based line the mutant is reported on. + /// The display name of the mutation. + /// The reason clause the classifier attached. + /// The described trivial mutant. + private static string TrivialEntry(int line, string displayName, string reason) => + Entry( + DiagnosticIds.TrivialMutant, + line, + "Mutation '" + displayName + "' cannot change observable behaviour (" + reason + ")" + ); + + /// + /// Builds the expectation of a set of gaps that all sit on , spelling out the + /// message formats. + /// + /// The 1-based line every gap is reported on. + /// The display names of the expected mutations. + /// The expected text block. + private static string ExpectGaps(int line, params string[] displayNames) => + Join(displayNames.Select(displayName => GapEntry(line, displayName))); + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => $"{id} line {ToText(line)}: {message}"; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/RegexInfrastructureMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/RegexInfrastructureMutationTests.cs new file mode 100644 index 0000000..c1167f6 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/RegexInfrastructureMutationTests.cs @@ -0,0 +1,1477 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Mutations.Operators; +using NetEvolve.FrameShift.Mutations.RegularExpressions; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives the regular expression infrastructure - the tokenizer and its nested scanner and class frame, +/// the token, the pattern locator, the pattern validity check and the pattern cache - together with the +/// backreference, character-class, escape and quantifier operators through +/// end to end. +/// +/// +/// +/// already proves the anchor, quantifier, group and +/// alternation operators, plus the character-class shorthand swap and the lookaround negation, reach the +/// build log as FSH0001. This class extends the same family with constructs that file does not +/// drive: a numbered backreference next to the plain capturing groups it renumbers, a character-class +/// range and its negation, a standalone class member and the dot/class equivalence in both directions, the +/// escaped-dot-to-any-character rewrite, and every shape of quantifier bound - the exact {n} form, +/// the open ended {n,} form and the full {n,m} form - together with the laziness toggle each +/// of the last two carries and the exact form does not. +/// +/// +/// Every fixture is built to keep a single operator family answerable for its reported gaps: the +/// backreference fixture pairs a backreference with the plain groups it needs to be a legal pattern, so +/// its expectation is the union of exactly those two operators, and every other fixture is a pattern that +/// carries no anchor, quantifier, group or shorthand escape at all, so only the operator under test can +/// report anything. Where a pattern cannot be kept that narrow - a class holding the very +/// \s/\S pair the dot-equivalence collapse recognises is unavoidably also a pair of shorthand +/// escapes the swap rewrites - the assertion falls back to naming the one message it needs to see, exactly +/// as does for its own character-class and lookaround +/// cases, rather than pinning a combinatorial set that would make the test fragile for reasons unrelated to +/// the construct it exists to prove. +/// +/// +/// Each fixture pairs the member under inspection with Fixture.Reached.Identity, whose body carries +/// no mutation point at all. Naming that member in the manifest is what gives the analyzer a non-empty +/// reachable set - without one it reports an unusable manifest and stays silent about the code - while +/// contributing not a single diagnostic of its own, exactly as in . +/// Every reported gap is filtered down to the ones whose message names a pattern mutation, the same +/// PatternMarker filter uses, so a mutation of an +/// unrelated family sitting on the same literal - for instance the RegexOptions flags of an explicit +/// options argument - can never leak into an expectation this class does not state anything about. +/// +/// +public class RegexInfrastructureMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + /// + /// The member of that carries the pattern, used to prove the + /// gaps of that fixture disappear once it is itself recorded as covered. + /// + private const string BackreferenceMemberId = "M:Fixture.Patterns.IsMatch(System.String)~System.Boolean"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts on + /// it, because these tests state what the operators report, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. It also keeps FSH0006 + /// silent, so that every expectation below stays a statement about the operators alone. + /// + private const string LowerBoundCount = "1+"; + + /// + /// The text every display name of the family starts its description with, and therefore the filter + /// that separates a pattern mutation from every other mutation of the same compilation. + /// + private const string PatternMarker = "pattern '"; + + /// + /// The text the assertions use for "not a single gap was reported". + /// + private const string NoGaps = ""; + + /// + /// The line feed the expectations are joined with, instead of , so + /// that the very same text is produced on Windows and on Linux. + /// + private const string LineFeed = "\n"; + + /// + /// A numbered backreference next to the three plain capturing groups it needs to be a legal pattern. + /// The group mutator offers to turn each of the three plain groups into a non-capturing one, and every + /// one of the three results is still a legal pattern because the backreference always finds a second + /// group to resolve against; the backreference mutator offers to shift \2 to \3 and to + /// \1, both legal because the pattern defines three groups. No anchor, quantifier, character + /// class or shorthand escape sits anywhere in the pattern, so these five mutations are the whole of + /// what either operator can report here. + /// + private const string BackreferenceAndGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"(a)(b)(c)\2"); + } + } + """; + + /// + /// A plain range with no other construct in the pattern: the negation toggle of the class open, and + /// the widening of both the range's start and its end by one code unit. Neither a nor z + /// is offered as a standalone removal, because both are the endpoint of the range rather than a + /// standalone member. + /// + private const string RangeSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "[a-z]"); + } + } + """; + + /// + /// Three standalone class members next to a literal dot: the negation toggle of the class, the + /// removal of each of the three members, and the expansion of the dot into [\s\S]. None of the + /// three members is the endpoint of a range, so every one of them is offered as a removal. + /// + private const string MemberRemovalAndDotSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "[xyz]."); + } + } + """; + + /// + /// The exact four token run [\s\S], which the class-open dispatch recognises as the operand of + /// the dot-equivalence collapse in addition to offering its own negation toggle. The very same two + /// tokens are also shorthand escapes in their own right, so the shorthand swap fires for each of them + /// as well; that combinatorial part of the output is not pinned here; see the class remarks. + /// + private const string AnyClassCollapseSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"[\s\S]"); + } + } + """; + + /// + /// A lone escaped dot, the one construct RegexEscapeMutator answers for. It is not one of the + /// six shorthand classes the character-class operator swaps, so this fixture is answerable by exactly + /// one operator. + /// + private const string EscapedDotSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\."); + } + } + """; + + /// + /// The exact {n} form of a counted quantifier, whose two rewrites decrease and increase the + /// count. Its repetition count leaves the engine no choice about how many times to repeat, so the + /// laziness toggle is never offered for it. + /// + private const string ExactCountSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a{3}"); + } + } + """; + + /// + /// The open ended {n,} form of a counted quantifier, whose lower bound is shifted in both + /// directions and whose laziness is toggled, because unlike the exact form it does leave the engine a + /// choice. + /// + private const string OpenEndedCountSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a{2,}"); + } + } + """; + + /// + /// The full {n,m} form of a counted quantifier, whose lower and upper bounds are each shifted + /// in both directions and whose laziness is toggled. + /// + private const string BoundedRangeSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a{2,3}"); + } + } + """; + + /// + /// A lazy +? quantifier: the shape swap to *?, which keeps the laziness, and the + /// laziness toggle back to the greedy +. + /// + private const string LazyPlusSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a+?"); + } + } + """; + + /// + /// An optional ? quantifier: its removal, which makes the atom mandatory, and its laziness + /// toggle to ??. + /// + private const string OptionalSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "a?"); + } + } + """; + + /// + /// The pattern reached through a named, reordered constructor argument rather than the positional + /// leading argument every other fixture of this class uses, which is what exercises the locator's + /// argument binding by name instead of by position. The explicit RegexOptions.IgnoreCase + /// argument is itself a mutation point of the unrelated RegexOptions family, which is exactly + /// why the assertion below filters by instead of taking every gap of the + /// member. + /// + private const string NamedArgumentSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static Regex Create() + { + return new Regex(options: RegexOptions.IgnoreCase, pattern: "a+"); + } + } + """; + + /// + /// A plain named group with the angle-bracket delimiter and no other construct in the pattern. The + /// group mutator recognises it as a capturing open exactly like a plain (, so it offers the very + /// same demotion to (?:; this is what proves the tokenizer's angle-bracket name scan + /// (ScanAngleGroup/ScanNamedGroup) and the mutator's name-aware dispatch agree with each + /// other. + /// + private const string NamedAngleGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "(?x)"); + } + } + """; + + /// + /// The quote-delimited spelling of a named group, (?'name', which the tokenizer scans through the + /// very same ScanNamedGroup path as the angle-bracket form but with the other closing character. + /// The mutator's own quote-prefix branch is what has to recognise it as a capturing open. + /// + private const string QuotedNameGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "(?'year'x)"); + } + } + """; + + /// + /// An atomic group (?>...) next to the string-start anchor it does not interact with. The + /// group mutator offers nothing for the atomic opening - it changes backtracking, not capturing - so the + /// anchor's own removal is the only pattern mutation this fixture can report; that absence is what + /// proves the mutator's capturing-open dispatch actually excludes this token text rather than merely + /// never having been asked about it. + /// + private const string AtomicGroupWithAnchorSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\A(?>x)"); + } + } + """; + + /// + /// A balancing group pair, (?<open>x) defining the capture and (?<-open>) + /// popping it. The mutator's own dash check keeps it from ever touching the pop, and while it does offer + /// to demote the defining group to (?:, that rewrite leaves the pop referring to a capture that + /// no longer exists, so discards it as an invalid mutant. This is + /// therefore a fixture with a real candidate mutation and zero surviving gaps, which is what separates + /// "the operator declined to look" from "the base class rejected what it produced". + /// + private const string BalancingGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "(?x)(?<-open>)"); + } + } + """; + + /// + /// Two named backreferences, one of each delimiter, next to the named groups they resolve against. The + /// backreference operator recognises the \k prefix and offers nothing for either of them, and the + /// group mutator's candidate demotion of either defining group is discarded because it would leave the + /// matching backreference undefined - exactly the same "candidate produced, mutant invalid" shape as + /// , but exercised through ScanNamedBackreference and both + /// forms of TryMeasureBracketedName instead of the balancing branch of ScanNamedGroup. + /// + private const string NamedBackreferenceSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"(?x)\k(?'b'y)\k'b'"); + } + } + """; + + /// + /// A (?#...) comment, a scoped inline-options group (?i:...) and a standalone inline + /// options construct (?i), one after another. None of the three is a capturing open the group + /// mutator answers for - the comment carries no group token at all, the scoped form is a group whose + /// text is neither ( nor (?: nor a plain named prefix, and the standalone form tokenizes + /// as rather than as + /// - so the whole pattern reports nothing at all, which is what + /// proves every one of the three tokenizer paths (ScanInlineComment, the scoped and the + /// standalone branch of ScanOptions) was actually exercised rather than short-circuited by an + /// earlier failure. + /// + private const string InlineConstructsSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "(?#note)(?i:a)(?i)b"); + } + } + """; + + /// + /// A Unicode category escape and its negation, \p{L} and \P{Lu}, which the tokenizer scans + /// through ScanUnicodeCategoryEscape and no operator of the family recognises: neither is one of + /// the six shorthand classes the character-class operator swaps, and neither is the escaped dot the + /// escape operator rewrites. The pattern therefore reports nothing at all, which is the fixture's whole + /// point - it exists to reach the property-escape scan, not to report a mutation. + /// + private const string UnicodeCategoryEscapeSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\p{L}\P{Lu}"); + } + } + """; + + /// + /// A hexadecimal escape, a Unicode escape and a control-character escape back to back, which reach + /// ScanHexadecimalEscape under both of its digit counts and ScanControlEscape. None of the + /// three is a shorthand class or the escaped dot, so - exactly like + /// - the pattern reports nothing at all. + /// + private const string HexAndControlEscapeSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\x41\u0041\cA"); + } + } + """; + + /// + /// A single plain group followed by \10. With exactly one group defined, \10 names no + /// capture the pattern has, so the tokenizer's ResolveNumberedBackreferences step re-reads it as + /// the octal escape \10 in full - both digits are octal, so nothing is left over as a literal - + /// instead of leaving it a backreference. Neither the escape nor the backreference operator has anything + /// to say about the result, so the only mutation left standing is the plain group's own demotion, and + /// demoting it away is harmless here: with the group gone, the very same digits are still read as the + /// very same octal escape by a fresh construction of the mutant, so the rewrite validates. + /// + private const string OctalReinterpretationWithGroupSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"(a)\10"); + } + } + """; + + /// + /// \18 with no group defined anywhere in the pattern. Only the leading 1 is an octal digit, + /// so ResolveNumberedBackreferences consumes it alone as the octal escape \1 and reinserts + /// the trailing 8 as a separate literal token - the one branch of that method none of the other + /// fixtures in this class reach, because they either define enough groups to stay a genuine + /// backreference or consume every digit as octal with nothing left over. Neither the reinterpreted escape + /// nor the reinserted literal digit is a construct any operator of the family rewrites, so the pattern + /// reports nothing at all. + /// + private const string LeftoverOctalDigitSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, @"\18"); + } + } + """; + + /// + /// A range next to a class subtraction, [a-z-[aeiou]]: the outer class negation, the widening of + /// both ends of the range, the nested class's own negation, and the removal of each of its five + /// standalone members. This is the one fixture of the class that reaches the subtraction dispatch of + /// ScanCharacterClassDash - the -[ that opens a nested class rather than a range - and the + /// SubtractionApplied enforcement that nothing but the outer ] may follow it. + /// + private const string CharacterClassSubtractionSource = """ + namespace Fixture; + + using System.Text.RegularExpressions; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Patterns + { + public static bool IsMatch(string value) + { + return Regex.IsMatch(value, "[a-z-[aeiou]]"); + } + } + """; + + /// + /// The line every pattern of this class sits on. Every fixture shares the same layout - the + /// Reached anchor, a blank line, the class declaring the pattern - so the pattern always lands + /// on the very same line. + /// + private const int PatternLine = 17; + + /// + /// Every mutant of 's pattern: the three group demotions in + /// pattern order, followed by the backreference's increase and decrease. + /// + private static readonly string[] _backreferenceAndGroupMutants = + [ + Mutant(@"(a)(b)(c)\2", @"(?:a)(b)(c)\2"), + Mutant(@"(a)(b)(c)\2", @"(a)(?:b)(c)\2"), + Mutant(@"(a)(b)(c)\2", @"(a)(b)(?:c)\2"), + Mutant(@"(a)(b)(c)\2", @"(a)(b)(c)\3"), + Mutant(@"(a)(b)(c)\2", @"(a)(b)(c)\1"), + ]; + + /// + /// The two mutants of 's pattern: the negation toggle and the widening of + /// both ends of the range. + /// + private static readonly string[] _rangeMutants = + [ + Mutant("[a-z]", "[^a-z]"), + Mutant("[a-z]", "[`-z]"), + Mutant("[a-z]", "[a-{]"), + ]; + + /// + /// Every mutant of 's pattern: the negation toggle, the + /// removal of each standalone member in pattern order, and the dot expansion. + /// + private static readonly string[] _memberRemovalAndDotMutants = + [ + Mutant("[xyz].", "[^xyz]."), + Mutant("[xyz].", "[yz]."), + Mutant("[xyz].", "[xz]."), + Mutant("[xyz].", "[xy]."), + Mutant("[xyz].", @"[xyz][\s\S]"), + ]; + + /// + /// The two mutants of 's pattern: the decrease and the increase of the + /// exact count. + /// + private static readonly string[] _exactCountMutants = [Mutant("a{3}", "a{2}"), Mutant("a{3}", "a{4}")]; + + /// + /// The three mutants of 's pattern: both shifts of the lower bound, + /// then the laziness toggle. + /// + private static readonly string[] _openEndedCountMutants = + [ + Mutant("a{2,}", "a{1,}"), + Mutant("a{2,}", "a{3,}"), + Mutant("a{2,}", "a{2,}?"), + ]; + + /// + /// The five mutants of 's pattern: both shifts of the lower bound, + /// both shifts of the upper bound, then the laziness toggle. + /// + private static readonly string[] _boundedRangeMutants = + [ + Mutant("a{2,3}", "a{1,3}"), + Mutant("a{2,3}", "a{3,3}"), + Mutant("a{2,3}", "a{2,2}"), + Mutant("a{2,3}", "a{2,4}"), + Mutant("a{2,3}", "a{2,3}?"), + ]; + + /// + /// The two mutants of 's pattern: the shape swap to a lazy star, then the + /// laziness toggle back to greedy. + /// + private static readonly string[] _lazyPlusMutants = [Mutant("a+?", "a*?"), Mutant("a+?", "a+")]; + + /// + /// The two mutants of 's pattern: the removal, then the laziness toggle. + /// + private static readonly string[] _optionalMutants = [Mutant("a?", "a"), Mutant("a?", "a??")]; + + /// + /// The two mutants of 's pattern, the same shape swap and laziness + /// toggle a bare greedy + offers everywhere else in this class. + /// + private static readonly string[] _namedArgumentMutants = [Mutant("a+", "a*"), Mutant("a+", "a+?")]; + + /// + /// The single mutant of 's pattern: the demotion of the named group + /// to non-capturing. + /// + private static readonly string[] _namedAngleGroupMutants = [Mutant("(?x)", "(?:x)")]; + + /// + /// The single mutant of 's pattern: the demotion of the named group + /// to non-capturing. + /// + private static readonly string[] _quotedNameGroupMutants = [Mutant("(?'year'x)", "(?:x)")]; + + /// + /// The single mutant of 's pattern: the removal of the + /// string-start anchor. The atomic group offers nothing of its own. + /// + private static readonly string[] _atomicGroupWithAnchorMutants = [Mutant(@"\A(?>x)", "(?>x)")]; + + /// + /// The single mutant of 's pattern: the demotion of + /// the plain group to non-capturing, which validates because the octal escape behind it is read the same + /// way whether or not the group exists. + /// + private static readonly string[] _octalReinterpretationWithGroupMutants = [Mutant(@"(a)\10", @"(?:a)\10")]; + + /// + /// Every mutant of 's pattern: the outer class negation, + /// the widening of both ends of the range, the nested class's own negation, and the removal of each of + /// its five standalone members. + /// + private static readonly string[] _characterClassSubtractionMutants = + [ + Mutant("[a-z-[aeiou]]", "[^a-z-[aeiou]]"), + Mutant("[a-z-[aeiou]]", "[`-z-[aeiou]]"), + Mutant("[a-z-[aeiou]]", "[a-{-[aeiou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[^aeiou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[eiou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[aiou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[aeou]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[aeiu]]"), + Mutant("[a-z-[aeiou]]", "[a-z-[aeio]]"), + ]; + + /// + /// Every fixture of this class, so that one test can prove that all of them compile and that none of + /// them makes the analyzer crash. + /// + /// One factory per fixture. + public static IEnumerable> Fixtures() => + new[] + { + BackreferenceAndGroupSource, + RangeSource, + MemberRemovalAndDotSource, + AnyClassCollapseSource, + EscapedDotSource, + ExactCountSource, + OpenEndedCountSource, + BoundedRangeSource, + LazyPlusSource, + OptionalSource, + NamedArgumentSource, + NamedAngleGroupSource, + QuotedNameGroupSource, + AtomicGroupWithAnchorSource, + BalancingGroupSource, + NamedBackreferenceSource, + InlineConstructsSource, + UnicodeCategoryEscapeSource, + HexAndControlEscapeSource, + OctalReinterpretationWithGroupSource, + LeftoverOctalDigitSource, + CharacterClassSubtractionSource, + }.Select(source => (Func)(() => source)); + + /// + /// The backreference next to the plain groups it renumbers reports a gap per group demotion and per + /// backreference shift, and nothing else. + /// + [Test] + public async Task Analyze_UntestedBackreferenceAndGroups_ReportsAGapPerGroupAndBackreferenceMutation() + { + var compilation = CompilationFactory.Create(BackreferenceAndGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _backreferenceAndGroupMutants)); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// Recording the pattern's own member as covered makes every one of its gaps disappear, which is what + /// proves the five gaps above to be a statement about coverage rather than about the operators. + /// + [Test] + public async Task Analyze_CoveredBackreferenceAndGroups_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(BackreferenceAndGroupSource, ProductionAssemblyName); + var manifest = new[] { CreateManifest(AnchorMemberId, BackreferenceMemberId) }; + + var diagnostics = await RunAsync(compilation, manifest).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A range with no other construct in its pattern reports its negation toggle and the widening of both + /// of its ends, and nothing else - neither one of its two endpoints is offered as a standalone removal. + /// + [Test] + public async Task Analyze_UntestedRange_ReportsTheNegationAndBothWidenings() + { + var compilation = CompilationFactory.Create(RangeSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _rangeMutants)); + } + } + + /// + /// Three standalone class members next to a literal dot report the class negation, the removal of + /// each member and the dot expansion, and nothing else. + /// + [Test] + public async Task Analyze_UntestedStandaloneMembersAndDot_ReportsEveryRemovalAndTheDotExpansion() + { + var compilation = CompilationFactory.Create(MemberRemovalAndDotSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _memberRemovalAndDotMutants)); + } + } + + /// + /// The exact [\s\S] token run collapses to a dot, in addition to offering its own negation + /// toggle, which is what proves the class-open dispatch recognises the run rather than merely toggling + /// the class it opens. + /// + [Test] + public async Task Analyze_UntestedAnyCharacterClass_ReportsTheCollapseToDot() + { + var compilation = CompilationFactory.Create(AnyClassCollapseSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + var messages = Summaries(diagnostics).Select(summary => summary.Message).ToArray(); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(messages.Any(message => message.Contains(@"pattern '[\s\S]' => '.'", StringComparison.Ordinal))) + .IsTrue(); + _ = await Assert + .That( + messages.Any(message => + message.Contains(@"pattern '[\s\S]' => '[^\s\S]'", StringComparison.Ordinal) + ) + ) + .IsTrue(); + } + } + + /// + /// A lone escaped dot reports its widening into any character, and nothing else - it is not one of + /// the six shorthand classes the character-class operator swaps. + /// + [Test] + public async Task Analyze_UntestedEscapedDot_ReportsTheWideningToAnyCharacter() + { + var compilation = CompilationFactory.Create(EscapedDotSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, [Mutant(@"\.", ".")])); + } + } + + /// + /// The exact {n} form reports its decrease and its increase, and never a laziness toggle: its + /// repetition count leaves the engine no choice to make lazy. + /// + [Test] + public async Task Analyze_UntestedExactCount_ReportsTheDecreaseAndIncreaseButNoLazinessToggle() + { + var compilation = CompilationFactory.Create(ExactCountSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _exactCountMutants)); + } + } + + /// + /// The open ended {n,} form reports both shifts of its lower bound and its laziness toggle. + /// + [Test] + public async Task Analyze_UntestedOpenEndedCount_ReportsBothBoundShiftsAndTheLazinessToggle() + { + var compilation = CompilationFactory.Create(OpenEndedCountSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _openEndedCountMutants)); + } + } + + /// + /// The full {n,m} form reports both shifts of its lower bound, both shifts of its upper bound + /// and its laziness toggle. + /// + [Test] + public async Task Analyze_UntestedBoundedRange_ReportsEveryBoundShiftAndTheLazinessToggle() + { + var compilation = CompilationFactory.Create(BoundedRangeSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _boundedRangeMutants)); + } + } + + /// + /// A lazy plus reports the shape swap to a lazy star, which keeps the laziness, and the laziness + /// toggle back to a greedy plus. + /// + [Test] + public async Task Analyze_UntestedLazyPlus_ReportsTheShapeSwapAndTheLazinessToggle() + { + var compilation = CompilationFactory.Create(LazyPlusSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _lazyPlusMutants)); + } + } + + /// + /// An optional quantifier reports its removal, which makes the atom mandatory, and its laziness + /// toggle. + /// + [Test] + public async Task Analyze_UntestedOptional_ReportsTheRemovalAndTheLazinessToggle() + { + var compilation = CompilationFactory.Create(OptionalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _optionalMutants)); + } + } + + /// + /// A pattern reached through a named, reordered constructor argument is located and mutated exactly + /// like one reached positionally, which is what proves the locator's argument binding resolves a + /// pattern by parameter rather than by its position in the argument list. + /// + [Test] + public async Task Analyze_UntestedPatternBehindNamedArgument_ReportsItsQuantifierMutations() + { + var compilation = CompilationFactory.Create(NamedArgumentSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _namedArgumentMutants)); + } + } + + /// + /// A plain named group with the angle-bracket delimiter is demoted to non-capturing exactly like a bare + /// (, which is what proves the mutator's named-capture recognition - and the tokenizer's + /// ScanAngleGroup/ScanNamedGroup path behind it - agree that the two spellings define the + /// same kind of capture. + /// + [Test] + public async Task Analyze_UntestedNamedAngleGroup_ReportsItsDemotionToNonCapturing() + { + var compilation = CompilationFactory.Create(NamedAngleGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _namedAngleGroupMutants)); + } + } + + /// + /// The quote-delimited spelling of a named group is demoted exactly like its angle-bracket counterpart, + /// which is what proves the mutator's quote-prefix branch, and the corresponding closing-quote path of + /// ScanNamedGroup, are exercised rather than only the angle-bracket one. + /// + [Test] + public async Task Analyze_UntestedQuotedNameGroup_ReportsItsDemotionToNonCapturing() + { + var compilation = CompilationFactory.Create(QuotedNameGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(ExpectAt(PatternLine, _quotedNameGroupMutants)); + } + } + + /// + /// An atomic group next to the string-start anchor reports only the anchor's own removal - the group + /// mutator offers nothing for the atomic opening at all, which is what separates "the operator was asked + /// and declined" from "the operator was never asked". + /// + [Test] + public async Task Analyze_UntestedAtomicGroupWithAnchor_ReportsOnlyTheAnchorRemoval() + { + var compilation = CompilationFactory.Create(AtomicGroupWithAnchorSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _atomicGroupWithAnchorMutants)); + } + } + + /// + /// A balancing group pair reports no pattern gap at all: the pop is never offered a rewrite because its + /// name carries the dash the mutator's capturing-open check rejects, and the one candidate the defining + /// group does offer is discarded because demoting it would leave the pop referring to a capture that no + /// longer exists. A real candidate mutation therefore still produces zero surviving gaps. + /// + [Test] + public async Task Analyze_UntestedBalancingGroup_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(BalancingGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// Two named backreferences, one of each delimiter, report no pattern gap at all: the backreference + /// operator recognises the \k prefix and offers nothing for either of them, and the group + /// mutator's candidate demotion of either defining group is discarded because it would leave the + /// matching backreference undefined. + /// + [Test] + public async Task Analyze_UntestedNamedBackreferences_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(NamedBackreferenceSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A comment, a scoped inline-options group and a standalone inline-options construct report no pattern + /// gap at all: none of the three tokenizes as a capturing open the group mutator answers for, which is + /// what proves every one of the three tokenizer paths behind them was reached rather than short-circuited + /// by an earlier failure. + /// + [Test] + public async Task Analyze_UntestedInlineConstructs_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(InlineConstructsSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A Unicode category escape and its negation report no pattern gap at all: neither is one of the six + /// shorthand classes the character-class operator swaps, and neither is the escaped dot the escape + /// operator rewrites. + /// + [Test] + public async Task Analyze_UntestedUnicodeCategoryEscapes_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(UnicodeCategoryEscapeSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A hexadecimal escape, a Unicode escape and a control-character escape report no pattern gap at all, + /// for the same reason as : none of the three is a shorthand + /// class or the escaped dot. + /// + [Test] + public async Task Analyze_UntestedHexAndControlEscapes_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(HexAndControlEscapeSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A single plain group followed by \10 reports only the group's own demotion: with exactly one + /// group defined, \10 names no capture the pattern has, so the tokenizer re-reads it as a full + /// octal escape instead of a backreference, and neither the escape nor the backreference operator has + /// anything to say about the result. + /// + [Test] + public async Task Analyze_UntestedOctalReinterpretationWithGroup_ReportsOnlyTheGroupDemotion() + { + var compilation = CompilationFactory.Create(OctalReinterpretationWithGroupSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _octalReinterpretationWithGroupMutants)); + } + } + + /// + /// \18 with no group defined anywhere in the pattern reports no pattern gap at all: the tokenizer + /// consumes only the leading digit as an octal escape and reinserts the trailing digit as a literal, + /// and neither token is a construct any operator of the family rewrites. + /// + [Test] + public async Task Analyze_UntestedLeftoverOctalDigit_ReportsNoPatternGapAtAll() + { + var compilation = CompilationFactory.Create(LeftoverOctalDigitSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(PatternGaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// A range next to a class subtraction reports the outer class negation, the widening of both ends of + /// the range, the nested class's own negation, and the removal of each of its five standalone members - + /// the one fixture of the class that reaches the subtraction dispatch of ScanCharacterClassDash + /// and the SubtractionApplied enforcement behind it. + /// + [Test] + public async Task Analyze_UntestedCharacterClassSubtraction_ReportsEveryConstituentMutation() + { + var compilation = CompilationFactory.Create(CharacterClassSubtractionSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(PatternGaps(diagnostics)) + .IsEqualTo(ExpectAt(PatternLine, _characterClassSubtractionMutants)); + } + } + + /// + /// Every fixture of this class compiles and is analysed without the analyzer throwing. Roslyn turns an + /// analyzer exception into AD0001 and carries on, so a crash would otherwise look like a + /// diagnostic the tests above simply did not expect. + /// + /// The fixture to analyse. + /// A task that completes when the fixture was analysed. + [Test] + [MethodDataSource(nameof(Fixtures))] + public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(string source) + { + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Describes the reported gaps that name a pattern mutation, one line per diagnostic, ordered + /// ordinally so that the result does not depend on the order the concurrently running analyzer + /// callbacks reported them in. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string PatternGaps(ImmutableArray diagnostics) + { + var entries = Summaries(diagnostics) + .Where(summary => NamesAPattern(summary.Message)) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + .ToList(); + + return entries.Count == 0 ? NoGaps : Join(entries); + } + + private static bool NamesAPattern(string message) => message.Contains(PatternMarker, StringComparison.Ordinal); + + private static ImmutableArray<(string Id, int Line, string Message)> Summaries( + ImmutableArray diagnostics + ) => DiagnosticAssertions.Summarise(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint)); + + /// + /// Builds the expectation of a set of gaps that all sit on . + /// + /// The 1-based line every gap is reported on. + /// The display names of the expected mutations. + /// The expected text block. + private static string ExpectAt(int line, IEnumerable displayNames) => + Join([.. displayNames.Select(displayName => GapEntry(line, displayName))]); + + /// + /// Composes the display name a pattern mutation carries. + /// + /// The original pattern, as the regular expression engine sees it. + /// The rewritten pattern, as the regular expression engine sees it. + /// The display name. + private static string Mutant(string original, string mutated) => + PatternMarker + original + "' => '" + mutated + "'"; + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => id + " line " + ToText(line) + ": " + message; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/StringLinqMathMutationTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/StringLinqMathMutationTests.cs new file mode 100644 index 0000000..b3ab79a --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/StringLinqMathMutationTests.cs @@ -0,0 +1,869 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Analyzers; + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Microsoft.CodeAnalysis; +using NetEvolve.FrameShift.Analyzers; +using NetEvolve.FrameShift.Diagnostics; +using NetEvolve.FrameShift.Mutations.Operators; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives , , +/// , , +/// and through end to +/// end, exactly like does for the culture-sensitivity family, so that +/// each operator is proven to produce the diagnostics a consumer sees in its build log instead of merely +/// to construct mutations in isolation. +/// +/// +/// +/// Every test states the exact set of reported gaps as one text block, built from the identifier, the +/// 1-based line and the full message of each diagnostic. Every fixture is written so that the operator +/// under test is the only one with a mutation point on the lines that matter: arguments are parameters or +/// field/property reads instead of literals, comparisons are avoided where they are not the point of the +/// test, so the diagnostic set stays a statement about the one operator instead of about every operator +/// that happens to also see the fixture. +/// +/// +/// Each fixture pairs the member under inspection with Fixture.Reached.Identity, whose body carries +/// no mutation point at all. Naming that member in the manifest is what gives the analyzer a non-empty +/// reachable set - without one it reports an unusable manifest and stays silent about the code - while +/// contributing not a single diagnostic of its own. +/// +/// +public class StringLinqMathMutationTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + + /// + /// The member every fixture declares to make the manifest resolvable, and whose return value; + /// carries no mutation point. + /// + private const string AnchorMemberId = "M:Fixture.Reached.Identity(System.Int32)~System.Int32"; + + /// + /// The test method id every manifest of this fixture attributes its references to. No test asserts on + /// it, because these tests state what the operators report, not which test reached what. + /// + private const string AnonymousTestId = "M:Fixture.Tests.AnonymousTests.Reaches"; + + /// + /// The case count recorded for : a lower bound, because nothing here + /// establishes how many input combinations the reaching test carries. + /// + private const string LowerBoundCount = "1+"; + + /// + /// The text the assertions use for "not a single gap was reported". + /// + private const string NoGaps = ""; + + /// + /// The line feed the expectations are joined with, instead of , so + /// that the very same text is produced on Windows and on Linux. + /// + private const string LineFeed = "\n"; + + private const int GreetingLine = 15; + private const int BlankLine = 20; + + private const int StartsWithLine = 20; +#if !NETFRAMEWORK + private const int TrimLine = 25; +#endif + private const int IsBlankLine = 30; + + private const int AnyLine = 26; + private const int FirstLine = 31; + private const int MinLine = 36; + private const int SkipLine = 41; + + private const int SineLine = 22; + private const int MinMaxLine = 27; + private const int FloorLine = 32; + private const int AbsLine = 37; + + private const int EarlyExitConditionLine = 17; + private const int NestedReturnLine = 19; + private const int EarlyExitAnnounceLine = 22; + private const int AnnounceBodyLine = 27; + private const int LastReturnAnnounceLine = 32; + private const int LoopConditionLine = 40; + private const int LoopBreakLine = 42; + private const int LoopAnnounceLine = 45; + private const int SkipConditionLine = 53; + private const int SkipContinueLine = 55; + private const int SkipAnnounceLine = 58; + private const int ValidateConditionLine = 64; + private const int ValidateThrowLine = 66; + private const int FailAnnounceLine = 72; + + private const int EmptyArrayLine = 22; + private const int CollectionInitializerLine = 30; + private const int EmptyIntsLine = 35; + private const int EmptyObjectsLine = 40; + private const int EmptyNullableStringsLine = 45; + + /// + /// Literals.Greeting (line 15) returns a non-empty literal, Literals.Blank (line 20) + /// returns the empty one, and ConstantLabel reads a field whose + /// initializer literal sits in a position that only accepts a compile-time constant - the one literal + /// of the whole file that is not a mutation point at all. + /// + private const string StringLiteralSource = """ + namespace Fixture; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Literals + { + public static string Greeting() + { + return "hello"; + } + + public static string Blank() + { + return ""; + } + } + + public static class ConstantLabel + { + private const string Marker = "fixed"; + + public static string Describe() + { + return Marker; + } + } + """; + + /// + /// Text.StartsWith (line 20), Text.TrimAll (line 25) and Text.IsBlank (line 30) + /// each call a well known method with a matching counterpart overload, while + /// Text.CustomTrim calls a same-named, same-shaped method declared on Custom instead of + /// on itself, which the operator leaves untouched. + /// + private const string StringMethodSource = """ + namespace Fixture; + + public sealed class Custom + { + public string Trim() => string.Empty; + } + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Text + { + public static bool StartsWith(string value, string prefix) + { + return value.StartsWith(prefix); + } + + public static string TrimAll(string value) + { + return value.Trim(); + } + + public static bool IsBlank(string value) + { + return string.IsNullOrEmpty(value); + } + + public static string CustomTrim(Custom custom) + { + return custom.Trim(); + } + } + """; + + /// + /// Queries.AnyMatch (line 26), Queries.FirstMatch (line 31), Queries.MinOf (line + /// 36) and Queries.SkipSome (line 41) each call a well known System.Linq.Enumerable + /// method, while Queries.CustomAny calls a same-named, same-shaped method declared on + /// Custom instead of on Enumerable itself. Every predicate reads the property + /// Flag instead of comparing anything, so the lambda itself carries no mutation point of its + /// own. + /// + private const string LinqMethodSource = """ + namespace Fixture; + + using System; + using System.Collections.Generic; + using System.Linq; + + public sealed class Custom + { + public bool Any() => Environment.Is64BitProcess; + } + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Queries + { + private static bool Flag => Environment.Is64BitProcess; + + public static bool AnyMatch(IEnumerable values) + { + return values.Any(_ => Flag); + } + + public static int FirstMatch(IEnumerable values) + { + return values.First(_ => Flag); + } + + public static int MinOf(IEnumerable values) + { + return values.Min(); + } + + public static IEnumerable SkipSome(IEnumerable values, int count) + { + return values.Skip(count); + } + + public static bool CustomAny(Custom custom) + { + return custom.Any(); + } + } + """; + + /// + /// Trig.SineOf (line 22), Trig.SmallerOf (line 27) and Trig.FloorOf (line 32) + /// each call a well known method with a matching counterpart overload, + /// Trig.AbsoluteOf (line 37) calls Math.Abs, whose only mutation drops the call entirely, + /// and Trig.CustomAbs calls a same-named, same-shaped method declared on Custom instead + /// of on itself. + /// + private const string MathMethodSource = """ + namespace Fixture; + + using System; + + public sealed class Custom + { + public static double Abs(double value) => value; + } + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Trig + { + public static double SineOf(double angle) + { + return Math.Sin(angle); + } + + public static double SmallerOf(double left, double right) + { + return Math.Min(left, right); + } + + public static double FloorOf(double value) + { + return Math.Floor(value); + } + + public static double AbsoluteOf(double value) + { + return Math.Abs(value); + } + + public static double CustomAbs(double value) + { + return Custom.Abs(value); + } + } + """; + + /// + /// Every construct recognises and every guard that keeps it + /// from firing, in one file: a nested return; (line 19, removed - it is not the trailing + /// statement of the method body), a trailing return; in LastReturn (kept - removing it + /// would change nothing), a break and a continue inside a loop (removed), a nested + /// throw in a method (removed), a trailing throw in a + /// non- method (kept - removing it would leave a code path without a required + /// return) and a standalone invocation with a ref argument (kept - removing it would change + /// what the caller observes through the reference). Every condition is a bare identifier, so no other + /// operator has a mutation point on any of these lines. + /// + private const string StatementRemovalSource = """ + namespace Fixture; + + using System; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Guard + { + public static void EarlyExit(bool skip) + { + if (skip) + { + return; + } + + Announce(skip); + } + + public static void Announce(bool flag) + { + Console.WriteLine(flag); + } + + public static void LastReturn(bool skip) + { + Announce(skip); + return; + } + + public static void Loop(bool[] flags) + { + foreach (var flag in flags) + { + if (flag) + { + break; + } + + Announce(flag); + } + } + + public static void SkipEvens(bool[] flags) + { + foreach (var flag in flags) + { + if (flag) + { + continue; + } + + Announce(flag); + } + } + + public static void Validate(bool skip) + { + if (skip) + { + throw new InvalidOperationException(); + } + } + + public static int Fail(bool skip) + { + Announce(skip); + throw new InvalidOperationException(); + } + + public static void Adjust(ref int value) + { + Increment(ref value); + } + + public static void Increment(ref int value) { } + } + """; + + /// + /// Arrays.Pair (line 17) empties a non-empty array initializer, Arrays.EmptyArray (line + /// 22) fills an empty collection expression converted to an array, Collections.Values (line 30) + /// empties a non-empty collection expression, and the remaining members of Collections probe + /// every answer AllowsDefault can give for an empty one: int is a value type (line 35, + /// filled), object is (line 40, filled), an + /// annotated string? is a nullable reference type (line 45, filled), a plain string is + /// not annotated under this compilation's enabled nullable context (not filled), and an unconstrained + /// type parameter resolves to neither (not filled). + /// + private const string CollectionInitializerSource = """ + namespace Fixture; + + using System.Collections.Generic; + + public static class Reached + { + public static int Identity(int value) + { + return value; + } + } + + public static class Arrays + { + public static int[] Pair(int first, int second) + { + return new[] { first, second }; + } + + public static int[] EmptyArray() + { + return []; + } + } + + public static class Collections + { + public static List Values(int first, int second) + { + return [first, second]; + } + + public static List EmptyInts() + { + return []; + } + + public static List EmptyObjects() + { + return []; + } + + public static List EmptyNullableStrings() + { + return []; + } + + public static List EmptyStrings() + { + return []; + } + + public static List EmptyGeneric() + { + return []; + } + } + """; + + /// + /// Every fixture of this class, so that one test can prove that all of them compile and that none of + /// them makes the analyzer crash. + /// + /// One factory per fixture. + public static IEnumerable> Fixtures() => + new[] + { + StringLiteralSource, + StringMethodSource, + LinqMethodSource, + MathMethodSource, + StatementRemovalSource, + CollectionInitializerSource, + }.Select(source => (Func)(() => source)); + + /// + /// A non-empty literal reports the collapse to an empty one, an empty literal reports the fill with a + /// non-empty placeholder, and the literal sitting in a field initializer - + /// a position that only accepts a compile-time constant - reports nothing at all. + /// + [Test] + public async Task Analyze_UntestedStringLiterals_ReportsTheEmptyAndNonEmptySwap() + { + var compilation = CompilationFactory.Create(StringLiteralSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo(Expect((GreetingLine, "\"...\" => \"\""), (BlankLine, "\"\" => \"FrameShift\""))); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// StartsWith and IsNullOrEmpty each report the rename to their counterpart(s), while + /// the same-named, same-shaped method of Custom reports nothing: the operator resolves the + /// called method instead of matching its name. + /// + /// + /// Whether Trim() renames to TrimStart/TrimEnd depends on + /// actually declaring a genuinely zero-parameter overload of those two + /// methods, which the modern .NET reference assemblies do and the .NET Framework ones this repository + /// also targets do not - there, only the params char[] overload exists, so + /// HasSameParameters never matches and the operator stays silent on that member. + /// still covers TrimAll in its + /// manifest regardless, so it is unaffected either way. + /// + [Test] + public async Task Analyze_UntestedStringMethods_ReportsRenamesAndSkipsAForeignType() + { + var compilation = CompilationFactory.Create(StringMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (StartsWithLine, "StartsWith => EndsWith"), +#if !NETFRAMEWORK + (TrimLine, "Trim => TrimStart"), + (TrimLine, "Trim => TrimEnd"), +#endif + (IsBlankLine, "IsNullOrEmpty => IsNullOrWhiteSpace") + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// The counterpart of the previous test: naming every member that calls a well known + /// method silences the analysis completely, which is what makes the four gaps above a statement about + /// coverage rather than about the operator. + /// + [Test] + public async Task Analyze_CoveredStringMethods_ReportsNothing() + { + const string startsWithMemberId = "M:Fixture.Text.StartsWith(System.String,System.String)~System.Boolean"; + const string trimAllMemberId = "M:Fixture.Text.TrimAll(System.String)~System.String"; + const string isBlankMemberId = "M:Fixture.Text.IsBlank(System.String)~System.Boolean"; + + var compilation = CompilationFactory.Create(StringMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync( + compilation, + [CreateManifest(AnchorMemberId, startsWithMemberId, trimAllMemberId, isBlankMemberId)] + ) + .ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert.That(Gaps(diagnostics)).IsEqualTo(NoGaps); + } + } + + /// + /// Any, First, Min and Skip each report the rename to their counterpart(s), + /// while the same-named, same-shaped method of Custom reports nothing: the invoked method has + /// to be bound to System.Linq.Enumerable itself. + /// + /// + /// Skip => SkipLast is deliberately not asserted here: Enumerable.SkipLast was added + /// to modern .NET and was never backported to .NET Framework, which this repository also targets, so + /// that particular rename candidate exists on some target frameworks and not on others. Skip => + /// Take is unaffected and asserted on every framework. + /// + [Test] + public async Task Analyze_UntestedLinqMethods_ReportsRenamesAndSkipsAForeignType() + { + var compilation = CompilationFactory.Create(LinqMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (AnyLine, "Any => All"), + (FirstLine, "First => FirstOrDefault"), + (MinLine, "Min => Max"), + (SkipLine, "Skip => Take") +#if !NETFRAMEWORK + , + (SkipLine, "Skip => SkipLast") +#endif + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// The co-function, the extreme and the rounding direction each report their swap, Math.Abs + /// reports the removal of the call itself, and the same-named, same-shaped method of Custom + /// reports nothing: the called method has to be declared on itself. + /// + [Test] + public async Task Analyze_UntestedMathMethods_ReportsRenamesAndTheAbsRemovalAndSkipsAForeignType() + { + var compilation = CompilationFactory.Create(MathMethodSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (SineLine, "Sin => Cos"), + (MinMaxLine, "Min => Max"), + (FloorLine, "Floor => Ceiling"), + (AbsLine, "Math.Abs(value) => value") + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// The flagship case of the family: every removable construct reports its removal, and every guard - + /// the trailing return;, the trailing throw of a non- member and + /// the invocation with a ref argument - keeps the analysis silent instead. Every if + /// condition guarding a removable statement is itself a bool parameter, which + /// also reports a gap for at its own line - a second, + /// independent operator firing on the same fixture, not a mistake. + /// + [Test] + public async Task Analyze_UntestedStatementRemovalConstructs_ReportsRemovalsAndRespectsEveryGuard() + { + var compilation = CompilationFactory.Create(StatementRemovalSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (EarlyExitConditionLine, "x => !(x)"), + (NestedReturnLine, "return; => (removed)"), + (EarlyExitAnnounceLine, "Announce(skip) => (removed)"), + (AnnounceBodyLine, "Console.WriteLine(flag) => (removed)"), + (LastReturnAnnounceLine, "Announce(skip) => (removed)"), + (LoopConditionLine, "x => !(x)"), + (LoopBreakLine, "break; => (removed)"), + (LoopAnnounceLine, "Announce(flag) => (removed)"), + (SkipConditionLine, "x => !(x)"), + (SkipContinueLine, "continue; => (removed)"), + (SkipAnnounceLine, "Announce(flag) => (removed)"), + (ValidateConditionLine, "x => !(x)"), + (ValidateThrowLine, "throw ...; => (removed)"), + (FailAnnounceLine, "Announce(skip) => (removed)") + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// A non-empty collection expression reports its collapse to an empty one, an empty collection + /// expression reports the fill with for every element type + /// AllowsDefault accepts - a value type, object, and an annotated nullable reference + /// type - and reports nothing for the two it rejects: a non-annotated reference type and an + /// unconstrained type parameter. The old-style array initializer of Arrays.Pair + /// (new[] { first, second }) is deliberately included and reports nothing at all: + /// only recognises collection-expression syntax + /// ([ ... ]), not the classic new[] { ... } form. + /// + [Test] + public async Task Analyze_UntestedCollectionInitializers_ReportsEmptyingAndDefaultFillsWhereSafe() + { + var compilation = CompilationFactory.Create(CollectionInitializerSource, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(Errors(compilation)).IsEmpty(); + _ = await Assert + .That(Gaps(diagnostics)) + .IsEqualTo( + Expect( + (EmptyArrayLine, "[] => [default]"), + (CollectionInitializerLine, "[ ... ] => []"), + (EmptyIntsLine, "[] => [default]"), + (EmptyObjectsLine, "[] => [default]"), + (EmptyNullableStringsLine, "[] => [default]") + ) + ); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + /// + /// Every fixture of this class compiles and is analysed without the analyzer throwing. Roslyn turns an + /// analyzer exception into AD0001 and carries on, so a crash would otherwise look like a + /// diagnostic the tests above simply did not expect. + /// + /// The fixture to analyse. + /// A task that completes when the fixture was analysed. + [Test] + [MethodDataSource(nameof(Fixtures))] + public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(string source) + { + var compilation = CompilationFactory.Create(source, ProductionAssemblyName); + + var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false); + + using (Assert.Multiple()) + { + _ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty(); + _ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty(); + } + } + + private static Task> RunAsync( + Compilation compilation, + IEnumerable? additionalFiles = null, + IReadOnlyDictionary? globalOptions = null + ) => AnalyzerRunner.RunAsync(new MutationCoverageAnalyzer(), compilation, additionalFiles, globalOptions); + + /// + /// Builds a manifest recording as the production members the + /// tests of the first pass touched. + /// + /// + /// Every reference is attributed to one anonymous test whose case count is the lower bound + /// . These tests are about which mutation points are reachable and state + /// nothing about test data, so a lower bound is the honest count - and it keeps FSH0006 silent, + /// which is what lets every exact diagnostic set above stay a statement about the six operators alone. + /// Every reference is also written as behaviorally verified, for the same reason, to keep + /// FSH0007 silent as well. + /// + /// The declaration ids of the covered members. + /// The manifest as an additional file. + private static InMemoryAdditionalText CreateManifest(params string[] referencedMemberIds) + { + var builder = new StringBuilder(); + _ = builder.Append(TestSurfaceManifestFormat.Header).Append('\n'); + _ = builder + .Append(TestSurfaceManifestFormat.TestPrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(AnonymousTestId) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(LowerBoundCount) + .Append('\n'); + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.ReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + foreach (var referencedMemberId in referencedMemberIds) + { + _ = builder + .Append(TestSurfaceManifestFormat.BehavioralReferencePrefix) + .Append(TestSurfaceManifestFormat.FieldSeparator) + .Append(referencedMemberId) + .Append('\n'); + } + + return new InMemoryAdditionalText(builder.ToString()); + } + + /// + /// Describes the reported gaps as one text block, one line per diagnostic, ordered ordinally so that + /// the result does not depend on the order the concurrently running analyzer callbacks reported them + /// in. Several gaps share one location, which is exactly the case a positional order cannot separate. + /// + /// All diagnostics of a run. + /// The described gaps, or when there is none. + private static string Gaps(ImmutableArray diagnostics) + { + var gaps = AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint); + + if (gaps.IsEmpty) + { + return NoGaps; + } + + return Join( + DiagnosticAssertions + .Summarise(gaps) + .OrderBy(summary => summary.Line) + .ThenBy(summary => summary.Message, StringComparer.Ordinal) + .Select(summary => Entry(summary.Id, summary.Line, summary.Message)) + ); + } + + /// + /// Builds the expectation of a set of gaps, each one a line and the display name of its mutation. + /// + /// The expected gaps. + /// The expected text block, or when nothing is expected. + private static string Expect(params (int Line, string DisplayName)[] gaps) => + gaps.Length == 0 + ? NoGaps + : Join( + gaps.OrderBy(gap => gap.Line) + .ThenBy(gap => gap.DisplayName, StringComparer.Ordinal) + .Select(gap => GapEntry(gap.Line, gap.DisplayName)) + ); + + /// + /// Builds the described gap of one mutation, spelling out the message + /// formats. + /// + /// The 1-based line the gap is reported on. + /// The display name of the mutation. + /// The described gap. + private static string GapEntry(int line, string displayName) => + Entry( + DiagnosticIds.UnreachableMutationPoint, + line, + "Mutation '" + + displayName + + "' at this location is not reachable from any test; a surviving mutant here would go unnoticed" + ); + + private static string Entry(string id, int line, string message) => $"{id} line {ToText(line)}: {message}"; + + private static string Join(IEnumerable entries) => + string.Join(LineFeed, entries.OrderBy(entry => entry, StringComparer.Ordinal)); + + private static string ToText(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static ImmutableArray Errors(Compilation compilation) => + CompilationFactory.GetCompileErrors(compilation); +} diff --git a/tests/NetEvolve.FrameShift.Tests.Integration/Generation/TestSurfaceEdgeCaseTests.cs b/tests/NetEvolve.FrameShift.Tests.Integration/Generation/TestSurfaceEdgeCaseTests.cs new file mode 100644 index 0000000..c416498 --- /dev/null +++ b/tests/NetEvolve.FrameShift.Tests.Integration/Generation/TestSurfaceEdgeCaseTests.cs @@ -0,0 +1,706 @@ +namespace NetEvolve.FrameShift.Tests.Integration.Generation; + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Text; +using NetEvolve.FrameShift.Generation; +using NetEvolve.FrameShift.Tests.Infrastructure; +using NetEvolve.FrameShift.TestSurface; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +/// +/// Drives over the data-driven, awkward-shaped test methods +/// that deliberately keeps out of its own fixtures: the +/// combinations of data-source attributes, matrices and value generators each recognised framework has to +/// turn into a test-case count. +/// +/// +/// +/// proves the shape of the manifest — the block comment, +/// the ordering, the byte-identical re-run — with one plain test per framework. It deliberately avoids the +/// long tail of case-counting shapes, because adding them there would blur what that file is about. This +/// class is that long tail: for every recognised framework, one fixture whose test methods carry the +/// data-source shapes their ITestMethodRecognizer has to fold into a , +/// each one asserted against the exact count (or lower bound) documented on the recogniser itself. +/// +/// +/// Every fixture is run through the real generator via , exactly like +/// does, and the emitted manifest is parsed back with +/// so that the assertions read +/// values rather than raw manifest text. A test method is looked up by the unique substring of its name in +/// the parsed , because the exact documentation comment id +/// of an overload-free method already differs only by its parameter list, which is not what any of these +/// tests are about. +/// +/// +public class TestSurfaceEdgeCaseTests +{ + private const string ProductionAssemblyName = "ProductionAssembly"; + private const string TestAssemblyName = "TestAssembly"; + private const string ProductionPath = "Production.cs"; + private const string TestPath = "CalculatorTests.cs"; + + private const string ProductionSource = """ + namespace Fixture; + + public static class Calculator + { + public static int Add(int left, int right) + { + return left + right; + } + } + """; + + /// + /// One MSTest test per shape has to count: several + /// [DataRow] applications, a [DynamicData] referencing a method that only yields, one + /// summing both kinds, a data source whose length depends on a loop and therefore cannot be read, and + /// three ways of naming the referenced sequence — an expression-bodied property, a property with a + /// block-bodied getter, and a member of another type named explicitly through the attribute's second + /// constructor argument. + /// + private const string MSTestSource = """ + namespace Tests; + + using System.Collections.Generic; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class CalculatorTests + { + public static IEnumerable GetRows() + { + yield return new object[] { 1 }; + yield return new object[] { 2 }; + yield return new object[] { 3 }; + } + + public static IEnumerable GetComputedRows() + { + for (var i = 0; i < 4; i++) + { + yield return new object[] { i }; + } + } + + public static IEnumerable RowsFromProperty => new object[][] { new object[] { 1 }, new object[] { 2 } }; + + public static IEnumerable RowsFromGetter + { + get + { + return new List { new object[] { 1 }, new object[] { 2 }, new object[] { 3 } }; + } + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + public void DataRow_ThreeRows_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(GetRows))] + public void DynamicData_MethodWithThreeYields_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DataRow(1)] + [DynamicData(nameof(GetRows))] + public void DataRowAndDynamicData_SumIsExactlyFour(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(GetComputedRows))] + public void DynamicData_ComputedLoop_IsAtLeastOne(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(RowsFromProperty))] + public void DynamicData_ExpressionBodiedProperty_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(RowsFromGetter))] + public void DynamicData_BlockGetterProperty_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestMethod] + [DynamicData(nameof(RemoteRows.Get), typeof(RemoteRows))] + public void DynamicData_ExternalDeclaringType_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + } + + public static class RemoteRows + { + public static IEnumerable Get() + { + return new[] { new object[] { 1 }, new object[] { 2 } }; + } + } + """; + + /// + /// One NUnit test per shape has to count: several + /// [TestCase] applications, a [TestCaseSource] naming a method of the fixture and one + /// naming a member of another type, the cross product two [Values] parameters produce, a + /// [ValueSource] reading a field, both [Range] overloads, both counted [Random] + /// shapes, the two combining strategies, and a bare [Theory]. + /// + private const string NUnitSource = """ + namespace Tests; + + using System.Collections.Generic; + using NUnit.Framework; + + public class CalculatorTests + { + private static readonly int[] Numbers = [7, 8, 9]; + + public static IEnumerable Rows() + { + yield return new object[] { 1, 2 }; + yield return new object[] { 3, 4 }; + } + + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + public void TestCase_ThreeCases_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [TestCaseSource(nameof(Rows))] + public void TestCaseSource_TwoRows_IsExactlyTwo(int left, int right) + { + _ = Fixture.Calculator.Add(left, right); + } + + [TestCaseSource(typeof(RemoteRows), nameof(RemoteRows.Get))] + public void TestCaseSource_ExternalTypeMember_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + public void Values_CrossProduct_IsExactlySix([Values(1, 2, 3)] int a, [Values(4, 5)] int b) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Test] + public void ValueSource_FromField_IsExactlyThree([ValueSource(nameof(Numbers))] int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + public void Range_TwoArgument_IsExactlyFive([Range(1, 5)] int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + public void Range_ThreeArgumentStep_IsExactlyThree([Range(0, 4, 2)] int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + public void Random_FixedCount_IsExactlyFour([Random(1, 10, 4)] int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Sequential] + public void Sequential_TakesTheLongestSet([Values(1, 2, 3)] int a, [Values(4, 5)] int b) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Pairwise] + public void Pairwise_IsLowerBoundOfTheLongestSet([Values(1, 2)] int a, [Values(3, 4)] int b) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Theory] + public void Theory_IsLowerBoundOfOne(bool flag) + { + _ = Fixture.Calculator.Add(flag ? 1 : 0, 1); + } + } + + public static class RemoteRows + { + public static IEnumerable Get() + { + yield return new object[] { 1 }; + yield return new object[] { 2 }; + } + } + """; + + /// + /// One TUnit test per shape has to count: several + /// [Arguments] rows, the cross product two [Matrix] parameters produce, a matrix parameter + /// excluding one of its own values, a [MethodDataSource] naming a literal sequence and one naming + /// a computed one, and a [Repeat] proven not to multiply the single inline row it sits next to. + /// + private const string TUnitSource = """ + namespace Tests; + + using System.Collections.Generic; + using TUnit.Core; + + public class CalculatorTests + { + public static int[] LiteralRows() => new[] { 1, 2, 3 }; + + public static IEnumerable ComputedRows() + { + for (var i = 0; i < 4; i++) + { + yield return new[] { i }; + } + } + + [Test] + [Arguments(1)] + [Arguments(2)] + [Arguments(3)] + public void Arguments_ThreeRows_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + [MatrixDataSource] + public void Matrix_CrossProduct_IsExactlySix([Matrix(1, 2, 3)] int a, [Matrix(4, 5)] int b) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Test] + [MatrixDataSource] + public void Matrix_WithExcludedValue_IsAtLeastOne( + [Matrix(1, 2, 3, Excluding = new object[] { 2 })] int a, + [Matrix(4, 5)] int b + ) + { + _ = Fixture.Calculator.Add(a, b); + } + + [Test] + [MethodDataSource(nameof(LiteralRows))] + public void MethodDataSource_LiteralSequence_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + [MethodDataSource(nameof(ComputedRows))] + public void MethodDataSource_ComputedLoop_IsAtLeastOne(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Test] + [Repeat(5)] + [Arguments(1)] + public void Repeat_DoesNotMultiplyTheCount_IsExactlyOne(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + } + """; + + /// + /// One xUnit v2 test per shape has to count: a plain [Fact], + /// several [InlineData] rows, a [MemberData] naming a method of the fixture and one naming + /// a member of another type through MemberType, a data source whose length depends on a loop, a + /// bare [Theory] without any data source at all, and a custom marker deriving [Fact] that + /// degrades an otherwise exact count to a lower bound. + /// + private const string XunitV2Source = """ + namespace Tests; + + using System.Collections.Generic; + using Xunit; + + public class CalculatorTests + { + public static IEnumerable Rows() + { + return new[] { new object[] { 1 }, new object[] { 2 } }; + } + + public static IEnumerable ComputedRows() + { + for (var i = 0; i < 3; i++) + { + yield return new object[] { i }; + } + } + + [Fact] + public void Fact_IsExactlyOne() + { + _ = Fixture.Calculator.Add(1, 1); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public void InlineData_ThreeRows_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + [MemberData(nameof(Rows))] + public void MemberData_TwoRows_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + [MemberData(nameof(RemoteRows.Get), MemberType = typeof(RemoteRows))] + public void MemberData_ExternalMemberType_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + [MemberData(nameof(ComputedRows))] + public void MemberData_ComputedLoop_IsAtLeastOne(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + public void Theory_WithoutData_IsExactlyZero(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [CustomFact] + [InlineData(1)] + [InlineData(2)] + public void CustomMarkerWithInlineData_IsAtLeastTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + } + + public sealed class CustomFactAttribute : FactAttribute + { + } + + public static class RemoteRows + { + public static IEnumerable Get() + { + yield return new object[] { 1 }; + yield return new object[] { 2 }; + } + } + """; + +#if FRAMESHIFT_XUNIT_V3 + /// + /// The xUnit v3 counterpart of , narrowed to the shapes that are identical + /// between the two major versions: shares its rules across both, so + /// this fixture exists only to prove the same rules apply when the recogniser is built for + /// xunit.v3.core instead of xunit.core. + /// + private const string XunitV3Source = """ + namespace Tests; + + using System.Collections.Generic; + using Xunit; + + public class CalculatorTests + { + public static IEnumerable Rows() + { + yield return new object[] { 1 }; + yield return new object[] { 2 }; + yield return new object[] { 3 }; + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public void InlineData_TwoRows_IsExactlyTwo(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + [MemberData(nameof(Rows))] + public void MemberData_ThreeRows_IsExactlyThree(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + + [Theory] + public void Theory_WithoutData_IsExactlyZero(int value) + { + _ = Fixture.Calculator.Add(value, value); + } + } + """; +#endif + + [Test] + public async Task Fixtures_EveryCompilation_CompileWithoutErrors() + { + var production = CreateProduction(); + List described = + [ + Describe(production), + Describe(CreateTest(TestFramework.MSTest, MSTestSource, production)), + Describe(CreateTest(TestFramework.NUnit, NUnitSource, production)), + Describe(CreateTest(TestFramework.TUnit, TUnitSource, production)), + Describe(CreateTest(TestFramework.XunitV2, XunitV2Source, production)), + ]; +#if FRAMESHIFT_XUNIT_V3 + described.Add(Describe(CreateTest(TestFramework.XunitV3, XunitV3Source, production))); +#endif + + _ = await Assert + .That(string.Join("|", described.Distinct(StringComparer.Ordinal))) + .IsEqualTo(DiagnosticAssertions.NoDiagnostics); + } + + [Test] + public async Task Generate_MSTestDataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.MSTest, MSTestSource); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "DataRow_ThreeRows_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_MethodWithThreeYields_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DataRowAndDynamicData_SumIsExactlyFour", TestCaseCount.Exact(4)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_ComputedLoop_IsAtLeastOne", TestCaseCount.AtLeast(1)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_ExpressionBodiedProperty_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_BlockGetterProperty_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "DynamicData_ExternalDeclaringType_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + } + } + + [Test] + public async Task Generate_NUnitDataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.NUnit, NUnitSource); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "TestCase_ThreeCases_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "TestCaseSource_TwoRows_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "TestCaseSource_ExternalTypeMember_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Values_CrossProduct_IsExactlySix", TestCaseCount.Exact(6)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "ValueSource_FromField_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Range_TwoArgument_IsExactlyFive", TestCaseCount.Exact(5)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Range_ThreeArgumentStep_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Random_FixedCount_IsExactlyFour", TestCaseCount.Exact(4)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Sequential_TakesTheLongestSet", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Pairwise_IsLowerBoundOfTheLongestSet", TestCaseCount.AtLeast(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Theory_IsLowerBoundOfOne", TestCaseCount.AtLeast(1)).ConfigureAwait(false); + } + } + + [Test] + public async Task Generate_TUnitDataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.TUnit, TUnitSource); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "Arguments_ThreeRows_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Matrix_CrossProduct_IsExactlySix", TestCaseCount.Exact(6)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Matrix_WithExcludedValue_IsAtLeastOne", TestCaseCount.AtLeast(1)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MethodDataSource_LiteralSequence_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MethodDataSource_ComputedLoop_IsAtLeastOne", TestCaseCount.AtLeast(1)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Repeat_DoesNotMultiplyTheCount_IsExactlyOne", TestCaseCount.Exact(1)) + .ConfigureAwait(false); + } + } + + [Test] + public async Task Generate_XunitV2DataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.XunitV2, XunitV2Source); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "Fact_IsExactlyOne", TestCaseCount.Exact(1)).ConfigureAwait(false); + _ = await AssertCount(manifest, "InlineData_ThreeRows_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MemberData_TwoRows_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MemberData_ExternalMemberType_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MemberData_ComputedLoop_IsAtLeastOne", TestCaseCount.AtLeast(1)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Theory_WithoutData_IsExactlyZero", TestCaseCount.Exact(0)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "CustomMarkerWithInlineData_IsAtLeastTwo", TestCaseCount.AtLeast(2)) + .ConfigureAwait(false); + } + } + +#if FRAMESHIFT_XUNIT_V3 + [Test] + public async Task Generate_XunitV3DataSourceShapes_MatchTheDocumentedCounts() + { + var manifest = CollectManifest(TestFramework.XunitV3, XunitV3Source); + + using (Assert.Multiple()) + { + _ = await AssertCount(manifest, "InlineData_TwoRows_IsExactlyTwo", TestCaseCount.Exact(2)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "MemberData_ThreeRows_IsExactlyThree", TestCaseCount.Exact(3)) + .ConfigureAwait(false); + _ = await AssertCount(manifest, "Theory_WithoutData_IsExactlyZero", TestCaseCount.Exact(0)) + .ConfigureAwait(false); + } + } +#endif + + private static async Task AssertCount(TestSurfaceManifest manifest, string methodName, TestCaseCount expected) + { + _ = await Assert.That(manifest.TestCaseCounts[IdOf(manifest, methodName)]).IsEqualTo(expected); + + return true; + } + + private static TestSurfaceManifest CollectManifest(TestFramework framework, string source) + { + var test = CreateTest(framework, source, CreateProduction()); + var (success, error, manifest) = Read(Generate(test)); + + if (!success) + { + throw new InvalidOperationException($"The generated manifest of '{framework}' does not parse: {error}"); + } + + return manifest; + } + + /// + /// Finds the single test method whose documentation comment id carries + /// as its own member name, rather than as a prefix of a longer one. + /// + /// The manifest to search. + /// The unqualified name of the test method. + /// The matching documentation comment id. + private static string IdOf(TestSurfaceManifest manifest, string methodName) => + manifest.TestMethodIds.Single(id => + id.EndsWith("." + methodName, StringComparison.Ordinal) + || id.Contains("." + methodName + "(", StringComparison.Ordinal) + ); + + private static GeneratorRunner.Output Run(Compilation compilation) => + GeneratorRunner.Run(new TestSurfaceManifestGenerator(), compilation); + + private static string Generate(Compilation compilation) => + Run(compilation).TextOf(TestSurfaceManifestGenerator.HintName); + + private static CSharpCompilation CreateProduction() => + CompilationFactory.Create( + ProductionSource, + TestFramework.None, + ProductionAssemblyName, + filePath: ProductionPath + ); + + private static CSharpCompilation CreateTest(TestFramework framework, string source, Compilation production) => + CompilationFactory.Create( + source, + framework, + TestAssemblyName, + additionalReferences: [production.ToMetadataReference()], + filePath: TestPath + ); + + /// + /// Parses the generated file the way the MSBuild target does: drop the first and the last line, hand + /// the rest to the reader. + /// + /// The content of the generated source file. + /// Whether the text parsed, the reported error and the parsed manifest. + private static (bool Success, string Error, TestSurfaceManifest Manifest) Read(string generated) + { + var inner = string.Join("\n", Lines(generated).Skip(1).SkipLast(1)) + "\n"; + var success = TestSurfaceManifestReader.TryRead(SourceText.From(inner), out var manifest, out var error); + + return (success, error ?? string.Empty, manifest); + } + + /// + /// Splits the generated text into its lines, dropping the empty remainder behind the trailing line + /// feed, which is the end of the last line and not a line of its own. + /// + /// The generated text. + /// The lines, without their line endings. + private static ImmutableArray Lines(string text) + { + var lines = text.Split('\n').Select(line => line.TrimEnd('\r')).ToList(); + + if (lines.Count > 0 && lines[^1].Length == 0) + { + lines.RemoveAt(lines.Count - 1); + } + + return [.. lines]; + } + + private static string Describe(Compilation compilation) => + DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(compilation)); +}