From fe33671194e340927db18f6fd94eefbdab13fb54 Mon Sep 17 00:00:00 2001 From: Laurents Meyer Date: Thu, 12 Oct 2023 17:44:35 +0200 Subject: [PATCH] Finalize implementation and add crash detection support. --- .../EFCore.Jet.FunctionalTests.csproj | 6 +- .../Properties/AssemblyInfo.cs | 2 +- .../Xunit/JetXunitTestCaseRunner.cs | 74 ---------- .../Xunit/JetXunitTestFramework.cs | 14 -- test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj | 4 - .../Attributes/AccessProviderTypeVariation.cs | 14 ++ .../Attributes/TestRunnerCrashAttribute.cs | 43 ++++++ .../Xunit/AscendingTestCaseOrderer.cs | 0 .../Xunit/AscendingTestCollectionOrderer.cs | 0 .../Xunit/ExceptionTestCaseOrderer.cs | 0 .../Xunit/JetConditionalFactDiscoverer.cs | 0 .../Xunit/JetConditionalFactTestCase.cs | 0 .../Xunit/JetConditionalTheoryDiscoverer.cs | 0 .../Xunit/JetConditionalTheoryTestCase.cs | 0 .../Xunit/JetXunitTestCaseRunner.cs | 135 ++++++++++++++++++ .../Xunit/JetXunitTestFramework.cs | 44 ++++++ .../Xunit/JetXunitTestFrameworkDiscoverer.cs | 5 +- .../TestUtilities/Xunit/JetXunitTestRunner.cs | 39 +++-- .../Xunit/JetXunitTheoryTestCaseRunner.cs | 0 .../Xunit/RandomTestCaseOrderer.cs | 0 20 files changed, 272 insertions(+), 108 deletions(-) delete mode 100644 test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestCaseRunner.cs delete mode 100644 test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestFramework.cs create mode 100644 test/Shared/TestUtilities/Attributes/AccessProviderTypeVariation.cs create mode 100644 test/Shared/TestUtilities/Attributes/TestRunnerCrashAttribute.cs rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/AscendingTestCaseOrderer.cs (100%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs (100%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs (100%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/JetConditionalFactDiscoverer.cs (100%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/JetConditionalFactTestCase.cs (100%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/JetConditionalTheoryDiscoverer.cs (100%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/JetConditionalTheoryTestCase.cs (100%) create mode 100644 test/Shared/TestUtilities/Xunit/JetXunitTestCaseRunner.cs create mode 100644 test/Shared/TestUtilities/Xunit/JetXunitTestFramework.cs rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs (88%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/JetXunitTestRunner.cs (75%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/JetXunitTheoryTestCaseRunner.cs (100%) rename test/{EFCore.Jet.FunctionalTests => Shared}/TestUtilities/Xunit/RandomTestCaseOrderer.cs (100%) diff --git a/test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj b/test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj index 9f0453bf..3a4cd267 100644 --- a/test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj +++ b/test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj @@ -8,9 +8,11 @@ - + + %(RecursiveDir)%(Filename)%(Extension) + - + diff --git a/test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs b/test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs index ddba66f6..9783d73a 100644 --- a/test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs +++ b/test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs @@ -10,4 +10,4 @@ [assembly: CollectionBehavior(DisableTestParallelization = true)] [assembly: TestCollectionOrderer("EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit." + nameof(AscendingTestCollectionOrderer), "EntityFrameworkCore.Jet.FunctionalTests")] [assembly: TestCaseOrderer("EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit." + nameof(AscendingTestCaseOrderer), "EntityFrameworkCore.Jet.FunctionalTests")] -[assembly: TestFramework("EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit.JetXunitTestFramework", "EntityFrameworkCore.Jet.FunctionalTests")] \ No newline at end of file +[assembly: TestFramework("EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit." + nameof(JetXunitTestFramework), "EntityFrameworkCore.Jet.FunctionalTests")] \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestCaseRunner.cs b/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestCaseRunner.cs deleted file mode 100644 index a09f4d88..00000000 --- a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestCaseRunner.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; - -public class JetXunitTestCaseRunner : XunitTestCaseRunner -{ - public JetXunitTestCaseRunner( - IXunitTestCase testCase, - string displayName, - string skipReason, - object[] constructorArguments, - object[] testMethodArguments, - IMessageBus messageBus, - ExceptionAggregator aggregator, - CancellationTokenSource cancellationTokenSource) - : base( - testCase, - displayName, - skipReason, - constructorArguments, - testMethodArguments, - messageBus, - aggregator, - cancellationTokenSource) - { - } - - protected override XunitTestRunner CreateTestRunner( - ITest test, - IMessageBus messageBus, - Type testClass, - object[] constructorArguments, - MethodInfo testMethod, - object[] testMethodArguments, - string skipReason, - IReadOnlyList beforeAfterAttributes, - ExceptionAggregator aggregator, - CancellationTokenSource cancellationTokenSource) - => new JetXunitTestRunner( - test, - messageBus, - testClass, - constructorArguments, - testMethod, - testMethodArguments, - skipReason, - beforeAfterAttributes, - new ExceptionAggregator(aggregator), - cancellationTokenSource); - - /// - /// `TestRunner<TTestCase>.RunAsync()` is not virtual, so we need to override this method here to call our own - /// `JetXunitTestRunner.RunAsync()` implementation. - /// > - protected override Task RunTestAsync() - => ((JetXunitTestRunner)CreateTestRunner( - CreateTest(TestCase, DisplayName), - MessageBus, - TestClass, - ConstructorArguments, - TestMethod, - TestMethodArguments, - SkipReason, - BeforeAfterAttributes, - Aggregator, - CancellationTokenSource)) - .RunAsync(); -} \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestFramework.cs b/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestFramework.cs deleted file mode 100644 index 98c13013..00000000 --- a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestFramework.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; - -public class JetXunitTestFramework : XunitTestFramework -{ - public JetXunitTestFramework(IMessageSink messageSink) : base(messageSink) - { - } - - protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) - => new JetXunitTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); -} \ No newline at end of file diff --git a/test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj b/test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj index 89a2c4aa..89bbb6b0 100644 --- a/test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj +++ b/test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj @@ -7,10 +7,6 @@ AnyCPU;x86 - - - - PreserveNewest diff --git a/test/Shared/TestUtilities/Attributes/AccessProviderTypeVariation.cs b/test/Shared/TestUtilities/Attributes/AccessProviderTypeVariation.cs new file mode 100644 index 00000000..b4b7a76f --- /dev/null +++ b/test/Shared/TestUtilities/Attributes/AccessProviderTypeVariation.cs @@ -0,0 +1,14 @@ +using System; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities; + +[Flags] +public enum AccessProviderTypeVariation +{ + None = 0, + X86 = 1 << 0, + X64 = 1 << 1, + Odbc = 1 << 2, + OleDb = 1 << 3, + All = -1, +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Attributes/TestRunnerCrashAttribute.cs b/test/Shared/TestUtilities/Attributes/TestRunnerCrashAttribute.cs new file mode 100644 index 00000000..b9d61e2a --- /dev/null +++ b/test/Shared/TestUtilities/Attributes/TestRunnerCrashAttribute.cs @@ -0,0 +1,43 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.TestUtilities.Xunit; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities; + +/// +/// Marks a test method or class that is known to crash the test runner. +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = true)] +public class TestRunnerCrashAttribute : Attribute, ITestCondition +{ + public const string DefaultSkipReason = "The test is known to crash the test runner."; + + protected AccessProviderTypeVariation[] AccessProviderTypeVariations { get; } + + public TestRunnerCrashAttribute(params AccessProviderTypeVariation[] accessProviderTypeVariations) + { + AccessProviderTypeVariations = accessProviderTypeVariations.Length > 0 + ? accessProviderTypeVariations + : new[] { AccessProviderTypeVariation.All }; + } + + public virtual ValueTask IsMetAsync() + { + // Implement and enable if we want to filter tests by specific runtime scenarios. + var currentVariation = AccessProviderTypeVariation.All; // AppConfig.AccessProviderTypeVariation; + var isMet = AccessProviderTypeVariations.Any(v => v.HasFlag(currentVariation)); + + if (!isMet && string.IsNullOrEmpty(Skip)) + { + Skip = DefaultSkipReason; + } + + return new ValueTask(isMet); + } + + public virtual string SkipReason + => Skip; + + public virtual string Skip { get; set; } +} \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCaseOrderer.cs b/test/Shared/TestUtilities/Xunit/AscendingTestCaseOrderer.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCaseOrderer.cs rename to test/Shared/TestUtilities/Xunit/AscendingTestCaseOrderer.cs diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs b/test/Shared/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs rename to test/Shared/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs b/test/Shared/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs rename to test/Shared/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetConditionalFactDiscoverer.cs b/test/Shared/TestUtilities/Xunit/JetConditionalFactDiscoverer.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetConditionalFactDiscoverer.cs rename to test/Shared/TestUtilities/Xunit/JetConditionalFactDiscoverer.cs diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetConditionalFactTestCase.cs b/test/Shared/TestUtilities/Xunit/JetConditionalFactTestCase.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetConditionalFactTestCase.cs rename to test/Shared/TestUtilities/Xunit/JetConditionalFactTestCase.cs diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetConditionalTheoryDiscoverer.cs b/test/Shared/TestUtilities/Xunit/JetConditionalTheoryDiscoverer.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetConditionalTheoryDiscoverer.cs rename to test/Shared/TestUtilities/Xunit/JetConditionalTheoryDiscoverer.cs diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetConditionalTheoryTestCase.cs b/test/Shared/TestUtilities/Xunit/JetConditionalTheoryTestCase.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetConditionalTheoryTestCase.cs rename to test/Shared/TestUtilities/Xunit/JetConditionalTheoryTestCase.cs diff --git a/test/Shared/TestUtilities/Xunit/JetXunitTestCaseRunner.cs b/test/Shared/TestUtilities/Xunit/JetXunitTestCaseRunner.cs new file mode 100644 index 00000000..71eb8785 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetXunitTestCaseRunner.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetXunitTestCaseRunner : XunitTestCaseRunner +{ + public const string TestRunnerCrashCacheDirectory = "TestRunnerCrashCache"; + public const string AutoSkipPrefix = "[AutoSkip]"; + public const string AutoSkipTestRunnerCrashingTestsEnvironmentVariableName = "EFCoreJet_AutoSkipTestRunnerCrashingTests"; + + public virtual bool EnableAutoSkipTestsKnownToCrashTestRunner + => (Environment.GetEnvironmentVariable(AutoSkipTestRunnerCrashingTestsEnvironmentVariableName)?.ToLowerInvariant() ?? "true") != "false"; + + public JetXunitTestCaseRunner(IXunitTestCase testCase, + string displayName, + string skipReason, + object[] constructorArguments, + object[] testMethodArguments, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base( + testCase, + displayName, + skipReason, + constructorArguments, + testMethodArguments, + messageBus, + aggregator, + cancellationTokenSource) + { + } + + protected override XunitTestRunner CreateTestRunner(ITest test, + IMessageBus messageBus, + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] testMethodArguments, + string skipReason, + IReadOnlyList beforeAfterAttributes, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + => new JetXunitTestRunner( + test, + messageBus, + testClass, + constructorArguments, + testMethod, + testMethodArguments, + skipReason, + beforeAfterAttributes, + new ExceptionAggregator(aggregator), + cancellationTokenSource); + + /// + /// `TestRunner<TTestCase>.RunAsync()` is not virtual, so we need to override this method here to call our own + /// `JetXunitTestRunner.RunAsync()` implementation. + /// > + protected override async Task RunTestAsync() + { + if (EnableAutoSkipTestsKnownToCrashTestRunner) + { + AutoSkipTestsKnownToCrashTestRunner(); + } + + return await RunWithCrashDetection( + () => ((JetXunitTestRunner)CreateTestRunner( + CreateTest(TestCase, DisplayName), + MessageBus, + TestClass, + ConstructorArguments, + TestMethod, + TestMethodArguments, + SkipReason, + BeforeAfterAttributes, + Aggregator, + CancellationTokenSource)) + .RunAsync()); + } + + protected virtual async Task RunWithCrashDetection(Func> func) + { + Directory.CreateDirectory(TestRunnerCrashCacheDirectory); + + var filePath = Path.Combine(TestRunnerCrashCacheDirectory, $"{DateTime.UtcNow:yyyyMMdd'_'HHmmss.fffffff}_{(Environment.Is64BitProcess ? "x64" : "x86")}_{Guid.NewGuid()}.txt"); + var contents = $"{TestCase.TestMethod.TestClass.Class.Name}\t{TestCase.TestMethod.Method.Name}"; + await File.WriteAllTextAsync(filePath, contents); + + var result = await func(); + + File.Delete(filePath); + + return result; + } + + protected virtual void AutoSkipTestsKnownToCrashTestRunner() + { + if (IsTestKnownToCrashTestRunner(TestCase)) + { + SkipReason = $"{AutoSkipPrefix} {TestRunnerCrashAttribute.DefaultSkipReason}"; + } + } + + protected virtual bool IsTestKnownToCrashTestRunner(ITestCase testCase) + { + if (File.Exists(JetXunitTestFramework.TestsKnownToCrashTestRunnerFilePath)) + { + foreach (var line in File.ReadLines(JetXunitTestFramework.TestsKnownToCrashTestRunnerFilePath)) + { + var parts = line.Split('\t'); + if (parts.Length >= 2) + { + var testClass = parts[^2]; + var testMethod = parts[^1]; + + if (testClass == testCase.TestMethod.TestClass.Class.Name && + testMethod == testCase.TestMethod.Method.Name) + { + return true; + } + } + } + } + + return false; + } +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetXunitTestFramework.cs b/test/Shared/TestUtilities/Xunit/JetXunitTestFramework.cs new file mode 100644 index 00000000..bb6cf3a7 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetXunitTestFramework.cs @@ -0,0 +1,44 @@ +using System; +using System.IO; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetXunitTestFramework : XunitTestFramework +{ + public const string TestsKnownToCrashTestRunnerFilePath = "./../../../TestsKnownToCrashTestRunner.txt"; + public const string DetectCrashesOfPreviousRunsEnvironmentVariableName = "EFCoreJet_DetectCrashesOfPreviousRuns"; + + public virtual bool EnableDetectCrashesOfPreviousRuns + => Environment.GetEnvironmentVariable(DetectCrashesOfPreviousRunsEnvironmentVariableName)?.ToLowerInvariant() == "true"; + + public JetXunitTestFramework(IMessageSink messageSink) : base(messageSink) + { + } + + protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) + { + if (EnableDetectCrashesOfPreviousRuns) + { + DetectCrashesOfPreviousRuns(); + } + + return new JetXunitTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); + } + + protected virtual void DetectCrashesOfPreviousRuns() + { + if (!Directory.Exists(JetXunitTestCaseRunner.TestRunnerCrashCacheDirectory)) + return; + + foreach (var filePath in Directory.EnumerateFiles(JetXunitTestCaseRunner.TestRunnerCrashCacheDirectory, "*.txt")) + { + var contents = File.ReadAllText(filePath); + contents = $"{string.Join('\t', Path.GetFileNameWithoutExtension(filePath).Split('_'))}\t{contents}\n"; + File.AppendAllText(TestsKnownToCrashTestRunnerFilePath, contents); + + File.Delete(filePath); + } + } +} \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs b/test/Shared/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs similarity index 88% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs rename to test/Shared/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs index c5a599f6..f790be7e 100644 --- a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs +++ b/test/Shared/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs @@ -27,9 +27,8 @@ public class JetXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer } protected override bool IsValidTestClass(ITypeInfo type) - => base.IsValidTestClass(type) /* && - IsTestConditionMet(type) && - IsTestConditionMet(type)*/; + => base.IsValidTestClass(type) && + IsTestConditionMet(type); protected virtual bool IsTestConditionMet(ITypeInfo type) where TType : ITestCondition => GetTestConditions(type).Aggregate(true, (current, next) => current && next.IsMetAsync().Result); diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestRunner.cs b/test/Shared/TestUtilities/Xunit/JetXunitTestRunner.cs similarity index 75% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestRunner.cs rename to test/Shared/TestUtilities/Xunit/JetXunitTestRunner.cs index 45c1644b..752dc5c6 100644 --- a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestRunner.cs +++ b/test/Shared/TestUtilities/Xunit/JetXunitTestRunner.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.IO; using System.Reflection; +using System.Text; using System.Threading; using System.Threading.Tasks; using Xunit.Abstractions; @@ -80,14 +82,14 @@ public new async Task RunAsync() } #region Customized - /// This is what we are after. Mark failed tests as 'Skipped', if their failure is expected. + /// This is what we are after. Mark failed tests as 'Skipped', if the failure is expected. else if (SkipFailedTest(exception)) { testResultMessage = new TestSkipped(Test, exception.Message); ++runSummary.Skipped; } #endregion Customized - + else { testResultMessage = new TestFailed(Test, runSummary.Time, output, exception); @@ -121,12 +123,14 @@ public new async Task RunAsync() } /// - /// Mark failed tests as 'Skipped', it they failed because they use an expression, that is not supported by the underlying database - /// server version. + /// Mark failed tests as 'Skipped', if they failed because they use an expression, that we explicitly marked as + /// supported by Jet. /// protected virtual bool SkipFailedTest(Exception exception) { var skip = true; + var unexpectedUnsupportedTranslation = false; + var aggregateException = exception as AggregateException ?? new AggregateException(exception); @@ -138,15 +142,30 @@ protected virtual bool SkipFailedTest(Exception exception) return false; } - if (innerException.Message.StartsWith("The LINQ expression '") || + if (innerException.Message.StartsWith("Jet does not support ")) + { + var expectedUnsupportedTranslation = innerException.Message.Contains("APPLY statements") || + innerException.Message.Contains("skipping rows"); + + skip &= expectedUnsupportedTranslation; + unexpectedUnsupportedTranslation = !expectedUnsupportedTranslation; + } + else if (innerException.Message.StartsWith("The LINQ expression '") && innerException.Message.Contains("' could not be translated.")) { - skip &= /*innerException.Message.Contains("OUTER APPLY") || - innerException.Message.Contains("CROSS APPLY") ||*/ - innerException.Message.Contains("RowNumberExpression"); + var expectedUnsupportedTranslation = innerException.Message.Contains("RowNumberExpression"); + + skip &= expectedUnsupportedTranslation; + unexpectedUnsupportedTranslation = !expectedUnsupportedTranslation; + } + + if (unexpectedUnsupportedTranslation) + { + var sb = new StringBuilder(); + sb.AppendLine(innerException.Message.ReplaceLineEndings(" ")); + sb.AppendLine("-----"); - var message = innerException.Message; - skip = skip && true; + File.AppendAllText("UnsupportedTranslations.txt", sb.ToString()); } } diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTheoryTestCaseRunner.cs b/test/Shared/TestUtilities/Xunit/JetXunitTheoryTestCaseRunner.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTheoryTestCaseRunner.cs rename to test/Shared/TestUtilities/Xunit/JetXunitTheoryTestCaseRunner.cs diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/RandomTestCaseOrderer.cs b/test/Shared/TestUtilities/Xunit/RandomTestCaseOrderer.cs similarity index 100% rename from test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/RandomTestCaseOrderer.cs rename to test/Shared/TestUtilities/Xunit/RandomTestCaseOrderer.cs