From 5976446953365569ab68ab03e1ebd50217621476 Mon Sep 17 00:00:00 2001 From: Brad Wilson Date: Mon, 18 Jul 2022 20:06:48 -0700 Subject: [PATCH] Add ITheoryDataRow.Timeout and DataAttribute.Timeout (and more code restructuring) --- .../CulturedTheoryAttributeDiscoverer.cs | 6 +- .../v3/Messages/_TestStarting.cs | 3 + .../v3/Metadata/_ITestMetadata.cs | 11 ++++ .../Acceptance/Xunit3TheoryAcceptanceTests.cs | 51 ++++++++++++++++ src/xunit.v3.core/FactAttribute.cs | 4 +- src/xunit.v3.core/ITheoryDataRow.cs | 12 ++++ .../Internal/XunitRunnerHelper.cs | 1 + src/xunit.v3.core/Sdk/DataAttribute.cs | 56 ++++++++++++++++- .../Sdk/Frameworks/TestIntrospectionHelper.cs | 61 +++++++++++-------- .../Sdk/Frameworks/TheoryDiscoverer.cs | 10 +-- src/xunit.v3.core/Sdk/InlineDataDiscoverer.cs | 11 ++++ src/xunit.v3.core/Sdk/TheoryDataRow.cs | 3 + .../Runners/ExecutionErrorTestCaseRunner.cs | 3 +- .../Sdk/v3/Runners/TestInvoker.cs | 2 +- .../Sdk/v3/Runners/TestRunner.cs | 1 + ...unitDelayEnumeratedTheoryTestCaseRunner.cs | 8 ++- .../Sdk/v3/Runners/XunitTestCaseRunnerBase.cs | 8 ++- .../Sdk/v3/Runners/XunitTestInvoker.cs | 31 ++++++---- .../Sdk/v3/TestCases/TestMethodTestCase.cs | 6 +- .../Sdk/v3/TestCases/XunitTest.cs | 12 +++- .../Frameworks/v2/Xunit2.cs | 1 + .../Frameworks/v2/Xunit2MessageAdapter.cs | 1 + 22 files changed, 237 insertions(+), 65 deletions(-) diff --git a/src/common.tests/CultureAwareTesting/CulturedTheoryAttributeDiscoverer.cs b/src/common.tests/CultureAwareTesting/CulturedTheoryAttributeDiscoverer.cs index 73e8b8508..4ebdc85bf 100644 --- a/src/common.tests/CultureAwareTesting/CulturedTheoryAttributeDiscoverer.cs +++ b/src/common.tests/CultureAwareTesting/CulturedTheoryAttributeDiscoverer.cs @@ -13,14 +13,12 @@ public class CulturedTheoryAttributeDiscoverer : TheoryDiscoverer _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute, - _IAttributeInfo dataAttribute, ITheoryDataRow dataRow, object?[] testMethodArguments) { var cultures = GetCultures(theoryAttribute); - var dataAttributeDisplayName = dataAttribute.GetNamedArgument(nameof(DataAttribute.TestDisplayName)); - var details = TestIntrospectionHelper.GetTestCaseDetails(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments, baseDisplayName: dataAttributeDisplayName); - var traits = TestIntrospectionHelper.GetTraits(testMethod, dataAttribute, dataRow); + var details = TestIntrospectionHelper.GetTestCaseDetailsForTheoryDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments); + var traits = TestIntrospectionHelper.GetTraits(testMethod, dataRow); var result = cultures.Select( // TODO: How do we get source information in here? diff --git a/src/xunit.v3.common/v3/Messages/_TestStarting.cs b/src/xunit.v3.common/v3/Messages/_TestStarting.cs index 453ae368a..c08bbba60 100644 --- a/src/xunit.v3.common/v3/Messages/_TestStarting.cs +++ b/src/xunit.v3.common/v3/Messages/_TestStarting.cs @@ -22,6 +22,9 @@ public string TestDisplayName set => testDisplayName = Guard.ArgumentNotNullOrEmpty(value, nameof(TestDisplayName)); } + /// + public int Timeout { get; set; } + /// public IReadOnlyDictionary> Traits { diff --git a/src/xunit.v3.common/v3/Metadata/_ITestMetadata.cs b/src/xunit.v3.common/v3/Metadata/_ITestMetadata.cs index c45774c09..4a11771fa 100644 --- a/src/xunit.v3.common/v3/Metadata/_ITestMetadata.cs +++ b/src/xunit.v3.common/v3/Metadata/_ITestMetadata.cs @@ -17,6 +17,17 @@ public interface _ITestMetadata /// string TestDisplayName { get; } + /// + /// A value greater than zero marks the test as having a timeout, and gets or sets the + /// timeout (in milliseconds). + /// + /// + /// WARNING: Using this with parallelization turned on will result in undefined behavior. + /// Timeout is only supported when parallelization is disabled, either globally or with + /// a parallelization-disabled test collection. + /// + int Timeout { get; } + /// /// Gets the trait values associated with this test case. If /// there are none, or the framework does not support traits, diff --git a/src/xunit.v3.core.tests/Acceptance/Xunit3TheoryAcceptanceTests.cs b/src/xunit.v3.core.tests/Acceptance/Xunit3TheoryAcceptanceTests.cs index 9e1331e55..937357c22 100644 --- a/src/xunit.v3.core.tests/Acceptance/Xunit3TheoryAcceptanceTests.cs +++ b/src/xunit.v3.core.tests/Acceptance/Xunit3TheoryAcceptanceTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -868,6 +869,56 @@ public void TestMethod(int x) } } + [Collection("Timeout Tests")] + public class DataAttributeTimeoutTests : AcceptanceTestV3 + { + [Theory] + [InlineData(true)] + [InlineData(false)] + public async void TimeoutAcceptanceTest(bool preEnumerateTheories) + { + var stopwatch = Stopwatch.StartNew(); + var results = await RunForResultsAsync(typeof(ClassUnderTest), preEnumerateTheories); + stopwatch.Stop(); + + Assert.Collection( + results.OfType().OrderBy(x => x.TestDisplayName), + passed => Assert.Equal($"{typeof(ClassUnderTest).FullName}.{nameof(ClassUnderTest.LongRunningTask)}(delay: 10)", passed.TestDisplayName), + passed => Assert.Equal($"{typeof(ClassUnderTest).FullName}.{nameof(ClassUnderTest.LongRunningTask)}(delay: 100)", passed.TestDisplayName) + ); + Assert.Collection( + results.OfType().OrderBy(f => f.TestDisplayName), + failed => + { + Assert.Equal($"{typeof(ClassUnderTest).FullName}.{nameof(ClassUnderTest.LongRunningTask)}(delay: 10000)", failed.TestDisplayName); + Assert.Equal("Test execution timed out after 42 milliseconds", failed.Messages.Single()); + }, + failed => + { + Assert.Equal($"{typeof(ClassUnderTest).FullName}.{nameof(ClassUnderTest.LongRunningTask)}(delay: 11000)", failed.TestDisplayName); + Assert.Equal("Test execution timed out after 10 milliseconds", failed.Messages.Single()); + } + ); + + Assert.True(stopwatch.ElapsedMilliseconds < 10000, "Elapsed time should be less than 10 seconds"); + } + + class ClassUnderTest + { + public static List MemberDataSource = new() + { + new TheoryDataRow(11000), + new TheoryDataRow(100) { Timeout = 10000 }, + }; + + [Theory(Timeout = 42)] + [InlineData(10000)] + [InlineData(10, Timeout = 10000)] + [MemberData(nameof(MemberDataSource), Timeout = 10)] + public Task LongRunningTask(int delay) => Task.Delay(delay); + } + } + public class InlineDataTests : AcceptanceTestV3 { [Fact] diff --git a/src/xunit.v3.core/FactAttribute.cs b/src/xunit.v3.core/FactAttribute.cs index 33b6abfc9..438d4115f 100644 --- a/src/xunit.v3.core/FactAttribute.cs +++ b/src/xunit.v3.core/FactAttribute.cs @@ -35,9 +35,11 @@ public class FactAttribute : Attribute /// /// A value greater than zero marks the test as having a timeout, and gets or sets the /// timeout (in milliseconds). + /// + /// /// WARNING: Using this with parallelization turned on will result in undefined behavior. /// Timeout is only supported when parallelization is disabled, either globally or with /// a parallelization-disabled test collection. - /// + /// public virtual int Timeout { get; set; } } diff --git a/src/xunit.v3.core/ITheoryDataRow.cs b/src/xunit.v3.core/ITheoryDataRow.cs index e6b769e5f..6b0202848 100644 --- a/src/xunit.v3.core/ITheoryDataRow.cs +++ b/src/xunit.v3.core/ITheoryDataRow.cs @@ -29,6 +29,18 @@ public interface ITheoryDataRow /// string? TestDisplayName { get; } + /// + /// A value greater than zero marks the test as having a timeout, and gets or sets the + /// timeout (in milliseconds). A non-null value here overrides any inherited value + /// from the or the . + /// + /// + /// WARNING: Using this with parallelization turned on will result in undefined behavior. + /// Timeout is only supported when parallelization is disabled, either globally or with + /// a parallelization-disabled test collection. + /// + int? Timeout { get; } + /// /// Gets the trait values associated with this theory data row. If there are none, you may either /// return a null or empty dictionary. diff --git a/src/xunit.v3.core/Internal/XunitRunnerHelper.cs b/src/xunit.v3.core/Internal/XunitRunnerHelper.cs index 80f23e696..da91ac2e7 100644 --- a/src/xunit.v3.core/Internal/XunitRunnerHelper.cs +++ b/src/xunit.v3.core/Internal/XunitRunnerHelper.cs @@ -51,6 +51,7 @@ public static class XunitRunnerHelper TestDisplayName = testCase.TestCaseDisplayName, TestMethodUniqueID = testCase.TestMethod?.UniqueID, TestUniqueID = testUniqueID, + Timeout = 0, Traits = testCase.Traits, }; messageBus.QueueMessage(testStarting); diff --git a/src/xunit.v3.core/Sdk/DataAttribute.cs b/src/xunit.v3.core/Sdk/DataAttribute.cs index 750f5a40d..8189b9e3d 100644 --- a/src/xunit.v3.core/Sdk/DataAttribute.cs +++ b/src/xunit.v3.core/Sdk/DataAttribute.cs @@ -16,7 +16,6 @@ namespace Xunit.Sdk; [AttributeUsage(AttributeTargets.Method, AllowMultiple = true, Inherited = true)] public abstract class DataAttribute : Attribute { - static readonly Dictionary> emptyTraits = new(); static readonly MethodInfo? tupleIndexerGetter; static readonly MethodInfo? tupleLengthGetter; static readonly Type? tupleType; @@ -77,6 +76,34 @@ public bool Explicit /// public string? TestDisplayName { get; set; } + /// + /// A value greater than zero marks the test as having a timeout, and gets or sets the + /// timeout (in milliseconds). Setting a value here overrides any inherited value + /// from the . + /// + /// + /// WARNING: Using this with parallelization turned on will result in undefined behavior. + /// Timeout is only supported when parallelization is disabled, either globally or with + /// a parallelization-disabled test collection. + /// + public int Timeout + { + get => TimeoutWithoutDefaultValue ?? 0; + set => TimeoutWithoutDefaultValue = value; + } + + /// + /// Gets the value for , except that it keeps track of whether it has been set + /// or not, and returns null if it hasn't been set. + /// + /// + /// Since attribute initializers cannot accept nullable integer values, this secondary attribute value + /// (that cannot be externally set) is required to keep track of whether has been + /// set by the user or not. At reflection time, we can peer into the , + /// but once the attribute instance has been created, this is the only way to know for sure. + /// + protected int? TimeoutWithoutDefaultValue { get; set; } + /// /// Gets or sets a set of traits for the associated data. The data is pushed as an array of /// string that are key/value pairs (f.e., new[] { "key1", "value1", "key2", "value2" }). @@ -114,19 +141,36 @@ public bool Explicit Guard.ArgumentNotNull(dataRow); if (dataRow is ITheoryDataRow theoryDataRow) + { + var dataRowTraits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + if (theoryDataRow.Traits != null) + foreach (var kvp in theoryDataRow.Traits) + dataRowTraits.GetOrAdd(kvp.Key).AddRange(kvp.Value); + + MergeTraitsInto(dataRowTraits); + return new TheoryDataRow(theoryDataRow.GetData()) { Explicit = theoryDataRow.Explicit ?? ExplicitWithoutDefaultValue, Skip = theoryDataRow.Skip ?? Skip, - TestDisplayName = theoryDataRow.TestDisplayName, - Traits = theoryDataRow.Traits ?? emptyTraits, + TestDisplayName = theoryDataRow.TestDisplayName ?? TestDisplayName, + Timeout = theoryDataRow.Timeout ?? TimeoutWithoutDefaultValue, + Traits = dataRowTraits, }; + } + + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + MergeTraitsInto(traits); if (dataRow is object?[] array) return new TheoryDataRow(array) { Explicit = ExplicitWithoutDefaultValue, Skip = Skip, + TestDisplayName = TestDisplayName, + Timeout = TimeoutWithoutDefaultValue, + Traits = traits, }; if (tupleType != null && tupleIndexerGetter != null && tupleLengthGetter != null) @@ -145,6 +189,9 @@ public bool Explicit { Explicit = ExplicitWithoutDefaultValue, Skip = Skip, + TestDisplayName = TestDisplayName, + Timeout = TimeoutWithoutDefaultValue, + Traits = traits, }; } } @@ -165,4 +212,7 @@ public bool Explicit /// One or more rows of theory data. Each invocation of the test method /// is represented by a single instance of . public abstract ValueTask?> GetData(MethodInfo testMethod); + + void MergeTraitsInto(Dictionary> traits) => + TestIntrospectionHelper.MergeTraitsInto(traits, Traits); } diff --git a/src/xunit.v3.core/Sdk/Frameworks/TestIntrospectionHelper.cs b/src/xunit.v3.core/Sdk/Frameworks/TestIntrospectionHelper.cs index c0133c935..0138ec7e2 100644 --- a/src/xunit.v3.core/Sdk/Frameworks/TestIntrospectionHelper.cs +++ b/src/xunit.v3.core/Sdk/Frameworks/TestIntrospectionHelper.cs @@ -46,6 +46,7 @@ static IReadOnlyCollection<_IAttributeInfo> GetCachedTraitAttributes(_ITypeInfo /// The test method. /// The fact attribute that decorates the test method. /// The optional test method arguments. + /// The optional timeout; if not provided, will be looked up from the . /// The optional base display name for the test method. public static ( string TestCaseDisplayName, @@ -59,6 +60,7 @@ _ITestMethod ResolvedTestMethod _ITestMethod testMethod, _IAttributeInfo factAttribute, object?[]? testMethodArguments = null, + int? timeout = null, string? baseDisplayName = null) { Guard.ArgumentNotNull(discoveryOptions); @@ -72,7 +74,7 @@ _ITestMethod ResolvedTestMethod baseDisplayName ??= factAttribute.GetNamedArgument(nameof(FactAttribute.DisplayName)); var factExplicit = factAttribute.GetNamedArgument(nameof(FactAttribute.Explicit)); var factSkipReason = factAttribute.GetNamedArgument(nameof(FactAttribute.Skip)); - var factTimeout = factAttribute.GetNamedArgument(nameof(FactAttribute.Timeout)); + timeout ??= factAttribute.GetNamedArgument(nameof(FactAttribute.Timeout)); if (baseDisplayName == null) { @@ -93,8 +95,7 @@ _ITestMethod ResolvedTestMethod var testCaseDisplayName = testMethod.Method.GetDisplayNameWithArguments(baseDisplayName, testMethodArguments, methodGenericTypes); var uniqueID = UniqueIDGenerator.ForTestCase(testMethod.UniqueID, methodGenericTypes, testMethodArguments); - - return (testCaseDisplayName, factExplicit, factSkipReason, factTimeout, uniqueID, testMethod); + return (testCaseDisplayName, factExplicit, factSkipReason, timeout.Value, uniqueID, testMethod); } /// @@ -107,7 +108,6 @@ _ITestMethod ResolvedTestMethod /// The theory attribute that decorates the test method. /// The data row for the test. /// The test method arguments obtained from the after being type-resolved. - /// The optional base display name (typically from the data attribute). public static ( string TestCaseDisplayName, bool Explicit, @@ -115,32 +115,33 @@ _ITestMethod ResolvedTestMethod int Timeout, string UniqueID, _ITestMethod ResolvedTestMethod - ) GetTestCaseDetails( + ) GetTestCaseDetailsForTheoryDataRow( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute, ITheoryDataRow dataRow, - object?[] testMethodArguments, - string? baseDisplayName = null) + object?[] testMethodArguments) { - var result = GetTestCaseDetails(discoveryOptions, testMethod, theoryAttribute, testMethodArguments, dataRow.TestDisplayName ?? baseDisplayName); - - if (dataRow.Skip != null) - result.SkipReason = dataRow.Skip; + var result = GetTestCaseDetails(discoveryOptions, testMethod, theoryAttribute, testMethodArguments, dataRow.Timeout, dataRow.TestDisplayName); if (dataRow.Explicit.HasValue) result.Explicit = dataRow.Explicit.Value; + if (dataRow.Skip != null) + result.SkipReason = dataRow.Skip; + return result; } /// - /// + /// Retrieve the traits for a test method (merging in the traits from the optional data row, which is + /// assumed to already have traits that were merged from the data row itself and the data attribute). /// + /// The test method to get traits from. + /// The data row to get traits from. /// The traits dictionary public static Dictionary> GetTraits( _ITestMethod testMethod, - _IAttributeInfo? dataAttribute = null, ITheoryDataRow? dataRow = null) { Guard.ArgumentNotNull(testMethod); @@ -192,19 +193,6 @@ _ITestMethod ResolvedTestMethod result.GetOrAdd(kvp.Key).Add(kvp.Value); } - // Traits from the data attribute - var traitsArray = dataAttribute?.GetNamedArgument(nameof(DataAttribute.Traits)); - if (traitsArray != null) - { - var idx = 0; - - while (idx < traitsArray.Length - 1) - { - result.GetOrAdd(traitsArray[idx]).Add(traitsArray[idx + 1]); - idx += 2; - } - } - // Traits from the data row if (dataRow?.Traits != null) foreach (var kvp in dataRow.Traits) @@ -212,4 +200,25 @@ _ITestMethod ResolvedTestMethod return result; } + + /// + /// Merges string-array traits (like from ) into an existing traits dictionary. + /// + /// The existing traits dictionary. + /// The additional traits to merge. + public static void MergeTraitsInto( + Dictionary> traits, + string[]? additionalTraits) + { + if (additionalTraits == null) + return; + + var idx = 0; + + while (idx < additionalTraits.Length - 1) + { + traits.GetOrAdd(additionalTraits[idx]).Add(additionalTraits[idx + 1]); + idx += 2; + } + } } diff --git a/src/xunit.v3.core/Sdk/Frameworks/TheoryDiscoverer.cs b/src/xunit.v3.core/Sdk/Frameworks/TheoryDiscoverer.cs index 74deb4469..12c89f7e4 100644 --- a/src/xunit.v3.core/Sdk/Frameworks/TheoryDiscoverer.cs +++ b/src/xunit.v3.core/Sdk/Frameworks/TheoryDiscoverer.cs @@ -20,7 +20,6 @@ public class TheoryDiscoverer : IXunitTestCaseDiscoverer /// The discovery options to be used. /// The test method the test cases belong to. /// The theory attribute attached to the test method. - /// The data attribute that discovered the data. /// The data row that generated . /// The arguments for the test method. /// The test cases @@ -28,20 +27,17 @@ public class TheoryDiscoverer : IXunitTestCaseDiscoverer _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute, - _IAttributeInfo dataAttribute, ITheoryDataRow dataRow, object?[] testMethodArguments) { Guard.ArgumentNotNull(discoveryOptions); Guard.ArgumentNotNull(testMethod); Guard.ArgumentNotNull(theoryAttribute); - Guard.ArgumentNotNull(dataAttribute); Guard.ArgumentNotNull(dataRow); Guard.ArgumentNotNull(testMethodArguments); - var dataAttributeDisplayName = dataAttribute.GetNamedArgument(nameof(DataAttribute.TestDisplayName)); - var details = TestIntrospectionHelper.GetTestCaseDetails(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments, dataAttributeDisplayName); - var traits = TestIntrospectionHelper.GetTraits(testMethod, dataAttribute, dataRow); + var details = TestIntrospectionHelper.GetTestCaseDetailsForTheoryDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments); + var traits = TestIntrospectionHelper.GetTraits(testMethod, dataRow); // TODO: How do we get source information in here? var testCase = new XunitTestCase( @@ -260,7 +256,7 @@ public class TheoryDiscoverer : IXunitTestCaseDiscoverer try { - results.AddRange(await CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataAttribute, dataRow, resolvedData)); + results.AddRange(await CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, resolvedData)); } catch (Exception ex) { diff --git a/src/xunit.v3.core/Sdk/InlineDataDiscoverer.cs b/src/xunit.v3.core/Sdk/InlineDataDiscoverer.cs index e2a372d2b..532678663 100644 --- a/src/xunit.v3.core/Sdk/InlineDataDiscoverer.cs +++ b/src/xunit.v3.core/Sdk/InlineDataDiscoverer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -25,10 +26,20 @@ public class InlineDataDiscoverer : IDataDiscoverer // in Xunit3TheoryAcceptanceTests.InlineDataTests.SingleNullValuesWork). var args = dataAttribute.GetConstructorArguments().Single() as IEnumerable ?? new object?[] { null }; + var testDisplayName = dataAttribute.GetNamedArgument(nameof(DataAttribute.TestDisplayName)); + var timeout = dataAttribute.GetNamedArgument(nameof(DataAttribute.Timeout)); + + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var traitsArray = dataAttribute.GetNamedArgument(nameof(DataAttribute.Traits)); + TestIntrospectionHelper.MergeTraitsInto(traits, traitsArray); + var theoryDataRow = new TheoryDataRow(args.ToArray()) { Explicit = dataAttribute.GetNamedArgument(nameof(DataAttribute.Explicit)), Skip = dataAttribute.GetNamedArgument(nameof(DataAttribute.Skip)), + TestDisplayName = testDisplayName, + Timeout = timeout, + Traits = traits, }; return new(new[] { theoryDataRow }); diff --git a/src/xunit.v3.core/Sdk/TheoryDataRow.cs b/src/xunit.v3.core/Sdk/TheoryDataRow.cs index e0ed4e141..ec70cdbe0 100644 --- a/src/xunit.v3.core/Sdk/TheoryDataRow.cs +++ b/src/xunit.v3.core/Sdk/TheoryDataRow.cs @@ -29,6 +29,9 @@ public TheoryDataRow(params object?[] data) /// public string? TestDisplayName { get; set; } + /// + public int? Timeout { get; set; } + /// /// Gets or sets the traits for the theory data row. /// diff --git a/src/xunit.v3.core/Sdk/v3/Runners/ExecutionErrorTestCaseRunner.cs b/src/xunit.v3.core/Sdk/v3/Runners/ExecutionErrorTestCaseRunner.cs index 885756d57..894318644 100644 --- a/src/xunit.v3.core/Sdk/v3/Runners/ExecutionErrorTestCaseRunner.cs +++ b/src/xunit.v3.core/Sdk/v3/Runners/ExecutionErrorTestCaseRunner.cs @@ -41,7 +41,7 @@ protected ExecutionErrorTestCaseRunner() protected override ValueTask RunTestsAsync(TestCaseRunnerContext ctxt) { // Use -1 for the index here so we don't collide with any legitimate test case IDs that might've been used - var test = new XunitTest(ctxt.TestCase, @explicit: null, ctxt.TestCase.TestCaseDisplayName, testIndex: -1, ctxt.TestCase.Traits.ToReadOnly()); + var test = new XunitTest(ctxt.TestCase, @explicit: null, ctxt.TestCase.TestCaseDisplayName, testIndex: -1, ctxt.TestCase.Traits.ToReadOnly(), timeout: 0); var summary = new RunSummary { Total = 1 }; var testAssemblyUniqueID = ctxt.TestCase.TestMethod.TestClass.TestCollection.TestAssembly.UniqueID; @@ -60,6 +60,7 @@ protected override ValueTask RunTestsAsync(TestCaseRunnerContext RunAsync(TContext ctxt) SetTestContext(ctxt, TestEngineStatus.Running); if (!ctxt.CancellationTokenSource.IsCancellationRequested && !ctxt.Aggregator.HasExceptions) - await InvokeTestMethodAsync(ctxt, testClassInstance); + elapsedTime += TimeSpan.FromSeconds((double)await InvokeTestMethodAsync(ctxt, testClassInstance)); SetTestContext(ctxt, TestEngineStatus.CleaningUp, TestResultState.FromException((decimal)elapsedTime.TotalSeconds, ctxt.Aggregator.ToException())); diff --git a/src/xunit.v3.core/Sdk/v3/Runners/TestRunner.cs b/src/xunit.v3.core/Sdk/v3/Runners/TestRunner.cs index 1efbedc7e..249aab210 100644 --- a/src/xunit.v3.core/Sdk/v3/Runners/TestRunner.cs +++ b/src/xunit.v3.core/Sdk/v3/Runners/TestRunner.cs @@ -74,6 +74,7 @@ protected async ValueTask RunAsync(TContext ctxt) TestDisplayName = ctxt.Test.TestDisplayName, TestMethodUniqueID = testMethodUniqueID, TestUniqueID = testUniqueID, + Timeout = ctxt.Test.Timeout, Traits = ctxt.Test.Traits, }; diff --git a/src/xunit.v3.core/Sdk/v3/Runners/XunitDelayEnumeratedTheoryTestCaseRunner.cs b/src/xunit.v3.core/Sdk/v3/Runners/XunitDelayEnumeratedTheoryTestCaseRunner.cs index 7d25692d2..faca19df8 100644 --- a/src/xunit.v3.core/Sdk/v3/Runners/XunitDelayEnumeratedTheoryTestCaseRunner.cs +++ b/src/xunit.v3.core/Sdk/v3/Runners/XunitDelayEnumeratedTheoryTestCaseRunner.cs @@ -103,8 +103,9 @@ protected override async ValueTask AfterTestCaseStartingAsync(XunitDelayEnumerat var baseDisplayName = dataRow.TestDisplayName ?? dataAttribute.GetNamedArgument(nameof(DataAttribute.TestDisplayName)) ?? ctxt.DisplayName; var theoryDisplayName = ctxt.TestCase.TestMethod.Method.GetDisplayNameWithArguments(baseDisplayName, convertedDataRow, resolvedTypes); - var traits = TestIntrospectionHelper.GetTraits(ctxt.TestCase.TestMethod, dataAttribute, dataRow); - var test = CreateTest(ctxt, dataRow.Explicit, theoryDisplayName, testIndex++, traits.ToReadOnly()); + var traits = TestIntrospectionHelper.GetTraits(ctxt.TestCase.TestMethod, dataRow); + var timeout = dataRow.Timeout ?? dataAttribute.GetNamedArgument(nameof(DataAttribute.Timeout)) ?? ctxt.TestCase.Timeout; + var test = CreateTest(ctxt, dataRow.Explicit, theoryDisplayName, testIndex++, traits.ToReadOnly(), timeout); var skipReason = dataRow.Skip ?? dataAttribute.GetNamedArgument(nameof(DataAttribute.Skip)) ?? ctxt.SkipReason; ctxt.DiscoveredTests.Add((test, methodToRun, convertedDataRow, skipReason)); @@ -206,7 +207,7 @@ protected override async ValueTask RunTestsAsync(XunitDelayEnumerate RunSummary RunTest_DataDiscoveryException(XunitDelayEnumeratedTheoryTestCaseRunnerContext ctxt) { // Use -1 for the index here so we don't collide with any legitimate test IDs that might've been used - var test = new XunitTest(ctxt.TestCase, @explicit: null, ctxt.DisplayName, testIndex: -1, ctxt.TestCase.Traits); + var test = new XunitTest(ctxt.TestCase, @explicit: null, ctxt.DisplayName, testIndex: -1, ctxt.TestCase.Traits, timeout: 0); var testAssemblyUniqueID = ctxt.TestCase.TestCollection.TestAssembly.UniqueID; var testCollectionUniqueID = ctxt.TestCase.TestCollection.UniqueID; @@ -224,6 +225,7 @@ RunSummary RunTest_DataDiscoveryException(XunitDelayEnumeratedTheoryTestCaseRunn TestDisplayName = test.TestDisplayName, TestMethodUniqueID = testMethodUniqueID, TestUniqueID = test.UniqueID, + Timeout = 0, Traits = test.Traits, }; diff --git a/src/xunit.v3.core/Sdk/v3/Runners/XunitTestCaseRunnerBase.cs b/src/xunit.v3.core/Sdk/v3/Runners/XunitTestCaseRunnerBase.cs index b259162b4..47ced4ff4 100644 --- a/src/xunit.v3.core/Sdk/v3/Runners/XunitTestCaseRunnerBase.cs +++ b/src/xunit.v3.core/Sdk/v3/Runners/XunitTestCaseRunnerBase.cs @@ -66,18 +66,20 @@ public class XunitTestCaseRunnerBase : TestCaseRunnerThe test index for the test. Multiple test per test case scenarios will need /// to use the test index to help construct the test unique ID. /// The traits for the test. + /// The timeout for the test. protected virtual _ITest CreateTest( TContext ctxt, bool? @explicit, string? displayName, int testIndex, - IReadOnlyDictionary> traits) => - new XunitTest(ctxt.TestCase, @explicit, displayName ?? ctxt.DisplayName, testIndex, traits); + IReadOnlyDictionary> traits, + int timeout) => + new XunitTest(ctxt.TestCase, @explicit, displayName ?? ctxt.DisplayName, testIndex, traits, timeout); /// protected override ValueTask RunTestsAsync(TContext ctxt) => XunitTestRunner.Instance.RunAsync( - CreateTest(ctxt, @explicit: null, displayName: null, testIndex: 0, ctxt.TestCase.Traits), + CreateTest(ctxt, @explicit: null, displayName: null, testIndex: 0, ctxt.TestCase.Traits, ctxt.TestCase.Timeout), ctxt.MessageBus, ctxt.TestClass, ctxt.ConstructorArguments, diff --git a/src/xunit.v3.core/Sdk/v3/Runners/XunitTestInvoker.cs b/src/xunit.v3.core/Sdk/v3/Runners/XunitTestInvoker.cs index 4e3c47a62..0bce617b5 100644 --- a/src/xunit.v3.core/Sdk/v3/Runners/XunitTestInvoker.cs +++ b/src/xunit.v3.core/Sdk/v3/Runners/XunitTestInvoker.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Threading; @@ -199,11 +200,9 @@ protected override ValueTask BeforeTestMethodInvokedAsync(XunitTestInvokerContex XunitTestInvokerContext ctxt, object? testClassInstance) { - var testCase = (IXunitTestCase)ctxt.Test.TestCase; - return - testCase.Timeout > 0 - ? InvokeTimeoutTestMethodAsync(ctxt, testClassInstance, testCase.Timeout) + ctxt.Test.Timeout > 0 + ? InvokeTimeoutTestMethodAsync(ctxt, testClassInstance, ctxt.Test.Timeout) : base.InvokeTestMethodAsync(ctxt, testClassInstance); } @@ -212,16 +211,26 @@ protected override ValueTask BeforeTestMethodInvokedAsync(XunitTestInvokerContex object? testClassInstance, int timeout) { - if (!AsyncUtility.IsAsync(ctxt.TestMethod)) - throw TestTimeoutException.ForIncompatibleTest(); + var stopwatch = Stopwatch.StartNew(); + + try + { + if (!AsyncUtility.IsAsync(ctxt.TestMethod)) + throw TestTimeoutException.ForIncompatibleTest(); - var baseTask = base.InvokeTestMethodAsync(ctxt, testClassInstance).AsTask(); - var resultTask = await Task.WhenAny(baseTask, Task.Delay(timeout)); + var baseTask = base.InvokeTestMethodAsync(ctxt, testClassInstance).AsTask(); + var resultTask = await Task.WhenAny(baseTask, Task.Delay(timeout)); - if (resultTask != baseTask) - throw TestTimeoutException.ForTimedOutTest(timeout); + if (resultTask != baseTask) + throw TestTimeoutException.ForTimedOutTest(timeout); - return await baseTask; + return await baseTask; + } + catch (Exception ex) + { + ctxt.Aggregator.Add(ex); + return (decimal)stopwatch.Elapsed.TotalSeconds; + } } /// diff --git a/src/xunit.v3.core/Sdk/v3/TestCases/TestMethodTestCase.cs b/src/xunit.v3.core/Sdk/v3/TestCases/TestMethodTestCase.cs index 11f74d466..26ba55a4b 100644 --- a/src/xunit.v3.core/Sdk/v3/TestCases/TestMethodTestCase.cs +++ b/src/xunit.v3.core/Sdk/v3/TestCases/TestMethodTestCase.cs @@ -56,10 +56,10 @@ protected TestMethodTestCase() this.testCaseDisplayName = Guard.ArgumentNotNull(testCaseDisplayName); this.uniqueID = Guard.ArgumentNotNull(uniqueID); + this.traits = new(StringComparer.OrdinalIgnoreCase); if (traits != null) - this.traits = new Dictionary>(traits, StringComparer.OrdinalIgnoreCase); - else - this.traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var kvp in traits) + this.traits.GetOrAdd(kvp.Key).AddRange(kvp.Value); foreach (var testMethodArgument in TestMethodArguments) disposalTracker.Add(testMethodArgument); diff --git a/src/xunit.v3.core/Sdk/v3/TestCases/XunitTest.cs b/src/xunit.v3.core/Sdk/v3/TestCases/XunitTest.cs index 231e7b42c..81274ec87 100644 --- a/src/xunit.v3.core/Sdk/v3/TestCases/XunitTest.cs +++ b/src/xunit.v3.core/Sdk/v3/TestCases/XunitTest.cs @@ -21,17 +21,20 @@ public class XunitTest : _ITest /// The display name for this test. /// The index of this test inside the test case. Used for computing . /// The traits for the given test. + /// The timeout for the test. public XunitTest( IXunitTestCase testCase, bool? @explicit, string testDisplayName, int testIndex, - IReadOnlyDictionary> traits) + IReadOnlyDictionary> traits, + int timeout) { TestCase = Guard.ArgumentNotNull(testCase); this.@explicit = @explicit; TestDisplayName = Guard.ArgumentNotNull(testDisplayName); UniqueID = UniqueIDGenerator.ForTest(testCase.UniqueID, testIndex); + Timeout = timeout; Guard.ArgumentNotNull(traits); @@ -49,12 +52,14 @@ public class XunitTest : _ITest bool? @explicit, string testDisplayName, string uniqueID, - IReadOnlyDictionary>? traits = null) + IReadOnlyDictionary>? traits = null, + int timeout = 0) { TestCase = Guard.ArgumentNotNull(testCase); this.@explicit = @explicit; TestDisplayName = Guard.ArgumentNotNull(testDisplayName); UniqueID = Guard.ArgumentNotNull(uniqueID); + Timeout = timeout; if (traits == null) Traits = EmptyDictionary; @@ -81,6 +86,9 @@ public class XunitTest : _ITest /// public string TestDisplayName { get; } + /// + public int Timeout { get; } + /// public IReadOnlyDictionary> Traits { get; } diff --git a/src/xunit.v3.runner.utility/Frameworks/v2/Xunit2.cs b/src/xunit.v3.runner.utility/Frameworks/v2/Xunit2.cs index b0de93432..70152657d 100644 --- a/src/xunit.v3.runner.utility/Frameworks/v2/Xunit2.cs +++ b/src/xunit.v3.runner.utility/Frameworks/v2/Xunit2.cs @@ -452,6 +452,7 @@ static AssemblyName GetTestFrameworkAssemblyName(string xunitExecutionAssemblyPa TestDisplayName = testCase.DisplayName, TestMethodUniqueID = testMethodUniqueID, TestUniqueID = testUniqueID, + Timeout = 0, Traits = testCaseTraits, }); diff --git a/src/xunit.v3.runner.utility/Frameworks/v2/Xunit2MessageAdapter.cs b/src/xunit.v3.runner.utility/Frameworks/v2/Xunit2MessageAdapter.cs index 5187ed10f..525753e52 100644 --- a/src/xunit.v3.runner.utility/Frameworks/v2/Xunit2MessageAdapter.cs +++ b/src/xunit.v3.runner.utility/Frameworks/v2/Xunit2MessageAdapter.cs @@ -728,6 +728,7 @@ _TestStarting AdaptTestStarting(ITestStarting message) TestDisplayName = message.Test.DisplayName, TestMethodUniqueID = testMethodUniqueID, TestUniqueID = testUniqueID, + Timeout = 0, Traits = message.TestCase.Traits.ToReadOnly(), }; }