Add experimental Timeout attibute to [FactAttribute]

src/xunit.core/FactAttribute.cs

@@ -22,5 +22,13 @@ public class FactAttribute : Attribute
         /// Marks the test so that it will not be run, and gets or sets the skip reason
         /// </summary>
         public virtual string Skip { get; set; }
+
+        /// <summary>
+        /// Marks the test as having a timeout, and gets or sets the timeout (in milliseconds).
+        /// WARNING: Using this with parallelization turned on will result in undefined behavior.
+        /// Timeout is only supported when parallelization is disabled, either globally or with
+        /// a parallelization-disabled test collection.
+        /// </summary>
+        public virtual int Timeout { get; set; }
     }
-}
+}

src/xunit.core/Sdk/IXunitTestCase.cs

@@ -22,6 +22,11 @@ public interface IXunitTestCase : ITestCase
         /// </summary>
         IMethodInfo Method { get; }
 
+        /// <summary>
+        /// Gets the timeout of the test, in milliseconds; if zero or negative, means the test case has no timeout.
+        /// </summary>
+        int Timeout { get; }
+
         /// <summary>
         /// Executes the test case, returning 0 or more result messages through the message sink.
         /// </summary>

src/xunit.core/TheoryAttribute.cs

@@ -14,4 +14,4 @@ namespace Xunit
     [XunitTestCaseDiscoverer("Xunit.Sdk.TheoryDiscoverer", "xunit.execution.{Platform}")]
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
     public class TheoryAttribute : FactAttribute { }
-}
+}

src/xunit.execution/Sdk/Frameworks/Runners/XunitTestInvoker.cs

@@ -109,7 +109,20 @@ protected override Task<decimal> InvokeTestMethodAsync(object testClassInstance)
                 return tcs.Task;
             }
 
-            return base.InvokeTestMethodAsync(testClassInstance);
+            return TestCase.Timeout > 0
+                ? InvokeTimeoutTestMethodAsync(testClassInstance)
+                : base.InvokeTestMethodAsync(testClassInstance);
+        }
+
+        async Task<decimal> InvokeTimeoutTestMethodAsync(object testClassInstance)
+        {
+            var baseTask = base.InvokeTestMethodAsync(testClassInstance);
+            var resultTask = await Task.WhenAny(baseTask, Task.Delay(TestCase.Timeout));
+
+            if (resultTask != baseTask)
+                throw new TestTimeoutException(TestCase.Timeout);
+
+            return baseTask.Result;
         }
     }
 }

src/xunit.execution/Sdk/Frameworks/TestTimeoutException.cs

@@ -0,0 +1,17 @@
+using System;
+
+namespace Xunit.Sdk
+{
+    /// <summary>
+    /// Thrown if a test exceeds the specified timeout.
+    /// </summary>
+    public class TestTimeoutException : Exception
+    {
+        /// <summary>
+        /// Initializes a new instance of <see cref="TestTimeoutException"/>.
+        /// </summary>
+        /// <param name="timeout">The timeout that was exceeded, in milliseconds</param>
+        public TestTimeoutException(int timeout)
+            : base($"Test execution timed out after {timeout} milliseconds") { }
+    }
+}

src/xunit.execution/Sdk/Frameworks/XunitTestCase.cs

@@ -20,6 +20,8 @@ public class XunitTestCase : TestMethodTestCase, IXunitTestCase
         static ConcurrentDictionary<string, IEnumerable<IAttributeInfo>> assemblyTraitAttributeCache = new ConcurrentDictionary<string, IEnumerable<IAttributeInfo>>(StringComparer.OrdinalIgnoreCase);
         static ConcurrentDictionary<string, IEnumerable<IAttributeInfo>> typeTraitAttributeCache = new ConcurrentDictionary<string, IEnumerable<IAttributeInfo>>(StringComparer.OrdinalIgnoreCase);
 
+        int timeout;
+
         /// <summary/>
         [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
@@ -67,6 +69,21 @@ public XunitTestCase()
         /// </summary>
         protected IMessageSink DiagnosticMessageSink { get; }
 
+        /// <inheritdoc/>
+        public int Timeout
+        {
+            get
+            {
+                EnsureInitialized();
+                return timeout;
+            }
+            protected set
+            {
+                EnsureInitialized();
+                timeout = value;
+            }
+        }
+
         /// <summary>
         /// Gets the display name for the test case. Calls <see cref="TypeUtility.GetDisplayNameWithArguments"/>
         /// with the given base display name (which is itself either derived from <see cref="FactAttribute.DisplayName"/>,
@@ -87,6 +104,15 @@ protected virtual string GetDisplayName(IAttributeInfo factAttribute, string dis
         protected virtual string GetSkipReason(IAttributeInfo factAttribute)
             => factAttribute.GetNamedArgument<string>("Skip");
 
+        /// <summary>
+        /// Gets the timeout for the test case. By default, pulls the skip reason from the
+        /// <see cref="FactAttribute.Timeout"/> property.
+        /// </summary>
+        /// <param name="factAttribute">The fact attribute the decorated the test case.</param>
+        /// <returns>The timeout in milliseconds, if set; 0, if unset.</returns>
+        protected virtual int GetTimeout(IAttributeInfo factAttribute)
+            => factAttribute.GetNamedArgument<int>("Timeout");
+
         /// <inheritdoc/>
         protected override void Initialize()
         {
@@ -97,6 +123,7 @@ protected override void Initialize()
 
             DisplayName = GetDisplayName(factAttribute, baseDisplayName);
             SkipReason = GetSkipReason(factAttribute);
+            Timeout = GetTimeout(factAttribute);
 
             foreach (var traitAttribute in GetTraitAttributesData(TestMethod))
             {
@@ -133,5 +160,21 @@ static IEnumerable<IAttributeInfo> GetTraitAttributesData(ITestMethod testMethod
                                                  ExceptionAggregator aggregator,
                                                  CancellationTokenSource cancellationTokenSource)
             => new XunitTestCaseRunner(this, DisplayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource).RunAsync();
+
+        /// <inheritdoc/>
+        public override void Serialize(IXunitSerializationInfo data)
+        {
+            base.Serialize(data);
+
+            data.AddValue("Timeout", Timeout);
+        }
+
+        /// <inheritdoc/>
+        public override void Deserialize(IXunitSerializationInfo data)
+        {
+            base.Deserialize(data);
+
+            Timeout = data.GetValue<int>("Timeout");
+        }
     }
 }

test/test.utility/TestDoubles/Mocks.cs

@@ -96,12 +96,13 @@ public static ExecutionErrorTestCase ExecutionErrorTestCase(string message, IMes
         return new ExecutionErrorTestCase(diagnosticMessageSink ?? new Xunit.NullMessageSink(), TestMethodDisplay.ClassAndMethod, TestMethodDisplayOptions.None, testMethod, message);
     }
 
-    public static IReflectionAttributeInfo FactAttribute(string displayName = null, string skip = null)
+    public static IReflectionAttributeInfo FactAttribute(string displayName = null, string skip = null, int timeout = 0)
     {
         var result = Substitute.For<IReflectionAttributeInfo, InterfaceProxy<IReflectionAttributeInfo>>();
-        result.Attribute.Returns(new FactAttribute { DisplayName = displayName, Skip = skip });
+        result.Attribute.Returns(new FactAttribute { DisplayName = displayName, Skip = skip, Timeout = timeout });
         result.GetNamedArgument<string>("DisplayName").Returns(displayName);
         result.GetNamedArgument<string>("Skip").Returns(skip);
+        result.GetNamedArgument<int>("Timeout").Returns(timeout);
         return result;
     }
 
@@ -434,6 +435,7 @@ public static IReflectionAttributeInfo TestFrameworkAttribute(Type type)
                                          string methodName = null,
                                          string displayName = null,
                                          string skip = null,
+                                         int timeout = 0,
                                          IEnumerable<IParameterInfo> parameters = null,
                                          IEnumerable<IReflectionAttributeInfo> classAttributes = null,
                                          IEnumerable<IReflectionAttributeInfo> methodAttributes = null)
@@ -448,7 +450,7 @@ public static IReflectionAttributeInfo TestFrameworkAttribute(Type type)
         var factAttribute = methodAttributes.FirstOrDefault(attr => typeof(FactAttribute).IsAssignableFrom(attr.Attribute.GetType()));
         if (factAttribute == null)
         {
-            factAttribute = FactAttribute(displayName, skip);
+            factAttribute = FactAttribute(displayName, skip, timeout);
             methodAttributes = methodAttributes.Concat(new[] { factAttribute });
         }
 
@@ -541,12 +543,13 @@ public static ITestStarting TestStarting(string displayName)
         return result;
     }
 
-    public static IReflectionAttributeInfo TheoryAttribute(string displayName = null, string skip = null)
+    public static IReflectionAttributeInfo TheoryAttribute(string displayName = null, string skip = null, int timeout = 0)
     {
         var result = Substitute.For<IReflectionAttributeInfo, InterfaceProxy<IReflectionAttributeInfo>>();
         result.Attribute.Returns(new TheoryAttribute { DisplayName = displayName, Skip = skip });
         result.GetNamedArgument<string>("DisplayName").Returns(displayName);
         result.GetNamedArgument<string>("Skip").Returns(skip);
+        result.GetNamedArgument<int>("Timeout").Returns(timeout);
         return result;
     }
 

test/test.xunit.execution/Acceptance/Xunit2AcceptanceTests.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
 using System.Threading.Tasks;
@@ -148,6 +149,43 @@ public void SingleSkippedTest()
         }
     }
 
+    [CollectionDefinition("Timeout Tests", DisableParallelization = true)]
+    public class TimeoutTestsCollection { }
+
+    [Collection("Timeout Tests")]
+    public class TimeoutTests : AcceptanceTestV2
+    {
+        // This test is a little sketchy, because it relies on the execution of the acceptance test to happen in less time
+        // than the timeout. The timeout is set arbitrarily high in order to give some padding to the timing, but even on
+        // a Core i7-7820HK, the execution time is ~ 400 milliseconds for what should be about 10 milliseconds of wait
+        // time. If this test becomes flaky, a higher value than 10000 could be considered.
+        [Fact]
+        public void TimedOutTest()
+        {
+            var stopwatch = Stopwatch.StartNew();
+            var results = Run(typeof(ClassUnderTest));
+            stopwatch.Stop();
+
+            var passedMessage = Assert.Single(results.OfType<ITestPassed>());
+            Assert.Equal("Xunit2AcceptanceTests+TimeoutTests+ClassUnderTest.ShortRunningTest", passedMessage.Test.DisplayName);
+
+            var failedMessage = Assert.Single(results.OfType<ITestFailed>());
+            Assert.Equal("Xunit2AcceptanceTests+TimeoutTests+ClassUnderTest.LongRunningTest", failedMessage.Test.DisplayName);
+            Assert.Equal("Test execution timed out after 10 milliseconds", failedMessage.Messages.Single());
+
+            Assert.True(stopwatch.ElapsedMilliseconds < 10000, "Elapsed time should be less than 10 seconds");
+        }
+
+        class ClassUnderTest
+        {
+            [Fact(Timeout = 10)]
+            public Task LongRunningTest() => Task.Delay(10000);
+
+            [Fact(Timeout = 10000)]
+            public void ShortRunningTest() => Task.Delay(10);
+        }
+    }
+
     public class FailingTests : AcceptanceTestV2
     {
         [Fact]
@@ -398,6 +436,7 @@ class TestClassNonParallelCollection
         {
             [Fact]
             public void IShouldBeLast2() { }
+
             [Fact]
             public void IShouldBeLast1() { }
         }

test/test.xunit.execution/Sdk/Frameworks/XunitTestCaseTests.cs

@@ -34,6 +34,16 @@ public static void SkipReason()
         Assert.Equal("Skip Reason", testCase.SkipReason);
     }
 
+    [Fact]
+    public static void Timeout()
+    {
+        var testMethod = Mocks.TestMethod(timeout: 42);
+
+        var testCase = new XunitTestCase(SpyMessageSink.Create(), TestMethodDisplay.ClassAndMethod, TestMethodDisplayOptions.None, testMethod);
+
+        Assert.Equal(42, testCase.Timeout);
+    }
+
     public class Traits : AcceptanceTestV2
     {
         [Fact]

test/test.xunit.execution/Sdk/TestCaseSerializerTests.cs

@@ -1,13 +1,12 @@
 using System;
-using System.Runtime.Serialization;
 using Xunit;
 using Xunit.Sdk;
 
 public class TestCaseSerializerTests
 {
     class ClassUnderTest
     {
-        [Fact(Skip = "Skip me", DisplayName = "Hi there")]
+        [Fact(Skip = "Skip me", DisplayName = "Hi there", Timeout = 2112)]
         [Trait("name", "value")]
         public void FactMethod()
         {
@@ -42,13 +41,14 @@ public static void DeserializedTestCaseContainsSameDataAsOriginalTestCase()
             Assert.Equal(testCase.TestMethod.Method.Name, result.TestMethod.Method.Name);
             Assert.Equal(testCase.DisplayName, result.DisplayName);
             Assert.Equal(testCase.SkipReason, result.SkipReason);
+            Assert.Equal(testCase.Timeout, result.Timeout);
             Assert.Null(result.TestMethodArguments);
             Assert.Collection(result.Traits.Keys,
                 key =>
                 {
                     Assert.Equal("Assembly", key);
                     Assert.Equal("Trait", Assert.Single(result.Traits[key]));
-                }, 
+                },
                 key =>
                 {
                     Assert.Equal("name", key);
@@ -110,6 +110,7 @@ public static void DeserializedTestCaseContainsSameDataAsOriginalTestCase()
             Assert.Equal(testCase.TestMethod.Method.Name, result.TestMethod.Method.Name);
             Assert.Equal(testCase.DisplayName, result.DisplayName);
             Assert.Equal(testCase.SkipReason, result.SkipReason);
+            Assert.Equal(testCase.Timeout, result.Timeout);
             Assert.Null(result.TestMethodArguments);
             Assert.Collection(result.Traits.Keys,
                 key =>