diff --git a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorServerTemplateTest.cs b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorServerTemplateTest.cs index db87158ffbf3..88548db7715b 100644 --- a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorServerTemplateTest.cs +++ b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorServerTemplateTest.cs @@ -16,6 +16,7 @@ namespace Templates.Test { + [Retry] public class BlazorServerTemplateTest : BlazorTemplateTest { public BlazorServerTemplateTest(ProjectFactoryFixture projectFactory) diff --git a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs index 24ff97cc2779..76b1605b85bf 100644 --- a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs +++ b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs @@ -23,6 +23,7 @@ namespace Templates.Test { + [Retry] public class BlazorWasmTemplateTest : BlazorTemplateTest { public BlazorWasmTemplateTest(ProjectFactoryFixture projectFactory) diff --git a/src/ProjectTemplates/Shared/Project.cs b/src/ProjectTemplates/Shared/Project.cs index 84417ced0375..dfb6129f6aaf 100644 --- a/src/ProjectTemplates/Shared/Project.cs +++ b/src/ProjectTemplates/Shared/Project.cs @@ -102,6 +102,13 @@ public class Project : IDisposable try { Output.WriteLine("Acquired DotNetNewLock"); + + if (Directory.Exists(TemplateOutputDir)) + { + Output.WriteLine($"Template directory already exists, deleting contents of {TemplateOutputDir}"); + Directory.Delete(TemplateOutputDir, recursive: true); + } + // Temporary while investigating why this process occasionally never runs or exits on Debian 9 environmentVariables.Add("COREHOST_TRACE", "1"); using var execution = ProcessEx.Run(Output, AppContext.BaseDirectory, DotNetMuxer.MuxerPathOrDefault(), argString, environmentVariables); diff --git a/src/Testing/src/RetryAttribute.cs b/src/Testing/src/RetryAttribute.cs new file mode 100644 index 000000000000..2fb3ce51b337 --- /dev/null +++ b/src/Testing/src/RetryAttribute.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.ComponentModel; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Runs a test multiple times when it fails + /// This can be used on an assembly, class, or method name. Requires using the AspNetCore test framework. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = false)] + public sealed class RetryAttribute : Attribute + { + public RetryAttribute(int maxRetries = 3) + { + MaxRetries = maxRetries; + } + + /// + /// The maximum number of times to retry a failed test. Defaults to 3. + /// + public int MaxRetries { get; } + } +} diff --git a/src/Testing/src/xunit/AspNetTestInvoker.cs b/src/Testing/src/xunit/AspNetTestInvoker.cs index ab1a7c9694d3..68c42a698626 100644 --- a/src/Testing/src/xunit/AspNetTestInvoker.cs +++ b/src/Testing/src/xunit/AspNetTestInvoker.cs @@ -45,7 +45,16 @@ protected override async Task InvokeTestMethodAsync(object testClassIns } }); - var time = await base.InvokeTestMethodAsync(testClassInstance); + var retryAttribute = GetRetryAttribute(TestMethod); + var time = 0.0M; + if (retryAttribute == null) + { + time = await base.InvokeTestMethodAsync(testClassInstance); + } + else + { + time = await RetryAsync(retryAttribute, testClassInstance); + } await Aggregator.RunAsync(async () => { @@ -59,6 +68,45 @@ protected override async Task InvokeTestMethodAsync(object testClassIns return time; } + protected async Task RetryAsync(RetryAttribute retryAttribute, object testClassInstance) + { + var attempts = 0; + var timeTaken = 0.0M; + for (attempts = 0; attempts < retryAttribute.MaxRetries; attempts++) + { + timeTaken = await base.InvokeTestMethodAsync(testClassInstance); + if (!Aggregator.HasExceptions) + { + return timeTaken; + } + else if (attempts < retryAttribute.MaxRetries - 1) + { + _testOutputHelper.WriteLine($"Retrying test, attempt {attempts} of {retryAttribute.MaxRetries} failed."); + await Task.Delay(5000); + Aggregator.Clear(); + } + } + + return timeTaken; + } + + private RetryAttribute GetRetryAttribute(MethodInfo methodInfo) + { + var attributeCandidate = methodInfo.GetCustomAttribute(); + if (attributeCandidate != null) + { + return attributeCandidate; + } + + attributeCandidate = methodInfo.DeclaringType.GetCustomAttribute(); + if (attributeCandidate != null) + { + return attributeCandidate; + } + + return methodInfo.DeclaringType.Assembly.GetCustomAttribute(); + } + private static IEnumerable GetLifecycleHooks(object testClassInstance, Type testClass, MethodInfo testMethod) { foreach (var attribute in testMethod.GetCustomAttributes(inherit: true).OfType()) diff --git a/src/Testing/test/RetryTest.cs b/src/Testing/test/RetryTest.cs new file mode 100644 index 000000000000..3f62c87ca7af --- /dev/null +++ b/src/Testing/test/RetryTest.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Xunit; + +namespace Microsoft.AspNetCore.Testing +{ + [Retry] + public class RetryTest + { + private static int _retryFailsUntil3 = 0; + + [Fact] + public void RetryFailsUntil3() + { + _retryFailsUntil3++; + if (_retryFailsUntil3 != 2) throw new Exception("NOOOOOOOO"); + } + + private static int _canOverrideRetries = 0; + + [Fact] + [Retry(5)] + public void CanOverrideRetries() + { + _canOverrideRetries++; + if (_canOverrideRetries != 5) throw new Exception("NOOOOOOOO"); + } + } +}