Merge pull request #5 from skbkontur/nunit-retries

add retries
skbkontur · Feb 21, 2024 · ed290dd · ed290dd
2 parents a9a8c6c + ad94489
commit ed290dd
Show file tree

Hide file tree

Showing 29 changed files with 658 additions and 39 deletions.
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
@@ -17,16 +17,16 @@ jobs:
           global-json-file: global.json
 
       - name: Install dependencies
-        run: dotnet restore ./NUnit.Middlewares.sln --verbosity minimal && dotnet tool restore
+        run: dotnet restore ./nunit-extensions.sln --verbosity minimal && dotnet tool restore
 
       - name: Build
-        run: dotnet build --configuration Release ./NUnit.Middlewares.sln
+        run: dotnet build --configuration Release ./nunit-extensions.sln
 
       - name: Check codestyle
-        run: dotnet jb cleanupcode NUnit.Middlewares.sln --profile=CatalogueCleanup --verbosity=WARN && git diff --exit-code
+        run: dotnet jb cleanupcode nunit-extensions.sln --profile=CatalogueCleanup --verbosity=WARN && git diff --exit-code
 
       - name: Run tests
-        run: dotnet test --no-build --configuration Release ./NUnit.Middlewares.Tests/NUnit.Middlewares.Tests.csproj
+        run: dotnet test --no-build --configuration Release ./NUnit.Extensions.Tests/NUnit.Extensions.Tests.csproj
   publish:
     runs-on: windows-2019
     needs: test
@@ -60,7 +60,7 @@ jobs:
 
           Write-Host "Will create $release for package $packageName ($version)" -ForegroundColor "Green"
 
-          echo "RELEASE_NOTE=https://github.com/skbkontur/nunit-middlewares/releases/tag/$tagName" >> $env:GITHUB_ENV
+          echo "RELEASE_NOTE=https://github.com/skbkontur/nunit-extensions/releases/tag/$tagName" >> $env:GITHUB_ENV
           echo "PACKAGE_NAME=$packageName" >> $env:GITHUB_ENV
           echo "VERSION=$version" >> $env:GITHUB_ENV
           echo "PRE=$pre" >> $env:GITHUB_ENV

diff --git a/Directory.Build.props b/Directory.Build.props
@@ -1,28 +1,28 @@
 <Project>
 
-    <PropertyGroup>
-        <LangVersion>9</LangVersion>
-        <Nullable>enable</Nullable>
-        <NoWarn>8618</NoWarn>
-    </PropertyGroup>
+  <PropertyGroup>
+    <LangVersion>9</LangVersion>
+    <Nullable>enable</Nullable>
+    <NoWarn>8618</NoWarn>
+  </PropertyGroup>
 
-    <!-- source line mappings are not supported in portable pdb format yet (https://github.com/dotnet/core/blob/master/Documentation/diagnostics/portable_pdb.md) -->
-    <PropertyGroup>
-        <DebugType>full</DebugType>
-        <DebugSymbols>true</DebugSymbols>
-    </PropertyGroup>
+  <!-- source line mappings are not supported in portable pdb format yet (https://github.com/dotnet/core/blob/master/Documentation/diagnostics/portable_pdb.md) -->
+  <PropertyGroup>
+    <DebugType>full</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+  </PropertyGroup>
 
-    <!-- include pdbs into nuget package -->
-    <PropertyGroup>
-        <RepositoryType>git</RepositoryType>
-        <RepositoryUrl>https://github.com/skbkontur/nunit-middlewares</RepositoryUrl>
-        <PackageProjectUrl>$(RepositoryUrl)</PackageProjectUrl>
-        <PublishRepositoryUrl>true</PublishRepositoryUrl>
-        <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
-    </PropertyGroup>
+  <!-- include pdbs into nuget package -->
+  <PropertyGroup>
+    <RepositoryType>git</RepositoryType>
+    <RepositoryUrl>https://github.com/skbkontur/nunit-extensions</RepositoryUrl>
+    <PackageProjectUrl>$(RepositoryUrl)</PackageProjectUrl>
+    <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
+  </PropertyGroup>
 
-    <ItemGroup>
-        <PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All"/>
-    </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All"/>
+  </ItemGroup>
 
 </Project>
diff --git a/...ts/ParallelParametrizedTestFixtureTest.cs → ...es/ParallelParametrizedTestFixtureTest.cs b/...ts/ParallelParametrizedTestFixtureTest.cs → ...es/ParallelParametrizedTestFixtureTest.cs
@@ -4,7 +4,9 @@
 
 using NUnit.Framework;
 
-namespace SkbKontur.NUnit.Middlewares.Tests
+using SkbKontur.NUnit.Middlewares;
+
+namespace SkbKontur.NUnit.Extensions.Tests.Middlewares
 {
     [TestFixture(-1)]
     [TestFixture(-2)]

diff --git a/...res.Tests/ParallelTestContextUsageTest.cs → ...ddlewares/ParallelTestContextUsageTest.cs b/...res.Tests/ParallelTestContextUsageTest.cs → ...ddlewares/ParallelTestContextUsageTest.cs
@@ -6,7 +6,9 @@
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
 
-namespace SkbKontur.NUnit.Middlewares.Tests
+using SkbKontur.NUnit.Middlewares;
+
+namespace SkbKontur.NUnit.Extensions.Tests.Middlewares
 {
     public class Counter
     {

diff --git a/...res.Tests/ParallelTestFixtureSetUpTest.cs → ...ddlewares/ParallelTestFixtureSetUpTest.cs b/...res.Tests/ParallelTestFixtureSetUpTest.cs → ...ddlewares/ParallelTestFixtureSetUpTest.cs
@@ -2,7 +2,9 @@
 
 using NUnit.Framework;
 
-namespace SkbKontur.NUnit.Middlewares.Tests
+using SkbKontur.NUnit.Middlewares;
+
+namespace SkbKontur.NUnit.Extensions.Tests.Middlewares
 {
     [Parallelizable(ParallelScope.Self)]
     public class FirstTestFixtureSetUpTest : SimpleTestBase

diff --git a/...es.Tests/TestWithRetriesHasItsOwnSetup.cs → ...dlewares/TestWithRetriesHasItsOwnSetup.cs b/...es.Tests/TestWithRetriesHasItsOwnSetup.cs → ...dlewares/TestWithRetriesHasItsOwnSetup.cs
@@ -5,7 +5,9 @@
 using NUnit.Framework;
 using NUnit.Framework.Internal;
 
-namespace SkbKontur.NUnit.Middlewares.Tests
+using SkbKontur.NUnit.Middlewares;
+
+namespace SkbKontur.NUnit.Extensions.Tests.Middlewares
 {
     public class DisposableCounter : IDisposable
     {

diff --git a/...ares.Tests/NUnit.Middlewares.Tests.csproj → ...sions.Tests/NUnit.Extensions.Tests.csproj b/...ares.Tests/NUnit.Middlewares.Tests.csproj → ...sions.Tests/NUnit.Extensions.Tests.csproj
@@ -2,8 +2,8 @@
 
   <PropertyGroup>
     <TargetFrameworks>net48;net8.0</TargetFrameworks>
-    <AssemblyName>SkbKontur.NUnit.Middlewares.Tests</AssemblyName>
-    <RootNamespace>SkbKontur.NUnit.Middlewares.Tests</RootNamespace>
+    <AssemblyName>SkbKontur.NUnit.Extensions.Tests</AssemblyName>
+    <RootNamespace>SkbKontur.NUnit.Extensions.Tests</RootNamespace>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
   </PropertyGroup>
@@ -20,6 +20,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\NUnit.Middlewares\NUnit.Middlewares.csproj" />
+    <ProjectReference Include="..\NUnit.Retries\NUnit.Retries.csproj" />
   </ItemGroup>
 
 </Project>
diff --git a/NUnit.Extensions.Tests/Retries/FixtureWithRetries.cs b/NUnit.Extensions.Tests/Retries/FixtureWithRetries.cs
@@ -0,0 +1,39 @@
+using System;
+
+using NUnit.Framework;
+using NUnit.Framework.Internal;
+
+using SkbKontur.NUnit.Retries;
+
+namespace SkbKontur.NUnit.Extensions.Tests.Retries
+{
+    [RetryOnError(3)]
+    public class FixtureWithRetries
+    {
+        [Test]
+        public void Test1()
+        {
+            if (TestExecutionContext.CurrentContext.CurrentRepeatCount is 0 or 1)
+            {
+                Assert.Fail("Third time's a Charm");
+            }
+        }
+
+        [Test]
+        [RetryOnError(4)]
+        public void Test2()
+        {
+            if (TestExecutionContext.CurrentContext.CurrentRepeatCount is 0 or 1 or 2)
+            {
+                throw new Exception("Forth time's a Charm, now with exception");
+            }
+        }
+
+        [Test]
+        [NoRetry]
+        public void Test3()
+        {
+            Assert.Pass();
+        }
+    }
+}
diff --git a/NUnit.Extensions.Tests/Retries/RetrySuite.cs b/NUnit.Extensions.Tests/Retries/RetrySuite.cs
@@ -0,0 +1,12 @@
+using NUnit.Framework;
+
+using SkbKontur.NUnit.Retries;
+
+namespace SkbKontur.NUnit.Extensions.Tests.Retries
+{
+    [SetUpFixture]
+    [RetryOnError(2)]
+    public class RetrySuite
+    {
+    }
+}
diff --git a/NUnit.Extensions.Tests/Retries/TestWithRetries.cs b/NUnit.Extensions.Tests/Retries/TestWithRetries.cs
@@ -0,0 +1,29 @@
+using NUnit.Framework;
+using NUnit.Framework.Internal;
+
+using SkbKontur.NUnit.Retries;
+
+namespace SkbKontur.NUnit.Extensions.Tests.Retries
+{
+    public class TestWithRetries
+    {
+        [Test]
+        public void Test1()
+        {
+            if (TestExecutionContext.CurrentContext.CurrentRepeatCount is 0)
+            {
+                Assert.Fail("Second time's a Charm");
+            }
+        }
+
+        [Test]
+        [RetryOnError(3)]
+        public void Test2()
+        {
+            if (TestExecutionContext.CurrentContext.CurrentRepeatCount is 0 or 1)
+            {
+                Assert.Fail("Third time's a Charm");
+            }
+        }
+    }
+}
diff --git a/NUnit.Middlewares/README.md b/NUnit.Middlewares/README.md
@@ -1,21 +1,21 @@
 # NUnit.Middlewares
 
 [![NuGet Status](https://img.shields.io/nuget/v/SkbKontur.NUnit.Middlewares.svg)](https://www.nuget.org/packages/SkbKontur.NUnit.Middlewares/)
-[![Build status](https://github.com/skbkontur/nunit-middlewares/actions/workflows/actions.yml/badge.svg)](https://github.com/skbkontur/nunit-middlewares/actions)
+[![Build status](https://github.com/skbkontur/nunit-extensions/actions/workflows/actions.yml/badge.svg)](https://github.com/skbkontur/nunit-extensions/actions)
 
 Use middleware pattern to write tests in concise and comprehensive manner. And ditch test bases.
 
 ## Test setup middlewares
 
 Inspired by ASP.NET Core [middlewares](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/middleware), the main idea of test middlewares can be summarized by this image:
 
-![nunit-middlewares](https://github.com/skbkontur/nunit-middlewares/assets/5417867/9707428f-11ec-4353-ac96-7fdf70200a47)
+![nunit-middlewares](https://github.com/skbkontur/nunit-extensions/assets/5417867/9707428f-11ec-4353-ac96-7fdf70200a47)
 
 Here we focus on *behaviours* that we want to add to our test rather than focusing on implementing test lifecycle methods provided by NUnit.
 
 `suite`, `fixture` and `test` in the image above are just `ISetupBuilder` that can accept either raw setup functions or anything that implements simple `ISetup` interface:
 
-![setup-builder](https://github.com/skbkontur/nunit-middlewares/assets/5417867/e4adb7c6-2078-401e-9bac-539f89ffec54)
+![setup-builder](https://github.com/skbkontur/nunit-extensions/assets/5417867/e4adb7c6-2078-401e-9bac-539f89ffec54)
 
 ## Simple test base
 
@@ -126,7 +126,7 @@ To ensure everything is working as intended, parent's *context item*s should be
 
 In our example from first image, test context will look something like this:
 
-![test-context](https://github.com/skbkontur/nunit-middlewares/assets/5417867/c70b41d6-5f3f-485a-9e9d-7616b3797232)
+![test-context](https://github.com/skbkontur/nunit-extensions/assets/5417867/c70b41d6-5f3f-485a-9e9d-7616b3797232)
 
 Both `SimpleTestContext` and `GetFromThisOrParentContext` are just `ITest` wrappers that search for context value in `ITest`'s `Properties` recursively
 

diff --git a/NUnit.Retries/CustomAttributeMethodWrapper.cs b/NUnit.Retries/CustomAttributeMethodWrapper.cs
@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+
+using NUnit.Framework.Interfaces;
+
+namespace SkbKontur.NUnit.Retries
+{
+    public sealed class CustomAttributeMethodWrapper : IMethodInfo
+    {
+        public CustomAttributeMethodWrapper(IMethodInfo baseInfo, params Attribute[] extraAttributes)
+        {
+            this.baseInfo = baseInfo;
+            this.extraAttributes = extraAttributes;
+        }
+
+        public ITypeInfo TypeInfo => baseInfo.TypeInfo;
+        public MethodInfo MethodInfo => baseInfo.MethodInfo;
+        public string Name => baseInfo.Name;
+        public bool IsAbstract => baseInfo.IsAbstract;
+        public bool IsPublic => baseInfo.IsPublic;
+        public bool IsStatic => baseInfo.IsStatic;
+        public bool ContainsGenericParameters => baseInfo.ContainsGenericParameters;
+        public bool IsGenericMethod => baseInfo.IsGenericMethod;
+        public bool IsGenericMethodDefinition => baseInfo.IsGenericMethodDefinition;
+        public ITypeInfo ReturnType => baseInfo.ReturnType;
+
+        public T[] GetCustomAttributes<T>(bool inherit) where T : class
+        {
+            var bases = baseInfo.GetCustomAttributes<T>(inherit);
+            var extras = extraAttributes.OfType<T>().ToArray();
+
+            return !extras.Any()
+                       ? bases
+                       : !bases.Any()
+                           ? extras
+                           : MergeAttributes(bases, extras);
+        }
+
+        public bool IsDefined<T>(bool inherit) where T : class
+            => baseInfo.IsDefined<T>(inherit) || extraAttributes.OfType<T>().Any();
+
+        public Type[] GetGenericArguments() => baseInfo.GetGenericArguments();
+        public IParameterInfo[] GetParameters() => baseInfo.GetParameters();
+        public object? Invoke(object? fixture, params object?[]? args) => baseInfo.Invoke(fixture, args);
+        public IMethodInfo MakeGenericMethod(params Type[] typeArguments) => baseInfo.MakeGenericMethod(typeArguments);
+
+        private static T[] MergeAttributes<T>(T[] bases, T[] extras) where T : class
+        {
+            var baseTypes = new HashSet<Type>(bases.Select(x => x.GetType()));
+            return bases
+                   .Concat(extras.Where(e => !baseTypes.Contains(e.GetType())))
+                   .ToArray();
+        }
+
+        private readonly IMethodInfo baseInfo;
+        private readonly Attribute[] extraAttributes;
+    }
+}
diff --git a/NUnit.Retries/IRetryStrategy.cs b/NUnit.Retries/IRetryStrategy.cs
@@ -0,0 +1,13 @@
+using System;
+
+using NUnit.Framework.Internal;
+
+namespace SkbKontur.NUnit.Retries
+{
+    public interface IRetryStrategy
+    {
+        public int TryCount { get; }
+        public bool ShouldRetry(TestResult result);
+        public void OnTestFailed(TestExecutionContext context, DateTimeOffset start);
+    }
+}
diff --git a/NUnit.Retries/NUnit.Retries.csproj b/NUnit.Retries/NUnit.Retries.csproj
@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <AssemblyName>SkbKontur.NUnit.Retries</AssemblyName>
+    <RootNamespace>SkbKontur.NUnit.Retries</RootNamespace>
+    <PackageId>SkbKontur.NUnit.Retries</PackageId>
+    <PackageDescription>Retries for NUnit</PackageDescription>
+    <PackageReadmeFile>README.md</PackageReadmeFile>
+    <PackageTags>NUnit Retry</PackageTags>
+    <Authors>Pavel Vostretsov</Authors>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NUnit" Version="3.13.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Include="README.md" Pack="true" PackagePath="\" />
+  </ItemGroup>
+
+</Project>
diff --git a/NUnit.Retries/NoRetryAttribute.cs b/NUnit.Retries/NoRetryAttribute.cs
@@ -0,0 +1,9 @@
+using System;
+
+namespace SkbKontur.NUnit.Retries
+{
+    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]
+    public class NoRetryAttribute : Attribute
+    {
+    }
+}
diff --git a/NUnit.Retries/README.md b/NUnit.Retries/README.md
@@ -0,0 +1,19 @@
+# NUnit.Retries
+
+[![NuGet Status](https://img.shields.io/nuget/v/SkbKontur.NUnit.Retries.svg)](https://www.nuget.org/packages/SkbKontur.NUnit.Retries/)
+[![Build status](https://github.com/skbkontur/nunit-extensions/actions/workflows/actions.yml/badge.svg)](https://github.com/skbkontur/nunit-extensions/actions)
+
+Couple of helpful attributes for test retries:
+- `RetryOnErrorAttribute` is like NUnit's own `RetryAttribute`, but it can be applied to whole Fixture/Suite/Assembly, and supports retry after exceptions in test, not only assertion failures
+- On top of that, `RetryOnTeamCityAttribute` also supports TeamCity's [test retry](https://www.jetbrains.com/help/teamcity/2022.10/build-failure-conditions.html#test-retry) feature
+- `NoRetryAttribute` for disabling retries
+
+Attributes can be overriden on any level, e.g. 
+- MyAssembly.dll: `[RetryOnError(2)]`
+- MySuite.cs: `[NoRetry]`
+- MyTestFixture.cs: `[RetryOnTeamCity(3)]`
+- MyTestMethod(): `[RetryOnError(4)]`
+
+This means we have two retries on assembly level in `MyAssembly.dll`, but no retries in `MySuite`,
+if `MyTestFixture` is also in `MySuite`, previous attributes are overriden by `RetryOnTeamCity`,
+and method `MyTestMethod` in `MyTestFixture` is retried 4 times.