for .NET Core 1.1 we should run only the Backward Compatibility tests. …

…dotnet#539
alinasmirnova · Sep 22, 2018 · 0976951 · 0976951
1 parent f46ff64
commit 0976951
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 5 deletions.
diff --git a/build.cake b/build.cake
@@ -94,13 +94,37 @@ Task("FastTests")
         DotNetCoreTest("./tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj", GetTestSettings());
     });
 
-Task("SlowTests")
+Task("BackwardCompatibilityTests")
     .IsDependentOn("Restore")
     .WithCriteria(!skipTests)
     .Does(() =>
     {
-        DotNetCoreTest("./tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj", GetTestSettings());
+        DotNetCoreTest(
+            "./tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj", 
+            new DotNetCoreTestSettings
+            {
+                Configuration = "Release",
+                Framework = "netcoreapp1.1",
+                Filter = "Category=BackwardCompatibility"
+            };
+        );
     });
+
+Task("SlowTestsNet46")
+    .IsDependentOn("Restore")
+    .WithCriteria(!skipTests && isRunningOnWindows)
+    .Does(() =>
+    {
+        DotNetCoreTest("./tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj", GetTestSettings("net46"));
+    });    
+
+Task("SlowTestsNetCore2")
+    .IsDependentOn("Restore")
+    .WithCriteria(!skipTests)
+    .Does(() =>
+    {
+        DotNetCoreTest("./tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj", GetTestSettings("netcoreapp2.0"));
+    });       
 
 Task("Pack")
     .IsDependentOn("Restore")
@@ -125,13 +149,15 @@ Task("Default")
     .IsDependentOn("Restore")
     .IsDependentOn("Build")
     .IsDependentOn("FastTests")
-    .IsDependentOn("SlowTests")
+    .IsDependentOn("SlowTestsNet46")
+    .IsDependentOn("SlowTestsNetCore2")
+    .IsDependentOn("BackwardCompatibilityTests")
     .IsDependentOn("Pack");
 
 RunTarget(target);
 
 // HELPERS
-private DotNetCoreTestSettings GetTestSettings()
+private DotNetCoreTestSettings GetTestSettings(string tfm = null)
 {
     var settings = new DotNetCoreTestSettings
     {
@@ -140,7 +166,11 @@ private DotNetCoreTestSettings GetTestSettings()
 
     if (!IsRunningOnWindows())
     {
-        settings.Framework = "netcoreapp1.1";
+        settings.Framework = "netcoreapp2.0";
+    }
+    else if(tfm != null)
+    {
+        settings.Framework = tfm;
     }
 
     return settings;

diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs
@@ -7,6 +7,7 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.IntegrationTests.Xunit;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Tests.Loggers;
 using Xunit;
@@ -64,6 +65,7 @@ public void Recursive()
 
         [Theory]
         [MemberData(nameof(GetAllJits))]
+        [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime runtime)
         {
             var disassemblyDiagnoser = (IDisassemblyDiagnoser)DisassemblyDiagnoser.Create(

diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs
@@ -9,6 +9,7 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.IntegrationTests.Xunit;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Tests.Loggers;
@@ -45,6 +46,7 @@ public class AccurateAllocations
         }
 
         [Theory, MemberData(nameof(GetToolchains))]
+        [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void MemoryDiagnoserIsAccurate(IToolchain toolchain)
         {
             long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word
@@ -79,6 +81,7 @@ private void AllocateUntilGcWakesUp()
         }
 
         [Theory, MemberData(nameof(GetToolchains))]
+        [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void MemoryDiagnoserDoesNotIncludeAllocationsFromGlobalSetupAndCleanup(IToolchain toolchain)
         {
             AssertAllocations(toolchain, typeof(AllocatingGlobalSetupAndCleanup), new Dictionary<string, long>
@@ -93,6 +96,7 @@ public class NoAllocationsAtAll
         }
 
         [Theory, MemberData(nameof(GetToolchains))]
+        [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain)
         {
             AssertAllocations(toolchain, typeof(NoAllocationsAtAll), new Dictionary<string, long>
@@ -107,6 +111,7 @@ public class NoBoxing
         }
 
         [Theory, MemberData(nameof(GetToolchains))]
+        [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void EngineShouldNotIntroduceBoxing(IToolchain toolchain)
         {
             AssertAllocations(toolchain, typeof(NoBoxing), new Dictionary<string, long>
@@ -127,6 +132,7 @@ public class NonAllocatingAsynchronousBenchmarks
         }
 
         [Theory, MemberData(nameof(GetToolchains))]
+        [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain)
         {
             AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary<string, long>

diff --git a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs
@@ -25,6 +25,7 @@ public MultipleRuntimesTest(ITestOutputHelper outputHelper)
         }
 
         [FactWindowsOnly("CLR is a valid job only on Windows")]
+        [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void SingleBenchmarkCanBeExecutedForMultpleRuntimes()
         {
             var summary = BenchmarkRunner

diff --git a/tests/BenchmarkDotNet.IntegrationTests/Xunit/Constants.cs b/tests/BenchmarkDotNet.IntegrationTests/Xunit/Constants.cs
@@ -0,0 +1,8 @@
+namespace BenchmarkDotNet.IntegrationTests.Xunit
+{
+    public class Constants
+    {
+        public const string Category = "Category";
+        public const string BackwardCompatibilityCategory = "BackwardCompatibility";
+    }
+}