adding NBench 2.0 to 1.3 Akka.NET (#4293)

* per #4292 - adding NBench 2.0 to 1.3 Akka.NET

* added NBench 2.0 support to build system

* added ClusterStartupSpec

build.fsx

@@ -343,57 +343,38 @@ Target "MultiNodeTestsNetCore" (fun _ ->
         multiNodeTestAssemblies |> Seq.iter (runMultiNodeSpec)
 )
 
-Target "NBench" <| fun _ ->
-    if not skipBuild.Value then
-        CleanDir outputPerfTests
-
-        let nbenchTestPath = findToolInSubPath "NBench.Runner.exe" (toolsDir @@ "NBench.Runner*")
-        printfn "Using NBench.Runner: %s" nbenchTestPath
-
-        let projects = 
-            let rawProjects = match (isWindows) with 
-                                | true -> !! "./src/**/*.Tests.Peformance.csproj"
-                                | _ -> !! "./src/**/*.Tests.Performance.csproj" // if you need to filter specs for Linux vs. Windows, do it here
-            rawProjects |> Seq.choose filterProjects
+Target "NBench" (fun _ ->
+    ensureDirectory outputPerfTests
 
-        let nbenchTestAssemblies = 
-            projects |> Seq.choose (getTestAssembly Runtime.NetFramework)
-
-        let runNBench assembly =
-            let includes = getBuildParam "include"
-            let excludes = getBuildParam "exclude"
-            let teamcityStr = (getBuildParam "teamcity")
-            let enableTeamCity = 
-                match teamcityStr with
-                | null -> false
-                | "" -> false
-                | _ -> bool.Parse teamcityStr
+    let projects = 
+        let rawProjects = match (isWindows) with 
+                            | true -> !! "./src/**/*Tests.Performance.csproj"
+                            | _ -> !! "./src/**/*Tests.Performance.csproj" // if you need to filter specs for Linux vs. Windows, do it here
+        rawProjects |> Seq.choose filterProjects
 
-            let args = StringBuilder()
-                    |> append assembly
-                    |> append (sprintf "output-directory=\"%s\"" outputPerfTests)
-                    |> append (sprintf "concurrent=\"%b\"" true)
-                    |> append (sprintf "trace=\"%b\"" true)
-                    |> append (sprintf "teamcity=\"%b\"" enableTeamCity)
-                    |> appendIfNotNullOrEmpty includes "include="
-                    |> appendIfNotNullOrEmpty excludes "include="
-                    |> toText
+    projects |> Seq.iter(fun project -> 
+        let args = new StringBuilder()
+                |> append "run"
+                |> append "--no-build"
+                |> append "-c"
+                |> append configuration
+                |> append " -- "
+                |> append "--output"
+                |> append outputPerfTests
+                |> append "--concurrent" 
+                |> append "true"
+                |> append "--trace"
+                |> append "true"
+                |> append "--diagnostic"               
+                |> toText
 
-            let result = ExecProcess(fun info -> 
-                info.FileName <- nbenchTestPath
-                info.WorkingDirectory <- (Path.GetDirectoryName (FullName nbenchTestPath))
-                info.Arguments <- args) (System.TimeSpan.FromMinutes 45.0) (* Reasonably long-running task. *)
-            if result <> 0 then failwithf "%s %s \nexited with code %i" nbenchTestPath args result
-
-        let failedRuns =
-            nbenchTestAssemblies
-            |> Seq.map (fun asm -> try runNBench asm; None with e -> Some(e.ToString()))
-            |> Seq.filter Option.isSome
-            |> Seq.map Option.get
-            |> Seq.mapi (fun i s -> sprintf "%i: \"%s\"" (i + 1) s)
-            |> Seq.toArray
-        if failedRuns.Length > 0 then
-            failwithf "NBench.Runner failed for %i run(s):\n%s\n\n" failedRuns.Length (String.concat "\n\n" failedRuns)
+        let result = ExecProcess(fun info -> 
+            info.FileName <- "dotnet"
+            info.WorkingDirectory <- (Directory.GetParent project).FullName
+            info.Arguments <- args) (System.TimeSpan.FromMinutes 15.0) (* Reasonably long-running task. *)
+        if result <> 0 then failwithf "NBench.Runner failed. %s %s" "dotnet" args
+    )
+)
 
 //--------------------------------------------------------------------------------
 // Nuget targets 
@@ -670,6 +651,7 @@ Target "RunTestsNetCoreFull" DoNothing
 // tests dependencies
 "Build" ==> "RunTests"
 "Build" ==> "RunTestsNetCore"
+"Build" ==> "NBench"
 
 "BuildRelease" ==> "MultiNodeTestsNetCore"
 "BuildRelease" ==> "MultiNodeTests"

src/common.props

@@ -13,7 +13,7 @@
     <TestSdkVersion>15.9.0</TestSdkVersion>
     <HyperionVersion>0.9.11</HyperionVersion>
     <NewtonsoftJsonVersion>9.0.1</NewtonsoftJsonVersion>
-    <NBenchVersion>1.2.2</NBenchVersion>
+    <NBenchVersion>2.0.1</NBenchVersion>
     <ProtobufVersion>3.9.1</ProtobufVersion>
     <NetCoreTestVersion>netcoreapp2.1</NetCoreTestVersion>
     <NetFrameworkTestVersion>net461</NetFrameworkTestVersion>

src/core/Akka.Cluster.Tests.Performance/Akka.Cluster.Tests.Performance.csproj

@@ -1,22 +1,18 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
+<Import Project="..\..\common.props" />
   <PropertyGroup>
-    <AssemblyTitle>Akka.Cluster.Tests.Performance</AssemblyTitle>
-    <TargetFrameworks>net452</TargetFrameworks>
+    <TargetFramework>$(NetCoreTestVersion)</TargetFramework>
     <ServerGarbageCollection>true</ServerGarbageCollection>
+    <IsPackable>false</IsPackable>
+    <OutputType>Exe</OutputType>
   </PropertyGroup>
 
   <ItemGroup>
-    <Content Include="App.config" />
     <ProjectReference Include="..\Akka.Cluster\Akka.Cluster.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="NBench" Version="1.0.1" />
+    <PackageReference Include="NBench" Version="$(NBenchVersion)" />
   </ItemGroup>
 
-  <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
-    <DefineConstants>$(DefineConstants);RELEASE</DefineConstants>
-  </PropertyGroup>
-
 </Project>

src/core/Akka.Cluster.Tests.Performance/Program.cs

@@ -0,0 +1,12 @@
+using NBench;
+
+namespace Akka.Cluster.Tests.Performance
+{
+    class Program
+    {
+        static int Main(string[] args)
+        {
+            return NBenchRunner.Run<Program>();
+        }
+    }
+}

src/core/Akka.Cluster.Tests.Performance/Startup/ClusterStartupSpec.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Akka.Actor;
+using Akka.Configuration;
+using NBench;
+
+namespace Akka.Cluster.Tests.Performance.Startup
+{
+    /// <summary>
+    /// Measures end to end startup time, including HOCON parsing
+    /// </summary>
+    public class ClusterStartup2NodeSpec
+    {
+        public ActorSystem Sys { get; set; }
+        public ActorSystem OtherSys { get; set; }
+
+        public const string SystemConfig = @"akka.actor.provider = cluster
+                                             akka.remote.dot-netty.tcp.port = 8110
+                                             akka.remote.dot-netty.tcp.hostname = 0.0.0.0
+                                             akka.remote.dot-netty.tcp.public-hostname = localhost
+                                             akka.cluster.seed-nodes = [""akka.tcp://ClusterSys@localhost:8110""]";
+
+        public TaskCompletionSource<Done> _clusterMemberUp = new TaskCompletionSource<Done>();
+
+        [PerfSetup]
+        public void Setup(BenchmarkContext context)
+        {
+            var config = ConfigurationFactory.ParseString(SystemConfig);
+            OtherSys = ActorSystem.Create("ClusterSys", config);
+            context.Trace.Info($"Started ActorSystem1 on {Cluster.Get(OtherSys).SelfAddress}");
+        }
+
+        [PerfBenchmark(Description = "Measures end to end startup time to create a 2 node cluster", NumberOfIterations = 3, RunMode = RunMode.Iterations)]
+        [TimingMeasurement]
+        [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
+        [GcMeasurement(GcMetric.TotalCollections, GcGeneration.AllGc)]
+        public void Run(BenchmarkContext context)
+        {
+            context.Trace.Info("Configuring ActorSystem2");
+
+            // adding a fallback specifically here as a point of measurement
+            var config2 = ConfigurationFactory.ParseString("akka.remote.dot-netty.tcp.port = 0")
+                .WithFallback(OtherSys.Settings.Config);
+            Sys = ActorSystem.Create("ClusterSys", config2);
+
+            Cluster.Get(Sys).RegisterOnMemberUp(() => { _clusterMemberUp.TrySetResult(Done.Instance); });
+            context.Trace.Info($"Started ActorSystem1 on {Cluster.Get(OtherSys).SelfAddress}");
+            var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+            _clusterMemberUp.Task.Wait(cts.Token);
+            context.Trace.Info($"Successfully joined cluster.");
+        }
+
+        [PerfCleanup]
+        public void Cleanup()
+        {
+            OtherSys?.Terminate();
+            Sys?.Terminate().Wait(TimeSpan.FromSeconds(5));
+        }
+    }
+}

src/core/Akka.Remote.Tests.Performance/Akka.Remote.Tests.Performance.csproj

@@ -1,22 +1,19 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
+  <Import Project="..\..\common.props" />
   <PropertyGroup>
-    <AssemblyTitle>Akka.Remote.Tests.Performance</AssemblyTitle>
-    <TargetFrameworks>net452</TargetFrameworks>
+    <TargetFramework>$(NetCoreTestVersion)</TargetFramework>
     <ServerGarbageCollection>true</ServerGarbageCollection>
+    <IsPackable>false</IsPackable>
+    <OutputType>Exe</OutputType>
   </PropertyGroup>
 
+
   <ItemGroup>
-    <Content Include="App.config" />
     <ProjectReference Include="..\Akka.Remote\Akka.Remote.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="NBench" Version="1.0.1" />
+    <PackageReference Include="NBench" Version="$(NBenchVersion)" />
   </ItemGroup>
 
-  <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
-    <DefineConstants>$(DefineConstants);RELEASE</DefineConstants>
-  </PropertyGroup>
-
 </Project>

src/core/Akka.Remote.Tests.Performance/Program.cs

@@ -0,0 +1,12 @@
+using NBench;
+
+namespace Akka.Remote.Tests.Performance
+{
+    class Program
+    {
+        static int Main(string[] args)
+        {
+            return NBenchRunner.Run<Program>();
+        }
+    }
+}

src/core/Akka.Streams.Tests.Performance/Akka.Streams.Tests.Performance.csproj

@@ -1,30 +1,21 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
+  <Import Project="..\..\common.props" />
   <PropertyGroup>
-    <AssemblyTitle>Akka.Streams.Tests.Performance</AssemblyTitle>
-    <TargetFrameworks>net452</TargetFrameworks>
+    <TargetFramework>$(NetCoreTestVersion)</TargetFramework>
     <ServerGarbageCollection>true</ServerGarbageCollection>
+    <IsPackable>false</IsPackable>
+    <OutputType>Exe</OutputType>
   </PropertyGroup>
 
   <ItemGroup>
-    <Content Include="App.config" />
     <ProjectReference Include="..\Akka.Streams\Akka.Streams.csproj" />
     <ProjectReference Include="..\Akka.Streams.Tests\Akka.Streams.Tests.csproj" />
     <ProjectReference Include="..\Akka.Streams.TestKit\Akka.Streams.TestKit.csproj" />
     <ProjectReference Include="..\Akka.Streams.TestKit.Tests\Akka.Streams.TestKit.Tests.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="NBench" Version="1.0.1" />
-  </ItemGroup>
-
-  <ItemGroup Condition=" '$(TargetFramework)' == 'net452' ">
-    <Reference Include="System.Configuration" />
-    <Reference Include="System.Xml" />
+    <PackageReference Include="NBench" Version="$(NBenchVersion)" />
   </ItemGroup>
 
-  <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
-    <DefineConstants>$(DefineConstants);RELEASE</DefineConstants>
-  </PropertyGroup>
-
 </Project>

src/core/Akka.Streams.Tests.Performance/Program.cs

@@ -0,0 +1,12 @@
+using NBench;
+
+namespace Akka.Streams.Tests.Performance
+{
+    class Program
+    {
+        static int Main(string[] args)
+        {
+            return NBenchRunner.Run<Program>();
+        }
+    }
+}

src/core/Akka.Tests.Performance/Akka.Tests.Performance.csproj

@@ -1,28 +1,19 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
+  <Import Project="..\..\common.props" />
   <PropertyGroup>
-    <AssemblyTitle>Akka.Tests.Performance</AssemblyTitle>
-    <TargetFrameworks>net452</TargetFrameworks>
+    <TargetFramework>$(NetCoreTestVersion)</TargetFramework>
     <ServerGarbageCollection>true</ServerGarbageCollection>
+    <IsPackable>false</IsPackable>
+    <OutputType>Exe</OutputType>
   </PropertyGroup>
 
   <ItemGroup>
-    <Content Include="App.config" />
     <ProjectReference Include="..\Akka\Akka.csproj" />
     <ProjectReference Include="..\Akka.TestKit\Akka.TestKit.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="NBench" Version="1.0.1" />
-  </ItemGroup>
-
-  <ItemGroup Condition=" '$(TargetFramework)' == 'net452' ">
-    <Reference Include="System.Configuration" />
-    <Reference Include="System.Xml" />
+    <PackageReference Include="NBench" Version="$(NBenchVersion)" />
   </ItemGroup>
 
-  <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
-    <DefineConstants>$(DefineConstants);RELEASE</DefineConstants>
-  </PropertyGroup>
-
 </Project>

src/core/Akka.Tests.Performance/Program.cs

@@ -0,0 +1,12 @@
+using NBench;
+
+namespace Akka.Tests.Performance
+{
+    class Program
+    {
+        static int Main(string[] args)
+        {
+            return NBenchRunner.Run<Program>();
+        }
+    }
+}