-
Notifications
You must be signed in to change notification settings - Fork 289
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
#3 Adding reservoir sampling benchmarks which compares uniform, slidi…
…ng window, exponentially decaying and hdr histogram reservoirs
- Loading branch information
Showing
19 changed files
with
289 additions
and
42 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
benchmark-results/ReservoirSamplingBenchmarks-report-github.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| ``` ini | ||
|
|
||
| BenchmarkDotNet=v0.10.0 | ||
| OS=Windows | ||
| Processor=?, ProcessorCount=8 | ||
| Frequency=3312791 Hz, Resolution=301.8603 ns, Timer=TSC | ||
| Host Runtime=.NET Core 4.6.24410.01, Arch=64-bit [RyuJIT] | ||
| GC=Concurrent Workstation | ||
| dotnet cli version=1.0.0-preview2-1-003177 | ||
| Job Runtime(s): | ||
| .NET Core 4.6.24410.01, Arch=64-bit [RyuJIT] | ||
|
|
||
| Job=MediumRun Jit=RyuJit Platform=X64 | ||
| Runtime=Core LaunchCount=2 TargetCount=15 | ||
| WarmupCount=10 | ||
|
|
||
| ``` | ||
| Method | Mean | StdErr | StdDev | Median | Scaled | Scaled-StdDev | | ||
| ------------------------------- |-------------- |----------- |------------ |-------------- |------- |-------------- | | ||
| UniformReservoir | 717.8625 us | 4.4517 us | 23.1318 us | 706.8204 us | 1.00 | 0.00 | | ||
| SlidingWindowReservoir | 717.1000 us | 5.0570 us | 27.2327 us | 702.2431 us | 1.00 | 0.05 | | ||
| ExponentiallyDecayingReservoir | 8,931.7810 us | 91.0223 us | 498.5499 us | 8,912.4664 us | 12.45 | 0.78 | | ||
| HdrHistogramReservoir | 7,797.4069 us | 88.6230 us | 485.4081 us | 7,667.7400 us | 10.87 | 0.74 | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
21 changes: 21 additions & 0 deletions
21
benchmarks/App.Metrics.Reservoir.Benchmarks/App.Metrics.Reservoir.Benchmarks.xproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| <?xml version="1.0" encoding="utf-8"?> | ||
| <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> | ||
| <PropertyGroup> | ||
| <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion> | ||
| <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath> | ||
| </PropertyGroup> | ||
| <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.Props" Condition="'$(VSToolsPath)' != ''" /> | ||
| <PropertyGroup Label="Globals"> | ||
| <ProjectGuid>EC34B3BA-A925-4C95-8074-30257C18EDD2</ProjectGuid> | ||
| <RootNamespace>App.Metrics.Reservoir.Benchmarks</RootNamespace> | ||
| <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">.\obj</BaseIntermediateOutputPath> | ||
| <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath> | ||
| </PropertyGroup> | ||
| <PropertyGroup> | ||
| <SchemaVersion>2.0</SchemaVersion> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" /> | ||
| </ItemGroup> | ||
| <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.targets" Condition="'$(VSToolsPath)' != ''" /> | ||
| </Project> |
14 changes: 14 additions & 0 deletions
14
benchmarks/App.Metrics.Reservoir.Benchmarks/Harness/BenchmarkHarness.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| using BenchmarkDotNet.Running; | ||
| using Xunit; | ||
|
|
||
| namespace App.Metrics.Reservoir.Benchmarks.Harness | ||
| { | ||
| public class BenchmarkHarness | ||
| { | ||
| [Fact, Trait("MediumRun", "ReservoirSamplingCompare")] | ||
| public void AtomicLongCompareBenchmark() | ||
| { | ||
| BenchmarkRunner.Run<ReservoirSamplingBenchmarks>(); | ||
| } | ||
| } | ||
| } |
32 changes: 32 additions & 0 deletions
32
benchmarks/App.Metrics.Reservoir.Benchmarks/Jobs/MediumRunJobAttribute.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| using System; | ||
| using BenchmarkDotNet.Configs; | ||
| using BenchmarkDotNet.Environments; | ||
| using BenchmarkDotNet.Exporters; | ||
| using BenchmarkDotNet.Jobs; | ||
|
|
||
| namespace App.Metrics.Reservoir.Benchmarks.Jobs | ||
| { | ||
| [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)] | ||
| public class MediumRunJobAttribute : Attribute, IConfigSource | ||
| { | ||
| /// <summary> | ||
| /// https://github.com/dotnet/BenchmarkDotNet/blob/307b7250210fc3a7b7ff2c454bb79f793aae57ea/docs/guide/Configs/Jobs.md | ||
| /// </summary> | ||
| public MediumRunJobAttribute() | ||
| { | ||
| var job = Job.MediumRun | ||
| .With(Platform.X64) | ||
| .With(Jit.RyuJit) | ||
| .With(Runtime.Core) | ||
| .WithId("MediumRun"); | ||
|
|
||
| var config = ManualConfig.CreateEmpty(); | ||
| config.Add(RPlotExporter.Default); | ||
| config.Add(MarkdownExporter.GitHub); | ||
|
|
||
| Config = config.With(job); | ||
| } | ||
|
|
||
| public IConfig Config { get; } | ||
| } | ||
| } |
35 changes: 35 additions & 0 deletions
35
benchmarks/App.Metrics.Reservoir.Benchmarks/Jobs/QuickRunJobAttribute.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| using System; | ||
| using BenchmarkDotNet.Configs; | ||
| using BenchmarkDotNet.Environments; | ||
| using BenchmarkDotNet.Exporters; | ||
| using BenchmarkDotNet.Jobs; | ||
|
|
||
| namespace App.Metrics.Reservoir.Benchmarks.Jobs | ||
| { | ||
| [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)] | ||
| public class QuickRunJobAttribute : Attribute, IConfigSource | ||
| { | ||
| /// <summary> | ||
| /// https://github.com/dotnet/BenchmarkDotNet/blob/307b7250210fc3a7b7ff2c454bb79f793aae57ea/docs/guide/Configs/Jobs.md | ||
| /// </summary> | ||
| public QuickRunJobAttribute() | ||
| { | ||
| var job = Job.Dry | ||
| .With(Platform.X64) | ||
| .With(Jit.RyuJit) | ||
| .With(Runtime.Core) | ||
| .WithWarmupCount(5) | ||
| .WithLaunchCount(1) | ||
| .WithTargetCount(5) | ||
| .WithId("QuickRun"); | ||
|
|
||
| var config = ManualConfig.CreateEmpty(); | ||
| config.Add(RPlotExporter.Default); | ||
| config.Add(MarkdownExporter.GitHub); | ||
|
|
||
| Config = config.With(job); | ||
| } | ||
|
|
||
| public IConfig Config { get; } | ||
| } | ||
| } |
19 changes: 19 additions & 0 deletions
19
benchmarks/App.Metrics.Reservoir.Benchmarks/Properties/AssemblyInfo.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| using System.Reflection; | ||
| using System.Runtime.CompilerServices; | ||
| using System.Runtime.InteropServices; | ||
|
|
||
| // General Information about an assembly is controlled through the following | ||
| // set of attributes. Change these attribute values to modify the information | ||
| // associated with an assembly. | ||
| [assembly: AssemblyConfiguration("")] | ||
| [assembly: AssemblyCompany("")] | ||
| [assembly: AssemblyProduct("App.Metrics.Reservoir.Benchmarks")] | ||
| [assembly: AssemblyTrademark("")] | ||
|
|
||
| // Setting ComVisible to false makes the types in this assembly not visible | ||
| // to COM components. If you need to access a type in this assembly from | ||
| // COM, set the ComVisible attribute to true on that type. | ||
| [assembly: ComVisible(false)] | ||
|
|
||
| // The following GUID is for the ID of the typelib if this project is exposed to COM | ||
| [assembly: Guid("c967ea87-325e-4c06-9cdc-1343c11cb62b")] |
75 changes: 75 additions & 0 deletions
75
benchmarks/App.Metrics.Reservoir.Benchmarks/ReservoirSamplingBenchmarks.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| using System.Collections.Generic; | ||
| using System.Threading; | ||
| using App.Metrics.Reservoir.Benchmarks.Jobs; | ||
| using App.Metrics.Sampling; | ||
| using App.Metrics.Sampling.Interfaces; | ||
| using BenchmarkDotNet.Attributes; | ||
|
|
||
| namespace App.Metrics.Reservoir.Benchmarks | ||
| { | ||
| [MediumRunJob] | ||
| public class ReservoirSamplingBenchmarks | ||
| { | ||
| private const int NumberOfRuns = 100; | ||
| private const int SampleSize = 100; | ||
| private const double ExponentialDecayFactor = 0.015; | ||
| private const int ThreadCount = 8; | ||
|
|
||
| [Benchmark(Baseline = true)] | ||
| public void UniformReservoir() | ||
| { | ||
| var reservoir = new UniformReservoir(SampleSize); | ||
| Run(reservoir); | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void SlidingWindowReservoir() | ||
| { | ||
| var reservoir = new SlidingWindowReservoir(SampleSize); | ||
| Run(reservoir); | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void ExponentiallyDecayingReservoir() | ||
| { | ||
| var reservoir = new ExponentiallyDecayingReservoir(SampleSize, ExponentialDecayFactor); | ||
| Run(reservoir); | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void HdrHistogramReservoir() | ||
| { | ||
| var reservoir = new HdrHistogramReservoir(); | ||
| Run(reservoir); | ||
| } | ||
|
|
||
| private static void Run(IReservoir reservoir) | ||
| { | ||
| var thread = new List<Thread>(); | ||
|
|
||
| for (var i = 0; i < ThreadCount; i++) | ||
| { | ||
| thread.Add(new Thread(() => | ||
| { | ||
| for (long j = 0; j < NumberOfRuns; j++) | ||
| { | ||
| reservoir.Update(1, $"user-value-{j}"); | ||
| if (j % 4 == 0) | ||
| { | ||
| reservoir.GetSnapshot(); | ||
| } | ||
| if (j == NumberOfRuns / 2) | ||
| { | ||
| reservoir.Reset(); | ||
| } | ||
| } | ||
| })); | ||
| } | ||
|
|
||
| thread.ForEach(t => t.Start()); | ||
| thread.ForEach(t => t.Join()); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| { | ||
| "buildOptions": { | ||
| "allowUnsafe": true | ||
| }, | ||
| "dependencies": { | ||
| "xunit": "2.1.0", | ||
| "dotnet-test-xunit": "1.0.0-rc2-build10025", | ||
| "BenchmarkDotNet": "0.10.0", | ||
| "BenchmarkDotNet.Core": "0.10.0", | ||
| "App.Metrics": "1.0.0-alpha" | ||
| }, | ||
| "frameworks": { | ||
| "netcoreapp1.0": { | ||
| "dependencies": { | ||
| "Microsoft.NETCore.App": { | ||
| "type": "platform", | ||
| "version": "1.0.0" | ||
| } | ||
| }, | ||
| "imports": [ | ||
| "dnxcore50", | ||
| "portable-net45+win8" | ||
| ] | ||
| } | ||
| }, | ||
| "testRunner": "xunit" | ||
| } |
Oops, something went wrong.