-
Notifications
You must be signed in to change notification settings - Fork 450
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add project for running microbenchmarks (#81)
* Add project for running microbenchmerks * Fix code style violations * Update README.md * Add link to BDN best practices to the benchmarking README * Add OperationsPerInvoke to allow more fair comparison * In the low-level read benchmarks, OperationsPerInvoke should amortize the small noise from the pinning * In write benchmarks, OperationsPerInvoke should allow the comparing the benchmarks which use random values to the other constant integer benches * Add default benchmark configuration with DPGO explicitly disabled for .NET 8 * Revert "Add OperationsPerInvoke to allow more fair comparison" This reverts commit 2625443. * Restore original TFMs to the benchmark project The TFM is required for running benchmarks against it after all. * Give the .NET 6 benchmark job a name * Reorder imports per the .editorconfig * Benchmark ascii length boundary values instead of random * Fix benchmark commands in the README * Rename BenchmarkDotNet.benchmark to BDN.benchmark
- Loading branch information
1 parent
4a88c5a
commit 934dad3
Showing
8 changed files
with
300 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -211,4 +211,7 @@ test/tmp/ | |
|
|
||
| # IDE files | ||
| .vscode/ | ||
| .idea/ | ||
| .idea/ | ||
|
|
||
| # BenchmarkDotNet Results | ||
| BenchmarkDotNet.Artifacts/ | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFrameworks>net6.0;net7.0;net8.0</TargetFrameworks> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <AllowUnsafeBlocks>true</AllowUnsafeBlocks> | ||
| <LangVersion>latest</LangVersion> | ||
| <Nullable>enable</Nullable> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <PackageReference Include="BenchmarkDotNet" Version="0.13.12" /> | ||
| </ItemGroup> | ||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\libs\common\Garnet.common.csproj" /> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT license. | ||
|
|
||
| using BenchmarkDotNet.Configs; | ||
| using BenchmarkDotNet.Environments; | ||
| using BenchmarkDotNet.Jobs; | ||
| using BenchmarkDotNet.Running; | ||
|
|
||
| var config = DefaultConfig.Instance | ||
| .AddJob(Job.Default | ||
| .WithRuntime(CoreRuntime.Core60) | ||
| .WithId(".NET 6")) | ||
| .AddJob(Job.Default | ||
| .WithRuntime(CoreRuntime.Core80) | ||
| .WithEnvironmentVariables(new EnvironmentVariable("DOTNET_TieredPGO", "0")) | ||
| .WithId(".NET 8")); | ||
|
|
||
| BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config); |
110 changes: 110 additions & 0 deletions
110
benchmark/BDN.benchmark/Resp/RespIntegerReadBenchmarks.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT license. | ||
|
|
||
| using System.Text; | ||
| using BenchmarkDotNet.Attributes; | ||
| using Garnet.common; | ||
|
|
||
| namespace BDN.benchmark.Resp | ||
| { | ||
| public unsafe class RespIntegerReadBenchmarks | ||
| { | ||
| [Benchmark] | ||
| [ArgumentsSource(nameof(SignedInt32EncodedValues))] | ||
| public int ReadInt32(AsciiTestCase testCase) | ||
| { | ||
| fixed (byte* inputPtr = testCase.Bytes) | ||
| { | ||
| var start = inputPtr; | ||
| RespReadUtils.ReadInt(out var value, ref start, start + testCase.Bytes.Length); | ||
| return value; | ||
| } | ||
| } | ||
|
|
||
| [Benchmark] | ||
| [ArgumentsSource(nameof(SignedInt64EncodedValues))] | ||
| public long ReadInt64(AsciiTestCase testCase) | ||
| { | ||
| fixed (byte* inputPtr = testCase.Bytes) | ||
| { | ||
| var start = inputPtr; | ||
| RespReadUtils.Read64Int(out var value, ref start, start + testCase.Bytes.Length); | ||
| return value; | ||
| } | ||
| } | ||
|
|
||
| [Benchmark] | ||
| [ArgumentsSource(nameof(SignedInt32EncodedValuesWithLengthHeader))] | ||
| public int ReadIntWithLengthHeader(AsciiTestCase testCase) | ||
| { | ||
| fixed (byte* inputPtr = testCase.Bytes) | ||
| { | ||
| var start = inputPtr; | ||
| RespReadUtils.ReadIntWithLengthHeader(out var value, ref start, start + testCase.Bytes.Length); | ||
| return value; | ||
| } | ||
| } | ||
|
|
||
| [Benchmark] | ||
| [ArgumentsSource(nameof(SignedInt64EncodedValuesWithLengthHeader))] | ||
| public long ReadLongWithLengthHeader(AsciiTestCase testCase) | ||
| { | ||
| fixed (byte* inputPtr = testCase.Bytes) | ||
| { | ||
| var start = inputPtr; | ||
| RespReadUtils.ReadLongWithLengthHeader(out var value, ref start, start + testCase.Bytes.Length); | ||
| return value; | ||
| } | ||
| } | ||
|
|
||
| [Benchmark] | ||
| [ArgumentsSource(nameof(UnsignedInt64EncodedValuesWithLengthHeader))] | ||
| public ulong ReadULongWithLengthHeader(AsciiTestCase testCase) | ||
| { | ||
| fixed (byte* inputPtr = testCase.Bytes) | ||
| { | ||
| var start = inputPtr; | ||
| RespReadUtils.ReadULongWithLengthHeader(out var value, ref start, start + testCase.Bytes.Length); | ||
| return value; | ||
| } | ||
| } | ||
|
|
||
| public static IEnumerable<object> SignedInt32EncodedValues | ||
| => ToRespIntegerTestCases(RespIntegerWriteBenchmarks.SignedInt32Values); | ||
|
|
||
| public static IEnumerable<object> SignedInt64EncodedValues | ||
| => ToRespIntegerTestCases(RespIntegerWriteBenchmarks.SignedInt64Values); | ||
|
|
||
| public static IEnumerable<object> UnsignedInt64EncodedValues | ||
| => ToRespIntegerTestCases(RespIntegerWriteBenchmarks.UnsignedInt64Values); | ||
|
|
||
| public static IEnumerable<object> SignedInt32EncodedValuesWithLengthHeader | ||
| => ToRespIntegerWithLengthHeader(RespIntegerWriteBenchmarks.SignedInt32Values); | ||
|
|
||
| public static IEnumerable<object> SignedInt64EncodedValuesWithLengthHeader | ||
| => ToRespIntegerWithLengthHeader(RespIntegerWriteBenchmarks.SignedInt64Values); | ||
|
|
||
| public static IEnumerable<object> UnsignedInt64EncodedValuesWithLengthHeader | ||
| => ToRespIntegerWithLengthHeader(RespIntegerWriteBenchmarks.UnsignedInt64Values); | ||
|
|
||
| public static IEnumerable<AsciiTestCase> ToRespIntegerTestCases<T>(T[] integerValues) where T : struct | ||
| => integerValues.Select(testCase => new AsciiTestCase($":{testCase}\r\n")); | ||
|
|
||
| public static IEnumerable<AsciiTestCase> ToRespIntegerWithLengthHeader<T>(T[] integerValues) where T : struct | ||
| => integerValues.Select(testCase => new AsciiTestCase($"${testCase.ToString()?.Length ?? 0}\r\n{testCase}\r\n")); | ||
|
|
||
| public sealed class AsciiTestCase | ||
| { | ||
| public byte[] Bytes { get; } | ||
| private string Text { get; } | ||
|
|
||
| public AsciiTestCase(string text) | ||
| { | ||
| Text = text; | ||
| Bytes = Encoding.ASCII.GetBytes(text); | ||
| } | ||
|
|
||
| public override string ToString() => Text; // displayed by BDN | ||
| } | ||
| } | ||
| } |
104 changes: 104 additions & 0 deletions
104
benchmark/BDN.benchmark/Resp/RespIntegerWriteBenchmarks.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT license. | ||
|
|
||
| using System.Runtime.InteropServices; | ||
| using BenchmarkDotNet.Attributes; | ||
| using Garnet.common; | ||
|
|
||
| namespace BDN.benchmark.Resp | ||
| { | ||
| public unsafe class RespIntegerWriteBenchmarks | ||
| { | ||
| // Big enough buffer for the benchmarks | ||
| private readonly byte[] _buffer = new byte[32]; | ||
|
|
||
| private byte* _bufferPtr; | ||
| private GCHandle _bufferHandle; | ||
|
|
||
| [GlobalSetup] | ||
| public void GlobalSetup() | ||
| { | ||
| // Pin the buffer for the benchmarks | ||
| _bufferHandle = GCHandle.Alloc(_buffer, GCHandleType.Pinned); | ||
| _bufferPtr = (byte*)_bufferHandle.AddrOfPinnedObject(); | ||
| } | ||
|
|
||
| [GlobalCleanup] | ||
| public void GlobalCleanup() => _bufferHandle.Free(); | ||
|
|
||
| [Benchmark] | ||
| [ArgumentsSource(nameof(SignedInt32Values))] | ||
| public bool WriteInt32(int value) | ||
| { | ||
| var startPtr = _bufferPtr; | ||
| return RespWriteUtils.WriteInteger(value, ref startPtr, _bufferPtr + _buffer.Length); | ||
| } | ||
|
|
||
| [Benchmark] | ||
| [ArgumentsSource(nameof(SignedInt64Values))] | ||
| public bool WriteInt64(long value) | ||
| { | ||
| var startPtr = _bufferPtr; | ||
| return RespWriteUtils.WriteInteger(value, ref startPtr, _bufferPtr + _buffer.Length); | ||
| } | ||
|
|
||
| [Benchmark] | ||
| [ArgumentsSource(nameof(SignedInt32Values))] | ||
| public bool WriteInt32AsBulkString(int value) | ||
| { | ||
| var startPtr = _bufferPtr; | ||
| return RespWriteUtils.WriteIntegerAsBulkString(value, ref startPtr, _bufferPtr + _buffer.Length); | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void WriteInt32_AllAsciiLengths() | ||
| { | ||
| for (int i = 0; i < SignedInt32MultiplesOfTen.Length; i++) | ||
| { | ||
| var startPtr = _bufferPtr; | ||
| RespWriteUtils.WriteInteger(SignedInt32MultiplesOfTen[i], ref startPtr, _bufferPtr + _buffer.Length); | ||
| } | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void WriteInt64_AllAsciiLengths() | ||
| { | ||
| for (int i = 0; i < SignedInt64MultiplesOfTen.Length; i++) | ||
| { | ||
| var startPtr = _bufferPtr; | ||
| RespWriteUtils.WriteInteger(SignedInt64MultiplesOfTen[i], ref startPtr, _bufferPtr + _buffer.Length); | ||
| } | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void WriteInt32BulkString_AllAsciiLengths() | ||
| { | ||
| for (int i = 0; i < SignedInt32MultiplesOfTen.Length; i++) | ||
| { | ||
| var startPtr = _bufferPtr; | ||
| RespWriteUtils.WriteIntegerAsBulkString(SignedInt32MultiplesOfTen[i], ref startPtr, _bufferPtr + _buffer.Length); | ||
| } | ||
| } | ||
|
|
||
| public const int UnsignedInt32MaxValueDigits = 10; // The number of digits in (u)int.MaxValue | ||
| public const int UnsignedInt64MaxValueDigits = 19; // The number of digits in (u)long.MaxValue | ||
|
|
||
| // All multiples of 10 from 10^-10 to 10^10 | ||
| public static int[] SignedInt32MultiplesOfTen => [ | ||
| ..UnsignedInt32MultiplesOfTen.Select(n => n * -1), | ||
| ..UnsignedInt32MultiplesOfTen | ||
| ]; | ||
| public static int[] UnsignedInt32MultiplesOfTen => Enumerable.Range(0, 10).Select(n => (int)Math.Pow(10, n)).ToArray(); | ||
|
|
||
| // All multiples of 10 from 10^-19 to 10^19 | ||
| public static long[] SignedInt64MultiplesOfTen => [ | ||
| ..UnsignedInt64MultiplesOfTen.Select(n => n * -1), | ||
| ..UnsignedInt64MultiplesOfTen | ||
| ]; | ||
| public static long[] UnsignedInt64MultiplesOfTen => Enumerable.Range(0, 19).Select(n => (long)Math.Pow(10, n)).ToArray(); | ||
|
|
||
| public static int[] SignedInt32Values => [int.MinValue, -1, 0, int.MaxValue]; | ||
| public static long[] SignedInt64Values => [long.MinValue, -1, 0, long.MaxValue]; | ||
| public static ulong[] UnsignedInt64Values => [0, int.MaxValue, ulong.MaxValue]; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| # Benchmarks | ||
|
|
||
| This directory contains projects for benchmarking Garnet. | ||
|
|
||
| ## Resp.benchmark | ||
|
|
||
| **Garnet** project contains a Benchmark tool for running RESP benchmarking using different clients, different workloads and different strategies for measuring throughput, performance and latency. | ||
|
|
||
| Please visit our website documentation about how to use it in the following link: [The Resp.benchmark tool](https://microsoft.github.io/garnet/docs/benchmarking/resp-bench) | ||
|
|
||
| ## BDN.benchmark | ||
|
|
||
| The `BDN.benchmark` command-line tool allows contributors to run precise and reproducible micro-benchmarks by utilizing the [BenchmarkDotNet](https://benchmarkdotnet.org/index.html). | ||
|
|
||
| ### Usage | ||
|
|
||
| You can list all available benchmarks using `--list flat` or `--list tree`, e.g. | ||
|
|
||
| ``` | ||
| dotnet run -c Release -f net8.0 --list flat | ||
| ``` | ||
|
|
||
| To run specific benchmarks, you can use `--filter`. For example, to run all RESP-protocol write benchmarks using the default configuration, which will run the benchmarks using both .NET 6 and .NET 8 runtimes (with the dynamic PGO disabled): | ||
|
|
||
| ``` | ||
| dotnet run -c Release -f net8.0 --filter *RespIntegerWriteBenchmarks* | ||
| ``` | ||
|
|
||
| See more command-line options at https://benchmarkdotnet.org/articles/guides/console-args.html | ||
|
|
||
| ### Writing microbenchmarks | ||
|
|
||
| Please see the [Microbenchmark Design Guidelines](https://github.com/dotnet/performance/blob/main/docs/microbenchmark-design-guidelines.md) for the best practices when writing microbenchmarks using BenchmarkDotNet. | ||
|
|
||
| ## Privacy | ||
|
|
||
| [Microsoft Privacy Statement](https://go.microsoft.com/fwlink/?LinkId=521839) |
This file was deleted.
Oops, something went wrong.