diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index d0dc2d475b..51451a5602 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -89,6 +89,10 @@ public string GetHighlightGroupKey(BenchmarkCase benchmarkCase) public string GetLogicalGroupKey(ImmutableArray allBenchmarksCases, BenchmarkCase benchmarkCase) { + // TODO: GetLogicalGroupKey is called for every benchmarkCase, so as the number of cases grows, this can get very expensive to recompute for each call. + // We should somehow amortize the cost by computing it only once per summary. + var paramSets = allBenchmarksCases.Select(benchmarkCase => benchmarkCase.Parameters).Distinct(ParameterEqualityComparer.Instance).ToArray(); + var explicitRules = benchmarkCase.Config.GetLogicalGroupRules().ToList(); var implicitRules = new List(); bool hasJobBaselines = allBenchmarksCases.Any(b => b.Job.Meta.Baseline); @@ -125,7 +129,7 @@ public string GetLogicalGroupKey(ImmutableArray allBenchmarksCase keys.Add(benchmarkCase.Job.DisplayInfo); break; case BenchmarkLogicalGroupRule.ByParams: - keys.Add(benchmarkCase.Parameters.ValueInfo); + keys.Add($"DistinctParamSet{Array.FindIndex(paramSets, (paramSet) => ParameterEqualityComparer.Instance.Equals(paramSet, benchmarkCase.Parameters))}"); break; case BenchmarkLogicalGroupRule.ByCategory: keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories)); diff --git a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs index 6cadf6a946..4f39311b82 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs @@ -1,47 +1,287 @@ using System; +using System.Collections; using System.Collections.Generic; +using System.Reflection; +using BenchmarkDotNet.Portability; + +#if NETSTANDARD2_0 +using System.Collections.Concurrent; +using System.Linq; +#endif namespace BenchmarkDotNet.Parameters { internal class ParameterComparer : IComparer { +#if NETSTANDARD2_0 + private static readonly ConcurrentDictionary s_tupleMembersCache = new(); + + private static MemberInfo[] GetTupleMembers(Type type) + => s_tupleMembersCache.GetOrAdd(type, t => + { + var members = type.FullName.StartsWith("System.Tuple`") + ? type.GetProperties(BindingFlags.Public | BindingFlags.Instance).Cast() + : type.GetFields(BindingFlags.Public | BindingFlags.Instance).Cast(); + return members + .Where(p => p.Name.StartsWith("Item")) + .OrderBy(p => p.Name) + .ToArray(); + }); +#endif + public static readonly ParameterComparer Instance = new ParameterComparer(); public int Compare(ParameterInstances x, ParameterInstances y) { if (x == null && y == null) return 0; - if (x != null && y == null) return 1; if (x == null) return -1; + if (y == null) return 1; + for (int i = 0; i < Math.Min(x.Count, y.Count); i++) { - var compareTo = CompareValues(x[i]?.Value, y[i]?.Value); - if (compareTo != 0) - return compareTo; + var comparison = CompareValues(x[i]?.Value, y[i]?.Value); + if (comparison != 0) + { + return comparison; + } } + return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo); } - private int CompareValues(object x, object y) + private static int CompareValues(T1 x, T2 y) { - // Detect IComparable implementations. - // This works for all primitive types in addition to user types that implement IComparable. - if (x != null && y != null && x.GetType() == y.GetType() && - x is IComparable xComparable) + if (x == null || y == null || x.GetType() != y.GetType()) + { + return string.CompareOrdinal(x?.ToString(), y?.ToString()); + } + + if (x is IStructuralComparable xStructuralComparable) { try { - return xComparable.CompareTo(y); + return StructuralComparisons.StructuralComparer.Compare(x, y); } - // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable. - // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346 - // For now, catch and ignore the exception, and fallback to string comparison below. - catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + // https://github.com/dotnet/BenchmarkDotNet/issues/2346 + // https://github.com/dotnet/runtime/issues/66472 + // Unfortunately we can't rely on checking the exception message because it may change per current culture. + catch (ArgumentException) { + if (TryFallbackStructuralCompareTo(x, y, out int comparison)) + { + return comparison; + } + // A complex user type did not handle a multi-dimensional array or tuple, just re-throw. + throw; } } - // Anything else. + if (x is IComparable xComparable) + { + // Tuples are already handled by IStructuralComparable case, if this throws, it's the user's own fault. + return xComparable.CompareTo(y); + } + + if (x is IEnumerable xEnumerable) // General collection comparison support + { + return CompareEnumerables(xEnumerable, (IEnumerable) y); + } + + // Anything else to differentiate between objects. return string.CompareOrdinal(x?.ToString(), y?.ToString()); } + + private static bool TryFallbackStructuralCompareTo(object x, object y, out int comparison) + { + // Check for multi-dimensional array and ITuple and re-try for each element recursively. + if (x is Array xArr) + { + Array yArr = (Array) y; + if (xArr.Rank != yArr.Rank) + { + comparison = xArr.Rank.CompareTo(yArr.Rank); + return true; + } + + for (int dim = 0; dim < xArr.Rank; dim++) + { + if (xArr.GetLength(dim) != yArr.GetLength(dim)) + { + comparison = xArr.GetLength(dim).CompareTo(yArr.GetLength(dim)); + return true; + } + } + + // Common 2D and 3D arrays are specialized to avoid expensive boxing where possible. + if (!RuntimeInformation.IsAot && xArr.Rank is 2 or 3) + { + string methodName = xArr.Rank == 2 + ? nameof(CompareTwoDArray) + : nameof(CompareThreeDArray); + comparison = (int) typeof(ParameterComparer) + .GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Static) + .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) + .Invoke(null, [xArr, yArr]); + return true; + } + + // 1D arrays will only hit this code path if a nested type is a multi-dimensional array. + // 4D and larger fall back to enumerable. + comparison = CompareEnumerables(xArr, yArr); + return true; + } + +#if NETSTANDARD2_0 + // ITuple does not exist in netstandard2.0, so we have to use reflection. ITuple does exist in net471 and newer, but the System.ValueTuple nuget package does not implement it. + string typeName = x.GetType().FullName; + if (typeName.StartsWith("System.Tuple`")) + { + comparison = CompareTuples(x, y); + return true; + } + else if (typeName.StartsWith("System.ValueTuple`")) + { + comparison = CompareValueTuples(x, y); + return true; + } +#else + if (x is System.Runtime.CompilerServices.ITuple xTuple) + { + comparison = CompareTuples(xTuple, (System.Runtime.CompilerServices.ITuple) y); + return true; + } +#endif + + if (x is IEnumerable xEnumerable) // General collection equality support + { + comparison = CompareEnumerables(xEnumerable, (IEnumerable) y); + return true; + } + + comparison = 0; + return false; + } + + private static int CompareEnumerables(IEnumerable x, IEnumerable y) + { + var xEnumerator = x.GetEnumerator(); + try + { + var yEnumerator = y.GetEnumerator(); + try + { + while (xEnumerator.MoveNext()) + { + if (!yEnumerator.MoveNext()) + { + return -1; + } + int comparison = CompareValues(xEnumerator.Current, yEnumerator.Current); + if (comparison != 0) + { + return comparison; + } + } + return yEnumerator.MoveNext() ? 1 : 0; + } + finally + { + if (yEnumerator is IDisposable disposable) + { + disposable.Dispose(); + } + } + } + finally + { + if (xEnumerator is IDisposable disposable) + { + disposable.Dispose(); + } + } + } + + private static int CompareTwoDArray(T1[,] arrOne, T2[,] arrTwo) + { + // Assumes that arrOne & arrTwo are the same length and width. + for (int i = 0; i < arrOne.GetLength(0); i++) + { + for (int j = 0; j < arrOne.GetLength(1); j++) + { + var comparison = CompareValues(arrOne[i, j], arrTwo[i, j]); + if (comparison != 0) + { + return comparison; + } + } + } + + return 0; + } + + private static int CompareThreeDArray(T1[,,] arrOne, T2[,,] arrTwo) + { + // Assumes that arrOne & arrTwo are the same length, width, and height. + for (int i = 0; i < arrOne.GetLength(0); i++) + { + for (int j = 0; j < arrOne.GetLength(1); j++) + { + for (int k = 0; k + { +#if NETSTANDARD2_0 + private static readonly ConcurrentDictionary s_tupleMembersCache = new(); + + private static MemberInfo[] GetTupleMembers(Type type) + => s_tupleMembersCache.GetOrAdd(type, t => + { + var members = type.FullName.StartsWith("System.Tuple`") + ? type.GetProperties(BindingFlags.Public | BindingFlags.Instance).Cast() + : type.GetFields(BindingFlags.Public | BindingFlags.Instance).Cast(); + return members + .Where(p => p.Name.StartsWith("Item")) + .OrderBy(p => p.Name) + .ToArray(); + }); +#endif + + public static readonly ParameterEqualityComparer Instance = new ParameterEqualityComparer(); + + public bool Equals(ParameterInstances x, ParameterInstances y) + { + if (x == null && y == null) return true; + if (x == null || y == null) return false; + + if (x.Count != y.Count) return false; + + for (int i = 0; i < x.Count; i++) + { + if (!ValuesEqual(x[i]?.Value, y[i]?.Value)) + { + return false; + } + } + + return true; + } + + private static bool ValuesEqual(T1 x, T2 y) + { + if (x == null && y == null) + { + return true; + } + + if (x == null || y == null || x.GetType() != y.GetType()) + { + // The objects are of different types or one is null, they cannot be equal + return false; + } + + if (x is IStructuralEquatable xStructuralEquatable) + { + try + { + return StructuralComparisons.StructuralEqualityComparer.Equals(xStructuralEquatable, y); + } + // https://github.com/dotnet/runtime/issues/66472 + // Unfortunately we can't rely on checking the exception message because it may change per current culture. + catch (ArgumentException) + { + if (TryFallbackStructuralEquals(x, y, out bool equals)) + { + return equals; + } + // A complex user type did not handle a multi-dimensional array, just re-throw. + throw; + } + } + + if (x is IEnumerable xEnumerable) // General collection equality support + { + return EnumerablesEqual(xEnumerable, (IEnumerable) y); + } + + return FallbackSimpleEquals(x, y); + } + + private static bool FallbackSimpleEquals(object x, object y) + { + if (x.Equals(y)) + { + return true; + } + // Anything else to differentiate between objects (match behavior of ParameterComparer). + return string.Equals(x.ToString(), y.ToString(), StringComparison.Ordinal); + } + + private static bool TryFallbackStructuralEquals(object x, object y, out bool equals) + { + // Check for multi-dimensional array and ITuple and re-try for each element recursively. + if (x is Array xArr) + { + Array yArr = (Array) y; + if (xArr.Rank != yArr.Rank) + { + equals = false; + return true; + } + + for (int dim = 0; dim < xArr.Rank; dim++) + { + if (xArr.GetLength(dim) != yArr.GetLength(dim)) + { + equals = false; + return true; + } + } + + // Common 2D and 3D arrays are specialized to avoid expensive boxing where possible. + if (!RuntimeInformation.IsAot && xArr.Rank is 2 or 3) + { + string methodName = xArr.Rank == 2 + ? nameof(TwoDArraysEqual) + : nameof(ThreeDArraysEqual); + equals = (bool) typeof(ParameterEqualityComparer) + .GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Static) + .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) + .Invoke(null, [xArr, yArr]); + return true; + } + + // 1D arrays will only hit this code path if a nested type is a multi-dimensional array. + // 4D and larger fall back to enumerable. + equals = EnumerablesEqual(xArr, yArr); + return true; + } + +#if NETSTANDARD2_0 + // ITuple does not exist in netstandard2.0, so we have to use reflection. ITuple does exist in net471 and newer, but the System.ValueTuple nuget package does not implement it. + string typeName = x.GetType().FullName; + if (typeName.StartsWith("System.Tuple`")) + { + equals = TuplesEqual(x, y); + return true; + } + else if (typeName.StartsWith("System.ValueTuple`")) + { + equals = ValueTuplesEqual(x, y); + return true; + } +#else + if (x is System.Runtime.CompilerServices.ITuple xTuple) + { + equals = TuplesEqual(xTuple, (System.Runtime.CompilerServices.ITuple) y); + return true; + } +#endif + + if (x is IEnumerable xEnumerable) // General collection equality support + { + equals = EnumerablesEqual(xEnumerable, (IEnumerable) y); + return true; + } + + equals = false; + return false; + } + + private static bool EnumerablesEqual(IEnumerable x, IEnumerable y) + { + var xEnumerator = x.GetEnumerator(); + try + { + var yEnumerator = y.GetEnumerator(); + try + { + while (xEnumerator.MoveNext()) + { + if (!(yEnumerator.MoveNext() && ValuesEqual(xEnumerator.Current, yEnumerator.Current))) + { + return false; + } + } + return !yEnumerator.MoveNext(); + } + finally + { + if (yEnumerator is IDisposable disposable) + { + disposable.Dispose(); + } + } + } + finally + { + if (xEnumerator is IDisposable disposable) + { + disposable.Dispose(); + } + } + } + + private static bool TwoDArraysEqual(T1[,] arrOne, T2[,] arrTwo) + { + // Assumes that arrOne & arrTwo are the same length and width. + for (int i = 0; i < arrOne.GetLength(0); i++) + { + for (int j = 0; j < arrOne.GetLength(1); j++) + { + if (!ValuesEqual(arrOne[i, j], arrTwo[i, j])) + { + return false; + } + } + } + + return true; + } + + private static bool ThreeDArraysEqual(T1[,,] arrOne, T2[,,] arrTwo) + { + // Assumes that arrOne & arrTwo are the same length, width, and height. + for (int i = 0; i < arrOne.GetLength(0); i++) + { + for (int j = 0; j < arrOne.GetLength(1); j++) + { + for (int k = 0; k Sources() public void Any(string name, IEnumerable source) => source.Any(); } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void IEnumerableArgumentsCanBeGrouped(IToolchain toolchain) + { + var summary = CanExecute(toolchain, (config) => config.AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByParams)); // We must group by params to test array argument grouping + + // There should be two logical groups, one for the first argument and one for the second argument + // Thus there should be two pairs per descriptor, and each pair should be distinct because it belongs to a different group + + var descriptorGroupPairs = summary.BenchmarksCases.Select(benchmarkCase => (benchmarkCase.Descriptor, summary.GetLogicalGroupKey(benchmarkCase))).GroupBy(benchmarkCase => benchmarkCase.Descriptor); + + Assert.True( + descriptorGroupPairs.All(group => group.Select(pair => pair.Item2).Distinct().Count() == 2) + ); + } + + public class EnumerableOnDifferentMethods + { + public IEnumerable> GetEnumerables() + { + yield return Enumerable.Range(1, 10); + yield return Enumerable.Range(2, 10); + } + + [Benchmark(Baseline = true)] + [ArgumentsSource(nameof(GetEnumerables))] + public void AcceptsEnumerables(IEnumerable arg) + { + if (arg.IsEmpty()) + throw new ArgumentException("Incorrect length"); + } + + [Benchmark] + [ArgumentsSource(nameof(GetEnumerables))] + public void AcceptsEnumerables2(IEnumerable arg) + { + if (arg.IsEmpty()) + throw new ArgumentException("Incorrect length"); + } + } + + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void IEnumerableArgumentsOfDifferentLengthsCanBeGrouped(IToolchain toolchain) + { + var summary = CanExecute(toolchain, (config) => config.AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByParams)); // We must group by params to test array argument grouping + + // There should be two logical groups, one for the first argument and one for the second argument + // Thus there should be two pairs per descriptor, and each pair should be distinct because it belongs to a different group + + var descriptorGroupPairs = summary.BenchmarksCases.Select(benchmarkCase => (benchmarkCase.Descriptor, summary.GetLogicalGroupKey(benchmarkCase))).GroupBy(benchmarkCase => benchmarkCase.Descriptor); + + Assert.True( + descriptorGroupPairs.All(group => group.Select(pair => pair.Item2).Distinct().Count() == 2) + ); + } + + public class EnumerableOfDiffLengthOnDifferentMethods + { + public IEnumerable> GetEnumerables() + { + yield return Enumerable.Range(1, 10); + yield return Enumerable.Range(1, 11); + } + + [Benchmark(Baseline = true)] + [ArgumentsSource(nameof(GetEnumerables))] + public void AcceptsEnumerables(IEnumerable arg) + { + if (arg.IsEmpty()) + throw new ArgumentException("Incorrect length"); + } + + [Benchmark] + [ArgumentsSource(nameof(GetEnumerables))] + public void AcceptsEnumerables2(IEnumerable arg) + { + if (arg.IsEmpty()) + throw new ArgumentException("Incorrect length"); + } + } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void JaggedArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); @@ -311,9 +393,9 @@ public void Test(int[][] array) throw new ArgumentNullException(nameof(array)); for (int i = 0; i < 10; i++) - for (int j = 0; j < i; j++) - if (array[i][j] != i) - throw new ArgumentException("Invalid value"); + for (int j = 0; j < i; j++) + if (array[i][j] != i) + throw new ArgumentException("Invalid value"); } public IEnumerable CreateMatrix() @@ -543,6 +625,46 @@ public void AcceptsArrays(int[] even, int[] notEven) } } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void ArrayArgumentsCanBeGrouped(IToolchain toolchain) + { + var summary = CanExecute(toolchain, (config) => config.AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByParams)); // We must group by params to test array argument grouping + + // There should be two logical groups, one for the first argument and one for the second argument + // Thus there should be two pairs per descriptor, and each pair should be distinct because it belongs to a different group + + var descriptorGroupPairs = summary.BenchmarksCases.Select(benchmarkCase => (benchmarkCase.Descriptor, summary.GetLogicalGroupKey(benchmarkCase))).GroupBy(benchmarkCase => benchmarkCase.Descriptor); + + Assert.True( + descriptorGroupPairs.All(group => group.Select(pair => pair.Item2).Distinct().Count() == 2) + ); + } + + public class ArrayOnDifferentMethods + { + public IEnumerable GetArrays() + { + yield return new int[] { 1, 2, 3 }; + yield return new int[] { 2, 3, 4 }; + } + + [Benchmark(Baseline = true)] + [ArgumentsSource(nameof(GetArrays))] + public void AcceptsArrays(int[] arr) + { + if (arr.Length != 3) + throw new ArgumentException("Incorrect length"); + } + + [Benchmark] + [ArgumentsSource(nameof(GetArrays))] + public void AcceptsArrays2(int[] arr) + { + if (arr.Length != 3) + throw new ArgumentException("Incorrect length"); + } + } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void VeryBigIntegersAreSupported(IToolchain toolchain) => CanExecute(toolchain); @@ -1026,6 +1148,16 @@ public Disposable(int id) } } - private void CanExecute(IToolchain toolchain) => CanExecute(CreateSimpleConfig(job: Job.Dry.WithToolchain(toolchain))); + private Reports.Summary CanExecute(IToolchain toolchain, Func furtherConfigure = null) + { + var config = CreateSimpleConfig(job: Job.Dry.WithToolchain(toolchain)); + + if (furtherConfigure is not null) + { + config = furtherConfigure(config); + } + + return CanExecute(config); + } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt index 314e1f16bd..e7fedb599a 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt @@ -8,14 +8,14 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|---------------------------- |--------- | - Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | Invalid_TwoJobBaselines.Foo | Yes | - Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 2 | Invalid_TwoJobBaselines.Foo | Yes | - | | | | | | | | | | - Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | Invalid_TwoJobBaselines.Bar | Yes | - Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.94 | 0.09 | 2 | Invalid_TwoJobBaselines.Bar | Yes | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- | + Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-Invalid_TwoJobBaselines.Foo | Yes | + Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 2 | DistinctParamSet0-Invalid_TwoJobBaselines.Foo | Yes | + | | | | | | | | | | + Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | DistinctParamSet0-Invalid_TwoJobBaselines.Bar | Yes | + Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.94 | 0.09 | 2 | DistinctParamSet0-Invalid_TwoJobBaselines.Bar | Yes | Errors: 2 -* Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Foo in class Invalid_TwoJobBaselines has 2 -* Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Bar in class Invalid_TwoJobBaselines has 2 +* Only 1 job in a group can have "Baseline = true" applied to it, group DistinctParamSet0-Invalid_TwoJobBaselines.Foo in class Invalid_TwoJobBaselines has 2 +* Only 1 job in a group can have "Baseline = true" applied to it, group DistinctParamSet0-Invalid_TwoJobBaselines.Bar in class Invalid_TwoJobBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt index 11fbd70a0b..61eb0a37dc 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt @@ -7,10 +7,10 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC DefaultJob : extra output line - Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DefaultJob | Yes | - Bar | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DefaultJob | Yes | + Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | + Foo | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-DefaultJob | Yes | + Bar | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-DefaultJob | Yes | Errors: 1 -* Only 1 benchmark method in a group can have "Baseline = true" applied to it, group DefaultJob in class Invalid_TwoMethodBaselines has 2 +* Only 1 benchmark method in a group can have "Baseline = true" applied to it, group DistinctParamSet0-DefaultJob in class Invalid_TwoMethodBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt index 5c8cbaad01..2bb1e98434 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt @@ -8,15 +8,15 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | - Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | JobBaseline_MethodsJobs.Base | Yes | - Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | JobBaseline_MethodsJobs.Base | No | - | | | | | | | | | | - Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | JobBaseline_MethodsJobs.Foo | Yes | - Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | JobBaseline_MethodsJobs.Foo | No | - | | | | | | | | | | - Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | JobBaseline_MethodsJobs.Bar | Yes | - Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | JobBaseline_MethodsJobs.Bar | No | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------------------------- |--------- | + Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-JobBaseline_MethodsJobs.Base | Yes | + Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | DistinctParamSet0-JobBaseline_MethodsJobs.Base | No | + | | | | | | | | | | + Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | DistinctParamSet0-JobBaseline_MethodsJobs.Foo | Yes | + Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | DistinctParamSet0-JobBaseline_MethodsJobs.Foo | No | + | | | | | | | | | | + Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | DistinctParamSet0-JobBaseline_MethodsJobs.Bar | Yes | + Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | DistinctParamSet0-JobBaseline_MethodsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt index 4f5f37b6c7..150f6b5d91 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt @@ -8,24 +8,24 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- | - Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No | - | | | | | | | | | | | - Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No | - | | | | | | | | | | | - Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No | - | | | | | | | | | | | - Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.42 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No | - | | | | | | | | | | | - Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.37 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No | - | | | | | | | | | | | - Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.33 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|----------------------------------------------------- |--------- | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Base | No | + | | | | | | | | | | | + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Foo | No | + | | | | | | | | | | | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Bar | No | + | | | | | | | | | | | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.42 | 0.02 | 2 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Base | No | + | | | | | | | | | | | + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.37 | 0.02 | 2 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Foo | No | + | | | | | | | | | | | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.33 | 0.02 | 2 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt index aa22c9caab..9bbfa03f47 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt @@ -7,10 +7,10 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC DefaultJob : extra output line - Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Base | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DefaultJob | Yes | - Foo | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DefaultJob | No | - Bar | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DefaultJob | No | + Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | + Base | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-DefaultJob | Yes | + Foo | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-DefaultJob | No | + Bar | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt index 297f782355..6741559eb2 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt @@ -8,14 +8,14 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | Job1 | Yes | - Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | Job1 | No | - Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | Job1 | No | - | | | | | | | | | | - Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | Job2 | Yes | - Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | Job2 | No | - Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | Job2 | No | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------- |--------- | + Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-Job1 | Yes | + Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-Job1 | No | + Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0-Job1 | No | + | | | | | | | | | | + Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | DistinctParamSet0-Job2 | Yes | + Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | DistinctParamSet0-Job2 | No | + Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | DistinctParamSet0-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt index e30ce8166a..eff7eb4cf2 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt @@ -7,14 +7,14 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC DefaultJob : extra output line - Method | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |------ |---------:|--------:|--------:|------:|--------:|-----:|---------------------- |--------- | - Base | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-DefaultJob | Yes | ^ - Foo | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2]-DefaultJob | No | - Bar | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2]-DefaultJob | No | - | | | | | | | | | | - Base | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | [Param=10]-DefaultJob | Yes | ^ - Foo | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | [Param=10]-DefaultJob | No | - Bar | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | [Param=10]-DefaultJob | No | + Method | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |------ |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | + Base | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-DefaultJob | Yes | ^ + Foo | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-DefaultJob | No | + Bar | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0-DefaultJob | No | + | | | | | | | | | | + Base | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | DistinctParamSet1-DefaultJob | Yes | ^ + Foo | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | DistinctParamSet1-DefaultJob | No | + Bar | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | DistinctParamSet1-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt index f85ce9e878..3b94cd3819 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt @@ -8,22 +8,22 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------- |--------- | - Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-Job1 | Yes | ^ - Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2]-Job1 | No | - Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2]-Job1 | No | - | | | | | | | | | | | - Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | [Param=2]-Job2 | Yes | - Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | [Param=2]-Job2 | No | - Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | [Param=2]-Job2 | No | - | | | | | | | | | | | - Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-Job1 | Yes | ^ - Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | [Param=10]-Job1 | No | - Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.28 | 0.02 | 3 | [Param=10]-Job1 | No | - | | | | | | | | | | | - Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | [Param=10]-Job2 | Yes | - Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.10 | 0.01 | 2 | [Param=10]-Job2 | No | - Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.20 | 0.01 | 3 | [Param=10]-Job2 | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|----------------------- |--------- | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-Job1 | Yes | ^ + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-Job1 | No | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0-Job1 | No | + | | | | | | | | | | | + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | DistinctParamSet0-Job2 | Yes | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | DistinctParamSet0-Job2 | No | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | DistinctParamSet0-Job2 | No | + | | | | | | | | | | | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | DistinctParamSet1-Job1 | Yes | ^ + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | DistinctParamSet1-Job1 | No | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.28 | 0.02 | 3 | DistinctParamSet1-Job1 | No | + | | | | | | | | | | | + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | DistinctParamSet1-Job2 | Yes | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.10 | 0.01 | 2 | DistinctParamSet1-Job2 | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.20 | 0.01 | 3 | DistinctParamSet1-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt index e71eeca6b6..1074de050f 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt @@ -8,11 +8,11 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | * | Yes | - Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | * | No | - Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | * | No | - Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | * | No | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------------ |--------- | + Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0 | Yes | + Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0 | No | + Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0 | No | + Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | DistinctParamSet0 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt index 30d92f0b9e..173aff0467 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt @@ -8,16 +8,16 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2] | Yes | ^ - Bar | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2] | No | - Foo | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2] | No | - Bar | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | [Param=2] | No | - | | | | | | | | | | | - Foo | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10] | Yes | ^ - Bar | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | [Param=10] | No | - Foo | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.39 | 0.03 | 3 | [Param=10] | No | - Bar | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.58 | 0.03 | 4 | [Param=10] | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |---------:|--------:|--------:|------:|--------:|-----:|------------------ |--------- | + Foo | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0 | Yes | ^ + Bar | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0 | No | + Foo | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0 | No | + Bar | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | DistinctParamSet0 | No | + | | | | | | | | | | | + Foo | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | DistinctParamSet1 | Yes | ^ + Bar | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | DistinctParamSet1 | No | + Foo | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.39 | 0.03 | 3 | DistinctParamSet1 | No | + Bar | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.58 | 0.03 | 4 | DistinctParamSet1 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt index a7a7350881..9012bdff5e 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt @@ -8,38 +8,38 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------------------------- |--------- | - A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^ - | | | | | | | | | | | - A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^ - | | | | | | | | | | | - B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^ - | | | | | | | | | | | - B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^ + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|----------------------------------------------------------------------- |--------- | + A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-DistinctParamSet0-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-DistinctParamSet1-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-DistinctParamSet0-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-DistinctParamSet1-CatA | Yes | ^ + | | | | | | | | | | | + A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-DistinctParamSet0-CatA | No | ^ + | | | | | | | | | | | + A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-DistinctParamSet1-CatA | No | ^ + | | | | | | | | | | | + A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-DistinctParamSet0-CatA | No | ^ + | | | | | | | | | | | + A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-DistinctParamSet1-CatA | No | ^ + | | | | | | | | | | | + B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-DistinctParamSet0-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-DistinctParamSet1-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-DistinctParamSet0-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-DistinctParamSet1-CatB | Yes | ^ + | | | | | | | | | | | + B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-DistinctParamSet0-CatB | No | ^ + | | | | | | | | | | | + B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-DistinctParamSet1-CatB | No | ^ + | | | | | | | | | | | + B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-DistinctParamSet0-CatB | No | ^ + | | | | | | | | | | | + B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-DistinctParamSet1-CatB | No | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt index bb98037dd3..af6dca36cd 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt @@ -8,30 +8,30 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|--------------------- |--------- | - A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | CatA-[Param=2]-Job1 | Yes | ^ - A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | CatA-[Param=2]-Job1 | No | - | | | | | | | | | | | - A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | CatA-[Param=2]-Job2 | Yes | - A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.32 | 0.05 | 2 | CatA-[Param=2]-Job2 | No | - | | | | | | | | | | | - A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-[Param=10]-Job1 | Yes | ^ - A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | CatA-[Param=10]-Job1 | No | - | | | | | | | | | | | - A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-[Param=10]-Job2 | Yes | - A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | CatA-[Param=10]-Job2 | No | - | | | | | | | | | | | - B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=2]-Job1 | Yes | ^ - B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.11 | 0.01 | 2 | CatB-[Param=2]-Job1 | No | - | | | | | | | | | | | - B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=2]-Job2 | Yes | - B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.09 | 0.01 | 2 | CatB-[Param=2]-Job2 | No | - | | | | | | | | | | | - B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=10]-Job1 | Yes | ^ - B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | 1.08 | 0.01 | 2 | CatB-[Param=10]-Job1 | No | - | | | | | | | | | | | - B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=10]-Job2 | Yes | - B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | 1.07 | 0.01 | 2 | CatB-[Param=10]-Job2 | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------- |--------- | + A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | CatA-DistinctParamSet0-Job1 | Yes | ^ + A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | CatA-DistinctParamSet0-Job1 | No | + | | | | | | | | | | | + A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | CatA-DistinctParamSet0-Job2 | Yes | + A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.32 | 0.05 | 2 | CatA-DistinctParamSet0-Job2 | No | + | | | | | | | | | | | + A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-DistinctParamSet1-Job1 | Yes | ^ + A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | CatA-DistinctParamSet1-Job1 | No | + | | | | | | | | | | | + A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-DistinctParamSet1-Job2 | Yes | + A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | CatA-DistinctParamSet1-Job2 | No | + | | | | | | | | | | | + B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-DistinctParamSet0-Job1 | Yes | ^ + B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.11 | 0.01 | 2 | CatB-DistinctParamSet0-Job1 | No | + | | | | | | | | | | | + B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-DistinctParamSet0-Job2 | Yes | + B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.09 | 0.01 | 2 | CatB-DistinctParamSet0-Job2 | No | + | | | | | | | | | | | + B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-DistinctParamSet1-Job1 | Yes | ^ + B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | 1.08 | 0.01 | 2 | CatB-DistinctParamSet1-Job1 | No | + | | | | | | | | | | | + B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-DistinctParamSet1-Job2 | Yes | + B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | 1.07 | 0.01 | 2 | CatB-DistinctParamSet1-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt index 6fa6a6534a..a481b0f6f9 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt @@ -8,20 +8,20 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | - Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | [Param=2] | No | ^ - Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | [Param=2] | No | - Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 3 | [Param=2] | No | - Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 4 | [Param=2] | No | - Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 5 | [Param=2] | No | - Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 6 | [Param=2] | No | - | | | | | | | | | - Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 1 | [Param=10] | No | ^ - Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 2 | [Param=10] | No | - Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 3 | [Param=10] | No | - Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 4 | [Param=10] | No | - Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 5 | [Param=10] | No | - Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 6 | [Param=10] | No | + Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|-----:|------------------ |--------- | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | DistinctParamSet0 | No | ^ + Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | DistinctParamSet0 | No | + Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 3 | DistinctParamSet0 | No | + Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 4 | DistinctParamSet0 | No | + Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 5 | DistinctParamSet0 | No | + Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 6 | DistinctParamSet0 | No | + | | | | | | | | | + Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 1 | DistinctParamSet1 | No | ^ + Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 2 | DistinctParamSet1 | No | + Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 3 | DistinctParamSet1 | No | + Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 4 | DistinctParamSet1 | No | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 5 | DistinctParamSet1 | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 6 | DistinctParamSet1 | No | Errors: 0