Merge pull request #902 from dotnet/moreCommandLineArgs

More command line args

docs/articles/guides/console-args.md

@@ -43,12 +43,57 @@ You can also filter the benchmarks by categories:
 
 ## Runtimes
 
-The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: Clr, Mono, Core and CoreRT.
+The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: Mono, CoreRT, net46, net461, net462, net47, net471, net472, netcoreapp2.0, netcoreapp2.1, netcoreapp2.2, netcoreapp3.0.
 
-Example: run the benchmarks for .NET and .NET Core:
+Example: run the benchmarks for .NET 4.7.2 and .NET Core 2.1:
 
 ```log
-dotnet run -c Release -- --runtimes clr core
+dotnet run -c Release -- --runtimes net472 netcoreapp2.1
+```
+
+## Number of invocations and iterations
+
+* `--launchCount` - how many times we should launch process with target benchmark. The default is 1.
+* `--warmupCount` - how many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic.
+* `--minWarmupCount` - minimum count of warmup iterations that should be performed. The default is 6.
+* `--maxWarmupCount` - maximum count of warmup iterations that should be performed. The default is 50.
+* `--iterationTime` - desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default.
+* `--iterationCount` - how many target iterations should be performed. By default calculated by the heuristic.
+* `--minIterationCount` - minimum number of iterations to run. The default is 15.
+* `--maxIterationCount` - maximum number of iterations to run. The default is 100.
+* `--invocationCount` - invocation count in a single iteration. By default calculated by the heuristic.
+* `--unrollFactor` - how many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default
+* `--runOncePerIteration` - run the benchmark exactly once per iteration. False by default.
+
+Example: run single warmup iteration, from 9 to 12 actual workload iterations.
+
+```log
+dotnet run -c Release -- --warmupCount 1 --minIterationCount 9 --maxIterationCount 12
+```
+
+## Specifying custom default settings for console argument parser
+
+If you want to have a possibility to specify custom default Job settings programmatically and optionally overwrite it with console line arguments, then you should create a global config with single job marked as `.AsDefault` and pass it to `BenchmarkSwitcher` together with the console line arguments.
+
+Example: run single warmup iteration by default.
+
+```cs
+static void Main(string[] args)
+    => BenchmarkSwitcher
+        .FromAssembly(typeof(Program).Assembly)
+        .Run(args, GetGlobalConfig());
+
+static IConfig GetGlobalConfig()
+    => DefaultConfig.Instance
+        .With(Job.Default
+            .WithWarmupCount(1)
+            .AsDefault()); // the KEY to get it working
+```
+
+Now, the default settings are: `WarmupCount=1` but you might still overwrite it from console args like in the example below: 
+
+```log
+dotnet run -c Release -- --warmupCount 2
 ```
 
 ## More
@@ -61,4 +106,7 @@ dotnet run -c Release -- --runtimes clr core
 * `--affinity` Affinity mask to set for the benchmark process
 * `--allStats` (Default: false) Displays all statistics (min, max & more)
 * `--attribute` Run all methods with given attribute (applied to class or method)
+* `--monoPath` custom Path for Mono
+* `--cliPath` custom Path for dotnet cli
+* `--coreRt` path to ILCompiler for CoreRT
 

src/BenchmarkDotNet/Configs/ConfigExtensions.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using BenchmarkDotNet.Analysers;
@@ -60,6 +61,38 @@ config is ReadOnlyConfig readOnly
 
         public static bool HasMemoryDiagnoser(this IConfig config) => config.GetDiagnosers().Any(diagnoser => diagnoser is MemoryDiagnoser);
 
+        /// <summary>
+        /// returns a set of unique jobs that are ready to run
+        /// </summary>
+        public static IReadOnlyList<Job> GetRunnableJobs(this IConfig config)
+        {
+            var unique = config.GetJobs().Distinct().ToArray();
+            var result = new List<Job>();
+
+            foreach (var standardJob in unique.Where(job => !job.Meta.IsMutator && !job.Meta.IsDefault))
+                result.Add(standardJob);
+
+            var customDefaultJob = unique.SingleOrDefault(job => job.Meta.IsDefault);
+            var defaultJob = customDefaultJob ?? Job.Default;
+
+            if (!result.Any())
+                result.Add(defaultJob);
+
+            foreach (var mutatorJob in unique.Where(job => job.Meta.IsMutator))
+            {
+                for (int i = 0; i < result.Count; i++)
+                {
+                    var copy = result[i].UnfreezeCopy();
+
+                    copy.Apply(mutatorJob);
+
+                    result[i] = copy.Freeze();
+                }
+            }
+
+            return result;
+        }
+
         private static IConfig With(this IConfig config, Action<ManualConfig> addAction)
         {
             var manualConfig = ManualConfig.Create(config);

src/BenchmarkDotNet/Configs/ManualConfig.cs

@@ -73,7 +73,7 @@ public void Add(IConfig config)
             loggers.AddRange(config.GetLoggers());
             diagnosers.AddRange(config.GetDiagnosers());
             analysers.AddRange(config.GetAnalysers());
-            AddJobs(config.GetJobs());
+            jobs.AddRange(config.GetJobs());
             validators.AddRange(config.GetValidators());
             hardwareCounters.AddRange(config.GetHardwareCounters());
             filters.AddRange(config.GetFilters());
@@ -161,29 +161,5 @@ public static ManualConfig Union(IConfig globalConfig, IConfig localConfig)
             }
             return manualConfig;
         }
-
-        private void AddJobs(IEnumerable<Job> toAdd)
-        {
-            var toAddList = toAdd.ToList();
-            foreach (var notMutator in toAddList.Where(job => !job.Meta.IsMutator))
-                jobs.Add(notMutator);
-
-            var mutators = toAddList.Where(job => job.Meta.IsMutator).ToArray();
-            if (!mutators.Any())
-                return;
-
-            if (!jobs.Any())
-                jobs.Add(Job.Default);
-
-            for (int i = 0; i < jobs.Count; i++)
-            {
-                var copy = jobs[i].UnfreezeCopy();
-
-                foreach (var mutator in mutators)
-                    copy.Apply(mutator);
-
-                jobs[i] = copy.Freeze();
-            }
-        }
     }
 }

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
+using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Mathematics;
 using BenchmarkDotNet.Portability;
 using CommandLine;
@@ -16,7 +17,7 @@ public class CommandLineOptions
         [Option('j', "job", Required = false, Default = "Default", HelpText = "Dry/Short/Medium/Long or Default")]
         public string BaseJob { get; set; }
 
-        [Option('r', "runtimes", Required = false, HelpText = "Clr/Core/Mono/CoreRt")]
+        [Option('r', "runtimes", Required = false, HelpText = "Full target framework moniker for .NET Core and .NET. For Mono just 'Mono', for CoreRT just 'CoreRT'. First one will be marked as baseline!")]
         public IEnumerable<string> Runtimes { get; set; }
 
         [Option('e', "exporters", Required = false, HelpText = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML")]
@@ -58,33 +59,87 @@ public class CommandLineOptions
         [Option("join", Required = false, Default = false, HelpText = "Prints single table with results for all benchmarks")]
         public bool Join { get; set; }
 
+        [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")]
+        public bool KeepBenchmarkFiles { get; set; }
+
+        [Option("counters", Required = false, HelpText = "Hardware Counters", Separator = '+')]
+        public IEnumerable<string> HardwareCounters { get; set; }
+
         [Option("cli", Required = false, HelpText = "Path to dotnet cli (optional).")]
         public FileInfo CliPath { get; set; }
 
         [Option("coreRun", Required = false, HelpText = "Path to CoreRun (optional).")]
         public FileInfo CoreRunPath { get; set; }
+
+        [Option("monoPath", Required = false, HelpText = "Optional path to Mono which should be used for running benchmarks.")]
+        public FileInfo MonoPath { get; set; }
+
+        [Option("clrVersion", Required = false, HelpText = "Optional version of private CLR build used as the value of COMPLUS_Version env var.")]
+        public string ClrVersion { get; set; }
 
-        [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")]
-        public bool KeepBenchmarkFiles { get; set; }
+        [Option("coreRtVersion", Required = false, HelpText = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with CoreRT. Example: \"1.0.0-alpha-26414-01\"")]
+        public string CoreRtVersion { get; set; }
+
+        [Option("ilcPath", Required = false, HelpText = "Optional IlcPath which should be used to run with private CoreRT build.")]
+        public DirectoryInfo CoreRtPath { get; set; }
+
+        [Option("launchCount", Required = false, HelpText = "How many times we should launch process with target benchmark. The default is 1.")]
+        public int? LaunchCount { get; set; }
+
+        [Option("warmupCount", Required = false, HelpText = "How many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic.")]
+        public int? WarmupIterationCount { get; set; }
+
+        [Option("minWarmupCount", Required = false, HelpText = "Minimum count of warmup iterations that should be performed. The default is 6.")]
+        public int? MinWarmupIterationCount { get; set; }
+
+        [Option("maxWarmupCount", Required = false, HelpText = "Maximum count of warmup iterations that should be performed. The default is 50.")]
+        public int? MaxWarmupIterationCount { get; set; }
+
+        [Option("iterationTime", Required = false, HelpText = "Desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default")]
+        public int? IterationTimeInMiliseconds { get; set; }
+
+        [Option("iterationCount", Required = false, HelpText = "How many target iterations should be performed. By default calculated by the heuristic.")]
+        public int? IterationCount { get; set; }
+
+        [Option("minIterationCount", Required = false, HelpText = "Minimum number of iterations to run. The default is 15.")]
+        public int? MinIterationCount { get; set; }
+
+        [Option("maxIterationCount", Required = false, HelpText = "Maximum number of iterations to run. The default is 100.")]
+        public int? MaxIterationCount { get; set; }
+
+        [Option("invocationCount", Required = false, HelpText = "Invocation count in a single iteration. By default calculated by the heuristic.")]
+        public int? InvocationCount { get; set; }
+
+        [Option("unrollFactor", Required = false, HelpText = "How many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default")]
+        public int? UnrollFactor { get; set; }
+
+        [Option("runOncePerIteration", Required = false, Default = false, HelpText = "Run the benchmark exactly once per iteration.")]
+        public bool RunOncePerIteration { get; set; }
 
         [Usage(ApplicationAlias = "")]
         [PublicAPI]
         public static IEnumerable<Example> Examples
         {
             get
             {
-                var style = new UnParserSettings { PreferShortName = true };
-
-                yield return new Example("Use Job.ShortRun for running the benchmarks", style, new CommandLineOptions { BaseJob = "short" });
-                yield return new Example("Run benchmarks in process", style, new CommandLineOptions { RunInProcess = true });
-                yield return new Example("Run benchmarks for Clr, Core and Mono", style, new CommandLineOptions { Runtimes = new[] { "Clr", "Core", "Mono" } });
-                yield return new Example("Use MemoryDiagnoser to get GC stats", style, new CommandLineOptions { UseMemoryDiagnoser = true });
-                yield return new Example("Use DisassemblyDiagnoser to get disassembly", style, new CommandLineOptions { UseDisassemblyDiagnoser = true });
-                yield return new Example("Run all benchmarks exactly once", style, new CommandLineOptions { BaseJob = "Dry", Filters = new[] { HandleWildcardsOnUnix("*") } });
-                yield return new Example("Run all benchmarks from System.Memory namespace", style, new CommandLineOptions { Filters = new[] { HandleWildcardsOnUnix("System.Memory*") } });
-                yield return new Example("Run all benchmarks from ClassA and ClassB using type names", style, new CommandLineOptions { Filters = new[] { "ClassA", "ClassB" } });
-                yield return new Example("Run all benchmarks from ClassA and ClassB using patterns", style, new CommandLineOptions { Filters = new[] { HandleWildcardsOnUnix("*.ClassA.*"), HandleWildcardsOnUnix("*.ClassB.*") } });
-                yield return new Example("Run all benchmarks called `BenchmarkName` and show the results in single summary", style, new CommandLineOptions { Join = true, Filters = new[] { HandleWildcardsOnUnix("*.BenchmarkName") } });
+                var shortName = new UnParserSettings { PreferShortName = true };
+                var longName = new UnParserSettings { PreferShortName = false };
+
+                yield return new Example("Use Job.ShortRun for running the benchmarks", shortName, new CommandLineOptions { BaseJob = "short" });
+                yield return new Example("Run benchmarks in process", shortName, new CommandLineOptions { RunInProcess = true });
+                yield return new Example("Run benchmarks for .NET 4.7.2, .NET Core 2.1 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "net472", "netcoreapp2.1", "Mono" } });
+                yield return new Example("Run benchmarks for .NET Core 2.0, .NET Core 2.1 and .NET Core 2.2. .NET Core 2.0 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "netcoreapp2.0", "netcoreapp2.1", "netcoreapp2.2" } });
+                yield return new Example("Use MemoryDiagnoser to get GC stats", shortName, new CommandLineOptions { UseMemoryDiagnoser = true });
+                yield return new Example("Use DisassemblyDiagnoser to get disassembly", shortName, new CommandLineOptions { UseDisassemblyDiagnoser = true });
+                yield return new Example("Use HardwareCountersDiagnoser to get hardware coutner info", longName, new CommandLineOptions { HardwareCounters = new [] { nameof(HardwareCounter.CacheMisses), nameof(HardwareCounter.InstructionRetired) } });
+                yield return new Example("Run all benchmarks exactly once", shortName, new CommandLineOptions { BaseJob = "Dry", Filters = new[] { HandleWildcardsOnUnix("*") } });
+                yield return new Example("Run all benchmarks from System.Memory namespace", shortName, new CommandLineOptions { Filters = new[] { HandleWildcardsOnUnix("System.Memory*") } });
+                yield return new Example("Run all benchmarks from ClassA and ClassB using type names", shortName, new CommandLineOptions { Filters = new[] { "ClassA", "ClassB" } });
+                yield return new Example("Run all benchmarks from ClassA and ClassB using patterns", shortName, new CommandLineOptions { Filters = new[] { HandleWildcardsOnUnix("*.ClassA.*"), HandleWildcardsOnUnix("*.ClassB.*") } });
+                yield return new Example("Run all benchmarks called `BenchmarkName` and show the results in single summary", longName, new CommandLineOptions { Join = true, Filters = new[] { HandleWildcardsOnUnix("*.BenchmarkName") } });
+                yield return new Example("Run selected benchmarks once per iteration", longName, new CommandLineOptions { RunOncePerIteration = true });
+                yield return new Example("Run selected benchmarks 100 times per iteration. Perform single warmup iteration and 5 actual workload iterations", longName, new CommandLineOptions { InvocationCount = 100, WarmupIterationCount = 1, IterationCount = 5});
+                yield return new Example("Run selected benchmarks 250ms per iteration. Perform from 9 to 15 iterations", longName, new CommandLineOptions { IterationTimeInMiliseconds = 250, MinIterationCount = 9, MaxIterationCount = 15});
             }
         }