allow the users to run benchmarks with CoreRun, #857

docs/articles/configs/toolchains.md

@@ -133,6 +133,16 @@ Frequency=2742185 Hz, Resolution=364.6727 ns, Timer=TSC
 Jit=RyuJit  
 ```
 
+This feature is now also exposed with the `--cli` console argument.
+
+Example: `dotnet run -c Release -- --cli "C:\Projects\machinelearning\Tools\dotnetcli\dotnet.exe"`
+
+## CoreRun
+
+To use CoreRun for running the benchmarks you need to use `--coreRun `command line argument. You can combine it with `--cli` described above. This is most probably the easiest and most reliable way of running benchmarks against local CoreFX/CoreCLR builds.
+
+Example: `dotnet run -c Release -- --coreRun "C:\Projects\corefx\bin\testhost\netcoreapp-Windows_NT-Release-x64\shared\Microsoft.NETCore.App\9.9.9\CoreRun.exe"`
+
 ## Custom CoreCLR and CoreFX
 
 BenchmarkDotNet allows the users to run their benchmarks against ANY CoreCLR and CoreFX builds. You can compare your local build vs MyGet feed or Debug vs Release or one version vs another.

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -56,6 +56,12 @@ public class CommandLineOptions
 
         [Option("join", Required = false, Default = false, HelpText = "Prints single table with results for all benchmarks")]
         public bool Join { get; set; }
+
+        [Option("cli", Required = false, HelpText = "Path to dotnet cli (optional).")]
+        public FileInfo CliPath { get; set; }
+
+        [Option("coreRun", Required = false, HelpText = "Path to CoreRun (optional).")]
+        public FileInfo CoreRunPath { get; set; }
 
         [Usage(ApplicationAlias = "")]
         [PublicAPI]

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -11,10 +11,14 @@
 using BenchmarkDotNet.Exporters.Csv;
 using BenchmarkDotNet.Exporters.Json;
 using BenchmarkDotNet.Exporters.Xml;
+using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Mathematics;
+using BenchmarkDotNet.Toolchains.CoreRun;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.DotNetCli;
 using BenchmarkDotNet.Toolchains.InProcess;
 using CommandLine;
 
@@ -112,6 +116,18 @@ private static bool Validate(CommandLineOptions options, ILogger logger)
                     logger.WriteLineError($"The provided exporter \"{exporter}\" is invalid. Available options are: {string.Join(", ", AvailableExporters.Keys)}.");
                     return false;
                 }
+
+            if (options.CliPath != null && !options.CliPath.Exists)
+            {
+                logger.WriteLineError($"The provided {nameof(options.CliPath)} \"{options.CliPath}\" does NOT exist.");
+                return false;
+            }
+
+            if (options.CoreRunPath != null && !options.CoreRunPath.Exists)
+            {
+                logger.WriteLineError($"The provided {nameof(options.CoreRunPath)} \"{options.CoreRunPath}\" does NOT exist.");
+                return false;
+            }
 
             return true;
         }
@@ -120,8 +136,11 @@ private static ReadOnlyConfig CreateConfig(CommandLineOptions options)
         {
             var config = new ManualConfig();
 
-            config.Add(Expand(GetBaseJob(options), options).ToArray());
-
+            var baseJob = GetBaseJob(options);
+            config.Add(Expand(baseJob, options).ToArray());
+            if (config.GetJobs().IsEmpty() && baseJob != Job.Default)
+                config.Add(baseJob);
+
             config.Add(options.Exporters.SelectMany(exporter => AvailableExporters[exporter]).ToArray());
 
             if (options.UseMemoryDiagnoser)
@@ -167,8 +186,10 @@ private static IEnumerable<Job> Expand(Job baseJob, CommandLineOptions options)
             foreach (string runtime in options.Runtimes)
                 yield return baseJob.With(AvailableRuntimes[runtime.ToLowerInvariant()]);
 
-            if (!options.RunInProcess && !options.Runtimes.Any() && baseJob != Job.Default)
-                yield return baseJob;
+            if (options.CoreRunPath != null)
+                yield return CreateCoreRunJob(baseJob, options);
+            else if (options.CliPath != null)
+                yield return baseJob.With(CsProjCoreToolchain.From(NetCoreAppSettings.GetCurrentVersion().WithCustomDotNetCliPath(options.CliPath.FullName, "cli")));
         }
 
         private static IEnumerable<IFilter> GetFilters(CommandLineOptions options)
@@ -194,5 +215,14 @@ private static int GetMaximumDisplayWidth()
                 return MinimumDisplayWidth;
             }
         }
+
+        private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options)
+            => baseJob
+                .With(Runtime.Core)
+                .With(new CoreRunToolchain(
+                    options.CoreRunPath,
+                    createCopy: true,
+                    targetFrameworkMoniker: NetCoreAppSettings.GetCurrentVersion().TargetFrameworkMoniker,
+                    customDotNetCliPath: options.CliPath));
     }
 }

src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunGenerator.cs

@@ -0,0 +1,55 @@
+using System;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Toolchains.CsProj;
+
+namespace BenchmarkDotNet.Toolchains.CoreRun
+{
+    public class CoreRunGenerator : CsProjGenerator
+    {
+        public CoreRunGenerator(FileInfo sourceCoreRun, FileInfo copyCoreRun, string targetFrameworkMoniker, Func<Platform, string> platformProvider)
+            : base(targetFrameworkMoniker, platformProvider)
+        {
+            SourceCoreRun = sourceCoreRun;
+            CopyCoreRun = copyCoreRun;
+        }
+
+        private FileInfo SourceCoreRun { get; }
+
+        private FileInfo CopyCoreRun { get; }
+
+        private bool NeedsCopy => SourceCoreRun != CopyCoreRun;
+
+        protected override string GetPackagesDirectoryPath(string buildArtifactsDirectoryPath) 
+            => null; // we don't want to restore to a dedicated folder
+
+        protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration)
+            => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, "publish");
+
+        protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths)
+        {
+            if (NeedsCopy)
+                CopyFilesRecursively(SourceCoreRun.Directory, CopyCoreRun.Directory);
+
+            base.CopyAllRequiredFiles(artifactsPaths);
+        }
+
+        protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) 
+            => NeedsCopy 
+                ? base.GetArtifactsToCleanup(artifactsPaths).Concat(new [] { CopyCoreRun.Directory.FullName }).ToArray() 
+                : base.GetArtifactsToCleanup(artifactsPaths);
+
+        // source: https://stackoverflow.com/a/58779/5852046
+        private static void CopyFilesRecursively(DirectoryInfo source, DirectoryInfo target) 
+        {
+            if (!target.Exists)
+                target.Create();
+
+            foreach (DirectoryInfo dir in source.GetDirectories())
+                CopyFilesRecursively(dir, target.CreateSubdirectory(dir.Name));
+            foreach (FileInfo file in source.GetFiles())
+                file.CopyTo(Path.Combine(target.FullName, file.Name));
+        }
+    }
+}

src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using BenchmarkDotNet.Toolchains.Results;
+
+namespace BenchmarkDotNet.Toolchains.CoreRun
+{
+    public class CoreRunPublisher : IBuilder
+    {
+        public CoreRunPublisher(FileInfo coreRun, FileInfo customDotNetCliPath = null)
+        {
+            CoreRun = coreRun;
+            DotNetCliPublisher = new DotNetCliPublisher(customDotNetCliPath?.FullName);
+        }
+
+        private FileInfo CoreRun { get; }
+
+        private DotNetCliPublisher DotNetCliPublisher { get; }
+
+        public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
+        {
+            var buildResult = DotNetCliPublisher.Build(generateResult, buildPartition, logger);
+
+            if (buildResult.IsBuildSuccess)
+                UpdateDuplicatedDependencies(buildResult.ArtifactsPaths, logger);
+
+            return buildResult;
+        }
+
+        /// <summary>
+        /// update CoreRun folder with newer versions of duplicated dependencies
+        /// </summary>
+        private void UpdateDuplicatedDependencies(ArtifactsPaths artifactsPaths, ILogger logger)
+        {
+            var publishedDirectory = new DirectoryInfo(artifactsPaths.BinariesDirectoryPath);
+            var coreRunDirectory =  CoreRun.Directory;
+
+            foreach (var publishedDependency in publishedDirectory
+                .EnumerateFileSystemInfos()
+                .Where(file => file.Extension == ".dll" || file.Extension == ".exe" ))
+            {
+                var coreRunDependency = new FileInfo(Path.Combine(coreRunDirectory.FullName, publishedDependency.Name));
+
+                if (!coreRunDependency.Exists)
+                    continue; // the file does not exist in CoreRun directory, we don't need to worry, it will be just loaded from publish directory by CoreRun
+
+                var publishedVersionInfo = FileVersionInfo.GetVersionInfo(publishedDependency.FullName);
+                var coreRunVersionInfo = FileVersionInfo.GetVersionInfo(coreRunDependency.FullName);
+
+                if(!Version.TryParse(publishedVersionInfo.FileVersion, out var publishedVersion) || !Version.TryParse(coreRunVersionInfo.FileVersion, out var coreRunVersion))
+                    continue;
+
+                if(publishedVersion > coreRunVersion) 
+                {
+                    File.Copy(publishedDependency.FullName, coreRunDependency.FullName, overwrite: true); // we need to ovwerite old things with their newer versions
+
+                    logger.WriteLineInfo($"Copying {publishedDependency.FullName} to {coreRunDependency.FullName}");
+                }
+            }
+        }
+    }
+}

src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs

@@ -0,0 +1,83 @@
+using System;
+using System.IO;
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+
+namespace BenchmarkDotNet.Toolchains.CoreRun
+{
+    public class CoreRunToolchain : IToolchain
+    {
+        /// <summary>
+        /// creates a CoreRunToolchain which is using provided CoreRun to execute .NET Core apps
+        /// </summary>
+        /// <param name="coreRun">the path to CoreRun</param>
+        /// /<param name="createCopy">should a copy of CoreRun be performed? True by default. <remarks>The toolchain replaces old dependencies in CoreRun folder with newer versions if used by the benchmarks.</remarks></param>
+        /// <param name="targetFrameworkMoniker">TFM, netcoreapp2.1 is the default</param>
+        /// <param name="customDotNetCliPath">path to dotnet cli, if not provided the one from PATH will be used</param>
+        /// <param name="displayName">display name, CoreRun is the default value</param>
+        public CoreRunToolchain(FileInfo coreRun, bool createCopy = true,
+            string targetFrameworkMoniker = "netcoreapp2.1", FileInfo customDotNetCliPath = null,
+            string displayName = "CoreRun") 
+        {
+            if (coreRun == null) throw new ArgumentNullException(nameof(coreRun));
+            if (!coreRun.Exists) throw new FileNotFoundException("Povided CoreRun path does not exist");
+
+            SourceCoreRun = coreRun;
+            CopyCoreRun = createCopy ? GetShadowCopyPath(coreRun) : coreRun;
+            CustomDotNetCliPath = customDotNetCliPath;
+
+            Name = displayName;
+            Generator = new CoreRunGenerator(SourceCoreRun, CopyCoreRun, targetFrameworkMoniker, platform => platform.ToConfig());
+            Builder = new CoreRunPublisher(CopyCoreRun, customDotNetCliPath);
+            Executor = new DotNetCliExecutor(customDotNetCliPath: CopyCoreRun.FullName); // instead of executing "dotnet $pathToDll" we do "CoreRun $pathToDll" 
+        }
+
+        public string Name { get; }
+
+        public IGenerator Generator { get; }
+
+        public IBuilder Builder { get; }
+
+        public IExecutor Executor { get; }
+
+        public FileInfo SourceCoreRun { get; }
+
+        public FileInfo CopyCoreRun { get; }
+
+        public FileInfo CustomDotNetCliPath { get; }
+
+        public override string ToString() => Name;
+
+        public bool IsSupported(BenchmarkCase benchmark, ILogger logger, IResolver resolver)
+        {
+            if (!SourceCoreRun.Exists)
+            {
+                logger.WriteLineError($"Povided CoreRun path does not exist, benchmark '{benchmark.DisplayInfo}' will not be executed");
+                return false;
+            }
+
+            if (CustomDotNetCliPath == null && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled())
+            {
+                logger.WriteLineError($"BenchmarkDotNet requires dotnet cli toolchain to be installed, benchmark '{benchmark.DisplayInfo}' will not be executed");
+                return false;
+            }
+
+            if (CustomDotNetCliPath != null && !CustomDotNetCliPath.Exists)
+            {
+                logger.WriteLineError($"Povided custom dotnet cli path does not exist, benchmark '{benchmark.DisplayInfo}' will not be executed");
+                return false;
+            }
+
+            return true;
+        }
+
+        private static FileInfo GetShadowCopyPath(FileInfo coreRunPath)
+            => coreRunPath.Directory.Parent != null
+                ? new FileInfo(Path.Combine(coreRunPath.Directory.Parent.FullName, Guid.NewGuid().ToString(), coreRunPath.Name))
+                : new FileInfo(Path.Combine(coreRunPath.Directory.FullName, Guid.NewGuid().ToString(), coreRunPath.Name)); // C:\CoreRun.exe case
+    }
+}

tests/BenchmarkDotNet.Tests/ConfigParserTests.cs

@@ -1,9 +1,13 @@
-using System.Linq;
+using System;
+using System.IO;
+using System.Linq;
 using BenchmarkDotNet.ConsoleArguments;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Exporters.Csv;
 using BenchmarkDotNet.Tests.Loggers;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.CoreRun;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -38,7 +42,7 @@ public void SimpleConfigParsedCorrectly(params string[] args)
         }
 
         [Fact]
-        public void SimpleConfigAlternativeVersionParserCorrectly()
+        public void SimpleConfigAlternativeVersionParsedCorrectly()
         {
             var config = ConfigParser.Parse(new[] { "--job=Dry" }, new OutputLogger(Output)).config;
 
@@ -47,9 +51,40 @@ public void SimpleConfigAlternativeVersionParserCorrectly()
         }
 
         [Fact]
-        public void UnknownConfigIsFailure()
+        public void UnknownConfigMeansFailure()
         {
             Assert.False(ConfigParser.Parse(new[] { "--unknown" }, new OutputLogger(Output)).isSuccess);
         }
+
+        [Fact]
+        public void EmptyArgsMeansConfigWithoutJobs()
+        {
+            var config = ConfigParser.Parse(Array.Empty<string>(), new OutputLogger(Output)).config;
+
+            Assert.Empty(config.GetJobs());
+        }
+
+        [Fact]
+        public void NonExistingPathMeansFailure()
+        {
+            string nonExistingFile = Path.Combine(Path.GetTempPath(), "veryUniqueFileName.exe");
+
+            Assert.False(ConfigParser.Parse(new[] { "--cli", nonExistingFile }, new OutputLogger(Output)).isSuccess);
+            Assert.False(ConfigParser.Parse(new[] { "--coreRun", nonExistingFile }, new OutputLogger(Output)).isSuccess);
+        }
+
+        [Fact]
+        public void CoreRunConfigParserCorrectly()
+        {
+            var fakeDotnetCliPath = typeof(object).Assembly.Location;
+            var fakeCoreRunPath = typeof(ConfigParserTests).Assembly.Location;
+            var config = ConfigParser.Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath }, new OutputLogger(Output)).config;
+
+            Assert.Single(config.GetJobs());
+            CoreRunToolchain toolchain = config.GetJobs().Single().GetToolchain() as CoreRunToolchain;
+            Assert.NotNull(toolchain);
+            Assert.Equal(fakeCoreRunPath, toolchain.SourceCoreRun.FullName);
+            Assert.Equal(fakeDotnetCliPath, toolchain.CustomDotNetCliPath.FullName);
+        }
     }
 }