Pedantic WASM improvements (#1498)

* remove unnecessary file

* composition over inheritance

* inline WasmAppBuilder into WasmBuilder

* add WASM to R# dictionary

* simplify WASM console line arguments

* add validation that prints errors

* merge WasmSettings into WasmRuntime

* extend Executor to support WasmRuntime, remove  WasmExecutor

* when toolchain was not specified, create it our of provided WasmRuntime

* expose WasmRuntime.Default that returns something consumable by RuntimeInformation.GetCurrentRuntime()

* final polishing

BenchmarkDotNet.sln.DotSettings

@@ -195,6 +195,7 @@
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Virtualization/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=vmware/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Warmup/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=WASM/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Welch/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Welch_0027s/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Wmic/@EntryIndexedValue">True</s:Boolean>

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -168,13 +168,13 @@ public class CommandLineOptions
         [Option("envVars", Required = false, HelpText = "Colon separated environment variables (key:value)")]
         public IEnumerable<string> EnvironmentVariables { get; set; }
 
-        [Option("wasmJavascriptEnginePath", Required = false, Default = "v8", HelpText = "Full path to a java script engine used to run the benchmarks, if using the WASM runtime.")]
-        public string WasmJavascriptEnginePath { get; set; }
+        [Option("wasmEngine", Required = false, HelpText = "Full path to a java script engine used to run the benchmarks, used by WASM toolchain.")]
+        public FileInfo WasmJavascriptEngine { get; set; }
 
-        [Option("wasmMainJS", Required = false, HelpText = "Path to the main.js file used for wasm apps.")]
-        public string WasmMainJS { get; set; }
+        [Option("wasmMainJS", Required = false, HelpText = "Path to the main.js file used by WASM toolchain. Mandatory when using \"--runtimes wasm\"")]
+        public FileInfo WasmMainJs { get; set; }
 
-        [Option("wasmJavaScriptEngineArguments", Required = false, Default = "--expose_wasm", HelpText = "Arguments for the javascript engine used by wasm.")]
+        [Option("wasmArgs", Required = false, Default = "--expose_wasm", HelpText = "Arguments for the javascript engine used by WASM toolchain.")]
         public string WasmJavaScriptEngineArguments { get; set; }
 
         internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any();

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -105,11 +105,18 @@ private static bool Validate(CommandLineOptions options, ILogger logger)
             }
 
             foreach (string runtime in options.Runtimes)
-                if (!Enum.TryParse<RuntimeMoniker>(runtime.Replace(".", string.Empty), ignoreCase: true, out _))
+            {
+                if (!Enum.TryParse<RuntimeMoniker>(runtime.Replace(".", string.Empty), ignoreCase: true, out var parsed))
                 {
                     logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}.");
                     return false;
                 }
+                else if (parsed == RuntimeMoniker.Wasm && (options.WasmMainJs == null || options.WasmMainJs.IsNotNullButDoesNotExist()))
+                {
+                    logger.WriteLineError($"The provided {nameof(options.WasmMainJs)} \"{options.WasmMainJs}\" does NOT exist. It MUST be provided.");
+                    return false;
+                }
+            }
 
             foreach (string exporter in options.Exporters)
                 if (!AvailableExporters.ContainsKey(exporter))
@@ -137,6 +144,12 @@ private static bool Validate(CommandLineOptions options, ILogger logger)
                 return false;
             }
 
+            if (options.WasmJavascriptEngine.IsNotNullButDoesNotExist())
+            {
+                logger.WriteLineError($"The provided {nameof(options.WasmJavascriptEngine)} \"{options.WasmJavascriptEngine}\" does NOT exist.");
+                return false;
+            }
+
             if (options.CoreRtPath.IsNotNullButDoesNotExist())
             {
                 logger.WriteLineError($"The provided {nameof(options.CoreRtPath)} \"{options.CoreRtPath}\" does NOT exist.");
@@ -361,18 +374,19 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
 
                     return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain());
                 case RuntimeMoniker.Wasm:
-                    var wasmRuntime = runtimeMoniker.GetRuntime();
-
-                    WasmSettings wasmSettings = new WasmSettings(wasmMainJS: options.WasmMainJS,
-                                                                 wasmJavaScriptEngine: options.WasmJavascriptEnginePath,
-                                                                 wasmjavaScriptEngineArguments: options.WasmJavaScriptEngineArguments);
-
-                    IToolchain toolChain = new WasmToolChain(name: "Wasm",
-                                                             targetFrameworkMoniker: wasmRuntime.MsBuildMoniker,
-                                                             cliPath: options.CliPath.FullName,
-                                                             packagesPath: options.RestorePath?.FullName,
-                                                             wasmSettings: wasmSettings,
-                                                             timeout: timeOut ?? NetCoreAppSettings.DefaultBuildTimeout);
+                    var wasmRuntime = new WasmRuntime(
+                        mainJs: options.WasmMainJs,
+                        msBuildMoniker: "net5.0",
+                        javaScriptEngine: options.WasmJavascriptEngine?.FullName ?? "v8",
+                        javaScriptEngineArguments: options.WasmJavaScriptEngineArguments);
+
+                    var toolChain = WasmToolChain.From(new NetCoreAppSettings(
+                        targetFrameworkMoniker: wasmRuntime.MsBuildMoniker,
+                        runtimeFrameworkVersion: null,
+                        name: wasmRuntime.Name,
+                        customDotNetCliPath: options.CliPath?.FullName,
+                        packagesPath: options.RestorePath?.FullName,
+                        timeout: timeOut ?? NetCoreAppSettings.DefaultBuildTimeout));
 
                         return baseJob.WithRuntime(wasmRuntime).WithToolchain(toolChain);
                 default:

src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs

@@ -1,22 +1,60 @@
 using System;
+using System.ComponentModel;
+using System.IO;
+using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Jobs;
 
 namespace BenchmarkDotNet.Environments
 {
     public class WasmRuntime : Runtime, IEquatable<WasmRuntime>
     {
-        public static readonly WasmRuntime Default = new WasmRuntime("Wasm");
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        internal static readonly WasmRuntime Default = new WasmRuntime();
 
-        public WasmRuntime(string name) : base(RuntimeMoniker.Wasm, "net5.0", name)
+        public FileInfo MainJs { get; }
+
+        public string JavaScriptEngine { get; }
+
+        public string JavaScriptEngineArguments { get;  }
+
+        /// <summary>
+        /// creates new instance of WasmRuntime
+        /// </summary>
+        /// <param name="mainJs">MANDATORY path to the main.js file.</param>
+        /// <param name="javaScriptEngine">Full path to a java script engine used to run the benchmarks. "v8" by default</param>
+        /// <param name="javaScriptEngineArguments">Arguments for the javascript engine. "--expose_wasm" by default</param>
+        /// <param name="msBuildMoniker">moniker, default: "net5.0"</param>
+        /// <param name="displayName">default: "WASM"</param>
+        /// <remarks>path to mainJs MUST be provided</remarks>
+        public WasmRuntime(FileInfo mainJs, string msBuildMoniker = "net5.0", string displayName = "WASM", string javaScriptEngine = "v8", string javaScriptEngineArguments = "--expose_wasm") : base(RuntimeMoniker.Wasm, msBuildMoniker, displayName)
         {
+            if (mainJs == null)
+                throw new ArgumentNullException(paramName: nameof(mainJs));
+            if (mainJs.IsNotNullButDoesNotExist())
+                throw new FileNotFoundException($"Provided {nameof(mainJs)} file: \"{mainJs.FullName}\" doest NOT exist");
+            if (!string.IsNullOrEmpty(javaScriptEngine) && javaScriptEngine != "v8" && !File.Exists(javaScriptEngine))
+                throw new FileNotFoundException($"Provided {nameof(javaScriptEngine)} file: \"{javaScriptEngine}\" doest NOT exist");
+
+            MainJs = mainJs;
+            JavaScriptEngine = javaScriptEngine;
+            JavaScriptEngineArguments = javaScriptEngineArguments;
         }
-
-        public WasmRuntime(string name, string msBuildMoniker) : base(RuntimeMoniker.Wasm, msBuildMoniker, name)
+
+        // this ctor exists only for the purpose of having .Default property that returns something consumable by RuntimeInformation.GetCurrentRuntime()
+        private WasmRuntime(string msBuildMoniker = "net5.0", string displayName = "WASM", string javaScriptEngine = "v8", string javaScriptEngineArguments = "--expose_wasm") : base(RuntimeMoniker.Wasm, msBuildMoniker, displayName)
         {
+            MainJs = new FileInfo("fake");
+            JavaScriptEngine = javaScriptEngine;
+            JavaScriptEngineArguments = javaScriptEngineArguments;
         }
 
-        public override bool Equals(object obj) => obj is WasmRuntime other && Equals(other);
+        public override bool Equals(object obj)
+            => obj is WasmRuntime other && Equals(other);
+
+        public bool Equals(WasmRuntime other)
+            => other != null && base.Equals(other) && other.MainJs == MainJs && other.JavaScriptEngine == JavaScriptEngine && other.JavaScriptEngineArguments == JavaScriptEngineArguments;
 
-        public bool Equals(WasmRuntime other) => base.Equals(other);
+        public override int GetHashCode()
+            => base.GetHashCode() ^ MainJs.GetHashCode() ^ (JavaScriptEngine?.GetHashCode() ?? 0) ^ (JavaScriptEngineArguments?.GetHashCode() ?? 0);
     }
 }

src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs

@@ -48,8 +48,6 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker)
                     return CoreRtRuntime.CoreRt31;
                 case RuntimeMoniker.CoreRt50:
                     return CoreRtRuntime.CoreRt50;
-                case RuntimeMoniker.Wasm:
-                    return WasmRuntime.Default;
                 default:
                     throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "Runtime Moniker not supported");
             }

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs

@@ -24,7 +24,7 @@ public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPat
             Timeout = timeout ?? NetCoreAppSettings.DefaultBuildTimeout;
         }
 
-        public virtual BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
+        public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
             => new DotNetCliCommand(
                     CustomDotNetCliPath,
                     string.Empty,

src/BenchmarkDotNet/Toolchains/Executor.cs

@@ -25,21 +25,22 @@ public class Executor : IExecutor
         public ExecuteResult Execute(ExecuteParameters executeParameters)
         {
             string exePath = executeParameters.BuildResult.ArtifactsPaths.ExecutablePath;
+            string programName = executeParameters.BuildResult.ArtifactsPaths.ProgramName;
             string args = executeParameters.BenchmarkId.ToArguments();
 
             if (!File.Exists(exePath))
             {
                 return new ExecuteResult(false, -1, default, Array.Empty<string>(), Array.Empty<string>());
             }
 
-            return Execute(executeParameters.BenchmarkCase, executeParameters.BenchmarkId, executeParameters.Logger, exePath, null, args, executeParameters.Diagnoser, executeParameters.Resolver);
+            return Execute(executeParameters.BenchmarkCase, executeParameters.BenchmarkId, executeParameters.Logger, exePath, null, args, executeParameters.Diagnoser, executeParameters.Resolver, programName);
         }
 
-        private ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, string exePath, string workingDirectory, string args, IDiagnoser diagnoser, IResolver resolver)
+        private ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, string exePath, string workingDirectory, string args, IDiagnoser diagnoser, IResolver resolver, string programName)
         {
             try
             {
-                using (var process = new Process { StartInfo = CreateStartInfo(benchmarkCase, exePath, args, workingDirectory, resolver) })
+                using (var process = new Process { StartInfo = CreateStartInfo(benchmarkCase, exePath, args, workingDirectory, resolver, programName) })
                 using (new ConsoleExitHandler(process, logger))
                 {
                     var loggerWithDiagnoser = new SynchronousProcessOutputLoggerWithDiagnoser(logger, process, diagnoser, benchmarkCase, benchmarkId);
@@ -82,7 +83,7 @@ private ExecuteResult Execute(Process process, BenchmarkCase benchmarkCase, Sync
             return new ExecuteResult(true, process.ExitCode, process.Id, Array.Empty<string>(), Array.Empty<string>());
         }
 
-        private ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, string exePath, string args, string workingDirectory, IResolver resolver)
+        private ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, string exePath, string args, string workingDirectory, IResolver resolver, string programName)
         {
             var start = new ProcessStartInfo
             {
@@ -113,6 +114,10 @@ private ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, string exe
                     start.FileName = mono.CustomPath ?? "mono";
                     start.Arguments = GetMonoArguments(benchmarkCase.Job, exePath, args, resolver);
                     break;
+                case WasmRuntime wasm:
+                    start.FileName = wasm.JavaScriptEngine;
+                    start.Arguments = $"{wasm.JavaScriptEngineArguments} runtime.js -- --run {programName}.dll {args} ";
+                    break;
                 default:
                     throw new NotSupportedException("Runtime = " + runtime);
             }