From b18693f8449974a625852bb2df4c69ce82294ba7 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 12 Nov 2018 17:17:48 +0100 Subject: [PATCH 01/10] Initial stab at incorporating Elastic.BenchmarkDotNetExporter --- build/scripts/Benchmarking.fsx | 214 +----------------- build/scripts/Targets.fsx | 3 - .../Tests.Benchmarking/BenchmarkProgram.cs | 68 +++++- .../Framework/BenchmarkConfig.cs | 67 +----- .../Tests.Benchmarking.csproj | 6 +- 5 files changed, 76 insertions(+), 282 deletions(-) diff --git a/build/scripts/Benchmarking.fsx b/build/scripts/Benchmarking.fsx index 68aee6475a8..63ed0a5cf15 100644 --- a/build/scripts/Benchmarking.fsx +++ b/build/scripts/Benchmarking.fsx @@ -30,114 +30,14 @@ module Benchmarker = let indexName = IndexName.op_Implicit("benchmark-reports") let typeName = TypeName.op_Implicit("benchmarkreport") - type Memory(gen0Collections:int, gen1Collections: int, gen2Collections: int, totalOperations:int64, bytesAllocatedPerOperation:int64) = - member val Gen0Collections=gen0Collections with get, set - member val Gen1Collections=gen1Collections with get, set - member val Gen2Collections=gen2Collections with get, set - member val TotalOperations=totalOperations with get, set - member val BytesAllocatedPerOperation=bytesAllocatedPerOperation with get, set - - type ChronometerFrequency(hertz:double) = - member val Hertz=hertz with get, set - - type HostEnvironmentInfo(benchmarkDotNetCaption:string, benchmarkDotNetVersion:string, osVersion: string, processorName:string, - processorCount:int, runtimeVersion:string, architecture:string, hasAttachedDebugger:bool, hasRyuJit:bool, - configuration:string, jitModules:string, dotnetCliVersion:string, chronometerFrequency:ChronometerFrequency, - hardwareTimerKind:string) = - member val BenchmarkDotNetCaption=benchmarkDotNetCaption with get, set - member val BenchmarkDotNetVersion=benchmarkDotNetVersion with get, set - member val OsVersion=osVersion with get, set - member val ProcessorName=processorName with get, set - member val ProcessorCount=processorCount with get, set - member val RuntimeVersion=runtimeVersion with get, set - member val Architecture=architecture with get, set - member val HasAttachedDebugger=hasAttachedDebugger with get, set - member val HasRyuJit=hasRyuJit with get, set - member val Configuration=configuration with get, set - member val JitModules=jitModules with get, set - member val DotNetCliVersion=dotnetCliVersion with get, set - member val ChronometerFrequency=chronometerFrequency with get, set - member val HardwareTimerKind=hardwareTimerKind with get, set - - type ConfidenceInterval(n:int, mean: double, standardError:double, level:int, margin:double, lower:double, upper:double) = - member val N=n with get, set - member val Mean=mean with get, set - member val StandardError=standardError with get, set - member val Level=level with get, set - member val Margin=margin with get, set - member val Lower=lower with get, set - member val Upper=upper with get, set - - type Percentiles (p0:double, p25:double, p50:double, p67:double, p80:double, p85:double, p90:double, p95:double, p100:double) = - member val P0=p0 with get, set - member val P25=p25 with get, set - member val P50=p50 with get, set - member val P67=p67 with get, set - member val P80=p80 with get, set - member val P85=p85 with get, set - member val P90=p90 with get, set - member val P95=p95 with get, set - member val P100=p100 with get, set - - type Statistics(n:int, min:double, lowerFence:double, q1:double, median:double, mean:double, q3:double, upperFence:double, max:double, - interquartileRange:double, outliers:double list, standardError:double, variance:double, standardDeviation:double, - skewness:double, kurtosis:double, confidenceInterval:ConfidenceInterval, percentiles:Percentiles) = - member val N=n with get, set - member val Min=min with get, set - member val LowerFence=lowerFence with get, set - member val Q1=q1 with get, set - member val Median=median with get, set - member val Mean=mean with get, set - member val Q3=q3 with get, set - member val UpperFence=upperFence with get, set - member val Max=max with get, set - member val InterquartileRange=interquartileRange with get, set - member val Outliers=outliers with get, set - member val StandardError=standardError with get, set - member val Variance=variance with get, set - member val StandardDeviation=standardDeviation with get, set - member val Skewness=skewness with get, set - member val Kurtosis=kurtosis with get, set - member val ConfidenceInterval=confidenceInterval with get, set - member val Percentiles=percentiles with get, set - - type Benchmark(displayInfo:string, namespyce:string, tipe:string, method:string, methodTitle:string, parameters:string, - statistics:Statistics, memory:Memory) = - member val DisplayInfo=displayInfo with get, set - member val Namespace=namespyce with get, set - member val Type=tipe with get, set - member val Method=method with get, set - member val MethodTitle=methodTitle with get, set - member val Parameters=parameters with get, set - member val Statistics=statistics with get, set - member val Memory=memory with get, set - - type BenchmarkReports(title: string, totalTime:TimeSpan, date:DateTime, commit:string, branchName:string, host:HostEnvironmentInfo, benchmarks:Benchmark list) = - member val Title = title with get, set - member val TotalTime = totalTime with get, set - member val Date = date with get, set - member val Commit = commit with get, set - member val BranchName = branchName with get, set - member val HostEnvironmentInfo = host with get, set - member val Benchmarks = benchmarks with get, set - - type BenchmarkReport(title: string, totalTime:TimeSpan, date:DateTime, commit:string, branchName:string, host:HostEnvironmentInfo, benchmark:Benchmark) = - member val Title = title with get, set - member val TotalTime = totalTime with get, set - member val Date = date with get, set - member val Commit = commit with get, set - member val BranchName = branchName with get, set - member val HostEnvironmentInfo = host with get, set - member val Benchmark = benchmark with get, set - let private testsProjectDirectory = Path.GetFullPath(Paths.TestsSource("Tests.Benchmarking")) - let private benchmarkOutput = Path.GetFullPath(Paths.Output("benchmarks")) |> directoryInfo - let private copyToOutput file = CopyFile benchmarkOutput.FullName file let Run(runInteractive:bool) = - ensureDirExists benchmarkOutput - + let url = getBuildParam "elasticsearch" + let username = getBuildParam "username" + let password = getBuildParam "password" + try if runInteractive then DotNetCli.RunCommand(fun p -> @@ -149,109 +49,3 @@ module Benchmarker = { p with WorkingDir = testsProjectDirectory }) "run -f netcoreapp2.1 -c Release non-interactive" - finally - // running benchmarks can timeout so clean up any generated benchmark files - let benchmarkOutputFiles = - let output = combinePaths testsProjectDirectory "BenchmarkDotNet.Artifacts" - Directory.EnumerateFiles(output, "*.*", SearchOption.AllDirectories) - |> Seq.toList - - for file in benchmarkOutputFiles do copyToOutput file - DeleteFiles benchmarkOutputFiles - - let IndexResult (client:ElasticClient, file:string, date:DateTime, commit:string, branchName:string, indexName, typeName) = - - trace (sprintf "Indexing benchmark results (class) %s" file) - - let benchmarkReports = JsonConvert.DeserializeObject(File.ReadAllText(file)) - benchmarkReports.Date <- date - benchmarkReports.Commit <- commit - benchmarkReports.BranchName <- branchName - - for benchmarkReportSingle in benchmarkReports.Benchmarks do - - trace (sprintf "Indexing benchmark result (method) %s" benchmarkReportSingle.MethodTitle) - - let document = new BenchmarkReport(benchmarkReports.Title, - benchmarkReports.TotalTime, - benchmarkReports.Date, - benchmarkReports.Commit, - benchmarkReports.BranchName, - benchmarkReports.HostEnvironmentInfo, - benchmarkReportSingle) - - let indexRequest = new IndexRequest(indexName, typeName) - indexRequest.Document <- document - indexRequest.Pipeline <- pipelineName - - let indexResponse = client.Index(indexRequest) - - if indexResponse.IsValid = false then - raise (Exception("Unable to index benchmark result (method): " + indexResponse.ServerError.Error.ToString())) - - let IndexResults (url, username, password) = - if (String.IsNullOrEmpty url = false) then - trace "Indexing benchmark reports" - - let date = DateTime.UtcNow - let commit = getSHA1 "." "HEAD" - let branchName = getBranchName "." - - let benchmarkJsonFiles = - Directory.EnumerateFiles(benchmarkOutput.FullName, "*-custom.json", SearchOption.AllDirectories) - |> Seq.toList - - let uri = new Uri(url) - let connectionSettings = new ConnectionSettings(uri); - - if (String.IsNullOrEmpty username = false && String.IsNullOrEmpty password = false) then - connectionSettings.BasicAuthentication(username, password) |> ignore - - let client = new ElasticClient(connectionSettings) - - let indexTemplateExists = client.IndexTemplateExists(Name.op_Implicit("benchmarks")).Exists - - if indexTemplateExists |> not then - - let typeMapping = new TypeMappingDescriptor() - typeMapping.AutoMap() |> ignore - - let mappings = new Mappings() - mappings.Add(typeName, typeMapping :> ITypeMapping) - - let indexSettings = new IndexSettings() - indexSettings.NumberOfShards <- Nullable 1 - - let putIndexTemplateRequest = new PutIndexTemplateRequest(Name.op_Implicit("benchmarks")) - putIndexTemplateRequest.IndexPatterns <- ["benchmark-reports-*"] - putIndexTemplateRequest.Mappings <- mappings - putIndexTemplateRequest.Settings <- indexSettings - - let putIndexTemplateResponse = client.PutIndexTemplate(putIndexTemplateRequest) - - if putIndexTemplateResponse.IsValid = false then - raise (Exception("Unable to create index template into Elasticsearch")) - - let grokProcessor = new GrokProcessor(); - grokProcessor.Field <- new Field("benchmark.displayInfo") - grokProcessor.Patterns <- ["%{WORD:class}.%{DATA:method}: Job-%{WORD:jobName}\\(Jit=%{WORD:jit}, Runtime=%{WORD:clr}, LaunchCount=%{NUMBER:launchCount}, RunStrategy=%{WORD:runStrategy}, TargetCount=%{NUMBER:targetCount}, UnrollFactor=%{NUMBER:unrollFactor}, WarmupCount=%{NUMBER:warmupCount}\\)"] - - let dateIndexProcessor = new DateIndexNameProcessor(); - dateIndexProcessor.Field <- new Field("date") - dateIndexProcessor.IndexNamePrefix <- "benchmark-reports-" - dateIndexProcessor.DateRounding <- new Nullable(DateRounding.Month) - dateIndexProcessor.DateFormats <- ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSZ"] - - let request = new PutPipelineRequest(Id.op_Implicit(pipelineName)) - request.Description <- "Benchmark settings pipeline" - request.Processors <- [dateIndexProcessor; grokProcessor] - - let createPipeline = client.PutPipeline(request) - - if createPipeline.IsValid = false then - raise (Exception("Unable to create pipeline")) - - for file in benchmarkJsonFiles - do IndexResult (client, file, date, commit, branchName, indexName, typeName) - - trace "Indexed benchmark reports" \ No newline at end of file diff --git a/build/scripts/Targets.fsx b/build/scripts/Targets.fsx index d897172070c..554bf49b119 100644 --- a/build/scripts/Targets.fsx +++ b/build/scripts/Targets.fsx @@ -62,9 +62,6 @@ Target "Integrate" Tests.RunIntegrationTests Target "Benchmark" <| fun _ -> let runInteractive = ((getBuildParam "nonInteractive") <> "1") Benchmarker.Run(runInteractive) - let url = getBuildParam "elasticsearch" - let username = getBuildParam "username" - let password = getBuildParam "password" Benchmarker.IndexResults (url, username, password) Target "InternalizeDependencies" Build.ILRepack diff --git a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs index 8ea77455bb5..b4254652582 100644 --- a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs +++ b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs @@ -1,31 +1,85 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Reflection; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; +using Elastic.BenchmarkDotNetExporter; +using LibGit2Sharp; +using Tests.Benchmarking.Framework; namespace Tests.Benchmarking { public static class Program { - public static int Main(string[] arguments) + private static string Commit { get; } + private static string CommitMessage { get; } + private static string Branch { get; } + + static Program() { - Console.WriteLine("Running Benchmarking."); - if (arguments.Count() >= 1 && arguments[0].Equals("non-interactive", StringComparison.OrdinalIgnoreCase)) + var dirInfo = new DirectoryInfo(Environment.CurrentDirectory); + while(dirInfo != dirInfo.Root && !Directory.Exists(Path.Combine(dirInfo.FullName, ".git"))) + dirInfo = dirInfo.Parent; + if (!Directory.Exists(Path.Combine(dirInfo.FullName, ".git"))) Environment.Exit(2); + + Console.WriteLine(dirInfo.FullName); + using (var repos = new Repository(dirInfo.FullName)) { - Console.WriteLine("Running in Non-Interactive mode."); - foreach (var benchmarkType in GetBenchmarkTypes()) BenchmarkRunner.Run(benchmarkType); + Commit = repos.Head.Tip.Sha; + CommitMessage = repos.Head.Tip.Message; + Branch = repos.Head.FriendlyName; + } + } - return 0; + public static int Main(string[] arguments) + { + Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) : {CommitMessage}"); + if (arguments.Any() && arguments[0].Equals("--all", StringComparison.OrdinalIgnoreCase)) + { + Console.WriteLine("Running all the benchmarks"); + return RunAllBenchmarks(arguments.Skip(1).ToArray()); } - Console.WriteLine("Running in Interactive mode."); + Console.WriteLine("Running the interactive benchmark switcher."); var benchmarkSwitcher = new BenchmarkSwitcher(GetBenchmarkTypes()); benchmarkSwitcher.Run(arguments); return 0; } + private static int RunAllBenchmarks(string [] arguments) + { + var url = arguments.Length > 0 ? arguments[0] : null; + var username = arguments.Length > 1 ? arguments[1] : null; + var password = arguments.Length > 2 ? arguments[2] : null; + + Console.WriteLine("Running in Non-Interactive mode."); + var exporter = !string.IsNullOrEmpty(url) ? new ElasticsearchBenchmarkExporter(url, username, password, Commit, Branch) : null; + foreach (var benchmarkType in GetBenchmarkTypes()) + { + var runCount = benchmarkType.GetCustomAttribute()?.RunCount ?? 1; + var jobs = new[] + { + Job.Dry.With(Runtime.Core).With(Jit.RyuJit).WithIterationCount(runCount), + Job.Dry.With(Runtime.Clr).With(Jit.RyuJit).WithIterationCount(runCount), + Job.Dry.With(Runtime.Clr).With(Jit.LegacyJit).WithIterationCount(runCount) + }; + var config = DefaultConfig.Instance + .With(jobs) + .With(MemoryDiagnoser.Default); + if (exporter != null) config = config.With(exporter); + BenchmarkRunner.Run(benchmarkType, config); + } + + return 0; + } + private static Type[] GetBenchmarkTypes() { diff --git a/src/Tests/Tests.Benchmarking/Framework/BenchmarkConfig.cs b/src/Tests/Tests.Benchmarking/Framework/BenchmarkConfig.cs index 87c153b4110..814554c9d3f 100644 --- a/src/Tests/Tests.Benchmarking/Framework/BenchmarkConfig.cs +++ b/src/Tests/Tests.Benchmarking/Framework/BenchmarkConfig.cs @@ -1,74 +1,19 @@ using System; -using System.Collections.Generic; -using System.Linq; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Reports; -using Newtonsoft.Json; namespace Tests.Benchmarking.Framework { - public class CustomJsonExporter : ExporterBase - { - protected override string FileExtension => "json"; - - protected override string FileNameSuffix => "-custom"; - - public override void ExportToLog(Summary summary, ILogger logger) - { - var environmentInfo = new - { - HostEnvironmentInfo.BenchmarkDotNetCaption, - summary.HostEnvironmentInfo.BenchmarkDotNetVersion, - OsVersion = summary.HostEnvironmentInfo.OsVersion.Value, - summary.HostEnvironmentInfo.CpuInfo.Value.ProcessorName, - summary.HostEnvironmentInfo.CpuInfo.Value.PhysicalCoreCount, - summary.HostEnvironmentInfo.RuntimeVersion, - summary.HostEnvironmentInfo.Architecture, - summary.HostEnvironmentInfo.HasAttachedDebugger, - summary.HostEnvironmentInfo.HasRyuJit, - summary.HostEnvironmentInfo.Configuration, - summary.HostEnvironmentInfo.JitModules, - DotNetCliVersion = summary.HostEnvironmentInfo.DotNetSdkVersion.Value, - summary.HostEnvironmentInfo.ChronometerFrequency, - HardwareTimerKind = summary.HostEnvironmentInfo.HardwareTimerKind.ToString() - }; - - var benchmarks = summary.Reports.Select(r => - { - var data = new Dictionary - { - { "DisplayInfo", r.BenchmarkCase.DisplayInfo }, - { "Namespace", r.BenchmarkCase.Descriptor.Type.Namespace }, - { "Type", r.BenchmarkCase.Descriptor.Type.Name }, - { "Method", r.BenchmarkCase.Descriptor.WorkloadMethod.Name }, - { "MethodTitle", r.BenchmarkCase.Descriptor.WorkloadMethod.Name }, - { "Parameters", r.BenchmarkCase.Parameters.PrintInfo }, - { "Statistics", r.ResultStatistics }, - { "Memory", r.GcStats } - }; - - return data; - }); - - logger.WriteLine(JsonConvert.SerializeObject(new Dictionary - { - { "Title", summary.Title }, - { "TotalTime", summary.TotalTime }, - { "HostEnvironmentInfo", environmentInfo }, - { "Benchmarks", benchmarks } - })); - } - } - public class BenchmarkConfigAttribute : Attribute, IConfigSource { + public int RunCount { get; } + public BenchmarkConfigAttribute(int runCount = 1) { + RunCount = 1; var jobs = new[] { Job.Dry.With(Runtime.Core).With(Jit.RyuJit).WithIterationCount(runCount), @@ -76,9 +21,9 @@ public BenchmarkConfigAttribute(int runCount = 1) Job.Dry.With(Runtime.Clr).With(Jit.LegacyJit).WithIterationCount(runCount) }; Config = DefaultConfig.Instance - .With(jobs) - .With(new CustomJsonExporter()) - .With(MemoryDiagnoser.Default); + .With(MemoryDiagnoser.Default) + .With(MarkdownExporter.GitHub) + .With(jobs); } public IConfig Config { get; } diff --git a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj index b23c57e1f1e..6af11d018eb 100644 --- a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj +++ b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj @@ -9,6 +9,10 @@ - + + + + + \ No newline at end of file From 8d999414cb0eec50900e03e0339c414467cad694 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 14 Nov 2018 11:01:16 +0100 Subject: [PATCH 02/10] WIP: use latest version of the exporter --- .gitignore | 1 + .../Tests.Benchmarking/BenchmarkProgram.cs | 35 ++++++++++++------- .../FieldResolverBenchmarkTests.cs | 2 +- .../Framework/BenchmarkConfig.cs | 24 ++----------- .../Tests.Benchmarking/PostDataBenchmarks.cs | 2 +- .../PropertyNameResolverBenchmarkTests.cs | 2 +- .../Tests.Benchmarking.csproj | 3 +- 7 files changed, 30 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 4a2cef28006..22d936056cd 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ packages/* paket.exe paket-files/*.cached +BenchmarkDotNet.Artifacts build/* !build/tools !build/keys diff --git a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs index b4254652582..22186fbcb9e 100644 --- a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs +++ b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs @@ -13,6 +13,7 @@ using Elastic.BenchmarkDotNetExporter; using LibGit2Sharp; using Tests.Benchmarking.Framework; +using RunMode = BenchmarkDotNet.Jobs.RunMode; namespace Tests.Benchmarking { @@ -40,20 +41,38 @@ static Program() public static int Main(string[] arguments) { + + Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) : {CommitMessage}"); + var config = CreateDefaultConfig(); if (arguments.Any() && arguments[0].Equals("--all", StringComparison.OrdinalIgnoreCase)) { Console.WriteLine("Running all the benchmarks"); - return RunAllBenchmarks(arguments.Skip(1).ToArray()); + return RunAllBenchmarks(config, arguments.Skip(1).ToArray()); } Console.WriteLine("Running the interactive benchmark switcher."); var benchmarkSwitcher = new BenchmarkSwitcher(GetBenchmarkTypes()); - benchmarkSwitcher.Run(arguments); + config = config.With(MarkdownExporter.GitHub); + benchmarkSwitcher.Run(arguments, config); return 0; } - private static int RunAllBenchmarks(string [] arguments) + private static IConfig CreateDefaultConfig() + { + var jobs = new[] + { + Job.ShortRun.With(Runtime.Core).With(Jit.RyuJit), + Job.ShortRun.With(Runtime.Clr).With(Jit.RyuJit), + Job.ShortRun.With(Runtime.Clr).With(Jit.LegacyJit), + }; + var config = DefaultConfig.Instance + .With(jobs) + .With(MemoryDiagnoser.Default); + return config; + } + + private static int RunAllBenchmarks(IConfig config, string[] arguments) { var url = arguments.Length > 0 ? arguments[0] : null; var username = arguments.Length > 1 ? arguments[1] : null; @@ -63,16 +82,6 @@ private static int RunAllBenchmarks(string [] arguments) var exporter = !string.IsNullOrEmpty(url) ? new ElasticsearchBenchmarkExporter(url, username, password, Commit, Branch) : null; foreach (var benchmarkType in GetBenchmarkTypes()) { - var runCount = benchmarkType.GetCustomAttribute()?.RunCount ?? 1; - var jobs = new[] - { - Job.Dry.With(Runtime.Core).With(Jit.RyuJit).WithIterationCount(runCount), - Job.Dry.With(Runtime.Clr).With(Jit.RyuJit).WithIterationCount(runCount), - Job.Dry.With(Runtime.Clr).With(Jit.LegacyJit).WithIterationCount(runCount) - }; - var config = DefaultConfig.Instance - .With(jobs) - .With(MemoryDiagnoser.Default); if (exporter != null) config = config.With(exporter); BenchmarkRunner.Run(benchmarkType, config); } diff --git a/src/Tests/Tests.Benchmarking/FieldResolverBenchmarkTests.cs b/src/Tests/Tests.Benchmarking/FieldResolverBenchmarkTests.cs index 575ece741df..7c15fb51378 100644 --- a/src/Tests/Tests.Benchmarking/FieldResolverBenchmarkTests.cs +++ b/src/Tests/Tests.Benchmarking/FieldResolverBenchmarkTests.cs @@ -5,7 +5,7 @@ namespace Tests.Benchmarking { - [BenchmarkConfig(100)] + //[BenchmarkConfig(100)] public class FieldResolverBenchmarkTests { private static readonly Field InferredField = Infer.Field(p => p.Name); diff --git a/src/Tests/Tests.Benchmarking/Framework/BenchmarkConfig.cs b/src/Tests/Tests.Benchmarking/Framework/BenchmarkConfig.cs index 814554c9d3f..22269897bb4 100644 --- a/src/Tests/Tests.Benchmarking/Framework/BenchmarkConfig.cs +++ b/src/Tests/Tests.Benchmarking/Framework/BenchmarkConfig.cs @@ -1,31 +1,11 @@ using System; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Exporters; -using BenchmarkDotNet.Jobs; namespace Tests.Benchmarking.Framework { - public class BenchmarkConfigAttribute : Attribute, IConfigSource + public class BenchmarkConfigAttribute : Attribute { public int RunCount { get; } - public BenchmarkConfigAttribute(int runCount = 1) - { - RunCount = 1; - var jobs = new[] - { - Job.Dry.With(Runtime.Core).With(Jit.RyuJit).WithIterationCount(runCount), - Job.Dry.With(Runtime.Clr).With(Jit.RyuJit).WithIterationCount(runCount), - Job.Dry.With(Runtime.Clr).With(Jit.LegacyJit).WithIterationCount(runCount) - }; - Config = DefaultConfig.Instance - .With(MemoryDiagnoser.Default) - .With(MarkdownExporter.GitHub) - .With(jobs); - } - - public IConfig Config { get; } + public BenchmarkConfigAttribute(int runCount = 1) => RunCount = runCount; } } diff --git a/src/Tests/Tests.Benchmarking/PostDataBenchmarks.cs b/src/Tests/Tests.Benchmarking/PostDataBenchmarks.cs index e0eac96196c..3964cf0b773 100644 --- a/src/Tests/Tests.Benchmarking/PostDataBenchmarks.cs +++ b/src/Tests/Tests.Benchmarking/PostDataBenchmarks.cs @@ -10,7 +10,7 @@ namespace Tests.Benchmarking { - [BenchmarkConfig(1000)] + //[BenchmarkConfig(1000)] public class PostDataBenchmarks { private readonly PostData _postDataOfBytes; diff --git a/src/Tests/Tests.Benchmarking/PropertyNameResolverBenchmarkTests.cs b/src/Tests/Tests.Benchmarking/PropertyNameResolverBenchmarkTests.cs index ced590b5906..d056e786391 100644 --- a/src/Tests/Tests.Benchmarking/PropertyNameResolverBenchmarkTests.cs +++ b/src/Tests/Tests.Benchmarking/PropertyNameResolverBenchmarkTests.cs @@ -5,7 +5,7 @@ namespace Tests.Benchmarking { - [BenchmarkConfig(100)] + //[BenchmarkConfig(100)] public class PropertyNameResolverBenchmarkTests { private static readonly PropertyName InferredPropertyName = Infer.Property(p => p.Name); diff --git a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj index 6af11d018eb..482583f27d9 100644 --- a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj +++ b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj @@ -3,6 +3,7 @@ netcoreapp2.1;net461 Exe + https://ci.appveyor.com/nuget/benchmarkdotnet;$(RestoreSources) true @@ -11,7 +12,7 @@ - + From e7e91a55bafff7a456288ba33cdd00d6e789d520 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 14 Nov 2018 12:37:10 +0100 Subject: [PATCH 03/10] update exporter --- src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj index 482583f27d9..3e51d591076 100644 --- a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj +++ b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj @@ -13,7 +13,7 @@ - + \ No newline at end of file From fd04c007744841412a4495cfaccaf237ebee6ada Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 15 Nov 2018 13:04:59 +0100 Subject: [PATCH 04/10] add es-net-abstractions appveyor myget feed as restore source --- src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj index 3e51d591076..797a02a2afb 100644 --- a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj +++ b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj @@ -3,7 +3,7 @@ netcoreapp2.1;net461 Exe - https://ci.appveyor.com/nuget/benchmarkdotnet;$(RestoreSources) + https://ci.appveyor.com/nuget/benchmarkdotnet;https://ci.appveyor.com/nuget/elasticsearch-net-abstractions;$(RestoreSources) true From 33511fb547fd290b1f55a90f6e869f183147def7 Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Tue, 20 Nov 2018 15:26:45 +0000 Subject: [PATCH 05/10] Update to latest exporter and temporary remove some of the benchmarks to quickly reduce the benchmark run time --- NuGet.config | 2 +- src/Elasticsearch.sln.DotSettings | 1 + .../Tests.Benchmarking/BenchmarkProgram.cs | 31 +- .../BulkDeserializationBenchmarkTests.cs | 76 +-- .../FieldResolverBenchmarkTests.cs | 50 -- .../NoncachingFieldResolver.cs | 167 ------- .../Tests.Benchmarking/PostDataBenchmarks.cs | 437 ------------------ .../PropertyNameResolverBenchmarkTests.cs | 50 -- .../Tests.Benchmarking.csproj | 5 +- .../Tests.ScratchPad/Tests.ScratchPad.csproj | 2 +- 10 files changed, 68 insertions(+), 753 deletions(-) delete mode 100644 src/Tests/Tests.Benchmarking/FieldResolverBenchmarkTests.cs delete mode 100644 src/Tests/Tests.Benchmarking/NoncachingFieldResolver.cs delete mode 100644 src/Tests/Tests.Benchmarking/PostDataBenchmarks.cs delete mode 100644 src/Tests/Tests.Benchmarking/PropertyNameResolverBenchmarkTests.cs diff --git a/NuGet.config b/NuGet.config index 71aff4ebe5c..ecdfb0fef76 100644 --- a/NuGet.config +++ b/NuGet.config @@ -2,6 +2,6 @@ - + \ No newline at end of file diff --git a/src/Elasticsearch.sln.DotSettings b/src/Elasticsearch.sln.DotSettings index c0aa08ce146..4bfa3de5193 100644 --- a/src/Elasticsearch.sln.DotSettings +++ b/src/Elasticsearch.sln.DotSettings @@ -470,6 +470,7 @@ True True True + True True True False diff --git a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs index 22186fbcb9e..efd01620ce3 100644 --- a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs +++ b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs @@ -22,6 +22,7 @@ public static class Program private static string Commit { get; } private static string CommitMessage { get; } private static string Branch { get; } + private static string Repository { get; } static Program() { @@ -36,14 +37,17 @@ static Program() Commit = repos.Head.Tip.Sha; CommitMessage = repos.Head.Tip.Message; Branch = repos.Head.FriendlyName; + var remoteName = repos.Head.RemoteName; + Repository = + repos.Network.Remotes.FirstOrDefault(r => r.Name == remoteName)?.Url + ?? repos.Network.Remotes.FirstOrDefault()?.Url; } } public static int Main(string[] arguments) { - - - Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) : {CommitMessage}"); + Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) on {Repository} : {CommitMessage} - "); + return 0; var config = CreateDefaultConfig(); if (arguments.Any() && arguments[0].Equals("--all", StringComparison.OrdinalIgnoreCase)) { @@ -63,8 +67,8 @@ private static IConfig CreateDefaultConfig() var jobs = new[] { Job.ShortRun.With(Runtime.Core).With(Jit.RyuJit), - Job.ShortRun.With(Runtime.Clr).With(Jit.RyuJit), - Job.ShortRun.With(Runtime.Clr).With(Jit.LegacyJit), +// Job.ShortRun.With(Runtime.Clr).With(Jit.RyuJit), +// Job.ShortRun.With(Runtime.Clr).With(Jit.LegacyJit), }; var config = DefaultConfig.Instance .With(jobs) @@ -79,7 +83,8 @@ private static int RunAllBenchmarks(IConfig config, string[] arguments) var password = arguments.Length > 2 ? arguments[2] : null; Console.WriteLine("Running in Non-Interactive mode."); - var exporter = !string.IsNullOrEmpty(url) ? new ElasticsearchBenchmarkExporter(url, username, password, Commit, Branch) : null; + + var exporter = CreateElasticsearchExporter(url, username, password); foreach (var benchmarkType in GetBenchmarkTypes()) { if (exporter != null) config = config.With(exporter); @@ -89,6 +94,20 @@ private static int RunAllBenchmarks(IConfig config, string[] arguments) return 0; } + private static ElasticsearchBenchmarkExporter CreateElasticsearchExporter(string url, string username, string password) + { + if (string.IsNullOrWhiteSpace(url)) return null; + var options = new ElasticsearchBenchmarkExporterOptions(url) + { + Username = username, + Password = password, + GitCommitSha = Commit, + GitBranch = Branch, + GitCommitMessage = CommitMessage + }; + return new ElasticsearchBenchmarkExporter(options); + } + private static Type[] GetBenchmarkTypes() { diff --git a/src/Tests/Tests.Benchmarking/BulkDeserializationBenchmarkTests.cs b/src/Tests/Tests.Benchmarking/BulkDeserializationBenchmarkTests.cs index 3e31c05681e..3de3613be73 100644 --- a/src/Tests/Tests.Benchmarking/BulkDeserializationBenchmarkTests.cs +++ b/src/Tests/Tests.Benchmarking/BulkDeserializationBenchmarkTests.cs @@ -46,44 +46,44 @@ public BulkResponse MediumResponse() return Client.RequestResponseSerializer.Deserialize(ms); } - [Benchmark(Description = "deserialize 1,000 items in bulk response")] - public BulkResponse LargeResponse() - { - using (var ms = new MemoryStream(_largeResponse)) - return Client.RequestResponseSerializer.Deserialize(ms); - } - - [Benchmark(Description = "deserialize 100,000 items in bulk response")] - public BulkResponse HugeResponse() - { - using (var ms = new MemoryStream(_hugeResponse)) - return Client.RequestResponseSerializer.Deserialize(ms); - } - - [Benchmark(Description = "deserialize 100,000 items in bulk response")] - public BulkResponse HugeResponseWithStream() - { - using (var ms = new JsonTextReader(new StreamReader(new MemoryStream(_hugeResponse)))) - return _jsonSerializer.Deserialize(ms); - } - - [Benchmark(Description = "deserialize 100,000 items in bulk string response")] - public BulkResponse HugeResponseWithString() - { - using (var reader = new JsonTextReader(new StringReader(Encoding.UTF8.GetString(_hugeResponse)))) - return _jsonSerializer.Deserialize(reader); - } - - [Benchmark(Description = "Baseline", Baseline = true)] - public BulkResponse Baseline() - { - using (var reader = new JsonTextReader(new StreamReader(new MemoryStream(_hugeResponse)))) - { - while (reader.Read()) { } - - return new BulkResponse(); - } - } +// [Benchmark(Description = "deserialize 1,000 items in bulk response")] +// public BulkResponse LargeResponse() +// { +// using (var ms = new MemoryStream(_largeResponse)) +// return Client.RequestResponseSerializer.Deserialize(ms); +// } +// +// [Benchmark(Description = "deserialize 100,000 items in bulk response")] +// public BulkResponse HugeResponse() +// { +// using (var ms = new MemoryStream(_hugeResponse)) +// return Client.RequestResponseSerializer.Deserialize(ms); +// } +// +// [Benchmark(Description = "deserialize 100,000 items in bulk response")] +// public BulkResponse HugeResponseWithStream() +// { +// using (var ms = new JsonTextReader(new StreamReader(new MemoryStream(_hugeResponse)))) +// return _jsonSerializer.Deserialize(ms); +// } +// +// [Benchmark(Description = "deserialize 100,000 items in bulk string response")] +// public BulkResponse HugeResponseWithString() +// { +// using (var reader = new JsonTextReader(new StringReader(Encoding.UTF8.GetString(_hugeResponse)))) +// return _jsonSerializer.Deserialize(reader); +// } +// +// [Benchmark(Description = "Baseline", Baseline = true)] +// public BulkResponse Baseline() +// { +// using (var reader = new JsonTextReader(new StreamReader(new MemoryStream(_hugeResponse)))) +// { +// while (reader.Read()) { } +// +// return new BulkResponse(); +// } +// } private static object BulkItemResponse() => new { diff --git a/src/Tests/Tests.Benchmarking/FieldResolverBenchmarkTests.cs b/src/Tests/Tests.Benchmarking/FieldResolverBenchmarkTests.cs deleted file mode 100644 index 7c15fb51378..00000000000 --- a/src/Tests/Tests.Benchmarking/FieldResolverBenchmarkTests.cs +++ /dev/null @@ -1,50 +0,0 @@ -using BenchmarkDotNet.Attributes; -using Nest; -using Tests.Benchmarking.Framework; -using Tests.Domain; - -namespace Tests.Benchmarking -{ - //[BenchmarkConfig(100)] - public class FieldResolverBenchmarkTests - { - private static readonly Field InferredField = Infer.Field(p => p.Name); - private static readonly Field PropertyField = typeof(Project).GetProperty(nameof(Project.Name)); - private static readonly Field StringField = "Name"; - private FieldResolver _expressionResolver; - private NoncachingFieldResolver _nonCachingExpressionResolver; - private NoncachingFieldResolver _nonCachingPropertyResolver; - private NoncachingFieldResolver _nonCachingStringResolver; - private FieldResolver _propertyResolver; - private FieldResolver _stringResolver; - - [GlobalSetup] - public void Setup() - { - _expressionResolver = new FieldResolver(new ConnectionSettings()); - _propertyResolver = new FieldResolver(new ConnectionSettings()); - _stringResolver = new FieldResolver(new ConnectionSettings()); - _nonCachingExpressionResolver = new NoncachingFieldResolver(new ConnectionSettings()); - _nonCachingPropertyResolver = new NoncachingFieldResolver(new ConnectionSettings()); - _nonCachingStringResolver = new NoncachingFieldResolver(new ConnectionSettings()); - } - - [Benchmark] - public string NonCachedFieldUsingExpression() => _nonCachingExpressionResolver.Resolve(InferredField); - - [Benchmark] - public string CachedFieldUsingExpression() => _expressionResolver.Resolve(InferredField); - - [Benchmark] - public string NonCachedFieldUsingPropertyInfo() => _nonCachingPropertyResolver.Resolve(PropertyField); - - [Benchmark] - public string CachedFieldUsingPropertyInfo() => _propertyResolver.Resolve(PropertyField); - - [Benchmark(Baseline = true)] - public string NonCachedFieldUsingString() => _nonCachingStringResolver.Resolve(StringField); - - [Benchmark] - public string CachedFieldUsingString() => _stringResolver.Resolve(StringField); - } -} diff --git a/src/Tests/Tests.Benchmarking/NoncachingFieldResolver.cs b/src/Tests/Tests.Benchmarking/NoncachingFieldResolver.cs deleted file mode 100644 index 8524768a044..00000000000 --- a/src/Tests/Tests.Benchmarking/NoncachingFieldResolver.cs +++ /dev/null @@ -1,167 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Globalization; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Text; -using Nest; - -namespace Tests.Benchmarking -{ - public class NoncachingFieldResolver - { - private readonly IConnectionSettingsValues _settings; - - public NoncachingFieldResolver(IConnectionSettingsValues settings) => _settings = settings; - - public string Resolve(Field field) - { - var name = ResolveFieldName(field); - if (field.Boost.HasValue) name += $"^{field.Boost.Value.ToString(CultureInfo.InvariantCulture)}"; - return name; - } - - internal static bool IsConditionless(Field field) => - field == null || string.IsNullOrEmpty(field.Name) && field.Expression == null && field.Property == null; - - internal static bool IsConditionless(PropertyName property) => - property == null || string.IsNullOrEmpty(property.Name) && property.Expression == null && property.Property == null; - - private string ResolveFieldName(Field field) - { - if (IsConditionless(field)) return null; - if (!string.IsNullOrEmpty(field.Name)) return field.Name; - if (field.Expression != null && !field.CachableExpression) return Resolve(field.Expression, field.Property); - - var fieldName = Resolve(field.Expression, field.Property); - return fieldName; - } - - public string Resolve(PropertyName property) - { - if (IsConditionless(property)) return null; - if (!string.IsNullOrEmpty(property.Name)) return property.Name; - - if (property.Expression != null && !property.CacheableExpression) return Resolve(property.Expression, property.Property); - - var propertyName = Resolve(property.Expression, property.Property, true); - return propertyName; - } - - private string Resolve(Expression expression, MemberInfo member, bool toLastToken = false) - { - var visitor = new FieldExpressionVisitor(_settings); - var name = expression != null - ? visitor.Resolve(expression, toLastToken) - : member != null - ? visitor.Resolve(member) - : null; - - if (name == null) - throw new ArgumentException("Name resolved to null for the given Expression or MemberInfo."); - - return name; - } - - internal class FieldExpressionVisitor : ExpressionVisitor - { - private readonly IConnectionSettingsValues _settings; - private readonly Stack _stack = new Stack(); - - public FieldExpressionVisitor(IConnectionSettingsValues settings) => _settings = settings; - - public string Resolve(Expression expression, bool toLastToken = false) - { - Visit(expression); - if (toLastToken) return Enumerable.Last(_stack); - - return Enumerable.Aggregate(_stack, new StringBuilder(), - (sb, name) => - (sb.Length > 0 ? sb.Append(".") : sb).Append(name)) - .ToString(); - } - - public string Resolve(MemberInfo info) - { - if (info == null) - return null; - - var name = info.Name; - - if (_settings.PropertyMappings.TryGetValue(info, out var propertyMapping)) - return propertyMapping.Name; - - var att = ElasticsearchPropertyAttributeBase.From(info); - if (att != null && !string.IsNullOrEmpty(att.Name)) - return att.Name; - - return _settings.PropertyMappingProvider?.CreatePropertyMapping(info)?.Name ?? _settings.DefaultFieldNameInferrer(name); - } - - protected override Expression VisitMember(MemberExpression expression) - { - if (_stack == null) return base.VisitMember(expression); - - var name = Resolve(expression.Member); - _stack.Push(name); - return base.VisitMember(expression); - } - - protected override Expression VisitMethodCall(MethodCallExpression methodCall) - { - if (methodCall.Method.Name == "Suffix" && methodCall.Arguments.Any()) - { - VisitConstantOrVariable(methodCall, _stack); - var callingMember = new ReadOnlyCollection( - new List { { methodCall.Arguments.First() } } - ); - Visit(callingMember); - return methodCall; - } - else if (methodCall.Method.Name == "get_Item" && methodCall.Arguments.Any()) - { - var t = methodCall.Object.Type; - var isDict = - typeof(IDictionary).IsAssignableFrom(t) - || typeof(IDictionary<,>).IsAssignableFrom(t) - || t.GetTypeInfo().IsGenericType && t.GetGenericTypeDefinition() == typeof(IDictionary<,>); - - if (!isDict) return base.VisitMethodCall(methodCall); - - VisitConstantOrVariable(methodCall, _stack); - Visit(methodCall.Object); - return methodCall; - } - else if (IsLinqOperator(methodCall.Method)) - { - for (var i = 1; i < methodCall.Arguments.Count; i++) Visit(methodCall.Arguments[i]); - Visit(methodCall.Arguments[0]); - return methodCall; - } - return base.VisitMethodCall(methodCall); - } - - private static void VisitConstantOrVariable(MethodCallExpression methodCall, Stack stack) - { - var lastArg = methodCall.Arguments.Last(); - var constantExpression = lastArg as ConstantExpression; - var value = constantExpression != null - ? constantExpression.Value.ToString() - : Expression.Lambda(lastArg).Compile().DynamicInvoke().ToString(); - stack.Push(value); - } - - private static bool IsLinqOperator(MethodInfo methodInfo) - { - if (methodInfo.DeclaringType != typeof(Queryable) && methodInfo.DeclaringType != typeof(Enumerable)) - return false; - - return methodInfo.GetCustomAttribute() != null; - } - } - } -} diff --git a/src/Tests/Tests.Benchmarking/PostDataBenchmarks.cs b/src/Tests/Tests.Benchmarking/PostDataBenchmarks.cs deleted file mode 100644 index 3964cf0b773..00000000000 --- a/src/Tests/Tests.Benchmarking/PostDataBenchmarks.cs +++ /dev/null @@ -1,437 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using BenchmarkDotNet.Attributes; -using Elasticsearch.Net; -using Nest; -using Newtonsoft.Json.Linq; -using Tests.Benchmarking.Framework; - -namespace Tests.Benchmarking -{ - //[BenchmarkConfig(1000)] - public class PostDataBenchmarks - { - private readonly PostData _postDataOfBytes; - - - private readonly PostData _postDataOfBytesDisableDirectStreaming; - private readonly PostData _postDataOfCollectionOfComplexObjects; - private readonly PostData _postDataOfCollectionOfComplexObjectsDisableDirectStreaming; - private readonly PostData _postDataOfCollectionOfSimpleObjects; - private readonly PostData _postDataOfCollectionOfSimpleObjectsDisableDirectStreaming; - private readonly PostData _postDataOfCollectionOfStrings; - private readonly PostData _postDataOfCollectionOfStringsDisableDirectStreaming; - private readonly PostData _postDataOfComplexObject; - private readonly PostData _postDataOfComplexObjectDisableDirectStreaming; - private readonly PostData _postDataOfSimpleObject; - private readonly PostData _postDataOfSimpleObjectDisableDirectStreaming; - private readonly PostData _postDataOfString; - private readonly PostData _postDataOfStringDisableDirectStreaming; - private readonly byte[] bytes = Encoding.UTF8.GetBytes("{my_property=\"value\"}"); - private readonly List collectionOfComplexObjects; - private readonly List collectionOfSimpleObjects; - private readonly List collectionOfStrings = Enumerable.Range(0, 5).Select(i => i.ToString()).ToList(); - private readonly object complexObject; - private readonly ConnectionSettings connectionSettings = new ConnectionSettings(); - - private readonly object simpleObject; - private readonly string @string = "{my_property=\"value\"}"; - - public PostDataBenchmarks() - { - simpleObject = new { my_property = "value" }; - complexObject = new - { - input = new - { - chain = new - { - inputs = new object[] - { - new - { - simple = new - { - simple = new - { - str = "val1", - num = 23, - obj = new - { - str = "val2" - } - } - } - }, - new - { - http = new - { - http = new - { - request = new - { - host = "localhost", - port = 8080, - method = "post", - path = "/path.html", - proxy = new - { - host = "proxy", - port = 6000 - }, - scheme = "https", - auth = new - { - basic = new - { - username = "Username123", - password = "Password123" - } - }, - body = - "{\"query\" : {\"range\": {\"@timestamp\" : {\"from\": \"{{ctx.trigger.triggered_time}}||-5m\",\"to\": \"{{ctx.trigger.triggered_time}}\"}}}}", - headers = new - { - header1 = "value1" - }, - @params = new - { - lat = "52.374031", - lon = "4.88969", - appid = "appid" - }, - connection_timeout = "3s", - read_timeout = "500ms" - }, - response_content_type = "text" - } - } - }, - new - { - search = new - { - search = new - { - request = new - { - indices = new[] { "project" }, - body = new - { - size = 0, - aggs = new - { - nested_tags = new - { - nested = new - { - path = "tags" - }, - aggs = new - { - top_project_tags = new - { - terms = new - { - field = "tags.name" - } - } - } - } - } - } - } - } - } - } - } - } - }, - transform = new - { - chain = new object[] - { - new - { - search = new - { - request = new - { - indices = new[] { "project" }, - indices_options = new - { - expand_wildcards = "open", - ignore_unavailable = true - }, - search_type = "dfs_query_then_fetch", - body = new - { - query = new - { - match = new - { - state = new - { - query = "stable" - } - } - } - } - }, - timeout = "10s" - } - }, - new - { - script = new - { - inline = "return [ 'time' : ctx.trigger.scheduled_time ]" - } - } - } - }, - condition = new - { - array_compare = new JObject - { - { - "ctx.payload.search.aggregations.top_project_tags.buckets", new JObject - { - { "path", "doc_count" }, - { "gte", new JObject { { "value", 1 } } } - } - } - } - }, - trigger = new - { - schedule = new - { - weekly = new[] - { - new { on = new[] { "monday" }, at = new[] { "noon" } }, - new { on = new[] { "friday" }, at = new[] { "17:00" } } - } - } - }, - actions = new - { - reminder_email = new - { - email = new - { - to = new[] { "me@example.com" }, - subject = "Something's strange in the neighbourhood", - body = new - { - text = "Dear {{ctx.payload.name}}, by the time you read these lines, I'll be gone" - }, - attachments = new - { - http_attachment = new - { - http = new - { - inline = true, - content_type = RequestData.MimeType, - request = new - { - url = "http://localhost:8080/http_attachment" - } - } - }, - data_attachment = new - { - data = new - { - format = "json" - } - } - } - } - }, - reminder_index = new - { - index = new - { - index = "put-watch-test-index", - doc_type = "reminder", - execution_time_field = "execution_time" - } - }, - reminder_pagerduty = new - { - pagerduty = new - { - account = "my_pagerduty_account", - description = "pager duty description", - attach_payload = true, - event_type = "trigger", - incident_key = "incident_key", - context = new object[] - { - new - { - type = "image", - src = "http://example.com/image" - }, - new - { - type = "link", - href = "http://example.com/link" - } - } - } - }, - reminder_slack = new - { - slack = new - { - account = "monitoring", - message = new - { - from = "nest integration test", - to = new[] { "#nest" }, - text = "slack message", - attachments = new[] - { - new - { - title = "Attachment 1", - author_name = "Russ Cam" - } - } - } - } - }, - reminder_hipchat = new - { - hipchat = new - { - account = "notify-monitoring", - message = new - { - body = "hipchat message", - color = "purple", - room = new[] { "nest" }, - notify = true - } - } - } - } - }; - - collectionOfSimpleObjects = Enumerable.Range(0, 5).Select(i => simpleObject).ToList(); - collectionOfComplexObjects = Enumerable.Range(0, 5).Select(i => complexObject).ToList(); - - _postDataOfString = PostData.String(@string); - _postDataOfBytes = PostData.Bytes(bytes); - _postDataOfCollectionOfStrings = PostData.MultiJson(collectionOfStrings); - _postDataOfCollectionOfSimpleObjects = PostData.MultiJson(collectionOfSimpleObjects); - _postDataOfCollectionOfComplexObjects = PostData.MultiJson(collectionOfComplexObjects); - _postDataOfSimpleObject = PostData.Serializable(simpleObject); - _postDataOfComplexObject = PostData.Serializable(complexObject); - - PostData DisableStreaming(PostData data) - { - data.DisableDirectStreaming = true; - return data; - } - - _postDataOfStringDisableDirectStreaming = DisableStreaming(PostData.String(@string)); - _postDataOfBytesDisableDirectStreaming = DisableStreaming(PostData.Bytes(bytes)); - _postDataOfCollectionOfStringsDisableDirectStreaming = DisableStreaming(PostData.MultiJson(collectionOfStrings)); - _postDataOfCollectionOfSimpleObjectsDisableDirectStreaming = DisableStreaming(PostData.MultiJson(collectionOfSimpleObjects)); - _postDataOfCollectionOfComplexObjectsDisableDirectStreaming = DisableStreaming(PostData.MultiJson(collectionOfComplexObjects)); - _postDataOfSimpleObjectDisableDirectStreaming = DisableStreaming(PostData.Serializable(simpleObject)); - _postDataOfComplexObjectDisableDirectStreaming = DisableStreaming(PostData.Serializable(complexObject)); - } - - [Benchmark] - public void PostString() - { - using (var ms = new MemoryStream()) _postDataOfString.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostBytes() - { - using (var ms = new MemoryStream()) _postDataOfBytes.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostCollectionOfStrings() - { - using (var ms = new MemoryStream()) _postDataOfCollectionOfStrings.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostCollectionOfSimpleObjects() - { - using (var ms = new MemoryStream()) _postDataOfCollectionOfSimpleObjects.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostCollectionOfComplexObjects() - { - using (var ms = new MemoryStream()) _postDataOfCollectionOfComplexObjects.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostSimpleObject() - { - using (var ms = new MemoryStream()) _postDataOfSimpleObject.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostComplexObject() - { - using (var ms = new MemoryStream()) _postDataOfComplexObject.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostStringDisableDirectStreaming() - { - using (var ms = new MemoryStream()) _postDataOfStringDisableDirectStreaming.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostBytesDisableDirectStreaming() - { - using (var ms = new MemoryStream()) _postDataOfBytesDisableDirectStreaming.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostCollectionOfStringsDisableDirectStreaming() - { - using (var ms = new MemoryStream()) _postDataOfCollectionOfStringsDisableDirectStreaming.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostCollectionOfSimpleObjectsDisableDirectStreaming() - { - using (var ms = new MemoryStream()) _postDataOfCollectionOfSimpleObjectsDisableDirectStreaming.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostCollectionOfComplexObjectsDisableDirectStreaming() - { - using (var ms = new MemoryStream()) _postDataOfCollectionOfComplexObjectsDisableDirectStreaming.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostSimpleObjectDisableDirectStreaming() - { - using (var ms = new MemoryStream()) _postDataOfSimpleObjectDisableDirectStreaming.Write(ms, connectionSettings); - } - - [Benchmark] - public void PostComplexObjectDisableDirectStreaming() - { - using (var ms = new MemoryStream()) _postDataOfComplexObjectDisableDirectStreaming.Write(ms, connectionSettings); - } - } -} diff --git a/src/Tests/Tests.Benchmarking/PropertyNameResolverBenchmarkTests.cs b/src/Tests/Tests.Benchmarking/PropertyNameResolverBenchmarkTests.cs deleted file mode 100644 index d056e786391..00000000000 --- a/src/Tests/Tests.Benchmarking/PropertyNameResolverBenchmarkTests.cs +++ /dev/null @@ -1,50 +0,0 @@ -using BenchmarkDotNet.Attributes; -using Nest; -using Tests.Benchmarking.Framework; -using Tests.Domain; - -namespace Tests.Benchmarking -{ - //[BenchmarkConfig(100)] - public class PropertyNameResolverBenchmarkTests - { - private static readonly PropertyName InferredPropertyName = Infer.Property(p => p.Name); - private static readonly PropertyName PropertyInfoPropertyName = typeof(Project).GetProperty(nameof(Project.Name)); - private static readonly PropertyName StringPropertyName = "Name"; - private FieldResolver _expressionResolver; - private NoncachingFieldResolver _nonCachingExpressionResolver; - private NoncachingFieldResolver _nonCachingPropertyResolver; - private NoncachingFieldResolver _nonCachingStringResolver; - private FieldResolver _propertyResolver; - private FieldResolver _stringResolver; - - [GlobalSetup] - public void Setup() - { - _expressionResolver = new FieldResolver(new ConnectionSettings()); - _propertyResolver = new FieldResolver(new ConnectionSettings()); - _stringResolver = new FieldResolver(new ConnectionSettings()); - _nonCachingExpressionResolver = new NoncachingFieldResolver(new ConnectionSettings()); - _nonCachingPropertyResolver = new NoncachingFieldResolver(new ConnectionSettings()); - _nonCachingStringResolver = new NoncachingFieldResolver(new ConnectionSettings()); - } - - [Benchmark] - public string NonCachedPropertyUsingExpression() => _nonCachingExpressionResolver.Resolve(InferredPropertyName); - - [Benchmark] - public string CachedPropertyUsingExpression() => _expressionResolver.Resolve(InferredPropertyName); - - [Benchmark] - public string NonCachedPropertyUsingPropertyInfo() => _nonCachingPropertyResolver.Resolve(PropertyInfoPropertyName); - - [Benchmark] - public string CachedPropertyUsingPropertyInfo() => _propertyResolver.Resolve(PropertyInfoPropertyName); - - [Benchmark(Baseline = true)] - public string NonCachedPropertyUsingString() => _nonCachingStringResolver.Resolve(StringPropertyName); - - [Benchmark] - public string CachedPropertyUsingString() => _stringResolver.Resolve(StringPropertyName); - } -} diff --git a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj index 797a02a2afb..51707d39a47 100644 --- a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj +++ b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj @@ -3,7 +3,6 @@ netcoreapp2.1;net461 Exe - https://ci.appveyor.com/nuget/benchmarkdotnet;https://ci.appveyor.com/nuget/elasticsearch-net-abstractions;$(RestoreSources) true @@ -12,8 +11,8 @@ - - + + \ No newline at end of file diff --git a/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj b/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj index 636587a5103..a0de4ab8206 100644 --- a/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj +++ b/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj @@ -9,6 +9,6 @@ - + \ No newline at end of file From d2fb141d4c05e9df5b03f1d8e914a7f51ec4478c Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Tue, 20 Nov 2018 15:27:08 +0000 Subject: [PATCH 06/10] re-enable benchmarking --- src/Tests/Tests.Benchmarking/BenchmarkProgram.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs index efd01620ce3..c5e8fe8c7e8 100644 --- a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs +++ b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs @@ -47,7 +47,6 @@ static Program() public static int Main(string[] arguments) { Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) on {Repository} : {CommitMessage} - "); - return 0; var config = CreateDefaultConfig(); if (arguments.Any() && arguments[0].Equals("--all", StringComparison.OrdinalIgnoreCase)) { From 74925e6ac2367b673dac322b81e65671b3aff0a7 Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Tue, 20 Nov 2018 15:43:50 +0000 Subject: [PATCH 07/10] make sure we report the repository as well --- src/Tests/Tests.Benchmarking/BenchmarkProgram.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs index c5e8fe8c7e8..305da888c8d 100644 --- a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs +++ b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs @@ -35,7 +35,7 @@ static Program() using (var repos = new Repository(dirInfo.FullName)) { Commit = repos.Head.Tip.Sha; - CommitMessage = repos.Head.Tip.Message; + CommitMessage = repos.Head.Tip.Message?.Trim(' ', '\t', '\r', '\n'); Branch = repos.Head.FriendlyName; var remoteName = repos.Head.RemoteName; Repository = @@ -102,7 +102,8 @@ private static ElasticsearchBenchmarkExporter CreateElasticsearchExporter(string Password = password, GitCommitSha = Commit, GitBranch = Branch, - GitCommitMessage = CommitMessage + GitCommitMessage = CommitMessage, + GitRepositoryIdentifier = Repository }; return new ElasticsearchBenchmarkExporter(options); } From 02983bae3f9349aed7237546bed306d5f8fe3c17 Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Wed, 21 Nov 2018 13:55:18 +0000 Subject: [PATCH 08/10] update paket targets --- .paket/Paket.Restore.targets | 67 ++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/.paket/Paket.Restore.targets b/.paket/Paket.Restore.targets index 7c6107011de..d93abfef97a 100644 --- a/.paket/Paket.Restore.targets +++ b/.paket/Paket.Restore.targets @@ -151,15 +151,17 @@ + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',').Length) $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0]) $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1]) $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[4]) - $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[5]) + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[5]) %(PaketReferencesFileLinesInfo.PackageVersion) All - runtime + runtime + runtime true @@ -192,19 +194,27 @@ false + $(MSBuildVersion) + 15.8.0 <_NuspecFilesNewLocation Include="$(BaseIntermediateOutputPath)$(Configuration)\*.nuspec"/> + + $(MSBuildProjectDirectory)/$(MSBuildProjectFile) true - false - true + false + true + false + true + false + true $(BaseIntermediateOutputPath)$(Configuration) $(BaseIntermediateOutputPath) @@ -219,9 +229,52 @@ - - + + - Date: Wed, 21 Nov 2018 13:56:27 +0000 Subject: [PATCH 09/10] update to latest exporter and update the build scripts to use the new arguments --- build/scripts/Benchmarking.fsx | 51 ++++++------------- build/scripts/Commandline.fsx | 23 ++------- build/scripts/Paths.fsx | 5 -- build/scripts/Targets.fsx | 5 +- .../Tests.Benchmarking/BenchmarkProgram.cs | 4 +- .../Tests.Benchmarking.csproj | 2 +- 6 files changed, 25 insertions(+), 65 deletions(-) diff --git a/build/scripts/Benchmarking.fsx b/build/scripts/Benchmarking.fsx index 63ed0a5cf15..9a5a499b302 100644 --- a/build/scripts/Benchmarking.fsx +++ b/build/scripts/Benchmarking.fsx @@ -1,51 +1,32 @@ -#r "../../packages/build/NEST/lib/net46/Nest.dll" -#r "../../packages/build/Elasticsearch.Net/lib/net46/Elasticsearch.Net.dll" -#r "../../packages/build/Newtonsoft.Json/lib/net45/Newtonsoft.Json.dll" -#r "../../packages/build/FSharp.Data/lib/net45/FSharp.Data.dll" -#I @"../../packages/build/FAKE/tools" +#I @"../../packages/build/FAKE/tools" #r @"FakeLib.dll" #nowarn "0044" //TODO sort out FAKE 5 -open Fake - +#load @"Commandline.fsx" #load @"Paths.fsx" -open System +open Fake open System.IO -open System.Linq -open System.Diagnostics open Paths - -open FSharp.Data - -open Nest -open Elasticsearch.Net -open Newtonsoft.Json -open Git.Branches -open Git.Information +open Commandline module Benchmarker = - let pipelineName = "benchmark-pipeline" - let indexName = IndexName.op_Implicit("benchmark-reports") - let typeName = TypeName.op_Implicit("benchmarkreport") - let private testsProjectDirectory = Path.GetFullPath(Paths.TestsSource("Tests.Benchmarking")) - let Run(runInteractive:bool) = - + let Run() = + let runInteractive = not Commandline.nonInteractive let url = getBuildParam "elasticsearch" let username = getBuildParam "username" let password = getBuildParam "password" + let hasUrl = not <| isNullOrEmpty url + let hasCredentials = not <| (isNullOrEmpty username && isNullOrEmpty password) + let runCommandPrefix = "run -f netcoreapp2.1 -c Release" + let runCommand = + match (runInteractive, hasUrl, hasCredentials) with + | (false, true, true) -> sprintf "%s -- --all \"%s\" \"%s\" \"%s\"" runCommandPrefix url username password + | (false, true, false) -> sprintf "%s -- --all \"%s\"" runCommandPrefix url + | (false, _, _) -> sprintf "%s -- --all" runCommandPrefix + | (true, _, _) -> runCommandPrefix - try - if runInteractive then - DotNetCli.RunCommand(fun p -> - { p with - WorkingDir = testsProjectDirectory - }) "run -f netcoreapp2.1 -c Release" - else - DotNetCli.RunCommand(fun p -> - { p with - WorkingDir = testsProjectDirectory - }) "run -f netcoreapp2.1 -c Release non-interactive" + DotNetCli.RunCommand(fun p -> { p with WorkingDir = testsProjectDirectory }) runCommand diff --git a/build/scripts/Commandline.fsx b/build/scripts/Commandline.fsx index 43ba81bf304..512a6421a38 100644 --- a/build/scripts/Commandline.fsx +++ b/build/scripts/Commandline.fsx @@ -32,7 +32,7 @@ Targets: * cluster [version] - Start a cluster defined in Tests.Core or Tests from the command line and leaves it running untill a key is pressed. Handy if you want to run the integration tests numerous times while developing -* benchmark [url] [username] [password] [non-interactive] +* benchmark [non-interactive] [url] [username] [password] - Runs a benchmark from Tests.Benchmarking and indexes the results to [url] when provided. If non-interactive runs all benchmarks without prompting @@ -41,6 +41,7 @@ NOTE: both the `test` and `integrate` targets can be suffixed with `-all` to for Execution hints can be provided anywhere on the command line - skiptests : skip running tests as part of the target chain - skipdocs : skip generating documentation +- non-interactive : make targets that run in interactive mode by default to run unassisted. - docs: : the branch name B to use when generating documentation - seed: : provide a seed to run the tests with. - random:<:B> : sets random K to bool B if if B is omitted will default to true @@ -52,6 +53,7 @@ module Commandline = let private args = getBuildParamOrDefault "cmdline" "build" |> split ' ' + let nonInteractive = args |> List.exists (fun x -> x = "non-interactive") let skipTests = args |> List.exists (fun x -> x = "skiptests") let skipDocs = args |> List.exists (fun x -> x = "skipdocs") || isMono let seed = @@ -75,6 +77,7 @@ module Commandline = fun x -> x <> "skiptests" && x <> "skipdocs" && + x <> "non-interactive" && not (x.StartsWith("seed:")) && not (x.StartsWith("random:")) && not (x.StartsWith("docs:")) @@ -152,29 +155,13 @@ module Commandline = | ["test"; testFilter] -> setBuildParam "testfilter" testFilter - | ["benchmark"; IsUrl elasticsearch; username; password; "non-interactive"] -> - setBuildParam "elasticsearch" elasticsearch - setBuildParam "nonInteractive" "1" - setBuildParam "username" username - setBuildParam "password" password - - | ["benchmark"; IsUrl elasticsearch; "non-interactive"] -> - setBuildParam "elasticsearch" elasticsearch - setBuildParam "nonInteractive" "1" - - | ["benchmark"; "non-interactive"] -> - setBuildParam "nonInteractive" "1" - | ["benchmark"; IsUrl elasticsearch; username; password] -> setBuildParam "elasticsearch" elasticsearch - setBuildParam "nonInteractive" "0" setBuildParam "username" username setBuildParam "password" password - | ["benchmark"; IsUrl elasticsearch] -> setBuildParam "elasticsearch" elasticsearch - setBuildParam "nonInteractive" "0" - + | ["profile"; IsUrl elasticsearch] -> setBuildParam "elasticsearch" elasticsearch diff --git a/build/scripts/Paths.fsx b/build/scripts/Paths.fsx index 6233e4bd345..6666bd24e47 100644 --- a/build/scripts/Paths.fsx +++ b/build/scripts/Paths.fsx @@ -4,11 +4,6 @@ #load @"Projects.fsx" -open System -open System.IO -open System.Diagnostics -open System.Net - open Fake open Projects diff --git a/build/scripts/Targets.fsx b/build/scripts/Targets.fsx index 554bf49b119..ac0ee5246a4 100644 --- a/build/scripts/Targets.fsx +++ b/build/scripts/Targets.fsx @@ -59,10 +59,7 @@ Target "Profile" <| fun _ -> Target "Integrate" Tests.RunIntegrationTests -Target "Benchmark" <| fun _ -> - let runInteractive = ((getBuildParam "nonInteractive") <> "1") - Benchmarker.Run(runInteractive) - Benchmarker.IndexResults (url, username, password) +Target "Benchmark" Benchmarker.Run Target "InternalizeDependencies" Build.ILRepack diff --git a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs index 305da888c8d..9b9973c5f3e 100644 --- a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs +++ b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs @@ -66,8 +66,8 @@ private static IConfig CreateDefaultConfig() var jobs = new[] { Job.ShortRun.With(Runtime.Core).With(Jit.RyuJit), -// Job.ShortRun.With(Runtime.Clr).With(Jit.RyuJit), -// Job.ShortRun.With(Runtime.Clr).With(Jit.LegacyJit), + Job.ShortRun.With(Runtime.Clr).With(Jit.RyuJit), + Job.ShortRun.With(Runtime.Clr).With(Jit.LegacyJit), }; var config = DefaultConfig.Instance .With(jobs) diff --git a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj index 51707d39a47..004d8b1faec 100644 --- a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj +++ b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj @@ -12,7 +12,7 @@ - + \ No newline at end of file From 38df161f9f12ab02b59d5024c733c20d6096e07d Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Wed, 21 Nov 2018 14:09:04 +0000 Subject: [PATCH 10/10] make sure Tests.ScratchPad uses the same BDNet version --- src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj b/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj index a0de4ab8206..ebbd46c4e51 100644 --- a/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj +++ b/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj @@ -9,6 +9,6 @@ - + \ No newline at end of file