1- #I @" ../../packages/build/FAKE/tools"
1+ #r " ../../packages/build/NEST/lib/net45/Nest.dll"
2+ #r " ../../packages/build/Elasticsearch.Net/lib/net45/Elasticsearch.Net.dll"
3+ #r " ../../packages/build/Newtonsoft.Json/lib/net45/Newtonsoft.Json.dll"
4+ #r " ../../packages/build/FSharp.Data/lib/net40/FSharp.Data.dll"
5+ #I @" ../../packages/build/FAKE/tools"
26#r @" FakeLib.dll"
7+ #nowarn " 0044" //TODO sort out FAKE 5
8+
39open Fake
410
511#load @" Paths.fsx"
@@ -10,31 +16,242 @@ open System.Linq
1016open System.Diagnostics
1117open Paths
1218
19+ open FSharp.Data
20+
21+ open Nest
22+ open Elasticsearch.Net
23+ open Newtonsoft.Json
24+ open Git.Branches
25+ open Git.Information
26+
1327module Benchmarker =
14- let private testsProjectDirectory = Path.GetFullPath( Paths.Source( " Tests" ))
15- let private benchmarkOutput = Path.GetFullPath( Paths.Output( " benchmarks" )) |> directoryInfo
1628
17- let private copyToOutput file = CopyFile benchmarkOutput.FullName file
29+ let pipelineName = " benchmark-pipeline"
30+ let indexName = IndexName.op_ Implicit( " benchmark-reports" )
31+ let typeName = TypeName.op_ Implicit( " benchmarkreport" )
32+
33+ type Memory ( gen0Collections : int , gen1Collections : int , gen2Collections : int , totalOperations : int64 , bytesAllocatedPerOperation : int64 ) =
34+ member val Gen0Collections = gen0Collections with get, set
35+ member val Gen1Collections = gen1Collections with get, set
36+ member val Gen2Collections = gen2Collections with get, set
37+ member val TotalOperations = totalOperations with get, set
38+ member val BytesAllocatedPerOperation = bytesAllocatedPerOperation with get, set
39+
40+ type ChronometerFrequency ( hertz : double ) =
41+ member val Hertz = hertz with get, set
42+
43+ type HostEnvironmentInfo ( benchmarkDotNetCaption : string , benchmarkDotNetVersion : string , osVersion : string , processorName : string ,
44+ processorCount: int, runtimeVersion: string, architecture: string, hasAttachedDebugger: bool, hasRyuJit: bool,
45+ configuration: string, jitModules: string, dotnetCliVersion: string, chronometerFrequency: ChronometerFrequency,
46+ hardwareTimerKind: string) =
47+ member val BenchmarkDotNetCaption = benchmarkDotNetCaption with get, set
48+ member val BenchmarkDotNetVersion = benchmarkDotNetVersion with get, set
49+ member val OsVersion = osVersion with get, set
50+ member val ProcessorName = processorName with get, set
51+ member val ProcessorCount = processorCount with get, set
52+ member val RuntimeVersion = runtimeVersion with get, set
53+ member val Architecture = architecture with get, set
54+ member val HasAttachedDebugger = hasAttachedDebugger with get, set
55+ member val HasRyuJit = hasRyuJit with get, set
56+ member val Configuration = configuration with get, set
57+ member val JitModules = jitModules with get, set
58+ member val DotNetCliVersion = dotnetCliVersion with get, set
59+ member val ChronometerFrequency = chronometerFrequency with get, set
60+ member val HardwareTimerKind = hardwareTimerKind with get, set
61+
62+ type ConfidenceInterval ( n : int , mean : double , standardError : double , level : int , margin : double , lower : double , upper : double ) =
63+ member val N = n with get, set
64+ member val Mean = mean with get, set
65+ member val StandardError = standardError with get, set
66+ member val Level = level with get, set
67+ member val Margin = margin with get, set
68+ member val Lower = lower with get, set
69+ member val Upper = upper with get, set
70+
71+ type Percentiles ( p0 : double , p25 : double , p50 : double , p67 : double , p80 : double , p85 : double , p90 : double , p95 : double , p100 : double ) =
72+ member val P0 = p0 with get, set
73+ member val P25 = p25 with get, set
74+ member val P50 = p50 with get, set
75+ member val P67 = p67 with get, set
76+ member val P80 = p80 with get, set
77+ member val P85 = p85 with get, set
78+ member val P90 = p90 with get, set
79+ member val P95 = p95 with get, set
80+ member val P100 = p100 with get, set
81+
82+ type Statistics ( n : int , min : double , lowerFence : double , q1 : double , median : double , mean : double , q3 : double , upperFence : double , max : double ,
83+ interquartileRange: double, outliers: double list, standardError: double, variance: double, standardDeviation: double,
84+ skewness: double, kurtosis: double, confidenceInterval: ConfidenceInterval, percentiles: Percentiles) =
85+ member val N = n with get, set
86+ member val Min = min with get, set
87+ member val LowerFence = lowerFence with get, set
88+ member val Q1 = q1 with get, set
89+ member val Median = median with get, set
90+ member val Mean = mean with get, set
91+ member val Q3 = q3 with get, set
92+ member val UpperFence = upperFence with get, set
93+ member val Max = max with get, set
94+ member val InterquartileRange = interquartileRange with get, set
95+ member val Outliers = outliers with get, set
96+ member val StandardError = standardError with get, set
97+ member val Variance = variance with get, set
98+ member val StandardDeviation = standardDeviation with get, set
99+ member val Skewness = skewness with get, set
100+ member val Kurtosis = kurtosis with get, set
101+ member val ConfidenceInterval = confidenceInterval with get, set
102+ member val Percentiles = percentiles with get, set
103+
104+ type Benchmark ( displayInfo : string , namespyce : string , tipe : string , method : string , methodTitle : string , parameters : string ,
105+ statistics: Statistics, memory: Memory) =
106+ member val DisplayInfo = displayInfo with get, set
107+ member val Namespace = namespyce with get, set
108+ member val Type = tipe with get, set
109+ member val Method = method with get, set
110+ member val MethodTitle = methodTitle with get, set
111+ member val Parameters = parameters with get, set
112+ member val Statistics = statistics with get, set
113+ member val Memory = memory with get, set
114+
115+ type BenchmarkReports ( title : string , totalTime : TimeSpan , date : DateTime , commit : string , branchName : string , host : HostEnvironmentInfo , benchmarks : Benchmark list ) =
116+ member val Title = title with get, set
117+ member val TotalTime = totalTime with get, set
118+ member val Date = date with get, set
119+ member val Commit = commit with get, set
120+ member val BranchName = branchName with get, set
121+ member val HostEnvironmentInfo = host with get, set
122+ member val Benchmarks = benchmarks with get, set
123+
124+ type BenchmarkReport ( title : string , totalTime : TimeSpan , date : DateTime , commit : string , branchName : string , host : HostEnvironmentInfo , benchmark : Benchmark ) =
125+ member val Title = title with get, set
126+ member val TotalTime = totalTime with get, set
127+ member val Date = date with get, set
128+ member val Commit = commit with get, set
129+ member val BranchName = branchName with get, set
130+ member val HostEnvironmentInfo = host with get, set
131+ member val Benchmark = benchmark with get, set
132+
133+ let private testsProjectDirectory = Path.GetFullPath( Paths.Source( " Tests" ))
134+ let private benchmarkOutput = Path.GetFullPath( Paths.Output( " benchmarks" )) |> directoryInfo
135+ let private copyToOutput file = CopyFile benchmarkOutput.FullName file
136+
137+ let Run ( runInteractive : bool ) =
18138
19- let Run () =
20139 ensureDirExists benchmarkOutput
21- let projectJson = testsProjectDirectory @@ " project.json"
22- // running benchmarks can timeout so clean up any generated benchmark files
140+
23141 try
24- DotNetCli.Restore( fun p ->
25- { p with
26- Project = projectJson
27- })
28-
29- DotNetCli.RunCommand( fun p ->
30- { p with
31- WorkingDir = testsProjectDirectory
32- }) " run Benchmark"
142+ if runInteractive then
143+ DotNetCli.RunCommand( fun p ->
144+ { p with
145+ WorkingDir = testsProjectDirectory
146+ }) " run -f net46 -c Release Benchmark"
147+ else
148+ DotNetCli.RunCommand( fun p ->
149+ { p with
150+ WorkingDir = testsProjectDirectory
151+ }) " run -f net46 -c Release Benchmark non-interactive"
33152 finally
153+ // running benchmarks can timeout so clean up any generated benchmark files
34154 let benchmarkOutputFiles =
35155 let output = combinePaths testsProjectDirectory " BenchmarkDotNet.Artifacts"
36156 Directory.EnumerateFiles( output, " *.*" , SearchOption.AllDirectories)
37157 |> Seq.toList
38158
39159 for file in benchmarkOutputFiles do copyToOutput file
40160 DeleteFiles benchmarkOutputFiles
161+
162+ let IndexResult ( client : ElasticClient , file : string , date : DateTime , commit : string , branchName : string , indexName , typeName ) =
163+
164+ trace ( sprintf " Indexing benchmark results (class) %s " )
165+
166+ let benchmarkReports = JsonConvert.DeserializeObject< BenchmarkReports>( File.ReadAllText( file))
167+ benchmarkReports.Date <- date
168+ benchmarkReports.Commit <- commit
169+ benchmarkReports.BranchName <- branchName
170+
171+ for benchmarkReportSingle in benchmarkReports.Benchmarks do
172+
173+ trace ( sprintf " Indexing benchmark result (method) %s " )
174+
175+ let document = new BenchmarkReport( benchmarkReports.Title,
176+ benchmarkReports.TotalTime,
177+ benchmarkReports.Date,
178+ benchmarkReports.Commit,
179+ benchmarkReports.BranchName,
180+ benchmarkReports.HostEnvironmentInfo,
181+ benchmarkReportSingle)
182+
183+ let indexRequest = new IndexRequest< BenchmarkReport>( indexName, typeName)
184+ indexRequest.Document <- document
185+ indexRequest.Pipeline <- pipelineName
186+
187+ let indexResponse = client.Index( indexRequest)
188+
189+ if indexResponse.IsValid = false then
190+ raise ( Exception( " Unable to index benchmark result (method): " + indexResponse.ServerError.Error.ToString()))
191+
192+ let IndexResults ( url , username , password ) =
193+ if ( String.IsNullOrEmpty url = false ) then
194+ trace " Indexing benchmark reports"
195+
196+ let date = DateTime.UtcNow
197+ let commit = getSHA1 " ." " HEAD"
198+ let branchName = getBranchName " ."
199+
200+ let benchmarkJsonFiles =
201+ Directory.EnumerateFiles( benchmarkOutput.FullName, " *-custom.json" , SearchOption.AllDirectories)
202+ |> Seq.toList
203+
204+ let uri = new Uri( url)
205+ let connectionSettings = new ConnectionSettings( uri);
206+
207+ if ( String.IsNullOrEmpty username = false && String.IsNullOrEmpty password = false ) then
208+ connectionSettings.BasicAuthentication( username, password) |> ignore
209+
210+ let client = new ElasticClient( connectionSettings)
211+
212+ let indexTemplateExists = client.IndexTemplateExists( Name.op_ Implicit( " benchmarks" )) .Exists
213+
214+ if indexTemplateExists |> not then
215+
216+ let typeMapping = new TypeMappingDescriptor< BenchmarkReport>()
217+ typeMapping.AutoMap() |> ignore
218+
219+ let mappings = new Mappings()
220+ mappings.Add( typeName, typeMapping :> ITypeMapping)
221+
222+ let indexSettings = new IndexSettings()
223+ indexSettings.NumberOfShards <- Nullable 1
224+
225+ let putIndexTemplateRequest = new PutIndexTemplateRequest( Name.op_ Implicit( " benchmarks" ))
226+ putIndexTemplateRequest.Template <- " benchmark-reports-*"
227+ putIndexTemplateRequest.Mappings <- mappings
228+ putIndexTemplateRequest.Settings <- indexSettings
229+
230+ let putIndexTemplateResponse = client.PutIndexTemplate( putIndexTemplateRequest)
231+
232+ if putIndexTemplateResponse.IsValid = false then
233+ raise ( Exception( " Unable to create index template into Elasticsearch" ))
234+
235+ let grokProcessor = new GrokProcessor();
236+ grokProcessor.Field <- new Field( " benchmark.displayInfo" )
237+ grokProcessor.Patterns <- [ " %{WORD:class}.%{DATA:method}: Job-%{WORD:jobName}\\ (Jit=%{WORD:jit}, Runtime=%{WORD:clr}, LaunchCount=%{NUMBER:launchCount}, RunStrategy=%{WORD:runStrategy}, TargetCount=%{NUMBER:targetCount}, UnrollFactor=%{NUMBER:unrollFactor}, WarmupCount=%{NUMBER:warmupCount}\\ )" ]
238+
239+ let dateIndexProcessor = new DateIndexNameProcessor();
240+ dateIndexProcessor.Field <- new Field( " date" )
241+ dateIndexProcessor.IndexNamePrefix <- " benchmark-reports-"
242+ dateIndexProcessor.DateRounding <- DateRounding.Month
243+ dateIndexProcessor.DateFormats <- [ " yyyy-MM-dd'T'HH:mm:ss.SSSSSSSZ" ]
244+
245+ let request = new PutPipelineRequest( Id.op_ Implicit( pipelineName))
246+ request.Description <- " Benchmark settings pipeline"
247+ request.Processors <- [ dateIndexProcessor; grokProcessor]
248+
249+ let createPipeline = client.PutPipeline( request)
250+
251+ if createPipeline.IsValid = false then
252+ raise ( Exception( " Unable to create pipeline" ))
253+
254+ for file in benchmarkJsonFiles
255+ do IndexResult ( client, file, date, commit, branchName, indexName, typeName)
256+
257+ trace " Indexed benchmark reports"
0 commit comments