-
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
/
ips.cr
211 lines (171 loc) · 5.89 KB
/
ips.cr
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
module Benchmark
# Benchmark IPS calculates the number of iterations per second for a given
# block of code. The strategy is to use two stages: a warmup stage and a
# calculation stage.
#
# The warmup phase defaults to 2 seconds. During this stage we figure out how
# many cycles are needed to run the block for roughly 100ms, and record it.
#
# The calculation defaults to 5 seconds. During this stage we run the block
# in sets of the size calculated in the warmup stage. The measurements for
# those sets are then used to calculate the mean and standard deviation,
# which are then reported. Additionally we compare the means to that of the
# fastest.
module IPS
class Job
# List of all entries in the benchmark.
# After #execute, these are populated with the resulting statistics.
property items : Array(Entry)
@warmup_time : Time::Span
@calculation_time : Time::Span
def initialize(calculation = 5, warmup = 2, interactive = STDOUT.tty?)
@interactive = !!interactive
@warmup_time = warmup.seconds
@calculation_time = calculation.seconds
@items = [] of Entry
end
# Adds code to be benchmarked
def report(label = "", &action) : Benchmark::IPS::Entry
item = Entry.new(label, action)
@items << item
item
end
def execute : Nil
run_warmup
run_calculation
run_comparison
end
def report : Nil
max_label = ran_items.max_of &.label.size
max_compare = ran_items.max_of &.human_compare.size
max_bytes_per_op = ran_items.max_of &.bytes_per_op.humanize(base: 1024).size
ran_items.each do |item|
printf "%s %s (%s) (±%5.2f%%) %sB/op %s\n",
item.label.rjust(max_label),
item.human_mean,
item.human_iteration_time,
item.relative_stddev,
item.bytes_per_op.humanize(base: 1024).rjust(max_bytes_per_op),
item.human_compare.rjust(max_compare)
end
end
# The warmup stage gathers information about the items that is later used
# in the calculation stage
private def run_warmup
@items.each do |item|
GC.collect
count = 0
elapsed = Time.measure do
target = Time.monotonic + @warmup_time
while Time.monotonic < target
item.call
count += 1
end
end
item.set_cycles(elapsed, count)
end
end
private def run_calculation
@items.each do |item|
GC.collect
measurements = [] of Time::Span
bytes = 0_i64
cycles = 0_i64
target = Time.monotonic + @calculation_time
loop do
elapsed = nil
bytes_taken = Benchmark.memory do
elapsed = Time.measure { item.call_for_100ms }
end
bytes += bytes_taken
cycles += item.cycles
measurements << elapsed.not_nil!
break if Time.monotonic >= target
end
ips = measurements.map { |m| item.cycles.to_f / m.total_seconds }
item.calculate_stats(ips)
item.bytes_per_op = (bytes.to_f / cycles.to_f).round.to_u64
if @interactive
run_comparison
report
print "\e[#{ran_items.size}A"
end
end
end
private def ran_items
@items.select(&.ran?)
end
private def run_comparison
fastest = ran_items.max_by { |i| i.mean }
ran_items.each do |item|
item.slower = (fastest.mean / item.mean).to_f
end
end
end
class Entry
# Label of the benchmark
property label : String
# Code to be benchmarked
property action : ->
# Number of cycles needed to run for approx 100ms
# Calculated during the warmup stage
property! cycles : Int32
# Number of 100ms runs during the calculation stage
property! size : Int32
# Statistical mean from calculation stage
property! mean : Float64
# Statistical variance from calculation stage
property! variance : Float64
# Statistical standard deviation from calculation stage
property! stddev : Float64
# Relative standard deviation as a percentage
property! relative_stddev : Float64
# Multiple slower than the fastest entry
property! slower : Float64
# Number of bytes allocated per operation
property! bytes_per_op : UInt64
@ran : Bool
@ran = false
def initialize(@label : String, @action : ->)
end
def ran? : Bool
@ran
end
def call : Nil
action.call
end
def call_for_100ms : Nil
cycles.times { action.call }
end
def set_cycles(duration, iterations)
@cycles = (iterations / duration.total_milliseconds * 100).to_i
@cycles = 1 if cycles <= 0
end
def calculate_stats(samples)
@ran = true
@size = samples.size
@mean = samples.sum.to_f / size.to_f
@variance = (samples.reduce(0) { |acc, i| acc + ((i - mean) ** 2) }).to_f / size.to_f
@stddev = Math.sqrt(variance)
@relative_stddev = 100.0 * (stddev / mean)
end
def human_mean : String
mean.humanize(precision: 2, significant: false, prefixes: Number::SI_PREFIXES_PADDED).rjust(7)
end
def human_iteration_time : String
iteration_time = 1.0 / mean
iteration_time.humanize(precision: 2, significant: false) do |magnitude, _|
magnitude = Number.prefix_index(magnitude).clamp(-9..0)
{magnitude, magnitude == 0 ? "s " : "#{Number.si_prefix(magnitude)}s"}
end.rjust(8)
end
def human_compare : String
if slower == 1.0
"fastest"
else
sprintf "%5.2f× slower", slower
end
end
end
end
end