-
Notifications
You must be signed in to change notification settings - Fork 58
/
stats.go
201 lines (163 loc) · 6.36 KB
/
stats.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
/*
Copyright 2024 The Vitess Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package macrobench
import (
"math"
"github.com/aclements/go-moremath/mathx"
"golang.org/x/perf/benchmath"
)
type (
Range struct {
Infinite bool `json:"infinite"`
Unknown bool `json:"unknown"`
Value float64 `json:"value"`
}
StatisticalSummary struct {
Center float64 `json:"center"`
Confidence float64 `json:"confidence"`
Range Range `json:"range"`
}
StatisticalResult struct {
Insignificant bool `json:"insignificant"`
Delta float64 `json:"delta"`
P float64 `json:"p"`
N1 int `json:"n1"`
N2 int `json:"n2"`
Old StatisticalSummary `json:"old"`
New StatisticalSummary `json:"new"`
}
ShortStatisticalSingleResult struct {
TotalQPS StatisticalSummary `json:"total_qps"`
}
// StatisticalSingleResult represents a single benchmark's statistical summary.
StatisticalSingleResult struct {
GitRef string `json:"git_ref"`
TotalQPS StatisticalSummary `json:"total_qps"`
ReadsQPS StatisticalSummary `json:"reads_qps"`
WritesQPS StatisticalSummary `json:"writes_qps"`
OtherQPS StatisticalSummary `json:"other_qps"`
TPS StatisticalSummary `json:"tps"`
Latency StatisticalSummary `json:"latency"`
Errors StatisticalSummary `json:"errors"`
TotalComponentsCPUTime StatisticalSummary `json:"total_components_cpu_time"`
ComponentsCPUTime map[string]StatisticalSummary `json:"components_cpu_time"`
TotalComponentsMemStatsAllocBytes StatisticalSummary `json:"total_components_mem_stats_alloc_bytes"`
ComponentsMemStatsAllocBytes map[string]StatisticalSummary `json:"components_mem_stats_alloc_bytes"`
}
// StatisticalCompareResults is the full representation of the results
// obtained by comparing two samples using the Mann Whitney U Test.
StatisticalCompareResults struct {
TotalQPS StatisticalResult `json:"total_qps"`
ReadsQPS StatisticalResult `json:"reads_qps"`
WritesQPS StatisticalResult `json:"writes_qps"`
OtherQPS StatisticalResult `json:"other_qps"`
TPS StatisticalResult `json:"tps"`
Latency StatisticalResult `json:"latency"`
Errors StatisticalResult `json:"errors"`
TotalComponentsCPUTime StatisticalResult `json:"total_components_cpu_time"`
ComponentsCPUTime map[string]StatisticalResult `json:"components_cpu_time"`
TotalComponentsMemStatsAllocBytes StatisticalResult `json:"total_components_mem_stats_alloc_bytes"`
ComponentsMemStatsAllocBytes map[string]StatisticalResult `json:"components_mem_stats_alloc_bytes"`
}
)
var (
// defaultThresholds sets the thresholds below which the null hypothesis that two samples
// come from the same distribution can be rejected.
defaultThresholds = benchmath.DefaultThresholds
// defaultConfidence sets the desired confidence interval when doing a summary of a sample.
defaultConfidence = 0.95
)
func getRangeFromSummary(s benchmath.Summary) Range {
if math.IsInf(s.Lo, 0) || math.IsInf(s.Hi, 0) {
return Range{Infinite: true}
}
// If the signs of the bounds differ from the center, we can't
// render it as a percent.
var csign = mathx.Sign(s.Center)
if csign != mathx.Sign(s.Lo) || csign != mathx.Sign(s.Hi) {
return Range{Unknown: true}
}
// If center is 0, avoid dividing by zero. But we can only get
// here if lo and hi are also 0, in which case is seems
// reasonable to call this 0%.
if s.Center == 0 {
return Range{Value: 0.00}
}
// Phew. Compute the range percent.
v := math.Max(s.Hi/s.Center-1, 1-s.Lo/s.Center)
return Range{Value: v * 100}
}
func getSummary(values []float64) (StatisticalSummary, *benchmath.Sample) {
sample := benchmath.NewSample(values, &defaultThresholds)
summary := benchmath.AssumeNothing.Summary(sample, defaultConfidence)
return StatisticalSummary{
Center: summary.Center,
Confidence: summary.Confidence,
Range: getRangeFromSummary(summary),
}, sample
}
func compare(old, new []float64) StatisticalResult {
var sr StatisticalResult
ssOld, s1 := getSummary(old)
ssNew, s2 := getSummary(new)
if math.IsNaN(ssOld.Center) {
ssOld.Center = 0.0
}
if math.IsNaN(ssNew.Center) {
ssNew.Center = 0.0
}
sr.Old = ssOld
sr.New = ssNew
c := benchmath.AssumeNothing.Compare(s1, s2)
sr.P = c.P
sr.N1 = c.N1
sr.N2 = c.N2
if c.P > c.Alpha {
sr.Insignificant = true
}
switch {
case sr.Old.Center == sr.New.Center || sr.Old.Center == 0:
sr.Delta = 0
default:
sr.Delta = ((sr.New.Center / sr.Old.Center) - 1.0) * 100.0
}
return sr
}
func performAnalysis(old, new resultAsSlice) StatisticalCompareResults {
scr := StatisticalCompareResults{
ComponentsCPUTime: map[string]StatisticalResult{
"vtgate": {Insignificant: true},
"vttablet": {Insignificant: true},
},
ComponentsMemStatsAllocBytes: map[string]StatisticalResult{
"vtgate": {Insignificant: true},
"vttablet": {Insignificant: true},
},
}
scr.TotalQPS = compare(old.qps.total, new.qps.total)
scr.ReadsQPS = compare(old.qps.reads, new.qps.reads)
scr.WritesQPS = compare(old.qps.writes, new.qps.writes)
scr.OtherQPS = compare(old.qps.other, new.qps.other)
scr.TPS = compare(old.tps, new.tps)
scr.Latency = compare(old.latency, new.latency)
scr.Errors = compare(old.errors, new.errors)
scr.TotalComponentsCPUTime = compare(old.metrics.totalComponentsCPUTime, new.metrics.totalComponentsCPUTime)
for name, values := range old.metrics.componentsCPUTime {
scr.ComponentsCPUTime[name] = compare(values, new.metrics.componentsCPUTime[name])
}
scr.TotalComponentsMemStatsAllocBytes = compare(old.metrics.totalComponentsMemStatsAllocBytes, new.metrics.totalComponentsMemStatsAllocBytes)
for name, values := range old.metrics.componentsMemStatsAllocBytes {
scr.ComponentsMemStatsAllocBytes[name] = compare(values, new.metrics.componentsMemStatsAllocBytes[name])
}
return scr
}