forked from winderica/kryptology
/
main.go
137 lines (123 loc) · 3.51 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
//
// Copyright Coinbase, Inc. All Rights Reserved.
//
// SPDX-License-Identifier: Apache-2.0
//
// benchcomp implements a command that receives two benchmarks files as input and flags the benchmarks that have
// degraded by more than a threshold amount. The main goal of this tool is to be used in CI
// to check each PR.
package main
import (
"bytes"
"flag"
"fmt"
"io"
"io/ioutil"
"github.com/pkg/errors"
"golang.org/x/tools/benchmark/parse"
)
const THRESHOLD = 1.1
func main() {
cReader, nReader, err := parseCmdArgs()
if err != nil {
panic(err)
}
if err := Compare(cReader, nReader); err != nil {
panic(err)
}
}
func parseCmdArgs() (io.Reader, io.Reader, error) {
cFlag := flag.String("current", "current-bench.log", "The patch to the log file containing the output of the current benchmark result.")
nFlag := flag.String("new", "new-bench.log", "The patch to the log file containing the output of the new benchmark result.")
flag.Parse()
cBytes, err := ioutil.ReadFile(*cFlag)
if err != nil {
return nil, nil, fmt.Errorf("reading current log file %v", err)
}
nBytes, err := ioutil.ReadFile(*nFlag)
if err != nil {
return nil, nil, fmt.Errorf("reading new log file %v", err)
}
return bytes.NewBuffer(cBytes), bytes.NewBuffer(nBytes), nil
}
// Compare expects two readers which contain the output of two runs of `go test -bench` command and throws an error if
// the performance has degraded by more than `THRESHOLD` amount.
func Compare(currBench, newBench io.Reader) error {
c, n, err := parseBenchmarks(currBench, newBench)
if err != nil {
return errors.Wrap(err, "parsing benchmark outputs")
}
perfDeviations := make([]string, 0)
for bench := range c {
if _, ok := n[bench]; !ok {
// New benchmark, skipping
continue
} else {
currB := c[bench]
newB := n[bench]
err = compareBenches(currB, newB)
if err != nil {
perfDeviations = append(perfDeviations, fmt.Sprintf("%v", err))
}
}
}
if len(perfDeviations) != 0 {
return fmt.Errorf("%#v", perfDeviations)
}
return nil
}
func parseBenchmarks(currBench, newBench io.Reader) (parse.Set, parse.Set, error) {
c, err := parse.ParseSet(currBench)
if err != nil {
return nil, nil, errors.Wrap(err, "parsing current benchmark output")
}
n, err := parse.ParseSet(newBench)
if err != nil {
return nil, nil, errors.Wrap(err, "parsing new benchmark output")
}
return c, n, nil
}
func compareBenches(currB, newB []*parse.Benchmark) error {
currMap := make(map[string]*parse.Benchmark)
newMap := make(map[string]*parse.Benchmark)
for _, b := range currB {
// TODO: double check what is in Name
currMap[b.Name] = b
}
for _, b := range newB {
// TODO: double check what is in Name
newMap[b.Name] = b
}
for name := range currMap {
if _, ok := newMap[name]; ok {
compare := []struct {
current float64
new float64
}{
{
current: float64(currMap[name].AllocedBytesPerOp),
new: float64(newMap[name].AllocedBytesPerOp),
},
{
current: float64(currMap[name].AllocsPerOp),
new: float64(newMap[name].AllocsPerOp),
},
{
current: currMap[name].NsPerOp,
new: newMap[name].NsPerOp,
},
{
current: currMap[name].MBPerS,
new: newMap[name].MBPerS,
},
}
for _, t := range compare {
if t.new > t.current*THRESHOLD {
percent := (t.new - t.current) * 100 / t.current
return fmt.Errorf("benchmark %s exceeded previous benchmark by %0.2f percent. Current: %0.2f, New: %0.2f", name, percent, t.current, t.new)
}
}
}
}
return nil
}