/
benchmark.go
162 lines (147 loc) · 5.21 KB
/
benchmark.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package mage
import (
"errors"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"github.com/elastic/elastic-agent-libs/dev-tools/mage/gotool"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)
const (
goBenchstat = "golang.org/x/perf/cmd/benchstat@v0.0.0-20230227161431-f7320a6d63e8"
)
var (
benchmarkCount = 8
)
// Benchmark namespace for mage to group all the related targets under this namespace
type Benchmark mg.Namespace
// Deps installs required plugins for reading benchmarks results
func (Benchmark) Deps() error {
err := gotool.Install(gotool.Install.Package(goBenchstat))
if err != nil {
return err
}
return nil
}
// Run execute the go benchmark tests for this repository, by defining the variable OUTPUT you write the results
// into a file. Optional you can set BENCH_COUNT to how many benchmark iteration you want to execute, default is 8
func (Benchmark) Run() error {
mg.Deps(Benchmark.Deps)
log.Println(">> go Test: Benchmark")
outputFile := os.Getenv("OUTPUT")
benchmarkCountOverride := os.Getenv("BENCH_COUNT")
if benchmarkCountOverride != "" {
var overrideErr error
benchmarkCount, overrideErr = strconv.Atoi(benchmarkCountOverride)
if overrideErr != nil {
return fmt.Errorf("failed to parse BENCH_COUNT, verify that you set the right value: , %w", overrideErr)
}
}
projectPackages, er := gotool.ListProjectPackages()
if er != nil {
return fmt.Errorf("failed to list package dependencies: %w", er)
}
cmdArg := fmt.Sprintf("test -count=%d -bench=Bench -run=Bench", benchmarkCount)
cmdArgs := strings.Split(cmdArg, " ")
for _, pkg := range projectPackages {
cmdArgs = append(cmdArgs, filepath.Join(pkg, "/..."))
}
_, err := runCommand(nil, "go", outputFile, cmdArgs...)
var goTestErr *exec.ExitError
switch {
case goTestErr == nil:
return nil
case errors.As(err, &goTestErr):
return fmt.Errorf("failed to execute go test -bench command: %w", err)
default:
return fmt.Errorf("failed to execute go test -bench command %w", err)
}
}
// Diff parse one or more benchmark outputs, Required environment variables are BASE for parsing results
// and NEXT to compare the base results with. Optional you can define OUTPUT to write the results into a file
func (Benchmark) Diff() error {
mg.Deps(Benchmark.Deps)
log.Println(">> running: benchstat")
outputFile := os.Getenv("OUTPUT")
baseFile := os.Getenv("BASE")
nextFile := os.Getenv("NEXT")
var args []string
if baseFile == "" {
log.Printf("Missing required parameter BASE parameter to parse the results. Please set this to a filepath of the benchmark results")
return fmt.Errorf("missing required parameter BASE parameter to parse the results. Please set this to a filepath of the benchmark results")
} else {
args = append(args, baseFile)
}
if nextFile == "" {
log.Printf("Missing NEXT parameter, we are not going to compare results")
} else {
args = append(args, nextFile)
}
_, err := runCommand(nil, "benchstat", outputFile, args...)
var goTestErr *exec.ExitError
switch {
case goTestErr == nil:
return nil
case errors.As(err, &goTestErr):
return fmt.Errorf("failed to execute benchstat command: %w", err)
default:
return fmt.Errorf("failed to execute benchstat command!! %w", err)
}
}
// runCommand is executing a command that is represented by cmd.
// when defining an outputFile it will write the stdErr, stdOut of that command to the output file
// otherwise it will capture it to stdErr, stdOut of the console used and return true, nil, if succeed
func runCommand(env map[string]string, cmd string, outputFile string, args ...string) (bool, error) {
var stdOut io.Writer
var stdErr io.Writer
if outputFile != "" {
fileOutput, err := os.Create(createDir(outputFile))
if err != nil {
return false, fmt.Errorf("failed to create %s output file: %w", cmd, err)
}
defer func(fileOutput *os.File) {
err := fileOutput.Close()
if err != nil {
log.Fatalf("Failed to close file %s", err)
}
}(fileOutput)
stdOut = io.MultiWriter(os.Stdout, fileOutput)
stdErr = io.MultiWriter(os.Stderr, fileOutput)
} else {
stdOut = os.Stdout
stdErr = os.Stderr
}
return sh.Exec(env, stdOut, stdErr, cmd, args...)
}
// createDir creates the parent directory for the given file.
func createDir(file string) string {
// Create the output directory.
if dir := filepath.Dir(file); dir != "." {
if err := os.MkdirAll(dir, 0755); err != nil {
log.Fatalf("Failed to create parent dir for %s", file)
}
}
return file
}