diff --git a/README.md b/README.md index 6ecbf80..6a39f23 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,60 @@ # benchdiff -[![godoc](https://godoc.org/github.com/WillAbides/benchdiff?status.svg)](https://godoc.org/github.com/WillAbides/benchdiff) +[![godoc](https://godoc.org/github.com/willabides/benchdiff?status.svg)](https://godoc.org/github.com/willabides/benchdiff) [![ci](https://github.com/WillAbides/benchdiff/workflows/ci/badge.svg?branch=main&event=push)](https://github.com/WillAbides/benchdiff/actions?query=workflow%3Aci+branch%3Amaster+event%3Apush) + +Benchdiff is a command line tool intended to help speed up the feedback loop for go benchmarks. + +The old workflow: +- `go test -bench MyBenchmark -run '^$' -count 10 . > tmp/bench.out` +- `git stash && git switch main` +- `go test -bench MyBenchmark -run '^$' -count 10 . > tmp/bench-main.out` +- `git switch - && git stash apply` +- `benchstat tmp/bench-main.out tmp/bench.out` + +The new workflow: +- `benchdiff --bench 'MyBenchmark'` + +## Usage + +``` +Usage: benchdiff + +benchdiff runs go benchmarks on your current git worktree and a base ref then +uses benchstat to show the delta. + +See https://github.com/willabides/benchdiff for more details. + +Flags: + -h, --help Show context-sensitive help. + --base-ref="HEAD" The git ref to be used as a baseline. + --bench="." Run only those benchmarks matching a regular + expression. + --bench-args="test -bench {{.Bench}} -run '^$' -benchmem -count {{.BenchCount}} {{.Packages}}" + Use these arguments to run benchmarks. It may + be a template. + --bench-cmd="go" The go command to use for benchmarks. + --bench-count=10 Run each benchmark n times. + --cache-dir="./tmp" The directory where benchmark output will + kept between runs. + --force-base Rerun benchmarks on the base reference even + if the output already exists. + --git-cmd="git" The executable to use for git commands. + --json-output Format output as JSON. When true the --csv + and --html flags affect only the + "benchstat_output" field. + --on-degrade=0 Exit code when there is a statistically + significant degradation in the results. + --packages="./..." Run benchmarks in these packages. + --alpha=0.05 consider change significant if p < α + --csv format benchstat results as CSV + --delta-test="utest" significance test to apply to delta: utest, + ttest, or none + --geomean print the geometric mean of each file + --html format benchstat results as CSV an HTML table + --norange suppress range columns (CSV only) + --reverse-sort reverse sort order + --sort="none" sort by order: delta, name, none + --split="pkg,goos,goarch" split benchmarks by labels + --version Output the benchdiff version and exit. +``` \ No newline at end of file diff --git a/cmd/benchdiff/benchdiff.go b/cmd/benchdiff/benchdiff.go index c692614..cd513ed 100644 --- a/cmd/benchdiff/benchdiff.go +++ b/cmd/benchdiff/benchdiff.go @@ -8,8 +8,8 @@ import ( "text/template" "github.com/alecthomas/kong" - "github.com/willabides/benchdiff" - pkgbenchstat "github.com/willabides/benchdiff/pkg/benchstat" + "github.com/willabides/benchdiff/cmd/benchdiff/internal" + "github.com/willabides/benchdiff/pkg/benchstatter" "golang.org/x/perf/benchstat" ) @@ -17,7 +17,7 @@ const defaultBenchArgsTmpl = `test -bench {{.Bench}} -run '^$' -benchmem -count var benchstatVars = kong.Vars{ "AlphaDefault": "0.05", - "AlphaHelp": `consider change significant if p < α (default 0.05)`, + "AlphaHelp": `consider change significant if p < α`, "CSVHelp": `format benchstat results as CSV`, "DeltaTestHelp": `significance test to apply to delta: utest, ttest, or none`, "DeltaTestDefault": `utest`, @@ -44,55 +44,72 @@ type benchstatOpts struct { Split string `kong:"help=${SplitHelp},default=${SplitDefault}"` } +var version string + var benchVars = kong.Vars{ - "BenchCmdDefault": `go`, - "BenchArgsDefault": defaultBenchArgsTmpl, - "ResultsDirDefault": filepath.FromSlash("./tmp"), - "BenchCountHelp": `Run each benchmark n times.`, - "BenchHelp": `Run only those benchmarks matching a regular expression.`, - "BenchArgsHelp": `Use these arguments to run benchmarks. It may be a template.`, - "PackagesHelp": `Run benchmarks in these packages.`, - "BenchCmdHelp": `The go command to use for benchmarks.`, - "ResultsDirHelp": `The directory where benchmark output will be deposited.`, - "BaseRefHelp": `The git ref to be used as a baseline.`, - "ForceBaseHelp": `Rerun benchmarks on the base reference even if the output already exists.`, - "DegradationExitHelp": `Exit code when there is a degradation in the results.`, - "JSONOutputHelp": `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`, + "version": version, + "BenchCmdDefault": `go`, + "BenchArgsDefault": defaultBenchArgsTmpl, + "CacheDirDefault": filepath.FromSlash("./tmp"), + "BenchCountHelp": `Run each benchmark n times.`, + "BenchHelp": `Run only those benchmarks matching a regular expression.`, + "BenchArgsHelp": `Use these arguments to run benchmarks. It may be a template.`, + "PackagesHelp": `Run benchmarks in these packages.`, + "BenchCmdHelp": `The go command to use for benchmarks.`, + "CacheDirHelp": `The directory where benchmark output will kept between runs.`, + "BaseRefHelp": `The git ref to be used as a baseline.`, + "ForceBaseHelp": `Rerun benchmarks on the base reference even if the output already exists.`, + "OnDegradeHelp": `Exit code when there is a statistically significant degradation in the results.`, + "JSONOutputHelp": `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`, + "GitCmdHelp": `The executable to use for git commands.`, + "VersionHelp": `Output the benchdiff version and exit.`, } var cli struct { - BaseRef string `kong:"default=HEAD,help=${BaseRefHelp}"` - Bench string `kong:"default='.',help=${BenchHelp}"` - BenchArgs string `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"` - BenchCmd string `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"` - BenchCount int `kong:"default=10,help=${BenchCountHelp}"` - ForceBase bool `kong:"help=${ForceBaseHelp}"` - Packages string `kong:"default='./...',help=${PackagesHelp}"` - ResultsDir string `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"` - DegradationExit int `kong:"name=on-degradation,default=0,help=${DegradationExitHelp}"` - JSONOutput bool `kong:"help=${JSONOutputHelp}"` - BenchstatOpts benchstatOpts `kong:"embed"` + BaseRef string `kong:"default=HEAD,help=${BaseRefHelp}"` + Bench string `kong:"default='.',help=${BenchHelp}"` + BenchArgs string `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"` + BenchCmd string `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"` + BenchCount int `kong:"default=10,help=${BenchCountHelp}"` + CacheDir string `kong:"type=dir,default=${CacheDirDefault},help=${CacheDirHelp}"` + ForceBase bool `kong:"help=${ForceBaseHelp}"` + GitCmd string `kong:"default=git,help=${GitCmdHelp}"` + JSONOutput bool `kong:"help=${JSONOutputHelp}"` + OnDegrade int `kong:"name=on-degrade,default=0,help=${OnDegradeHelp}"` + Packages string `kong:"default='./...',help=${PackagesHelp}"` + BenchstatOpts benchstatOpts `kong:"embed"` + Version kong.VersionFlag `kong:"help=${VersionHelp}"` } +const description = ` +benchdiff runs go benchmarks on your current git worktree and a base ref then +uses benchstat to show the delta. + +See https://github.com/willabides/benchdiff for more details. +` + func main() { - kctx := kong.Parse(&cli, benchstatVars, benchVars) + kctx := kong.Parse(&cli, benchstatVars, benchVars, + kong.Description(strings.TrimSpace(description)), + ) tmpl, err := template.New("").Parse(cli.BenchArgs) kctx.FatalIfErrorf(err) var benchArgs bytes.Buffer err = tmpl.Execute(&benchArgs, cli) kctx.FatalIfErrorf(err) - differ := &benchdiff.Benchdiff{ + bd := &internal.Benchdiff{ BenchCmd: cli.BenchCmd, BenchArgs: benchArgs.String(), - ResultsDir: cli.ResultsDir, + ResultsDir: cli.CacheDir, BaseRef: cli.BaseRef, Path: ".", Writer: os.Stdout, Benchstat: buildBenchstat(cli.BenchstatOpts), Force: cli.ForceBase, + GitCmd: cli.GitCmd, } - result, err := differ.Run() + result, err := bd.Run() kctx.FatalIfErrorf(err) outputFormat := "human" @@ -100,13 +117,13 @@ func main() { outputFormat = "json" } - err = result.WriteOutput(os.Stdout, &benchdiff.RunResultOutputOptions{ + err = result.WriteOutput(os.Stdout, &internal.RunResultOutputOptions{ BenchstatFormatter: buildBenchstat(cli.BenchstatOpts).OutputFormatter, OutputFormat: outputFormat, }) kctx.FatalIfErrorf(err) - if result.HasChangeType(benchdiff.DegradingChange) { - os.Exit(cli.DegradationExit) + if result.HasChangeType(internal.DegradingChange) { + os.Exit(cli.OnDegrade) } } @@ -122,23 +139,23 @@ var sortOpts = map[string]benchstat.Order{ "delta": benchstat.ByDelta, } -func buildBenchstat(opts benchstatOpts) *pkgbenchstat.Benchstat { +func buildBenchstat(opts benchstatOpts) *benchstatter.Benchstat { order := sortOpts[opts.Sort] reverse := opts.ReverseSort if order == nil { reverse = false } - formatter := pkgbenchstat.TextFormatter(nil) + formatter := benchstatter.TextFormatter(nil) if opts.CSV { - formatter = pkgbenchstat.CSVFormatter(&pkgbenchstat.CSVFormatterOptions{ + formatter = benchstatter.CSVFormatter(&benchstatter.CSVFormatterOptions{ NoRange: opts.Norange, }) } if opts.HTML { - formatter = pkgbenchstat.HTMLFormatter(nil) + formatter = benchstatter.HTMLFormatter(nil) } - return &pkgbenchstat.Benchstat{ + return &benchstatter.Benchstat{ DeltaTest: deltaTestOpts[opts.DeltaTest], Alpha: opts.Alpha, AddGeoMean: opts.Geomean, diff --git a/benchdiff.go b/cmd/benchdiff/internal/benchdiff.go similarity index 92% rename from benchdiff.go rename to cmd/benchdiff/internal/benchdiff.go index fd974cc..617849e 100644 --- a/benchdiff.go +++ b/cmd/benchdiff/internal/benchdiff.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "bytes" @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - pkgbenchstat "github.com/willabides/benchdiff/pkg/benchstat" + "github.com/willabides/benchdiff/pkg/benchstatter" "golang.org/x/perf/benchstat" ) @@ -21,8 +21,9 @@ type Benchdiff struct { ResultsDir string BaseRef string Path string + GitCmd string Writer io.Writer - Benchstat *pkgbenchstat.Benchstat + Benchstat *benchstatter.Benchstat Force bool JSONOutput bool } @@ -45,16 +46,16 @@ func fileExists(path string) bool { func (c *Benchdiff) gitRunner() *gitRunner { return &gitRunner{ - repoPath: c.Path, + gitExecutable: c.GitCmd, + repoPath: c.Path, } } func (c *Benchdiff) baseRefRunner() *refRunner { + gr := c.gitRunner() return &refRunner{ - ref: c.BaseRef, - gitRunner: gitRunner{ - repoPath: c.Path, - }, + ref: c.BaseRef, + gitRunner: *gr, } } @@ -90,6 +91,7 @@ func (c *Benchdiff) runBenchmarks() (result *runBenchmarksResults, err error) { } baseFilename := fmt.Sprintf("benchdiff-%s.out", baseSHA) + baseFilename = filepath.Join(c.ResultsDir, baseFilename) result.headSHA = headSHA result.baseSHA = baseSHA result.baseOutputFile = baseFilename @@ -161,7 +163,7 @@ type RunResult struct { // RunResultOutputOptions options for RunResult.WriteOutput type RunResultOutputOptions struct { - BenchstatFormatter pkgbenchstat.OutputFormatter // default benchstat.TextFormatter(nil) + BenchstatFormatter benchstatter.OutputFormatter // default benchstatter.TextFormatter(nil) OutputFormat string // one of json or human. default: human } @@ -171,7 +173,7 @@ func (r *RunResult) WriteOutput(w io.Writer, opts *RunResultOutputOptions) error opts = new(RunResultOutputOptions) } finalOpts := &RunResultOutputOptions{ - BenchstatFormatter: pkgbenchstat.TextFormatter(nil), + BenchstatFormatter: benchstatter.TextFormatter(nil), OutputFormat: "human", } if opts.BenchstatFormatter != nil { diff --git a/benchdiff_test.go b/cmd/benchdiff/internal/benchdiff_test.go similarity index 94% rename from benchdiff_test.go rename to cmd/benchdiff/internal/benchdiff_test.go index e050d5b..ac80ef8 100644 --- a/benchdiff_test.go +++ b/cmd/benchdiff/internal/benchdiff_test.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "io/ioutil" @@ -7,7 +7,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "github.com/willabides/benchdiff/pkg/benchstat" + "github.com/willabides/benchdiff/pkg/benchstatter" ) func setupTestRepo(t *testing.T, path string) { @@ -50,7 +50,7 @@ func TestBenchstat_Run(t *testing.T) { ResultsDir: "./tmp", BaseRef: "HEAD", Path: ".", - Benchstat: &benchstat.Benchstat{}, + Benchstat: &benchstatter.Benchstat{}, } _, err := differ.Run() require.NoError(t, err) diff --git a/gitrunner.go b/cmd/benchdiff/internal/gitrunner.go similarity index 99% rename from gitrunner.go rename to cmd/benchdiff/internal/gitrunner.go index 01cc3ce..2b054fb 100644 --- a/gitrunner.go +++ b/cmd/benchdiff/internal/gitrunner.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "bytes" diff --git a/gitrunner_test.go b/cmd/benchdiff/internal/gitrunner_test.go similarity index 98% rename from gitrunner_test.go rename to cmd/benchdiff/internal/gitrunner_test.go index f4ad58c..079145c 100644 --- a/gitrunner_test.go +++ b/cmd/benchdiff/internal/gitrunner_test.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "io/ioutil" diff --git a/testutil_test.go b/cmd/benchdiff/internal/testutil_test.go similarity index 93% rename from testutil_test.go rename to cmd/benchdiff/internal/testutil_test.go index dccafb2..da30deb 100644 --- a/testutil_test.go +++ b/cmd/benchdiff/internal/testutil_test.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "io/ioutil" @@ -13,7 +13,7 @@ var preserveTmpDir bool func tmpDir(t *testing.T) string { t.Helper() - projectTmp := filepath.FromSlash("./tmp") + projectTmp := filepath.FromSlash("../../../tmp") err := os.MkdirAll(projectTmp, 0o700) assert.NoError(t, err) diff --git a/pkg/benchstat/benchstat.go b/pkg/benchstatter/benchstat.go similarity index 98% rename from pkg/benchstat/benchstat.go rename to pkg/benchstatter/benchstat.go index 9e5fb64..a808ccf 100644 --- a/pkg/benchstat/benchstat.go +++ b/pkg/benchstatter/benchstat.go @@ -1,5 +1,5 @@ -// Package benchstat is used to run benchstat programmatically -package benchstat +// Package benchstatter is used to run benchstatter programmatically +package benchstatter import ( "bytes" diff --git a/pkg/benchstat/benchstat_test.go b/pkg/benchstatter/benchstat_test.go similarity index 97% rename from pkg/benchstat/benchstat_test.go rename to pkg/benchstatter/benchstat_test.go index a1e623b..a4e4ef5 100644 --- a/pkg/benchstat/benchstat_test.go +++ b/pkg/benchstatter/benchstat_test.go @@ -1,4 +1,4 @@ -package benchstat +package benchstatter import ( "bytes" diff --git a/pkg/benchstat/testdata/outputs/benchdiff-1.out b/pkg/benchstatter/testdata/outputs/benchdiff-1.out similarity index 100% rename from pkg/benchstat/testdata/outputs/benchdiff-1.out rename to pkg/benchstatter/testdata/outputs/benchdiff-1.out diff --git a/pkg/benchstat/testdata/outputs/benchdiff-worktree.out b/pkg/benchstatter/testdata/outputs/benchdiff-worktree.out similarity index 100% rename from pkg/benchstat/testdata/outputs/benchdiff-worktree.out rename to pkg/benchstatter/testdata/outputs/benchdiff-worktree.out