From f78fab257f2954c57dfc749dd83a13a9dd543040 Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 15:57:01 -0600 Subject: [PATCH 01/10] move packages around to make most of this internal --- .gitignore | 1 - .goreleaser.yml | 2 +- Makefile | 2 +- cmd/benchdiff/benchdiff.go | 17 ++++++++++------- .../benchdiff/internal/benchdiff.go | 13 +++++++------ .../benchdiff/internal/benchdiff_test.go | 2 +- .../benchdiff/internal/gitrunner.go | 2 +- .../benchdiff/internal/gitrunner_test.go | 2 +- .../benchdiff/internal/testutil_test.go | 4 ++-- 9 files changed, 24 insertions(+), 21 deletions(-) rename benchdiff.go => cmd/benchdiff/internal/benchdiff.go (97%) rename benchdiff_test.go => cmd/benchdiff/internal/benchdiff_test.go (99%) rename gitrunner.go => cmd/benchdiff/internal/gitrunner.go (99%) rename gitrunner_test.go => cmd/benchdiff/internal/gitrunner_test.go (98%) rename testutil_test.go => cmd/benchdiff/internal/testutil_test.go (93%) diff --git a/.gitignore b/.gitignore index 49ae4de..b1db43e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -tmp/ bin/ .bindown/ /dist diff --git a/.goreleaser.yml b/.goreleaser.yml index 4851d25..77819c7 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -7,7 +7,7 @@ builds: goos: - linux - darwin - main: ./cmd/benchdiff + main: ./cmd_benchdiff checksum: name_template: 'checksums.txt' snapshot: diff --git a/Makefile b/Makefile index 225a21c..eccaf18 100644 --- a/Makefile +++ b/Makefile @@ -24,4 +24,4 @@ bin/gofumpt: bin/gobin bin/gobin mvdan.cc/gofumpt@$(GOFUMPT_REV) bin/benchdiff: gobuildcache - ${GOBUILD} -o $@ ./cmd/benchdiff + ${GOBUILD} -o $@ ./cmd_benchdiff diff --git a/cmd/benchdiff/benchdiff.go b/cmd/benchdiff/benchdiff.go index c692614..92f88df 100644 --- a/cmd/benchdiff/benchdiff.go +++ b/cmd/benchdiff/benchdiff.go @@ -8,7 +8,7 @@ import ( "text/template" "github.com/alecthomas/kong" - "github.com/willabides/benchdiff" + "github.com/willabides/benchdiff/cmd/benchdiff/internal" pkgbenchstat "github.com/willabides/benchdiff/pkg/benchstat" "golang.org/x/perf/benchstat" ) @@ -58,6 +58,7 @@ var benchVars = kong.Vars{ "ForceBaseHelp": `Rerun benchmarks on the base reference even if the output already exists.`, "DegradationExitHelp": `Exit code when there is a degradation in the results.`, "JSONOutputHelp": `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`, + "GitCmdHelp": `The executable to use for git commands.`, } var cli struct { @@ -66,11 +67,12 @@ var cli struct { BenchArgs string `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"` BenchCmd string `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"` BenchCount int `kong:"default=10,help=${BenchCountHelp}"` + DegradationExit int `kong:"name=on-degradation,default=0,help=${DegradationExitHelp}"` ForceBase bool `kong:"help=${ForceBaseHelp}"` + GitCmd string `kong:"default=git,help=${GitCmdHelp}"` + JSONOutput bool `kong:"help=${JSONOutputHelp}"` Packages string `kong:"default='./...',help=${PackagesHelp}"` ResultsDir string `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"` - DegradationExit int `kong:"name=on-degradation,default=0,help=${DegradationExitHelp}"` - JSONOutput bool `kong:"help=${JSONOutputHelp}"` BenchstatOpts benchstatOpts `kong:"embed"` } @@ -82,7 +84,7 @@ func main() { err = tmpl.Execute(&benchArgs, cli) kctx.FatalIfErrorf(err) - differ := &benchdiff.Benchdiff{ + bd := &internal.Benchdiff{ BenchCmd: cli.BenchCmd, BenchArgs: benchArgs.String(), ResultsDir: cli.ResultsDir, @@ -91,8 +93,9 @@ func main() { Writer: os.Stdout, Benchstat: buildBenchstat(cli.BenchstatOpts), Force: cli.ForceBase, + GitCmd: cli.GitCmd, } - result, err := differ.Run() + result, err := bd.Run() kctx.FatalIfErrorf(err) outputFormat := "human" @@ -100,12 +103,12 @@ func main() { outputFormat = "json" } - err = result.WriteOutput(os.Stdout, &benchdiff.RunResultOutputOptions{ + err = result.WriteOutput(os.Stdout, &internal.RunResultOutputOptions{ BenchstatFormatter: buildBenchstat(cli.BenchstatOpts).OutputFormatter, OutputFormat: outputFormat, }) kctx.FatalIfErrorf(err) - if result.HasChangeType(benchdiff.DegradingChange) { + if result.HasChangeType(internal.DegradingChange) { os.Exit(cli.DegradationExit) } } diff --git a/benchdiff.go b/cmd/benchdiff/internal/benchdiff.go similarity index 97% rename from benchdiff.go rename to cmd/benchdiff/internal/benchdiff.go index fd974cc..0350860 100644 --- a/benchdiff.go +++ b/cmd/benchdiff/internal/benchdiff.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "bytes" @@ -21,6 +21,7 @@ type Benchdiff struct { ResultsDir string BaseRef string Path string + GitCmd string Writer io.Writer Benchstat *pkgbenchstat.Benchstat Force bool @@ -45,16 +46,16 @@ func fileExists(path string) bool { func (c *Benchdiff) gitRunner() *gitRunner { return &gitRunner{ - repoPath: c.Path, + gitExecutable: c.GitCmd, + repoPath: c.Path, } } func (c *Benchdiff) baseRefRunner() *refRunner { + gr := c.gitRunner() return &refRunner{ - ref: c.BaseRef, - gitRunner: gitRunner{ - repoPath: c.Path, - }, + ref: c.BaseRef, + gitRunner: *gr, } } diff --git a/benchdiff_test.go b/cmd/benchdiff/internal/benchdiff_test.go similarity index 99% rename from benchdiff_test.go rename to cmd/benchdiff/internal/benchdiff_test.go index e050d5b..354c3e5 100644 --- a/benchdiff_test.go +++ b/cmd/benchdiff/internal/benchdiff_test.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "io/ioutil" diff --git a/gitrunner.go b/cmd/benchdiff/internal/gitrunner.go similarity index 99% rename from gitrunner.go rename to cmd/benchdiff/internal/gitrunner.go index 01cc3ce..2b054fb 100644 --- a/gitrunner.go +++ b/cmd/benchdiff/internal/gitrunner.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "bytes" diff --git a/gitrunner_test.go b/cmd/benchdiff/internal/gitrunner_test.go similarity index 98% rename from gitrunner_test.go rename to cmd/benchdiff/internal/gitrunner_test.go index f4ad58c..079145c 100644 --- a/gitrunner_test.go +++ b/cmd/benchdiff/internal/gitrunner_test.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "io/ioutil" diff --git a/testutil_test.go b/cmd/benchdiff/internal/testutil_test.go similarity index 93% rename from testutil_test.go rename to cmd/benchdiff/internal/testutil_test.go index dccafb2..da30deb 100644 --- a/testutil_test.go +++ b/cmd/benchdiff/internal/testutil_test.go @@ -1,4 +1,4 @@ -package benchdiff +package internal import ( "io/ioutil" @@ -13,7 +13,7 @@ var preserveTmpDir bool func tmpDir(t *testing.T) string { t.Helper() - projectTmp := filepath.FromSlash("./tmp") + projectTmp := filepath.FromSlash("../../../tmp") err := os.MkdirAll(projectTmp, 0o700) assert.NoError(t, err) From 9f41f33a421ff04c812d1422830862c1a3c11f9a Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 16:00:41 -0600 Subject: [PATCH 02/10] revert unintentional changes --- .gitignore | 1 + .goreleaser.yml | 2 +- Makefile | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b1db43e..49ae4de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +tmp/ bin/ .bindown/ /dist diff --git a/.goreleaser.yml b/.goreleaser.yml index 77819c7..4851d25 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -7,7 +7,7 @@ builds: goos: - linux - darwin - main: ./cmd_benchdiff + main: ./cmd/benchdiff checksum: name_template: 'checksums.txt' snapshot: diff --git a/Makefile b/Makefile index eccaf18..225a21c 100644 --- a/Makefile +++ b/Makefile @@ -24,4 +24,4 @@ bin/gofumpt: bin/gobin bin/gobin mvdan.cc/gofumpt@$(GOFUMPT_REV) bin/benchdiff: gobuildcache - ${GOBUILD} -o $@ ./cmd_benchdiff + ${GOBUILD} -o $@ ./cmd/benchdiff From a4070c7b55ee2526121f27c710229b34a92d1e9e Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 16:10:10 -0600 Subject: [PATCH 03/10] reword help --- cmd/benchdiff/benchdiff.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/benchdiff/benchdiff.go b/cmd/benchdiff/benchdiff.go index 92f88df..3ad0646 100644 --- a/cmd/benchdiff/benchdiff.go +++ b/cmd/benchdiff/benchdiff.go @@ -17,7 +17,7 @@ const defaultBenchArgsTmpl = `test -bench {{.Bench}} -run '^$' -benchmem -count var benchstatVars = kong.Vars{ "AlphaDefault": "0.05", - "AlphaHelp": `consider change significant if p < α (default 0.05)`, + "AlphaHelp": `consider change significant if p < α`, "CSVHelp": `format benchstat results as CSV`, "DeltaTestHelp": `significance test to apply to delta: utest, ttest, or none`, "DeltaTestDefault": `utest`, From 09571503b01abf970fbb01b445dbcfcdf2d30031 Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 16:13:47 -0600 Subject: [PATCH 04/10] rename to --on-degrade --- cmd/benchdiff/benchdiff.go | 54 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/cmd/benchdiff/benchdiff.go b/cmd/benchdiff/benchdiff.go index 3ad0646..fb33d21 100644 --- a/cmd/benchdiff/benchdiff.go +++ b/cmd/benchdiff/benchdiff.go @@ -45,35 +45,35 @@ type benchstatOpts struct { } var benchVars = kong.Vars{ - "BenchCmdDefault": `go`, - "BenchArgsDefault": defaultBenchArgsTmpl, - "ResultsDirDefault": filepath.FromSlash("./tmp"), - "BenchCountHelp": `Run each benchmark n times.`, - "BenchHelp": `Run only those benchmarks matching a regular expression.`, - "BenchArgsHelp": `Use these arguments to run benchmarks. It may be a template.`, - "PackagesHelp": `Run benchmarks in these packages.`, - "BenchCmdHelp": `The go command to use for benchmarks.`, - "ResultsDirHelp": `The directory where benchmark output will be deposited.`, - "BaseRefHelp": `The git ref to be used as a baseline.`, - "ForceBaseHelp": `Rerun benchmarks on the base reference even if the output already exists.`, - "DegradationExitHelp": `Exit code when there is a degradation in the results.`, - "JSONOutputHelp": `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`, - "GitCmdHelp": `The executable to use for git commands.`, + "BenchCmdDefault": `go`, + "BenchArgsDefault": defaultBenchArgsTmpl, + "ResultsDirDefault": filepath.FromSlash("./tmp"), + "BenchCountHelp": `Run each benchmark n times.`, + "BenchHelp": `Run only those benchmarks matching a regular expression.`, + "BenchArgsHelp": `Use these arguments to run benchmarks. It may be a template.`, + "PackagesHelp": `Run benchmarks in these packages.`, + "BenchCmdHelp": `The go command to use for benchmarks.`, + "ResultsDirHelp": `The directory where benchmark output will be deposited.`, + "BaseRefHelp": `The git ref to be used as a baseline.`, + "ForceBaseHelp": `Rerun benchmarks on the base reference even if the output already exists.`, + "OnDegradeHelp": `Exit code when there is a statistically significant degradation in the results.`, + "JSONOutputHelp": `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`, + "GitCmdHelp": `The executable to use for git commands.`, } var cli struct { - BaseRef string `kong:"default=HEAD,help=${BaseRefHelp}"` - Bench string `kong:"default='.',help=${BenchHelp}"` - BenchArgs string `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"` - BenchCmd string `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"` - BenchCount int `kong:"default=10,help=${BenchCountHelp}"` - DegradationExit int `kong:"name=on-degradation,default=0,help=${DegradationExitHelp}"` - ForceBase bool `kong:"help=${ForceBaseHelp}"` - GitCmd string `kong:"default=git,help=${GitCmdHelp}"` - JSONOutput bool `kong:"help=${JSONOutputHelp}"` - Packages string `kong:"default='./...',help=${PackagesHelp}"` - ResultsDir string `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"` - BenchstatOpts benchstatOpts `kong:"embed"` + BaseRef string `kong:"default=HEAD,help=${BaseRefHelp}"` + Bench string `kong:"default='.',help=${BenchHelp}"` + BenchArgs string `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"` + BenchCmd string `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"` + BenchCount int `kong:"default=10,help=${BenchCountHelp}"` + OnDegrade int `kong:"name=on-degrade,default=0,help=${OnDegradeHelp}"` + ForceBase bool `kong:"help=${ForceBaseHelp}"` + GitCmd string `kong:"default=git,help=${GitCmdHelp}"` + JSONOutput bool `kong:"help=${JSONOutputHelp}"` + Packages string `kong:"default='./...',help=${PackagesHelp}"` + ResultsDir string `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"` + BenchstatOpts benchstatOpts `kong:"embed"` } func main() { @@ -109,7 +109,7 @@ func main() { }) kctx.FatalIfErrorf(err) if result.HasChangeType(internal.DegradingChange) { - os.Exit(cli.DegradationExit) + os.Exit(cli.OnDegrade) } } From 310743d8c9ed9c28b42779f1b8889705942f75d3 Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 16:20:10 -0600 Subject: [PATCH 05/10] rename benchstat to benchstatter --- cmd/benchdiff/benchdiff.go | 12 ++++++------ cmd/benchdiff/internal/benchdiff.go | 8 ++++---- cmd/benchdiff/internal/benchdiff_test.go | 4 ++-- pkg/{benchstat => benchstatter}/benchstat.go | 4 ++-- pkg/{benchstat => benchstatter}/benchstat_test.go | 2 +- .../testdata/outputs/benchdiff-1.out | 0 .../testdata/outputs/benchdiff-worktree.out | 0 7 files changed, 15 insertions(+), 15 deletions(-) rename pkg/{benchstat => benchstatter}/benchstat.go (98%) rename pkg/{benchstat => benchstatter}/benchstat_test.go (97%) rename pkg/{benchstat => benchstatter}/testdata/outputs/benchdiff-1.out (100%) rename pkg/{benchstat => benchstatter}/testdata/outputs/benchdiff-worktree.out (100%) diff --git a/cmd/benchdiff/benchdiff.go b/cmd/benchdiff/benchdiff.go index fb33d21..4f4d24c 100644 --- a/cmd/benchdiff/benchdiff.go +++ b/cmd/benchdiff/benchdiff.go @@ -9,7 +9,7 @@ import ( "github.com/alecthomas/kong" "github.com/willabides/benchdiff/cmd/benchdiff/internal" - pkgbenchstat "github.com/willabides/benchdiff/pkg/benchstat" + "github.com/willabides/benchdiff/pkg/benchstatter" "golang.org/x/perf/benchstat" ) @@ -125,23 +125,23 @@ var sortOpts = map[string]benchstat.Order{ "delta": benchstat.ByDelta, } -func buildBenchstat(opts benchstatOpts) *pkgbenchstat.Benchstat { +func buildBenchstat(opts benchstatOpts) *benchstatter.Benchstat { order := sortOpts[opts.Sort] reverse := opts.ReverseSort if order == nil { reverse = false } - formatter := pkgbenchstat.TextFormatter(nil) + formatter := benchstatter.TextFormatter(nil) if opts.CSV { - formatter = pkgbenchstat.CSVFormatter(&pkgbenchstat.CSVFormatterOptions{ + formatter = benchstatter.CSVFormatter(&benchstatter.CSVFormatterOptions{ NoRange: opts.Norange, }) } if opts.HTML { - formatter = pkgbenchstat.HTMLFormatter(nil) + formatter = benchstatter.HTMLFormatter(nil) } - return &pkgbenchstat.Benchstat{ + return &benchstatter.Benchstat{ DeltaTest: deltaTestOpts[opts.DeltaTest], Alpha: opts.Alpha, AddGeoMean: opts.Geomean, diff --git a/cmd/benchdiff/internal/benchdiff.go b/cmd/benchdiff/internal/benchdiff.go index 0350860..8d0ce34 100644 --- a/cmd/benchdiff/internal/benchdiff.go +++ b/cmd/benchdiff/internal/benchdiff.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - pkgbenchstat "github.com/willabides/benchdiff/pkg/benchstat" + "github.com/willabides/benchdiff/pkg/benchstatter" "golang.org/x/perf/benchstat" ) @@ -23,7 +23,7 @@ type Benchdiff struct { Path string GitCmd string Writer io.Writer - Benchstat *pkgbenchstat.Benchstat + Benchstat *benchstatter.Benchstat Force bool JSONOutput bool } @@ -162,7 +162,7 @@ type RunResult struct { // RunResultOutputOptions options for RunResult.WriteOutput type RunResultOutputOptions struct { - BenchstatFormatter pkgbenchstat.OutputFormatter // default benchstat.TextFormatter(nil) + BenchstatFormatter benchstatter.OutputFormatter // default benchstatter.TextFormatter(nil) OutputFormat string // one of json or human. default: human } @@ -172,7 +172,7 @@ func (r *RunResult) WriteOutput(w io.Writer, opts *RunResultOutputOptions) error opts = new(RunResultOutputOptions) } finalOpts := &RunResultOutputOptions{ - BenchstatFormatter: pkgbenchstat.TextFormatter(nil), + BenchstatFormatter: benchstatter.TextFormatter(nil), OutputFormat: "human", } if opts.BenchstatFormatter != nil { diff --git a/cmd/benchdiff/internal/benchdiff_test.go b/cmd/benchdiff/internal/benchdiff_test.go index 354c3e5..ac80ef8 100644 --- a/cmd/benchdiff/internal/benchdiff_test.go +++ b/cmd/benchdiff/internal/benchdiff_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "github.com/willabides/benchdiff/pkg/benchstat" + "github.com/willabides/benchdiff/pkg/benchstatter" ) func setupTestRepo(t *testing.T, path string) { @@ -50,7 +50,7 @@ func TestBenchstat_Run(t *testing.T) { ResultsDir: "./tmp", BaseRef: "HEAD", Path: ".", - Benchstat: &benchstat.Benchstat{}, + Benchstat: &benchstatter.Benchstat{}, } _, err := differ.Run() require.NoError(t, err) diff --git a/pkg/benchstat/benchstat.go b/pkg/benchstatter/benchstat.go similarity index 98% rename from pkg/benchstat/benchstat.go rename to pkg/benchstatter/benchstat.go index 9e5fb64..a808ccf 100644 --- a/pkg/benchstat/benchstat.go +++ b/pkg/benchstatter/benchstat.go @@ -1,5 +1,5 @@ -// Package benchstat is used to run benchstat programmatically -package benchstat +// Package benchstatter is used to run benchstatter programmatically +package benchstatter import ( "bytes" diff --git a/pkg/benchstat/benchstat_test.go b/pkg/benchstatter/benchstat_test.go similarity index 97% rename from pkg/benchstat/benchstat_test.go rename to pkg/benchstatter/benchstat_test.go index a1e623b..a4e4ef5 100644 --- a/pkg/benchstat/benchstat_test.go +++ b/pkg/benchstatter/benchstat_test.go @@ -1,4 +1,4 @@ -package benchstat +package benchstatter import ( "bytes" diff --git a/pkg/benchstat/testdata/outputs/benchdiff-1.out b/pkg/benchstatter/testdata/outputs/benchdiff-1.out similarity index 100% rename from pkg/benchstat/testdata/outputs/benchdiff-1.out rename to pkg/benchstatter/testdata/outputs/benchdiff-1.out diff --git a/pkg/benchstat/testdata/outputs/benchdiff-worktree.out b/pkg/benchstatter/testdata/outputs/benchdiff-worktree.out similarity index 100% rename from pkg/benchstat/testdata/outputs/benchdiff-worktree.out rename to pkg/benchstatter/testdata/outputs/benchdiff-worktree.out From 445478de86f97a717b8a983f86db537121d96778 Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 16:26:19 -0600 Subject: [PATCH 06/10] update godoc badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ecbf80..fe9b942 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ # benchdiff -[![godoc](https://godoc.org/github.com/WillAbides/benchdiff?status.svg)](https://godoc.org/github.com/WillAbides/benchdiff) +[![godoc](https://godoc.org/github.com/willabides/benchdiff?status.svg)](https://godoc.org/github.com/willabides/benchdiff) [![ci](https://github.com/WillAbides/benchdiff/workflows/ci/badge.svg?branch=main&event=push)](https://github.com/WillAbides/benchdiff/actions?query=workflow%3Aci+branch%3Amaster+event%3Apush) From 494f2dfece932444cbe058440390aae8e29deac2 Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 16:37:16 -0600 Subject: [PATCH 07/10] --version flag --- cmd/benchdiff/benchdiff.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/cmd/benchdiff/benchdiff.go b/cmd/benchdiff/benchdiff.go index 4f4d24c..bdf394e 100644 --- a/cmd/benchdiff/benchdiff.go +++ b/cmd/benchdiff/benchdiff.go @@ -44,7 +44,10 @@ type benchstatOpts struct { Split string `kong:"help=${SplitHelp},default=${SplitDefault}"` } +var version string + var benchVars = kong.Vars{ + "version": version, "BenchCmdDefault": `go`, "BenchArgsDefault": defaultBenchArgsTmpl, "ResultsDirDefault": filepath.FromSlash("./tmp"), @@ -59,21 +62,23 @@ var benchVars = kong.Vars{ "OnDegradeHelp": `Exit code when there is a statistically significant degradation in the results.`, "JSONOutputHelp": `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`, "GitCmdHelp": `The executable to use for git commands.`, + "VersionHelp": `Output the benchdiff version and exit.`, } var cli struct { - BaseRef string `kong:"default=HEAD,help=${BaseRefHelp}"` - Bench string `kong:"default='.',help=${BenchHelp}"` - BenchArgs string `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"` - BenchCmd string `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"` - BenchCount int `kong:"default=10,help=${BenchCountHelp}"` - OnDegrade int `kong:"name=on-degrade,default=0,help=${OnDegradeHelp}"` - ForceBase bool `kong:"help=${ForceBaseHelp}"` - GitCmd string `kong:"default=git,help=${GitCmdHelp}"` - JSONOutput bool `kong:"help=${JSONOutputHelp}"` - Packages string `kong:"default='./...',help=${PackagesHelp}"` - ResultsDir string `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"` - BenchstatOpts benchstatOpts `kong:"embed"` + BaseRef string `kong:"default=HEAD,help=${BaseRefHelp}"` + Bench string `kong:"default='.',help=${BenchHelp}"` + BenchArgs string `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"` + BenchCmd string `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"` + BenchCount int `kong:"default=10,help=${BenchCountHelp}"` + OnDegrade int `kong:"name=on-degrade,default=0,help=${OnDegradeHelp}"` + ForceBase bool `kong:"help=${ForceBaseHelp}"` + GitCmd string `kong:"default=git,help=${GitCmdHelp}"` + JSONOutput bool `kong:"help=${JSONOutputHelp}"` + Packages string `kong:"default='./...',help=${PackagesHelp}"` + ResultsDir string `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"` + BenchstatOpts benchstatOpts `kong:"embed"` + Version kong.VersionFlag `kong:"help=${VersionHelp}"` } func main() { From 458579baa175b5a09e100dcfd74065b28834a482 Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 16:56:37 -0600 Subject: [PATCH 08/10] add description --- cmd/benchdiff/benchdiff.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/benchdiff/benchdiff.go b/cmd/benchdiff/benchdiff.go index bdf394e..3c5316f 100644 --- a/cmd/benchdiff/benchdiff.go +++ b/cmd/benchdiff/benchdiff.go @@ -81,8 +81,17 @@ var cli struct { Version kong.VersionFlag `kong:"help=${VersionHelp}"` } +const description = ` +benchdiff runs go benchmarks on your current git worktree and a base ref then +uses benchstat to show the delta. + +See https://github.com/willabides/benchdiff for more details. +` + func main() { - kctx := kong.Parse(&cli, benchstatVars, benchVars) + kctx := kong.Parse(&cli, benchstatVars, benchVars, + kong.Description(strings.TrimSpace(description)), + ) tmpl, err := template.New("").Parse(cli.BenchArgs) kctx.FatalIfErrorf(err) var benchArgs bytes.Buffer From 87d9277602b16c718a5942be785649947324d358 Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 17:18:05 -0600 Subject: [PATCH 09/10] update readme --- README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/README.md b/README.md index fe9b942..4db68cd 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,59 @@ [![godoc](https://godoc.org/github.com/willabides/benchdiff?status.svg)](https://godoc.org/github.com/willabides/benchdiff) [![ci](https://github.com/WillAbides/benchdiff/workflows/ci/badge.svg?branch=main&event=push)](https://github.com/WillAbides/benchdiff/actions?query=workflow%3Aci+branch%3Amaster+event%3Apush) + +Benchdiff is a command line tool intended to help speed up the feedback loop for go benchmarks. + +The old workflow: +- `go test -bench MyBenchmark -run '^$' -count 10 . > tmp/bench.out` +- `git stash && git switch main` +- `go test -bench MyBenchmark -run '^$' -count 10 . > tmp/bench-main.out` +- `git switch - && git stash apply` +- `benchstat tmp/bench-main.out tmp/bench.out` + +The new workflow: +- `benchdiff --bench 'MyBenchmark'` + +## Usage + +``` +Usage: benchdiff + +benchdiff runs go benchmarks on your current git worktree and a base ref then +uses benchstat to show the delta. + +See https://github.com/willabides/benchdiff for more details. + +Flags: + -h, --help Show context-sensitive help. + --base-ref="HEAD" The git ref to be used as a baseline. + --bench="." Run only those benchmarks matching a regular + expression. + --bench-args="test -bench {{.Bench}} -run '^$' -benchmem -count {{.BenchCount}} {{.Packages}}" + Use these arguments to run benchmarks. It may + be a template. + --bench-cmd="go" The go command to use for benchmarks. + --bench-count=10 Run each benchmark n times. + --on-degrade=0 Exit code when there is a statistically + significant degradation in the results. + --force-base Rerun benchmarks on the base reference even + if the output already exists. + --git-cmd="git" The executable to use for git commands. + --json-output Format output as JSON. When true the --csv + and --html flags affect only the + "benchstat_output" field. + --packages="./..." Run benchmarks in these packages. + --results-dir="./tmp" The directory where benchmark output will be + deposited. + --alpha=0.05 consider change significant if p < α + --csv format benchstat results as CSV + --delta-test="utest" significance test to apply to delta: utest, + ttest, or none + --geomean print the geometric mean of each file + --html format benchstat results as CSV an HTML table + --norange suppress range columns (CSV only) + --reverse-sort reverse sort order + --sort="none" sort by order: delta, name, none + --split="pkg,goos,goarch" split benchmarks by labels + --version Output the benchdiff version and exit. +``` \ No newline at end of file From 4ab469cf01c1e419d1332901b093fc9eac5b61ec Mon Sep 17 00:00:00 2001 From: Will Roden Date: Tue, 29 Dec 2020 17:28:46 -0600 Subject: [PATCH 10/10] update cachedir --- README.md | 8 +++--- cmd/benchdiff/benchdiff.go | 38 ++++++++++++++--------------- cmd/benchdiff/internal/benchdiff.go | 1 + 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 4db68cd..6a39f23 100644 --- a/README.md +++ b/README.md @@ -35,17 +35,17 @@ Flags: be a template. --bench-cmd="go" The go command to use for benchmarks. --bench-count=10 Run each benchmark n times. - --on-degrade=0 Exit code when there is a statistically - significant degradation in the results. + --cache-dir="./tmp" The directory where benchmark output will + kept between runs. --force-base Rerun benchmarks on the base reference even if the output already exists. --git-cmd="git" The executable to use for git commands. --json-output Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field. + --on-degrade=0 Exit code when there is a statistically + significant degradation in the results. --packages="./..." Run benchmarks in these packages. - --results-dir="./tmp" The directory where benchmark output will be - deposited. --alpha=0.05 consider change significant if p < α --csv format benchstat results as CSV --delta-test="utest" significance test to apply to delta: utest, diff --git a/cmd/benchdiff/benchdiff.go b/cmd/benchdiff/benchdiff.go index 3c5316f..cd513ed 100644 --- a/cmd/benchdiff/benchdiff.go +++ b/cmd/benchdiff/benchdiff.go @@ -47,22 +47,22 @@ type benchstatOpts struct { var version string var benchVars = kong.Vars{ - "version": version, - "BenchCmdDefault": `go`, - "BenchArgsDefault": defaultBenchArgsTmpl, - "ResultsDirDefault": filepath.FromSlash("./tmp"), - "BenchCountHelp": `Run each benchmark n times.`, - "BenchHelp": `Run only those benchmarks matching a regular expression.`, - "BenchArgsHelp": `Use these arguments to run benchmarks. It may be a template.`, - "PackagesHelp": `Run benchmarks in these packages.`, - "BenchCmdHelp": `The go command to use for benchmarks.`, - "ResultsDirHelp": `The directory where benchmark output will be deposited.`, - "BaseRefHelp": `The git ref to be used as a baseline.`, - "ForceBaseHelp": `Rerun benchmarks on the base reference even if the output already exists.`, - "OnDegradeHelp": `Exit code when there is a statistically significant degradation in the results.`, - "JSONOutputHelp": `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`, - "GitCmdHelp": `The executable to use for git commands.`, - "VersionHelp": `Output the benchdiff version and exit.`, + "version": version, + "BenchCmdDefault": `go`, + "BenchArgsDefault": defaultBenchArgsTmpl, + "CacheDirDefault": filepath.FromSlash("./tmp"), + "BenchCountHelp": `Run each benchmark n times.`, + "BenchHelp": `Run only those benchmarks matching a regular expression.`, + "BenchArgsHelp": `Use these arguments to run benchmarks. It may be a template.`, + "PackagesHelp": `Run benchmarks in these packages.`, + "BenchCmdHelp": `The go command to use for benchmarks.`, + "CacheDirHelp": `The directory where benchmark output will kept between runs.`, + "BaseRefHelp": `The git ref to be used as a baseline.`, + "ForceBaseHelp": `Rerun benchmarks on the base reference even if the output already exists.`, + "OnDegradeHelp": `Exit code when there is a statistically significant degradation in the results.`, + "JSONOutputHelp": `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`, + "GitCmdHelp": `The executable to use for git commands.`, + "VersionHelp": `Output the benchdiff version and exit.`, } var cli struct { @@ -71,12 +71,12 @@ var cli struct { BenchArgs string `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"` BenchCmd string `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"` BenchCount int `kong:"default=10,help=${BenchCountHelp}"` - OnDegrade int `kong:"name=on-degrade,default=0,help=${OnDegradeHelp}"` + CacheDir string `kong:"type=dir,default=${CacheDirDefault},help=${CacheDirHelp}"` ForceBase bool `kong:"help=${ForceBaseHelp}"` GitCmd string `kong:"default=git,help=${GitCmdHelp}"` JSONOutput bool `kong:"help=${JSONOutputHelp}"` + OnDegrade int `kong:"name=on-degrade,default=0,help=${OnDegradeHelp}"` Packages string `kong:"default='./...',help=${PackagesHelp}"` - ResultsDir string `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"` BenchstatOpts benchstatOpts `kong:"embed"` Version kong.VersionFlag `kong:"help=${VersionHelp}"` } @@ -101,7 +101,7 @@ func main() { bd := &internal.Benchdiff{ BenchCmd: cli.BenchCmd, BenchArgs: benchArgs.String(), - ResultsDir: cli.ResultsDir, + ResultsDir: cli.CacheDir, BaseRef: cli.BaseRef, Path: ".", Writer: os.Stdout, diff --git a/cmd/benchdiff/internal/benchdiff.go b/cmd/benchdiff/internal/benchdiff.go index 8d0ce34..617849e 100644 --- a/cmd/benchdiff/internal/benchdiff.go +++ b/cmd/benchdiff/internal/benchdiff.go @@ -91,6 +91,7 @@ func (c *Benchdiff) runBenchmarks() (result *runBenchmarksResults, err error) { } baseFilename := fmt.Sprintf("benchdiff-%s.out", baseSHA) + baseFilename = filepath.Join(c.ResultsDir, baseFilename) result.headSHA = headSHA result.baseSHA = baseSHA result.baseOutputFile = baseFilename