move packages around to make most of this internal

README.md

@@ -1,4 +1,60 @@
 # benchdiff
 
-[![godoc](https://godoc.org/github.com/WillAbides/benchdiff?status.svg)](https://godoc.org/github.com/WillAbides/benchdiff)
+[![godoc](https://godoc.org/github.com/willabides/benchdiff?status.svg)](https://godoc.org/github.com/willabides/benchdiff)
 [![ci](https://github.com/WillAbides/benchdiff/workflows/ci/badge.svg?branch=main&event=push)](https://github.com/WillAbides/benchdiff/actions?query=workflow%3Aci+branch%3Amaster+event%3Apush)
+
+Benchdiff is a command line tool intended to help speed up the feedback loop for go benchmarks.
+
+The old workflow:
+- `go test -bench MyBenchmark -run '^$' -count 10 . > tmp/bench.out`
+- `git stash && git switch main`
+- `go test -bench MyBenchmark -run '^$' -count 10 . > tmp/bench-main.out`
+- `git switch - && git stash apply`
+- `benchstat tmp/bench-main.out tmp/bench.out`
+
+The new workflow:
+- `benchdiff --bench 'MyBenchmark'`
+
+## Usage
+
+```
+Usage: benchdiff
+
+benchdiff runs go benchmarks on your current git worktree and a base ref then
+uses benchstat to show the delta.
+
+See https://github.com/willabides/benchdiff for more details.
+
+Flags:
+  -h, --help                       Show context-sensitive help.
+      --base-ref="HEAD"            The git ref to be used as a baseline.
+      --bench="."                  Run only those benchmarks matching a regular
+                                   expression.
+      --bench-args="test -bench {{.Bench}} -run '^$' -benchmem -count {{.BenchCount}} {{.Packages}}"
+                                   Use these arguments to run benchmarks. It may
+                                   be a template.
+      --bench-cmd="go"             The go command to use for benchmarks.
+      --bench-count=10             Run each benchmark n times.
+      --cache-dir="./tmp"          The directory where benchmark output will
+                                   kept between runs.
+      --force-base                 Rerun benchmarks on the base reference even
+                                   if the output already exists.
+      --git-cmd="git"              The executable to use for git commands.
+      --json-output                Format output as JSON. When true the --csv
+                                   and --html flags affect only the
+                                   "benchstat_output" field.
+      --on-degrade=0               Exit code when there is a statistically
+                                   significant degradation in the results.
+      --packages="./..."           Run benchmarks in these packages.
+      --alpha=0.05                 consider change significant if p < α
+      --csv                        format benchstat results as CSV
+      --delta-test="utest"         significance test to apply to delta: utest,
+                                   ttest, or none
+      --geomean                    print the geometric mean of each file
+      --html                       format benchstat results as CSV an HTML table
+      --norange                    suppress range columns (CSV only)
+      --reverse-sort               reverse sort order
+      --sort="none"                sort by order: delta, name, none
+      --split="pkg,goos,goarch"    split benchmarks by labels
+      --version                    Output the benchdiff version and exit.
+```

cmd/benchdiff/benchdiff.go

@@ -8,16 +8,16 @@ import (
 	"text/template"
 
 	"github.com/alecthomas/kong"
-	"github.com/willabides/benchdiff"
-	pkgbenchstat "github.com/willabides/benchdiff/pkg/benchstat"
+	"github.com/willabides/benchdiff/cmd/benchdiff/internal"
+	"github.com/willabides/benchdiff/pkg/benchstatter"
 	"golang.org/x/perf/benchstat"
 )
 
 const defaultBenchArgsTmpl = `test -bench {{.Bench}} -run '^$' -benchmem -count {{.BenchCount}} {{.Packages}}`
 
 var benchstatVars = kong.Vars{
 	"AlphaDefault":     "0.05",
-	"AlphaHelp":        `consider change significant if p < α (default 0.05)`,
+	"AlphaHelp":        `consider change significant if p < α`,
 	"CSVHelp":          `format benchstat results as CSV`,
 	"DeltaTestHelp":    `significance test to apply to delta: utest, ttest, or none`,
 	"DeltaTestDefault": `utest`,
@@ -44,69 +44,86 @@ type benchstatOpts struct {
 	Split       string  `kong:"help=${SplitHelp},default=${SplitDefault}"`
 }
 
+var version string
+
 var benchVars = kong.Vars{
-	"BenchCmdDefault":     `go`,
-	"BenchArgsDefault":    defaultBenchArgsTmpl,
-	"ResultsDirDefault":   filepath.FromSlash("./tmp"),
-	"BenchCountHelp":      `Run each benchmark n times.`,
-	"BenchHelp":           `Run only those benchmarks matching a regular expression.`,
-	"BenchArgsHelp":       `Use these arguments to run benchmarks. It may be a template.`,
-	"PackagesHelp":        `Run benchmarks in these packages.`,
-	"BenchCmdHelp":        `The go command to use for benchmarks.`,
-	"ResultsDirHelp":      `The directory where benchmark output will be deposited.`,
-	"BaseRefHelp":         `The git ref to be used as a baseline.`,
-	"ForceBaseHelp":       `Rerun benchmarks on the base reference even if the output already exists.`,
-	"DegradationExitHelp": `Exit code when there is a degradation in the results.`,
-	"JSONOutputHelp":      `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`,
+	"version":          version,
+	"BenchCmdDefault":  `go`,
+	"BenchArgsDefault": defaultBenchArgsTmpl,
+	"CacheDirDefault":  filepath.FromSlash("./tmp"),
+	"BenchCountHelp":   `Run each benchmark n times.`,
+	"BenchHelp":        `Run only those benchmarks matching a regular expression.`,
+	"BenchArgsHelp":    `Use these arguments to run benchmarks. It may be a template.`,
+	"PackagesHelp":     `Run benchmarks in these packages.`,
+	"BenchCmdHelp":     `The go command to use for benchmarks.`,
+	"CacheDirHelp":     `The directory where benchmark output will kept between runs.`,
+	"BaseRefHelp":      `The git ref to be used as a baseline.`,
+	"ForceBaseHelp":    `Rerun benchmarks on the base reference even if the output already exists.`,
+	"OnDegradeHelp":    `Exit code when there is a statistically significant degradation in the results.`,
+	"JSONOutputHelp":   `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`,
+	"GitCmdHelp":       `The executable to use for git commands.`,
+	"VersionHelp":      `Output the benchdiff version and exit.`,
 }
 
 var cli struct {
-	BaseRef         string        `kong:"default=HEAD,help=${BaseRefHelp}"`
-	Bench           string        `kong:"default='.',help=${BenchHelp}"`
-	BenchArgs       string        `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"`
-	BenchCmd        string        `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"`
-	BenchCount      int           `kong:"default=10,help=${BenchCountHelp}"`
-	ForceBase       bool          `kong:"help=${ForceBaseHelp}"`
-	Packages        string        `kong:"default='./...',help=${PackagesHelp}"`
-	ResultsDir      string        `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"`
-	DegradationExit int           `kong:"name=on-degradation,default=0,help=${DegradationExitHelp}"`
-	JSONOutput      bool          `kong:"help=${JSONOutputHelp}"`
-	BenchstatOpts   benchstatOpts `kong:"embed"`
+	BaseRef       string           `kong:"default=HEAD,help=${BaseRefHelp}"`
+	Bench         string           `kong:"default='.',help=${BenchHelp}"`
+	BenchArgs     string           `kong:"default=${BenchArgsDefault},help=${BenchArgsHelp}"`
+	BenchCmd      string           `kong:"default=${BenchCmdDefault},help=${BenchCmdHelp}"`
+	BenchCount    int              `kong:"default=10,help=${BenchCountHelp}"`
+	CacheDir      string           `kong:"type=dir,default=${CacheDirDefault},help=${CacheDirHelp}"`
+	ForceBase     bool             `kong:"help=${ForceBaseHelp}"`
+	GitCmd        string           `kong:"default=git,help=${GitCmdHelp}"`
+	JSONOutput    bool             `kong:"help=${JSONOutputHelp}"`
+	OnDegrade     int              `kong:"name=on-degrade,default=0,help=${OnDegradeHelp}"`
+	Packages      string           `kong:"default='./...',help=${PackagesHelp}"`
+	BenchstatOpts benchstatOpts    `kong:"embed"`
+	Version       kong.VersionFlag `kong:"help=${VersionHelp}"`
 }
 
+const description = `
+benchdiff runs go benchmarks on your current git worktree and a base ref then
+uses benchstat to show the delta.
+
+See https://github.com/willabides/benchdiff for more details.
+`
+
 func main() {
-	kctx := kong.Parse(&cli, benchstatVars, benchVars)
+	kctx := kong.Parse(&cli, benchstatVars, benchVars,
+		kong.Description(strings.TrimSpace(description)),
+	)
 	tmpl, err := template.New("").Parse(cli.BenchArgs)
 	kctx.FatalIfErrorf(err)
 	var benchArgs bytes.Buffer
 	err = tmpl.Execute(&benchArgs, cli)
 	kctx.FatalIfErrorf(err)
 
-	differ := &benchdiff.Benchdiff{
+	bd := &internal.Benchdiff{
 		BenchCmd:   cli.BenchCmd,
 		BenchArgs:  benchArgs.String(),
-		ResultsDir: cli.ResultsDir,
+		ResultsDir: cli.CacheDir,
 		BaseRef:    cli.BaseRef,
 		Path:       ".",
 		Writer:     os.Stdout,
 		Benchstat:  buildBenchstat(cli.BenchstatOpts),
 		Force:      cli.ForceBase,
+		GitCmd:     cli.GitCmd,
 	}
-	result, err := differ.Run()
+	result, err := bd.Run()
 	kctx.FatalIfErrorf(err)
 
 	outputFormat := "human"
 	if cli.JSONOutput {
 		outputFormat = "json"
 	}
 
-	err = result.WriteOutput(os.Stdout, &benchdiff.RunResultOutputOptions{
+	err = result.WriteOutput(os.Stdout, &internal.RunResultOutputOptions{
 		BenchstatFormatter: buildBenchstat(cli.BenchstatOpts).OutputFormatter,
 		OutputFormat:       outputFormat,
 	})
 	kctx.FatalIfErrorf(err)
-	if result.HasChangeType(benchdiff.DegradingChange) {
-		os.Exit(cli.DegradationExit)
+	if result.HasChangeType(internal.DegradingChange) {
+		os.Exit(cli.OnDegrade)
 	}
 }
 
@@ -122,23 +139,23 @@ var sortOpts = map[string]benchstat.Order{
 	"delta": benchstat.ByDelta,
 }
 
-func buildBenchstat(opts benchstatOpts) *pkgbenchstat.Benchstat {
+func buildBenchstat(opts benchstatOpts) *benchstatter.Benchstat {
 	order := sortOpts[opts.Sort]
 	reverse := opts.ReverseSort
 	if order == nil {
 		reverse = false
 	}
-	formatter := pkgbenchstat.TextFormatter(nil)
+	formatter := benchstatter.TextFormatter(nil)
 	if opts.CSV {
-		formatter = pkgbenchstat.CSVFormatter(&pkgbenchstat.CSVFormatterOptions{
+		formatter = benchstatter.CSVFormatter(&benchstatter.CSVFormatterOptions{
 			NoRange: opts.Norange,
 		})
 	}
 	if opts.HTML {
-		formatter = pkgbenchstat.HTMLFormatter(nil)
+		formatter = benchstatter.HTMLFormatter(nil)
 	}
 
-	return &pkgbenchstat.Benchstat{
+	return &benchstatter.Benchstat{
 		DeltaTest:       deltaTestOpts[opts.DeltaTest],
 		Alpha:           opts.Alpha,
 		AddGeoMean:      opts.Geomean,

benchdiff.go → cmd/benchdiff/internal/benchdiff.go

@@ -1,4 +1,4 @@
-package benchdiff
+package internal
 
 import (
 	"bytes"
@@ -10,7 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	pkgbenchstat "github.com/willabides/benchdiff/pkg/benchstat"
+	"github.com/willabides/benchdiff/pkg/benchstatter"
 	"golang.org/x/perf/benchstat"
 )
 
@@ -21,8 +21,9 @@ type Benchdiff struct {
 	ResultsDir string
 	BaseRef    string
 	Path       string
+	GitCmd     string
 	Writer     io.Writer
-	Benchstat  *pkgbenchstat.Benchstat
+	Benchstat  *benchstatter.Benchstat
 	Force      bool
 	JSONOutput bool
 }
@@ -45,16 +46,16 @@ func fileExists(path string) bool {
 
 func (c *Benchdiff) gitRunner() *gitRunner {
 	return &gitRunner{
-		repoPath: c.Path,
+		gitExecutable: c.GitCmd,
+		repoPath:      c.Path,
 	}
 }
 
 func (c *Benchdiff) baseRefRunner() *refRunner {
+	gr := c.gitRunner()
 	return &refRunner{
-		ref: c.BaseRef,
-		gitRunner: gitRunner{
-			repoPath: c.Path,
-		},
+		ref:       c.BaseRef,
+		gitRunner: *gr,
 	}
 }
 
@@ -90,6 +91,7 @@ func (c *Benchdiff) runBenchmarks() (result *runBenchmarksResults, err error) {
 	}
 
 	baseFilename := fmt.Sprintf("benchdiff-%s.out", baseSHA)
+	baseFilename = filepath.Join(c.ResultsDir, baseFilename)
 	result.headSHA = headSHA
 	result.baseSHA = baseSHA
 	result.baseOutputFile = baseFilename
@@ -161,7 +163,7 @@ type RunResult struct {
 
 // RunResultOutputOptions options for RunResult.WriteOutput
 type RunResultOutputOptions struct {
-	BenchstatFormatter pkgbenchstat.OutputFormatter // default benchstat.TextFormatter(nil)
+	BenchstatFormatter benchstatter.OutputFormatter // default benchstatter.TextFormatter(nil)
 	OutputFormat       string                       // one of json or human. default: human
 }
 
@@ -171,7 +173,7 @@ func (r *RunResult) WriteOutput(w io.Writer, opts *RunResultOutputOptions) error
 		opts = new(RunResultOutputOptions)
 	}
 	finalOpts := &RunResultOutputOptions{
-		BenchstatFormatter: pkgbenchstat.TextFormatter(nil),
+		BenchstatFormatter: benchstatter.TextFormatter(nil),
 		OutputFormat:       "human",
 	}
 	if opts.BenchstatFormatter != nil {

benchdiff_test.go → cmd/benchdiff/internal/benchdiff_test.go

@@ -1,4 +1,4 @@
-package benchdiff
+package internal
 
 import (
 	"io/ioutil"
@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"github.com/willabides/benchdiff/pkg/benchstat"
+	"github.com/willabides/benchdiff/pkg/benchstatter"
 )
 
 func setupTestRepo(t *testing.T, path string) {
@@ -50,7 +50,7 @@ func TestBenchstat_Run(t *testing.T) {
 		ResultsDir: "./tmp",
 		BaseRef:    "HEAD",
 		Path:       ".",
-		Benchstat:  &benchstat.Benchstat{},
+		Benchstat:  &benchstatter.Benchstat{},
 	}
 	_, err := differ.Run()
 	require.NoError(t, err)

gitrunner.go → cmd/benchdiff/internal/gitrunner.go

@@ -1,4 +1,4 @@
-package benchdiff
+package internal
 
 import (
 	"bytes"

gitrunner_test.go → cmd/benchdiff/internal/gitrunner_test.go

@@ -1,4 +1,4 @@
-package benchdiff
+package internal
 
 import (
 	"io/ioutil"

testutil_test.go → cmd/benchdiff/internal/testutil_test.go

@@ -1,4 +1,4 @@
-package benchdiff
+package internal
 
 import (
 	"io/ioutil"
@@ -13,7 +13,7 @@ var preserveTmpDir bool
 
 func tmpDir(t *testing.T) string {
 	t.Helper()
-	projectTmp := filepath.FromSlash("./tmp")
+	projectTmp := filepath.FromSlash("../../../tmp")
 
 	err := os.MkdirAll(projectTmp, 0o700)
 	assert.NoError(t, err)

pkg/benchstat/benchstat.go → pkg/benchstatter/benchstat.go

@@ -1,5 +1,5 @@
-// Package benchstat is used to run benchstat programmatically
-package benchstat
+// Package benchstatter is used to run benchstatter programmatically
+package benchstatter
 
 import (
 	"bytes"

pkg/benchstat/benchstat_test.go → pkg/benchstatter/benchstat_test.go

@@ -1,4 +1,4 @@
-package benchstat
+package benchstatter
 
 import (
 	"bytes"