json out

benchdiff.go

@@ -2,6 +2,7 @@ package benchdiff
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -13,8 +14,8 @@ import (
 	"golang.org/x/perf/benchstat"
 )
 
-// Differ runs benchstats and outputs their deltas
-type Differ struct {
+// Benchdiff runs benchstats and outputs their deltas
+type Benchdiff struct {
 	BenchCmd   string
 	BenchArgs  string
 	ResultsDir string
@@ -23,23 +24,15 @@ type Differ struct {
 	Writer     io.Writer
 	Benchstat  *pkgbenchstat.Benchstat
 	Force      bool
-}
-
-func (c *Differ) baseOutputFile() (string, error) {
-	runner := &gitRunner{
-		repoPath: c.Path,
-	}
-	revision, err := runner.run("rev-parse", c.BaseRef)
-	if err != nil {
-		return "", err
-	}
-	revision = bytes.TrimSpace(revision)
-	name := fmt.Sprintf("benchstatter-%s.out", string(revision))
-	return filepath.Join(c.ResultsDir, name), nil
+	JSONOutput bool
 }
 
 type runBenchmarksResults struct {
-	worktreeOutputFile, baseOutputFile string
+	worktreeOutputFile string
+	baseOutputFile     string
+	benchmarkCmd       string
+	headSHA            string
+	baseSHA            string
 }
 
 func fileExists(path string) bool {
@@ -50,8 +43,24 @@ func fileExists(path string) bool {
 	return true
 }
 
-func (c *Differ) runBenchmarks() (result *runBenchmarksResults, err error) {
-	worktreeFilename := filepath.Join(c.ResultsDir, "benchstatter-worktree.out")
+func (c *Benchdiff) gitRunner() *gitRunner {
+	return &gitRunner{
+		repoPath: c.Path,
+	}
+}
+
+func (c *Benchdiff) baseRefRunner() *refRunner {
+	return &refRunner{
+		ref: c.BaseRef,
+		gitRunner: gitRunner{
+			repoPath: c.Path,
+		},
+	}
+}
+
+func (c *Benchdiff) runBenchmarks() (result *runBenchmarksResults, err error) {
+	result = new(runBenchmarksResults)
+	worktreeFilename := filepath.Join(c.ResultsDir, "benchdiff-worktree.out")
 	worktreeFile, err := os.Create(worktreeFilename)
 	if err != nil {
 		return nil, err
@@ -64,23 +73,28 @@ func (c *Differ) runBenchmarks() (result *runBenchmarksResults, err error) {
 	}()
 
 	cmd := exec.Command(c.BenchCmd, strings.Fields(c.BenchArgs)...) //nolint:gosec // this is fine
-	fmt.Println(c.BenchArgs)
+	result.benchmarkCmd = cmd.String()
 	cmd.Stdout = worktreeFile
 	err = cmd.Run()
 	if err != nil {
 		return nil, err
 	}
 
-	baseFilename, err := c.baseOutputFile()
+	headSHA, err := c.gitRunner().getRefSha("HEAD")
 	if err != nil {
 		return nil, err
 	}
-
-	result = &runBenchmarksResults{
-		worktreeOutputFile: worktreeFilename,
-		baseOutputFile:     baseFilename,
+	baseSHA, err := c.gitRunner().getRefSha(c.BaseRef)
+	if err != nil {
+		return nil, err
 	}
 
+	baseFilename := fmt.Sprintf("benchdiff-%s.out", baseSHA)
+	result.headSHA = headSHA
+	result.baseSHA = baseSHA
+	result.baseOutputFile = baseFilename
+	result.worktreeOutputFile = worktreeFilename
+
 	if fileExists(baseFilename) && !c.Force {
 		return result, nil
 	}
@@ -99,14 +113,8 @@ func (c *Differ) runBenchmarks() (result *runBenchmarksResults, err error) {
 	baseCmd := exec.Command(c.BenchCmd, strings.Fields(c.BenchArgs)...) //nolint:gosec // this is fine
 	baseCmd.Stdout = baseFile
 	var baseCmdErr error
-	runner := &refRunner{
-		ref: c.BaseRef,
-		gitRunner: gitRunner{
-			repoPath:      c.Path,
-			gitExecutable: "",
-		},
-	}
-	err = runner.run(func() {
+
+	err = c.baseRefRunner().run(func() {
 		baseCmdErr = baseCmd.Run()
 	})
 	if err != nil {
@@ -120,8 +128,8 @@ func (c *Differ) runBenchmarks() (result *runBenchmarksResults, err error) {
 	return result, nil
 }
 
-// Run runs the Differ
-func (c *Differ) Run() (*RunResult, error) {
+// Run runs the Benchdiff
+func (c *Benchdiff) Run() (*RunResult, error) {
 	err := os.MkdirAll(c.ResultsDir, 0o700)
 	if err != nil {
 		return nil, err
@@ -135,19 +143,100 @@ func (c *Differ) Run() (*RunResult, error) {
 		return nil, err
 	}
 	result := &RunResult{
-		tables: collection.Tables(),
+		headSHA:  res.headSHA,
+		baseSHA:  res.baseSHA,
+		benchCmd: res.benchmarkCmd,
+		tables:   collection.Tables(),
 	}
 	return result, nil
 }
 
-// OutputResult outputs a Run result
-func (c *Differ) OutputResult(runResult *RunResult) error {
-	return c.Benchstat.OutputTables(runResult.tables)
-}
-
 // RunResult is the result of a Run
 type RunResult struct {
-	tables []*benchstat.Table
+	headSHA  string
+	baseSHA  string
+	benchCmd string
+	tables   []*benchstat.Table
+}
+
+// RunResultOutputOptions options for RunResult.WriteOutput
+type RunResultOutputOptions struct {
+	BenchstatFormatter pkgbenchstat.OutputFormatter // default benchstat.TextFormatter(nil)
+	OutputFormat       string                       // one of json or human. default: human
+}
+
+// WriteOutput outputs the result
+func (r *RunResult) WriteOutput(w io.Writer, opts *RunResultOutputOptions) error {
+	if opts == nil {
+		opts = new(RunResultOutputOptions)
+	}
+	finalOpts := &RunResultOutputOptions{
+		BenchstatFormatter: pkgbenchstat.TextFormatter(nil),
+		OutputFormat:       "human",
+	}
+	if opts.BenchstatFormatter != nil {
+		finalOpts.BenchstatFormatter = opts.BenchstatFormatter
+	}
+
+	if opts.OutputFormat != "" {
+		finalOpts.OutputFormat = opts.OutputFormat
+	}
+
+	var benchstatBuf bytes.Buffer
+	err := finalOpts.BenchstatFormatter(&benchstatBuf, r.tables)
+	if err != nil {
+		return err
+	}
+
+	var fn func(io.Writer, string) error
+	switch finalOpts.OutputFormat {
+	case "human":
+		fn = r.writeHumanResult
+	case "json":
+		fn = r.writeJSONResult
+	default:
+		return fmt.Errorf("unknown OutputFormat")
+	}
+	return fn(w, benchstatBuf.String())
+}
+
+func (r *RunResult) writeJSONResult(w io.Writer, benchstatResult string) error {
+	type runResultJSON struct {
+		BenchCommand    string `json:"bench_command,omitempty"`
+		HeadSHA         string `json:"head_sha,omitempty"`
+		BaseSHA         string `json:"base_sha,omitempty"`
+		BenchstatOutput string `json:"benchstat_output,omitempty"`
+	}
+	encoder := json.NewEncoder(w)
+	encoder.SetIndent("", "  ")
+	return encoder.Encode(&runResultJSON{
+		BenchCommand:    r.benchCmd,
+		BenchstatOutput: benchstatResult,
+		HeadSHA:         r.headSHA,
+		BaseSHA:         r.baseSHA,
+	})
+}
+
+func (r *RunResult) writeHumanResult(w io.Writer, benchstatResult string) error {
+	var err error
+	_, err = fmt.Fprintf(w, "bench command:\n  %s\n", r.benchCmd)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprintf(w, "HEAD sha:\n  %s\n", r.headSHA)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprintf(w, "base sha:\n  %s\n", r.baseSHA)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprintf(w, "benchstat output:\n\n%s\n", benchstatResult)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // HasChangeType returns true if the result has at least one change with the given type

benchdiff_test.go

@@ -40,21 +40,19 @@ func testInDir(t *testing.T, dir string) {
 	})
 }
 
-func TestDiffer_Run(t *testing.T) {
+func TestBenchstat_Run(t *testing.T) {
 	dir := tmpDir(t)
 	setupTestRepo(t, dir)
 	testInDir(t, dir)
-	differ := Differ{
+	differ := Benchdiff{
 		BenchCmd:   "go",
 		BenchArgs:  "test -bench . -benchmem -count 10 -benchtime 10x .",
 		ResultsDir: "./tmp",
 		BaseRef:    "HEAD",
 		Path:       ".",
 		Benchstat:  &benchstat.Benchstat{},
 	}
-	result, err := differ.Run()
-	require.NoError(t, err)
-	err = differ.OutputResult(result)
+	_, err := differ.Run()
 	require.NoError(t, err)
 }
 

cmd/benchdiff/benchdiff.go

@@ -18,12 +18,12 @@ const defaultBenchArgsTmpl = `test -bench {{.Bench}} -run '^$' -benchmem -count
 var benchstatVars = kong.Vars{
 	"AlphaDefault":     "0.05",
 	"AlphaHelp":        `consider change significant if p < α (default 0.05)`,
-	"CSVHelp":          `print results in CSV form`,
+	"CSVHelp":          `format benchstat results as CSV`,
 	"DeltaTestHelp":    `significance test to apply to delta: utest, ttest, or none`,
 	"DeltaTestDefault": `utest`,
 	"DeltaTestEnum":    `utest,ttest,none`,
 	"GeomeanHelp":      `print the geometric mean of each file`,
-	"HTMLHelp":         `print results as an HTML table`,
+	"HTMLHelp":         `format benchstat results as CSV an HTML table`,
 	"NorangeHelp":      `suppress range columns (CSV only)`,
 	"ReverseSortHelp":  `reverse sort order`,
 	"SortHelp":         `sort by order: delta, name, none`,
@@ -57,6 +57,7 @@ var benchVars = kong.Vars{
 	"BaseRefHelp":         `The git ref to be used as a baseline.`,
 	"ForceBaseHelp":       `Rerun benchmarks on the base reference even if the output already exists.`,
 	"DegradationExitHelp": `Exit code when there is a degradation in the results.`,
+	"JSONOutputHelp":      `Format output as JSON. When true the --csv and --html flags affect only the "benchstat_output" field.`,
 }
 
 var cli struct {
@@ -68,7 +69,8 @@ var cli struct {
 	ForceBase       bool          `kong:"help=${ForceBaseHelp}"`
 	Packages        string        `kong:"default='./...',help=${PackagesHelp}"`
 	ResultsDir      string        `kong:"type=dir,default=${ResultsDirDefault},help=${ResultsDirHelp}"`
-	DegradationExit int           `kong:"type=on-degradation,default=0,help=${DegradationExitHelp}"`
+	DegradationExit int           `kong:"name=on-degradation,default=0,help=${DegradationExitHelp}"`
+	JSONOutput      bool          `kong:"help=${JSONOutputHelp}"`
 	BenchstatOpts   benchstatOpts `kong:"embed"`
 }
 
@@ -80,7 +82,7 @@ func main() {
 	err = tmpl.Execute(&benchArgs, cli)
 	kctx.FatalIfErrorf(err)
 
-	differ := &benchdiff.Differ{
+	differ := &benchdiff.Benchdiff{
 		BenchCmd:   cli.BenchCmd,
 		BenchArgs:  benchArgs.String(),
 		ResultsDir: cli.ResultsDir,
@@ -92,7 +94,16 @@ func main() {
 	}
 	result, err := differ.Run()
 	kctx.FatalIfErrorf(err)
-	err = differ.OutputResult(result)
+
+	outputFormat := "human"
+	if cli.JSONOutput {
+		outputFormat = "json"
+	}
+
+	err = result.WriteOutput(os.Stdout, &benchdiff.RunResultOutputOptions{
+		BenchstatFormatter: buildBenchstat(cli.BenchstatOpts).OutputFormatter,
+		OutputFormat:       outputFormat,
+	})
 	kctx.FatalIfErrorf(err)
 	if result.HasChangeType(benchdiff.DegradingChange) {
 		os.Exit(cli.DegradationExit)

gitrunner.go

@@ -12,6 +12,15 @@ type gitRunner struct {
 	gitExecutable string
 }
 
+func (r *gitRunner) getRefSha(ref string) (string, error) {
+	b, err := r.run("rev-parse", ref)
+	if err != nil {
+		return "", err
+	}
+	b = bytes.TrimSpace(b)
+	return string(b), nil
+}
+
 func (r *gitRunner) run(args ...string) ([]byte, error) {
 	executable := "git"
 	if r.gitExecutable != "" {
@@ -23,7 +32,12 @@ func (r *gitRunner) run(args ...string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	return cmd.Output()
+
+	b, err := cmd.Output()
+	if exitErr, ok := err.(*exec.ExitError); ok {
+		err = fmt.Errorf("error running git command: %s", string(exitErr.Stderr))
+	}
+	return b, err
 }
 
 type refRunner struct {

pkg/benchstat/benchstat.go

@@ -37,9 +37,6 @@ type Benchstat struct {
 
 	// OutputFormatter determines how the output will be formatted. Default is TextFormatter
 	OutputFormatter OutputFormatter
-
-	// Writer is where to write output. Default is stdout.
-	Writer io.Writer
 }
 
 // OutputFormatter formats benchstat output
@@ -72,11 +69,7 @@ func (b *Benchstat) Run(files ...string) (*benchstat.Collection, error) {
 }
 
 // OutputTables outputs the results from tables using b.OutputFormatter
-func (b *Benchstat) OutputTables(tables []*benchstat.Table) error {
-	writer := b.Writer
-	if writer == nil {
-		writer = os.Stdout
-	}
+func (b *Benchstat) OutputTables(writer io.Writer, tables []*benchstat.Table) error {
 	formatter := b.OutputFormatter
 	if formatter == nil {
 		formatter = TextFormatter(nil)