main.go

//
// Copyright Coinbase, Inc. All Rights Reserved.
//
// SPDX-License-Identifier: Apache-2.0
//

// benchcomp implements a command that receives two benchmarks files as input and flags the benchmarks that have
// degraded by more than a threshold amount. The main goal of this tool is to be used in CI
// to check each PR.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"

	"github.com/pkg/errors"
	"golang.org/x/tools/benchmark/parse"
)

const THRESHOLD = 1.1

func main() {
	cReader, nReader, err := parseCmdArgs()
	if err != nil {
		panic(err)
	}

	if err := Compare(cReader, nReader); err != nil {
		panic(err)
	}
}

func parseCmdArgs() (io.Reader, io.Reader, error) {
	cFlag := flag.String("current", "current-bench.log", "The patch to the log file containing the output of the current benchmark result.")
	nFlag := flag.String("new", "new-bench.log", "The patch to the log file containing the output of the new benchmark result.")
	flag.Parse()
	cBytes, err := ioutil.ReadFile(*cFlag)
	if err != nil {
		return nil, nil, fmt.Errorf("reading current log file %v", err)
	}
	nBytes, err := ioutil.ReadFile(*nFlag)
	if err != nil {
		return nil, nil, fmt.Errorf("reading new log file %v", err)
	}

	return bytes.NewBuffer(cBytes), bytes.NewBuffer(nBytes), nil
}

// Compare expects two readers which contain the output of two runs of `go test -bench` command and throws an error if
// the performance has degraded by more than `THRESHOLD` amount.
func Compare(currBench, newBench io.Reader) error {
	c, n, err := parseBenchmarks(currBench, newBench)
	if err != nil {
		return errors.Wrap(err, "parsing benchmark outputs")
	}

	perfDeviations := make([]string, 0)
	for bench := range c {
		if _, ok := n[bench]; !ok {
			// New benchmark, skipping
			continue
		} else {
			currB := c[bench]
			newB := n[bench]
			err = compareBenches(currB, newB)
			if err != nil {
				perfDeviations = append(perfDeviations, fmt.Sprintf("%v", err))
			}
		}
	}

	if len(perfDeviations) != 0 {
		return fmt.Errorf("%#v", perfDeviations)
	}
	return nil
}

func parseBenchmarks(currBench, newBench io.Reader) (parse.Set, parse.Set, error) {
	c, err := parse.ParseSet(currBench)
	if err != nil {
		return nil, nil, errors.Wrap(err, "parsing current benchmark output")
	}
	n, err := parse.ParseSet(newBench)
	if err != nil {
		return nil, nil, errors.Wrap(err, "parsing new benchmark output")
	}

	return c, n, nil
}

func compareBenches(currB, newB []*parse.Benchmark) error {
	currMap := make(map[string]*parse.Benchmark)
	newMap := make(map[string]*parse.Benchmark)
	for _, b := range currB {
		// TODO: double check what is in Name
		currMap[b.Name] = b
	}
	for _, b := range newB {
		// TODO: double check what is in Name
		newMap[b.Name] = b
	}
	for name := range currMap {
		if _, ok := newMap[name]; ok {
			compare := []struct {
				current float64
				new     float64
			}{
				{
					current: float64(currMap[name].AllocedBytesPerOp),
					new:     float64(newMap[name].AllocedBytesPerOp),
				},
				{
					current: float64(currMap[name].AllocsPerOp),
					new:     float64(newMap[name].AllocsPerOp),
				},
				{
					current: currMap[name].NsPerOp,
					new:     newMap[name].NsPerOp,
				},
				{
					current: currMap[name].MBPerS,
					new:     newMap[name].MBPerS,
				},
			}
			for _, t := range compare {
				if t.new > t.current*THRESHOLD {
					percent := (t.new - t.current) * 100 / t.current
					return fmt.Errorf("benchmark %s exceeded previous benchmark by %0.2f percent. Current: %0.2f, New: %0.2f", name, percent, t.current, t.new)
				}
			}
		}
	}
	return nil
}