Merge pull request #197 from Tantalor93/improverrors

Error improvements

cmd/plot.go

@@ -16,6 +16,10 @@ import (
 )
 
 func plotHistogramLatency(file string, times []Datapoint) {
+	if len(times) == 0 {
+		// nothing to plot
+		return
+	}
 	var values plotter.Values
 	for _, v := range times {
 		values = append(values, v.Duration)
@@ -28,9 +32,9 @@ func plotHistogramLatency(file string, times []Datapoint) {
 		panic(err)
 	}
 	p.X.Label.Text = "Latencies (ms)"
-	p.X.Tick.Marker = hplot.Ticks{N: 10, Format: "%.0f"}
+	p.X.Tick.Marker = hplot.Ticks{N: 3, Format: "%.0f"}
 	p.Y.Label.Text = "Number of requests"
-	p.Y.Tick.Marker = hplot.Ticks{N: 10, Format: "%.0f"}
+	p.Y.Tick.Marker = hplot.Ticks{N: 3, Format: "%.0f"}
 	hist.FillColor = color.RGBA{R: 175, G: 238, B: 238}
 	p.Add(hist)
 
@@ -40,14 +44,18 @@ func plotHistogramLatency(file string, times []Datapoint) {
 }
 
 func plotBoxPlotLatency(file, server string, times []Datapoint) {
+	if len(times) == 0 {
+		// nothing to plot
+		return
+	}
 	var values plotter.Values
 	for _, v := range times {
 		values = append(values, v.Duration)
 	}
 	p := plot.New()
 	p.Title.Text = "Latencies distribution"
 	p.Y.Label.Text = "Latencies (ms)"
-	p.Y.Tick.Marker = hplot.Ticks{N: 10, Format: "%.0f"}
+	p.Y.Tick.Marker = hplot.Ticks{N: 3, Format: "%.0f"}
 	p.NominalX(server)
 
 	boxplot, err := plotter.NewBoxPlot(vg.Length(120), 0, values)
@@ -63,6 +71,10 @@ func plotBoxPlotLatency(file, server string, times []Datapoint) {
 }
 
 func plotResponses(file string, rcodes map[int]int64) {
+	if len(rcodes) == 0 {
+		// nothing to plot
+		return
+	}
 	sortedKeys := make([]int, 0)
 	for k := range rcodes {
 		sortedKeys = append(sortedKeys, k)
@@ -102,7 +114,7 @@ func plotResponses(file string, rcodes map[int]int64) {
 	}
 
 	p.Y.Label.Text = "Number of requests"
-	p.Y.Tick.Marker = hplot.Ticks{N: 10, Format: "%.0f"}
+	p.Y.Tick.Marker = hplot.Ticks{N: 3, Format: "%.0f"}
 	p.Legend.Top = true
 
 	if err := p.Save(6*vg.Inch, 6*vg.Inch, file); err != nil {
@@ -111,6 +123,10 @@ func plotResponses(file string, rcodes map[int]int64) {
 }
 
 func plotLineThroughput(file string, times []Datapoint) {
+	if len(times) == 0 {
+		// nothing to plot
+		return
+	}
 	var values plotter.XYs
 	m := make(map[int64]int64)
 
@@ -137,9 +153,9 @@ func plotLineThroughput(file string, times []Datapoint) {
 	p := plot.New()
 	p.Title.Text = "Throughput per second"
 	p.X.Label.Text = "Time of test (s)"
-	p.X.Tick.Marker = hplot.Ticks{N: 10, Format: "%.0f"}
+	p.X.Tick.Marker = hplot.Ticks{N: 3, Format: "%.0f"}
 	p.Y.Label.Text = "Number of requests (per sec)"
-	p.Y.Tick.Marker = hplot.Ticks{N: 10, Format: "%.0f"}
+	p.Y.Tick.Marker = hplot.Ticks{N: 3, Format: "%.0f"}
 
 	l, err := plotter.NewLine(values)
 	l.FillColor = color.RGBA{R: 175, G: 238, B: 238}
@@ -148,6 +164,12 @@ func plotLineThroughput(file string, times []Datapoint) {
 	}
 	p.Add(l)
 
+	scatter, err := plotter.NewScatter(values)
+	if err != nil {
+		panic(err)
+	}
+	p.Add(scatter)
+
 	if err := p.Save(6*vg.Inch, 6*vg.Inch, file); err != nil {
 		fmt.Fprintln(os.Stderr, "Failed to save plot.", err)
 	}
@@ -161,6 +183,10 @@ type latencyMeasurements struct {
 }
 
 func plotLineLatencies(file string, times []Datapoint) {
+	if len(times) == 0 {
+		// nothing to plot
+		return
+	}
 	m := make(map[int64]latencyMeasurements)
 
 	timings := make([]float64, 0)
@@ -228,9 +254,7 @@ func plotLineLatencies(file string, times []Datapoint) {
 	p := plot.New()
 	p.Title.Text = "Response latencies"
 	p.X.Label.Text = "Time of test (s)"
-	p.X.Tick.Marker = hplot.Ticks{N: 10, Format: "%.0f"}
 	p.Y.Label.Text = "Latency (ms)"
-	p.Y.Tick.Marker = hplot.Ticks{N: 10, Format: "%.0f"}
 
 	plotLine(p, p99values, plotutil.DarkColors[0], plotutil.SoftColors[0], "p99")
 	plotLine(p, p95values, plotutil.DarkColors[1], plotutil.SoftColors[1], "p95")
@@ -244,6 +268,59 @@ func plotLineLatencies(file string, times []Datapoint) {
 	}
 }
 
+func plotErrorRate(file string, times []ErrorDatapoint) {
+	if len(times) == 0 {
+		// nothing to plot
+		return
+	}
+	var values plotter.XYs
+	m := make(map[int64]int64)
+
+	if len(times) != 0 {
+		first := times[0].Start.Unix()
+
+		for _, v := range times {
+			unix := v.Start.Unix() - first
+			if _, ok := m[unix]; !ok {
+				m[unix] = 0
+			}
+			m[unix]++
+		}
+	}
+
+	for k, v := range m {
+		values = append(values, plotter.XY{X: float64(k), Y: float64(v)})
+	}
+
+	sort.SliceStable(values, func(i, j int) bool {
+		return values[i].X < values[j].X
+	})
+
+	p := plot.New()
+	p.Title.Text = "Error rate over time"
+	p.X.Label.Text = "Time of test (s)"
+	p.X.Tick.Marker = hplot.Ticks{N: 3, Format: "%.0f"}
+	p.Y.Label.Text = "Number of errors (per sec)"
+	p.Y.Tick.Marker = hplot.Ticks{N: 3, Format: "%.0f"}
+
+	l, err := plotter.NewLine(values)
+	l.FillColor = plotutil.SoftColors[0]
+	if err != nil {
+		panic(err)
+	}
+	p.Add(l)
+
+	scatter, err := plotter.NewScatter(values)
+	if err != nil {
+		panic(err)
+	}
+	p.Add(scatter)
+
+	if err := p.Save(6*vg.Inch, 6*vg.Inch, file); err != nil {
+		fmt.Fprintln(os.Stderr, "Failed to save plot.", err)
+	}
+}
+
 func plotLine(p *plot.Plot, values plotter.XYs, color color.Color, fill color.Color, name string) {
 	l, err := plotter.NewLine(values)
 	l.Color = color
@@ -253,4 +330,10 @@ func plotLine(p *plot.Plot, values plotter.XYs, color color.Color, fill color.Co
 	l.FillColor = fill
 	p.Add(l)
 	p.Legend.Add(name, l)
+	scatter, err := plotter.NewScatter(values)
+	if err != nil {
+		panic(err)
+	}
+	scatter.Color = color
+	p.Add(scatter)
 }

cmd/plot_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var timings = []Datapoint{
+var testDatapoints = []Datapoint{
 	{Start: time.Now(), Duration: 100},
 	{Start: time.Now().Add(time.Second), Duration: 200},
 	{Start: time.Now().Add(2 * time.Second), Duration: 300},
@@ -23,7 +23,17 @@ var timings = []Datapoint{
 	{Start: time.Now().Add(10 * time.Second), Duration: 200},
 }
 
-var rcodes = map[int]int64{
+var testErrorDatapoints = []ErrorDatapoint{
+	{Start: time.Now()},
+	{Start: time.Now().Add(2 * time.Second)},
+	{Start: time.Now().Add(3 * time.Second)},
+	{Start: time.Now().Add(4 * time.Second)},
+	{Start: time.Now().Add(5 * time.Second)},
+	{Start: time.Now().Add(6 * time.Second)},
+	{Start: time.Now().Add(7 * time.Second)},
+}
+
+var testRcodes = map[int]int64{
 	0: 8,
 	2: 1,
 	3: 2,
@@ -33,7 +43,7 @@ func Test_plotHistogramLatency(t *testing.T) {
 	dir := t.TempDir()
 
 	file := dir + "/histogram-latency.png"
-	plotHistogramLatency(file, timings)
+	plotHistogramLatency(file, testDatapoints)
 
 	expected, err := os.ReadFile("test-histogram-latency.png")
 	require.NoError(t, err)
@@ -48,7 +58,7 @@ func Test_plotBoxPlotLatency(t *testing.T) {
 	dir := t.TempDir()
 
 	file := dir + "/boxplot-latency.png"
-	plotBoxPlotLatency(file, "127.0.0.1", timings)
+	plotBoxPlotLatency(file, "127.0.0.1", testDatapoints)
 
 	expected, err := os.ReadFile("test-boxplot-latency.png")
 	require.NoError(t, err)
@@ -63,43 +73,58 @@ func Test_plotResponses(t *testing.T) {
 	dir := t.TempDir()
 
 	file := dir + "/responses-barchart.png"
-	plotResponses(file, rcodes)
+	plotResponses(file, testRcodes)
 
 	expected, err := os.ReadFile("test-responses-barchart.png")
 	require.NoError(t, err)
 
 	actual, err := os.ReadFile(file)
 	require.NoError(t, err)
 
-	assert.Equal(t, expected, actual, "generated boxplot latency plot does not equal to expected 'test-responses-barchart.png'")
+	assert.Equal(t, expected, actual, "generated responses plot does not equal to expected 'test-responses-barchart.png'")
 }
 
 func Test_plotLineThroughput(t *testing.T) {
 	dir := t.TempDir()
 
 	file := dir + "/throughput-lineplot.png"
-	plotLineThroughput(file, timings)
+	plotLineThroughput(file, testDatapoints)
 
 	expected, err := os.ReadFile("test-throughput-lineplot.png")
 	require.NoError(t, err)
 
 	actual, err := os.ReadFile(file)
 	require.NoError(t, err)
 
-	assert.Equal(t, expected, actual, "generated boxplot latency plot does not equal to expected 'test-throughput-lineplot.png'")
+	assert.Equal(t, expected, actual, "generated line throughput plot does not equal to expected 'test-throughput-lineplot.png'")
 }
 
 func Test_plotLineLatencies(t *testing.T) {
 	dir := t.TempDir()
 
 	file := dir + "/latency-lineplot.png"
-	plotLineLatencies(file, timings)
+	plotLineLatencies(file, testDatapoints)
 
 	expected, err := os.ReadFile("test-latency-lineplot.png")
 	require.NoError(t, err)
 
 	actual, err := os.ReadFile(file)
 	require.NoError(t, err)
 
-	assert.Equal(t, expected, actual, "generated boxplot latency plot does not equal to expected 'test-latency-lineplot.png'")
+	assert.Equal(t, expected, actual, "generated line latencies plot does not equal to expected 'test-latency-lineplot.png'")
+}
+
+func Test_plotErrorRate(t *testing.T) {
+	dir := t.TempDir()
+
+	file := dir + "/errorrate-lineplot.png"
+	plotErrorRate(file, testErrorDatapoints)
+
+	expected, err := os.ReadFile("test-errorrate-lineplot.png")
+	require.NoError(t, err)
+
+	actual, err := os.ReadFile(file)
+	require.NoError(t, err)
+
+	assert.Equal(t, expected, actual, "generated error rate plot does not equal to expected 'test-errorrate-lineplot.png")
 }

cmd/report.go

@@ -1,8 +1,10 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"io"
+	"net"
 	"os"
 	"sort"
 	"time"
@@ -42,6 +44,7 @@ func (b *Benchmark) PrintReport(w io.Writer, stats []*ResultStats, benchmarkDura
 	codeTotals := make(map[int]int64)
 	qtypeTotals := make(map[string]int64)
 	times := make([]Datapoint, 0)
+	errTimes := make([]ErrorDatapoint, 0)
 
 	errs := make(map[string]int, 0)
 	top3errs := make(map[string]int)
@@ -51,11 +54,15 @@ func (b *Benchmark) PrintReport(w io.Writer, stats []*ResultStats, benchmarkDura
 
 	for _, s := range stats {
 		for _, err := range s.Errors {
-			if v, ok := errs[err.Error()]; ok {
-				errs[err.Error()] = v + 1
+			errorString := errString(err)
+
+			if v, ok := errs[errorString]; ok {
+				errs[errorString] = v + 1
 			} else {
-				errs[err.Error()] = 1
+				errs[errorString] = 1
 			}
+
+			errTimes = append(errTimes, s.Errors...)
 		}
 
 		timings.Merge(s.Hist)
@@ -96,11 +103,16 @@ func (b *Benchmark) PrintReport(w io.Writer, stats []*ResultStats, benchmarkDura
 		}
 	}
 
-	// sort data points from the oldest to the earliest so we can better plot time dependant graphs (like line)
+	// sort data points from the oldest to the earliest, so we can better plot time dependant graphs (like line)
 	sort.SliceStable(times, func(i, j int) bool {
 		return times[i].Start.Before(times[j].Start)
 	})
 
+	// sort error data points from the oldest to the earliest, so we can better plot time dependant graphs (like line)
+	sort.SliceStable(errTimes, func(i, j int) bool {
+		return errTimes[i].Start.Before(errTimes[j].Start)
+	})
+
 	if len(b.PlotDir) != 0 {
 		now := time.Now().Format(time.RFC3339)
 		dir := fmt.Sprintf("%s/graphs-%s", b.PlotDir, now)
@@ -112,6 +124,7 @@ func (b *Benchmark) PrintReport(w io.Writer, stats []*ResultStats, benchmarkDura
 		plotResponses(b.fileName(dir, "responses-barchart"), codeTotals)
 		plotLineThroughput(b.fileName(dir, "throughput-lineplot"), times)
 		plotLineLatencies(b.fileName(dir, "latency-lineplot"), times)
+		plotErrorRate(b.fileName(dir, "errorrate-lineplot"), errTimes)
 	}
 
 	var csv *os.File
@@ -166,6 +179,19 @@ func (b *Benchmark) PrintReport(w io.Writer, stats []*ResultStats, benchmarkDura
 	return s.print(params)
 }
 
+func errString(err ErrorDatapoint) string {
+	var errorString string
+	var netOpErr *net.OpError
+
+	switch {
+	case errors.As(err.Err, &netOpErr):
+		errorString = netOpErr.Op + " " + netOpErr.Net + " " + netOpErr.Addr.String()
+	default:
+		errorString = err.Err.Error()
+	}
+	return errorString
+}
+
 func (b *Benchmark) fileName(dir, name string) string {
 	return dir + "/" + name + "." + b.PlotFormat
 }