Merge pull request #462 from antekresic/feature/cli-trend-stats-option

CLI option to specify what stats to report for trend metrics

cmd/options.go

@@ -24,6 +24,7 @@ import (
 	"net"
 
 	"github.com/loadimpact/k6/lib"
+	"github.com/loadimpact/k6/ui"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 )
@@ -46,6 +47,7 @@ func optionFlagSet() *pflag.FlagSet {
 	flags.Bool("no-connection-reuse", false, "don't reuse connections between iterations")
 	flags.BoolP("throw", "w", false, "throw warnings (like failed http requests) as errors")
 	flags.StringSlice("blacklist-ip", nil, "blacklist an `ip range` from being called")
+	flags.StringSlice("summary-trend-stats", nil, "define `stats` for trend metrics (response times), one or more as 'avg,p(95),...'")
 	return flags
 }
 
@@ -92,5 +94,17 @@ func getOptions(flags *pflag.FlagSet) (lib.Options, error) {
 		opts.BlacklistIPs = append(opts.BlacklistIPs, net)
 	}
 
+	trendStatStrings, err := flags.GetStringSlice("summary-trend-stats")
+	if err != nil {
+		return opts, err
+	}
+	for _, s := range trendStatStrings {
+		if err := ui.VerifyTrendColumnStat(s); err != nil {
+			return opts, errors.Wrapf(err, "stat '%s'", s)
+		}
+
+		opts.SummaryTrendStats = append(opts.SummaryTrendStats, s)
+	}
+
 	return opts, nil
 }

cmd/run.go

@@ -145,6 +145,10 @@ a commandline interface for interacting with it.`,
 		if conf.Duration.Valid && conf.Duration.Duration == 0 {
 			conf.Duration = lib.NullDuration{}
 		}
+		// If summary trend stats are defined, update the UI to reflect them
+		if len(conf.SummaryTrendStats) > 0 {
+			ui.UpdateTrendColumns(conf.SummaryTrendStats)
+		}
 
 		// Write options back to the runner too.
 		r.SetOptions(conf.Options)

lib/options.go

@@ -191,6 +191,9 @@ type Options struct {
 	// These values are for third party collectors' benefit.
 	// Can't be set through env vars.
 	External map[string]interface{} `json:"ext" ignored:"true"`
+
+	// Summary trend stats for trend metrics (response times) in CLI output
+	SummaryTrendStats []string `json:"SummaryTrendStats" envconfig:"summary_trend_stats"`
 }
 
 // Returns the result of overwriting any fields with any that are set on the argument.
@@ -260,5 +263,8 @@ func (o Options) Apply(opts Options) Options {
 	if opts.External != nil {
 		o.External = opts.External
 	}
+	if opts.SummaryTrendStats != nil {
+		o.SummaryTrendStats = opts.SummaryTrendStats
+	}
 	return o
 }

ui/summary.go

@@ -21,6 +21,7 @@
 package ui
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"sort"
@@ -41,6 +42,12 @@ const (
 	FailMark = "✗"
 )
 
+var (
+	ErrStatEmptyString            = errors.New("Invalid stat, empty string")
+	ErrStatUnknownFormat          = errors.New("Invalid stat, unknown format")
+	ErrPercentileStatInvalidValue = errors.New("Invalid percentile stat value, accepts a number")
+)
+
 var TrendColumns = []TrendColumn{
 	{"avg", func(s *stats.TrendSink) float64 { return s.Avg }},
 	{"min", func(s *stats.TrendSink) float64 { return s.Min }},
@@ -55,6 +62,70 @@ type TrendColumn struct {
 	Get func(s *stats.TrendSink) float64
 }
 
+// VerifyTrendColumnStat checks if stat is a valid trend column
+func VerifyTrendColumnStat(stat string) error {
+	if stat == "" {
+		return ErrStatEmptyString
+	}
+
+	for _, col := range TrendColumns {
+		if col.Key == stat {
+			return nil
+		}
+	}
+
+	if _, err := generatePercentileTrendColumn(stat); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// UpdateTrendColumns updates the default trend columns with user defined ones
+func UpdateTrendColumns(stats []string) {
+	newTrendColumns := make([]TrendColumn, 0, len(stats))
+
+	for _, stat := range stats {
+		percentileTrendColumn, err := generatePercentileTrendColumn(stat)
+
+		if err == nil {
+			newTrendColumns = append(newTrendColumns, TrendColumn{stat, percentileTrendColumn})
+			continue
+		}
+
+		for _, col := range TrendColumns {
+			if col.Key == stat {
+				newTrendColumns = append(newTrendColumns, col)
+				break
+			}
+		}
+	}
+
+	if len(newTrendColumns) > 0 {
+		TrendColumns = newTrendColumns
+	}
+}
+
+func generatePercentileTrendColumn(stat string) (func(s *stats.TrendSink) float64, error) {
+	if stat == "" {
+		return nil, ErrStatEmptyString
+	}
+
+	if !strings.HasPrefix(stat, "p(") || !strings.HasSuffix(stat, ")") {
+		return nil, ErrStatUnknownFormat
+	}
+
+	percentile, err := strconv.ParseFloat(stat[2:len(stat)-1], 64)
+
+	if err != nil {
+		return nil, ErrPercentileStatInvalidValue
+	}
+
+	percentile = percentile / 100
+
+	return func(s *stats.TrendSink) float64 { return s.P(percentile) }, nil
+}
+
 // Returns the actual width of the string.
 func StrWidth(s string) (n int) {
 	var it norm.Iter

ui/summary_test.go

@@ -0,0 +1,160 @@
+/*
+ *
+ * k6 - a next-generation load testing tool
+ * Copyright (C) 2018 Load Impact
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package ui
+
+import (
+	"testing"
+
+	"github.com/loadimpact/k6/stats"
+	"github.com/stretchr/testify/assert"
+)
+
+var verifyTests = []struct {
+	in  string
+	out error
+}{
+	{"avg", nil},
+	{"min", nil},
+	{"med", nil},
+	{"max", nil},
+	{"p(0)", nil},
+	{"p(90)", nil},
+	{"p(95)", nil},
+	{"p(99)", nil},
+	{"p(99.9)", nil},
+	{"p(99.9999)", nil},
+	{"nil", ErrStatUnknownFormat},
+	{" avg", ErrStatUnknownFormat},
+	{"avg ", ErrStatUnknownFormat},
+	{"", ErrStatEmptyString},
+}
+
+var defaultTrendColumns = TrendColumns
+
+func createTestTrendSink(count int) *stats.TrendSink {
+	sink := stats.TrendSink{}
+
+	for i := 0; i < count; i++ {
+		sink.Add(stats.Sample{Value: float64(i)})
+	}
+
+	return &sink
+}
+
+func TestVerifyTrendColumnStat(t *testing.T) {
+	for _, testCase := range verifyTests {
+		err := VerifyTrendColumnStat(testCase.in)
+		assert.Equal(t, testCase.out, err)
+	}
+}
+
+func TestUpdateTrendColumns(t *testing.T) {
+	sink := createTestTrendSink(100)
+
+	t.Run("No stats", func(t *testing.T) {
+		TrendColumns = defaultTrendColumns
+
+		UpdateTrendColumns(make([]string, 0))
+
+		assert.Equal(t, defaultTrendColumns, TrendColumns)
+	})
+
+	t.Run("One stat", func(t *testing.T) {
+		TrendColumns = defaultTrendColumns
+
+		UpdateTrendColumns([]string{"avg"})
+
+		assert.Exactly(t, 1, len(TrendColumns))
+		assert.Exactly(t,
+			sink.Avg,
+			TrendColumns[0].Get(sink))
+	})
+
+	t.Run("Multiple stats", func(t *testing.T) {
+		TrendColumns = defaultTrendColumns
+
+		UpdateTrendColumns([]string{"med", "max"})
+
+		assert.Exactly(t, 2, len(TrendColumns))
+		assert.Exactly(t, sink.Med, TrendColumns[0].Get(sink))
+		assert.Exactly(t, sink.Max, TrendColumns[1].Get(sink))
+	})
+
+	t.Run("Ignore invalid stats", func(t *testing.T) {
+		TrendColumns = defaultTrendColumns
+
+		UpdateTrendColumns([]string{"med", "max", "invalid"})
+
+		assert.Exactly(t, 2, len(TrendColumns))
+		assert.Exactly(t, sink.Med, TrendColumns[0].Get(sink))
+		assert.Exactly(t, sink.Max, TrendColumns[1].Get(sink))
+	})
+
+	t.Run("Percentile stats", func(t *testing.T) {
+		TrendColumns = defaultTrendColumns
+
+		UpdateTrendColumns([]string{"p(99.9999)"})
+
+		assert.Exactly(t, 1, len(TrendColumns))
+		assert.Exactly(t, sink.P(0.999999), TrendColumns[0].Get(sink))
+	})
+}
+
+func TestGeneratePercentileTrendColumn(t *testing.T) {
+	sink := createTestTrendSink(100)
+
+	t.Run("Happy path", func(t *testing.T) {
+		colFunc, err := generatePercentileTrendColumn("p(99)")
+
+		assert.NotNil(t, colFunc)
+		assert.Exactly(t, sink.P(0.99), colFunc(sink))
+		assert.NotEqual(t, sink.P(0.98), colFunc(sink))
+		assert.Nil(t, err)
+	})
+
+	t.Run("Empty stat", func(t *testing.T) {
+		colFunc, err := generatePercentileTrendColumn("")
+
+		assert.Nil(t, colFunc)
+		assert.Exactly(t, err, ErrStatEmptyString)
+	})
+
+	t.Run("Invalid format", func(t *testing.T) {
+		colFunc, err := generatePercentileTrendColumn("p90")
+
+		assert.Nil(t, colFunc)
+		assert.Exactly(t, err, ErrStatUnknownFormat)
+	})
+
+	t.Run("Invalid format 2", func(t *testing.T) {
+		colFunc, err := generatePercentileTrendColumn("p(90")
+
+		assert.Nil(t, colFunc)
+		assert.Exactly(t, err, ErrStatUnknownFormat)
+	})
+
+	t.Run("Invalid float", func(t *testing.T) {
+		colFunc, err := generatePercentileTrendColumn("p(a)")
+
+		assert.Nil(t, colFunc)
+		assert.Exactly(t, err, ErrPercentileStatInvalidValue)
+	})
+}