Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Added metrics recorder and prometheus backend
Signed-off-by: Xabier Larrakoetxea <slok69@gmail.com>
  • Loading branch information
slok committed Dec 11, 2018
1 parent aafe79a commit 4f7be56
Show file tree
Hide file tree
Showing 7 changed files with 468 additions and 0 deletions.
1 change: 1 addition & 0 deletions go.mod
Expand Up @@ -3,6 +3,7 @@ module github.com/slok/goresilience
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v0.9.2
github.com/stretchr/objx v0.1.1 // indirect
github.com/stretchr/testify v1.2.2
)
17 changes: 17 additions & 0 deletions go.sum
@@ -1,8 +1,25 @@
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
18 changes: 18 additions & 0 deletions metrics/dummy.go
@@ -0,0 +1,18 @@
package metrics

import "time"

// Dummy is a dummy recorder.
var Dummy Recorder = &dummy{}

type dummy struct{}

func (d *dummy) WithID(id string) Recorder { return d }
func (dummy) ObserveCommandExecution(start time.Time, success bool) {}
func (dummy) IncRetry() {}
func (dummy) IncTimeout() {}
func (dummy) IncBulkheadQueued() {}
func (dummy) IncBulkheadProcessed() {}
func (dummy) IncBulkheadTimeout() {}
func (dummy) IncCircuitbreakerState(state string) {}
func (dummy) IncChaosInjectedFailure(kind string) {}
27 changes: 27 additions & 0 deletions metrics/metrics.go
@@ -0,0 +1,27 @@
package metrics

import "time"

// Recorder knows how to measure different kind of metrics.
type Recorder interface {
// WithID will set the ID name to the recorde and every metric
// measured with the obtained recorder will be identified with
// the name.
WithID(id string) Recorder
// ObserveCommandExecution will measure the exeuction of the runner chain.
ObserveCommandExecution(start time.Time, success bool)
// IncRetry will increment the number of retries.
IncRetry()
// IncTimeout will increment the number of timeouts.
IncTimeout()
// IncBulkheadQueued increments the number of queued Funcs to execute.
IncBulkheadQueued()
// IncBulkheadProcessed increments the number of processed Funcs to execute.
IncBulkheadProcessed()
// IncBulkheadProcessed increments the number of timeouts Funcs waiting to execute.
IncBulkheadTimeout()
// IncCircuitbreakerState increments the number of state change.
IncCircuitbreakerState(state string)
// IncChaosInjectedFailure increments the number of times injected failure.
IncChaosInjectedFailure(kind string)
}
162 changes: 162 additions & 0 deletions metrics/prometheus.go
@@ -0,0 +1,162 @@
package metrics

import (
"fmt"
"time"

"github.com/prometheus/client_golang/prometheus"
)

var (
promNamespace = "goresilience"

promCommandSubsystem = "command"
promRetrySubsystem = "retry"
promTimeoutSubsystem = "timeout"
promBulkheadSubsystem = "bulkhead"
promCBSubsystem = "circuitbreaker"
promChaosSubsystem = "chaos"
)

type prometheusRec struct {
// Metrics.
cmdExecutionDuration *prometheus.HistogramVec
retryRetries *prometheus.CounterVec
timeoutTimeouts *prometheus.CounterVec
bulkQueued *prometheus.CounterVec
bulkProcessed *prometheus.CounterVec
bulkTimeouts *prometheus.CounterVec
cbStateChanges *prometheus.CounterVec
chaosFailureInjections *prometheus.CounterVec

id string
reg prometheus.Registerer
}

// NewPrometheusRecorder returns a new Recorder that knows how to measure
// using Prometheus kind metrics.
func NewPrometheusRecorder(reg prometheus.Registerer) Recorder {
p := &prometheusRec{
reg: reg,
}

p.registerMetrics()
return p
}

func (p prometheusRec) WithID(id string) Recorder {
return &prometheusRec{
cmdExecutionDuration: p.cmdExecutionDuration,
retryRetries: p.retryRetries,
timeoutTimeouts: p.timeoutTimeouts,
bulkQueued: p.bulkQueued,
bulkProcessed: p.bulkProcessed,
bulkTimeouts: p.bulkTimeouts,
cbStateChanges: p.cbStateChanges,
chaosFailureInjections: p.chaosFailureInjections,

id: id,
reg: p.reg,
}
}

func (p *prometheusRec) registerMetrics() {
p.cmdExecutionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: promNamespace,
Subsystem: promCommandSubsystem,
Name: "execution_duration_seconds",
Help: "The duration of the command execution in seconds.",
}, []string{"id", "success"})

p.retryRetries = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: promNamespace,
Subsystem: promRetrySubsystem,
Name: "retries_total",
Help: "Total number of retries made by the retry runner.",
}, []string{"id"})

p.timeoutTimeouts = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: promNamespace,
Subsystem: promTimeoutSubsystem,
Name: "timeouts_total",
Help: "Total number of timeouts made by the timeout runner.",
}, []string{"id"})

p.bulkQueued = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: promNamespace,
Subsystem: promBulkheadSubsystem,
Name: "queued_total",
Help: "Total number of queued funcs made by the bulkhead runner.",
}, []string{"id"})

p.bulkProcessed = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: promNamespace,
Subsystem: promBulkheadSubsystem,
Name: "processed_total",
Help: "Total number of processed funcs made by the bulkhead runner.",
}, []string{"id"})

p.bulkTimeouts = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: promNamespace,
Subsystem: promBulkheadSubsystem,
Name: "timeouts_total",
Help: "Total number of timeouts funcs waiting for execution made by the bulkhead runner.",
}, []string{"id"})

p.cbStateChanges = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: promNamespace,
Subsystem: promCBSubsystem,
Name: "state_changes_total",
Help: "Total number of state changes made by the circuit breaker runner.",
}, []string{"id", "state"})

p.chaosFailureInjections = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: promNamespace,
Subsystem: promChaosSubsystem,
Name: "failure_injections_total",
Help: "Total number of failure injectionsmade by the chaos runner.",
}, []string{"id", "kind"})

p.reg.MustRegister(p.cmdExecutionDuration,
p.retryRetries,
p.timeoutTimeouts,
p.bulkQueued,
p.bulkProcessed,
p.bulkTimeouts,
p.cbStateChanges,
p.chaosFailureInjections,
)
}

func (p prometheusRec) ObserveCommandExecution(start time.Time, success bool) {
secs := time.Since(start).Seconds()
p.cmdExecutionDuration.WithLabelValues(p.id, fmt.Sprintf("%t", success)).Observe(secs)
}

func (p prometheusRec) IncRetry() {
p.retryRetries.WithLabelValues(p.id).Inc()
}

func (p prometheusRec) IncTimeout() {
p.timeoutTimeouts.WithLabelValues(p.id).Inc()
}

func (p prometheusRec) IncBulkheadQueued() {
p.bulkQueued.WithLabelValues(p.id).Inc()
}

func (p prometheusRec) IncBulkheadProcessed() {
p.bulkProcessed.WithLabelValues(p.id).Inc()
}

func (p prometheusRec) IncBulkheadTimeout() {
p.bulkTimeouts.WithLabelValues(p.id).Inc()
}

func (p prometheusRec) IncCircuitbreakerState(state string) {
p.cbStateChanges.WithLabelValues(p.id, state).Inc()
}

func (p prometheusRec) IncChaosInjectedFailure(kind string) {
p.chaosFailureInjections.WithLabelValues(p.id, kind).Inc()
}

0 comments on commit 4f7be56

Please sign in to comment.