Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
Added metrics recorder and prometheus backend
Signed-off-by: Xabier Larrakoetxea <slok69@gmail.com>
- Loading branch information
Showing
7 changed files
with
468 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,8 +1,25 @@ | ||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= | ||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= | ||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= | ||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= | ||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= | ||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= | ||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= | ||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | ||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= | ||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= | ||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= | ||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= | ||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= | ||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= | ||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= | ||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= | ||
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= | ||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | ||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= | ||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= | ||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | ||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= | ||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
package metrics | ||
|
||
import "time" | ||
|
||
// Dummy is a dummy recorder. | ||
var Dummy Recorder = &dummy{} | ||
|
||
type dummy struct{} | ||
|
||
func (d *dummy) WithID(id string) Recorder { return d } | ||
func (dummy) ObserveCommandExecution(start time.Time, success bool) {} | ||
func (dummy) IncRetry() {} | ||
func (dummy) IncTimeout() {} | ||
func (dummy) IncBulkheadQueued() {} | ||
func (dummy) IncBulkheadProcessed() {} | ||
func (dummy) IncBulkheadTimeout() {} | ||
func (dummy) IncCircuitbreakerState(state string) {} | ||
func (dummy) IncChaosInjectedFailure(kind string) {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
package metrics | ||
|
||
import "time" | ||
|
||
// Recorder knows how to measure different kind of metrics. | ||
type Recorder interface { | ||
// WithID will set the ID name to the recorde and every metric | ||
// measured with the obtained recorder will be identified with | ||
// the name. | ||
WithID(id string) Recorder | ||
// ObserveCommandExecution will measure the exeuction of the runner chain. | ||
ObserveCommandExecution(start time.Time, success bool) | ||
// IncRetry will increment the number of retries. | ||
IncRetry() | ||
// IncTimeout will increment the number of timeouts. | ||
IncTimeout() | ||
// IncBulkheadQueued increments the number of queued Funcs to execute. | ||
IncBulkheadQueued() | ||
// IncBulkheadProcessed increments the number of processed Funcs to execute. | ||
IncBulkheadProcessed() | ||
// IncBulkheadProcessed increments the number of timeouts Funcs waiting to execute. | ||
IncBulkheadTimeout() | ||
// IncCircuitbreakerState increments the number of state change. | ||
IncCircuitbreakerState(state string) | ||
// IncChaosInjectedFailure increments the number of times injected failure. | ||
IncChaosInjectedFailure(kind string) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,162 @@ | ||
package metrics | ||
|
||
import ( | ||
"fmt" | ||
"time" | ||
|
||
"github.com/prometheus/client_golang/prometheus" | ||
) | ||
|
||
var ( | ||
promNamespace = "goresilience" | ||
|
||
promCommandSubsystem = "command" | ||
promRetrySubsystem = "retry" | ||
promTimeoutSubsystem = "timeout" | ||
promBulkheadSubsystem = "bulkhead" | ||
promCBSubsystem = "circuitbreaker" | ||
promChaosSubsystem = "chaos" | ||
) | ||
|
||
type prometheusRec struct { | ||
// Metrics. | ||
cmdExecutionDuration *prometheus.HistogramVec | ||
retryRetries *prometheus.CounterVec | ||
timeoutTimeouts *prometheus.CounterVec | ||
bulkQueued *prometheus.CounterVec | ||
bulkProcessed *prometheus.CounterVec | ||
bulkTimeouts *prometheus.CounterVec | ||
cbStateChanges *prometheus.CounterVec | ||
chaosFailureInjections *prometheus.CounterVec | ||
|
||
id string | ||
reg prometheus.Registerer | ||
} | ||
|
||
// NewPrometheusRecorder returns a new Recorder that knows how to measure | ||
// using Prometheus kind metrics. | ||
func NewPrometheusRecorder(reg prometheus.Registerer) Recorder { | ||
p := &prometheusRec{ | ||
reg: reg, | ||
} | ||
|
||
p.registerMetrics() | ||
return p | ||
} | ||
|
||
func (p prometheusRec) WithID(id string) Recorder { | ||
return &prometheusRec{ | ||
cmdExecutionDuration: p.cmdExecutionDuration, | ||
retryRetries: p.retryRetries, | ||
timeoutTimeouts: p.timeoutTimeouts, | ||
bulkQueued: p.bulkQueued, | ||
bulkProcessed: p.bulkProcessed, | ||
bulkTimeouts: p.bulkTimeouts, | ||
cbStateChanges: p.cbStateChanges, | ||
chaosFailureInjections: p.chaosFailureInjections, | ||
|
||
id: id, | ||
reg: p.reg, | ||
} | ||
} | ||
|
||
func (p *prometheusRec) registerMetrics() { | ||
p.cmdExecutionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ | ||
Namespace: promNamespace, | ||
Subsystem: promCommandSubsystem, | ||
Name: "execution_duration_seconds", | ||
Help: "The duration of the command execution in seconds.", | ||
}, []string{"id", "success"}) | ||
|
||
p.retryRetries = prometheus.NewCounterVec(prometheus.CounterOpts{ | ||
Namespace: promNamespace, | ||
Subsystem: promRetrySubsystem, | ||
Name: "retries_total", | ||
Help: "Total number of retries made by the retry runner.", | ||
}, []string{"id"}) | ||
|
||
p.timeoutTimeouts = prometheus.NewCounterVec(prometheus.CounterOpts{ | ||
Namespace: promNamespace, | ||
Subsystem: promTimeoutSubsystem, | ||
Name: "timeouts_total", | ||
Help: "Total number of timeouts made by the timeout runner.", | ||
}, []string{"id"}) | ||
|
||
p.bulkQueued = prometheus.NewCounterVec(prometheus.CounterOpts{ | ||
Namespace: promNamespace, | ||
Subsystem: promBulkheadSubsystem, | ||
Name: "queued_total", | ||
Help: "Total number of queued funcs made by the bulkhead runner.", | ||
}, []string{"id"}) | ||
|
||
p.bulkProcessed = prometheus.NewCounterVec(prometheus.CounterOpts{ | ||
Namespace: promNamespace, | ||
Subsystem: promBulkheadSubsystem, | ||
Name: "processed_total", | ||
Help: "Total number of processed funcs made by the bulkhead runner.", | ||
}, []string{"id"}) | ||
|
||
p.bulkTimeouts = prometheus.NewCounterVec(prometheus.CounterOpts{ | ||
Namespace: promNamespace, | ||
Subsystem: promBulkheadSubsystem, | ||
Name: "timeouts_total", | ||
Help: "Total number of timeouts funcs waiting for execution made by the bulkhead runner.", | ||
}, []string{"id"}) | ||
|
||
p.cbStateChanges = prometheus.NewCounterVec(prometheus.CounterOpts{ | ||
Namespace: promNamespace, | ||
Subsystem: promCBSubsystem, | ||
Name: "state_changes_total", | ||
Help: "Total number of state changes made by the circuit breaker runner.", | ||
}, []string{"id", "state"}) | ||
|
||
p.chaosFailureInjections = prometheus.NewCounterVec(prometheus.CounterOpts{ | ||
Namespace: promNamespace, | ||
Subsystem: promChaosSubsystem, | ||
Name: "failure_injections_total", | ||
Help: "Total number of failure injectionsmade by the chaos runner.", | ||
}, []string{"id", "kind"}) | ||
|
||
p.reg.MustRegister(p.cmdExecutionDuration, | ||
p.retryRetries, | ||
p.timeoutTimeouts, | ||
p.bulkQueued, | ||
p.bulkProcessed, | ||
p.bulkTimeouts, | ||
p.cbStateChanges, | ||
p.chaosFailureInjections, | ||
) | ||
} | ||
|
||
func (p prometheusRec) ObserveCommandExecution(start time.Time, success bool) { | ||
secs := time.Since(start).Seconds() | ||
p.cmdExecutionDuration.WithLabelValues(p.id, fmt.Sprintf("%t", success)).Observe(secs) | ||
} | ||
|
||
func (p prometheusRec) IncRetry() { | ||
p.retryRetries.WithLabelValues(p.id).Inc() | ||
} | ||
|
||
func (p prometheusRec) IncTimeout() { | ||
p.timeoutTimeouts.WithLabelValues(p.id).Inc() | ||
} | ||
|
||
func (p prometheusRec) IncBulkheadQueued() { | ||
p.bulkQueued.WithLabelValues(p.id).Inc() | ||
} | ||
|
||
func (p prometheusRec) IncBulkheadProcessed() { | ||
p.bulkProcessed.WithLabelValues(p.id).Inc() | ||
} | ||
|
||
func (p prometheusRec) IncBulkheadTimeout() { | ||
p.bulkTimeouts.WithLabelValues(p.id).Inc() | ||
} | ||
|
||
func (p prometheusRec) IncCircuitbreakerState(state string) { | ||
p.cbStateChanges.WithLabelValues(p.id, state).Inc() | ||
} | ||
|
||
func (p prometheusRec) IncChaosInjectedFailure(kind string) { | ||
p.chaosFailureInjections.WithLabelValues(p.id, kind).Inc() | ||
} |
Oops, something went wrong.