From 3d64296ad3355df12438d6af54cc18dffe29a0d0 Mon Sep 17 00:00:00 2001 From: guanw Date: Tue, 28 Jan 2020 10:42:19 -0500 Subject: [PATCH] add metrics to retry handler --- main.go | 2 +- pkg/store/retry_handler.go | 8 +++- pkg/store/retry_handler_metrics.go | 33 ++++++++++++++++ pkg/store/retry_handler_test.go | 62 ++++++++++++++++++++---------- 4 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 pkg/store/retry_handler_metrics.go diff --git a/main.go b/main.go index f2b2893..9433bb6 100644 --- a/main.go +++ b/main.go @@ -43,7 +43,7 @@ func main() { } store := ctStore.NewStore(client) // TODO move 5 to config/from flag - retryStore := ctStore.NewRetryHandler(5, store) + retryStore := ctStore.NewRetryHandler(5, store, ctStore.InitializeMetrics()) dnsServer := dns.NewServer(retryStore, dns.InitializeMetrics()) lis, err := net.Listen("tcp", fmt.Sprintf(":%s", cfg.GRPCPort)) if err != nil { diff --git a/pkg/store/retry_handler.go b/pkg/store/retry_handler.go index 353b7ca..552b39d 100644 --- a/pkg/store/retry_handler.go +++ b/pkg/store/retry_handler.go @@ -7,13 +7,15 @@ import ( type retryHandler struct { Store Store MaximumRetryTimes int + Metrics *Metrics } // NewRetryHandler initializes RetryHandler -func NewRetryHandler(maximumRetryTimes int, store Store) Store { +func NewRetryHandler(maximumRetryTimes int, store Store, metrics *Metrics) Store { return &retryHandler{ MaximumRetryTimes: maximumRetryTimes, Store: store, + Metrics: metrics, } } @@ -25,7 +27,9 @@ func (r *retryHandler) GetService(serviceName string) ([]string, error) { if res, err = r.Store.GetService(serviceName); err == nil { return res, nil } + r.Metrics.GetServiceRetryAttempts.Inc() } + r.Metrics.GetServiceRetryExhausted.Inc() return nil, errors.Wrap(err, "Failed to GetService with RetryHandler") } @@ -36,6 +40,8 @@ func (r *retryHandler) UpdateService(serviceName, operation, Host string) error if err = r.Store.UpdateService(serviceName, operation, Host); err == nil { return nil } + r.Metrics.PostServiceRetryAttempts.Inc() } + r.Metrics.PostServiceRetryExhausted.Inc() return errors.Wrap(err, "Failed to PostService with RetryHandler") } diff --git a/pkg/store/retry_handler_metrics.go b/pkg/store/retry_handler_metrics.go new file mode 100644 index 0000000..8e2309b --- /dev/null +++ b/pkg/store/retry_handler_metrics.go @@ -0,0 +1,33 @@ +package store + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Metrics defines all metrics for retry handler +type Metrics struct { + GetServiceRetryAttempts prometheus.Counter + GetServiceRetryExhausted prometheus.Counter + + PostServiceRetryAttempts prometheus.Counter + PostServiceRetryExhausted prometheus.Counter +} + +// InitializeMetrics initialize retry metrics +func InitializeMetrics() *Metrics { + return &Metrics{ + GetServiceRetryAttempts: promauto.NewCounter(prometheus.CounterOpts{ + Name: "get_service_retry_attempts", + }), + GetServiceRetryExhausted: promauto.NewCounter(prometheus.CounterOpts{ + Name: "get_service_retry_exhausted", + }), + PostServiceRetryAttempts: promauto.NewCounter(prometheus.CounterOpts{ + Name: "update_service_retry_attempts", + }), + PostServiceRetryExhausted: promauto.NewCounter(prometheus.CounterOpts{ + Name: "update_service_retry_exhausted", + }), + } +} diff --git a/pkg/store/retry_handler_test.go b/pkg/store/retry_handler_test.go index 3608511..a23c717 100644 --- a/pkg/store/retry_handler_test.go +++ b/pkg/store/retry_handler_test.go @@ -5,25 +5,37 @@ import ( "testing" "github.com/guanw/ct-dns/pkg/store/mocks" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" ) +var ( + metrics = InitializeMetrics() + maximumRetry = 10 +) + func TestRetryHandler_GetService(t *testing.T) { mockStore := &mocks.Store{} mockStore.On("GetService", "valid-service").Return([]string{"192.0.0.1:8081"}, nil) mockStore.On("GetService", "error-service").Return(nil, errors.New("new error")) - retryHandler := NewRetryHandler(5, mockStore) + retryHandler := NewRetryHandler(maximumRetry, mockStore, metrics) tests := []struct { - ExpectError bool - ServiceName string + ExpectError bool + ServiceName string + ExpectedRetryAttempts float64 + ExpectedRetryExhausted float64 }{ { - ServiceName: "valid-service", - ExpectError: false, + ServiceName: "valid-service", + ExpectError: false, + ExpectedRetryAttempts: 0.0, + ExpectedRetryExhausted: 0.0, }, { - ServiceName: "error-service", - ExpectError: true, + ServiceName: "error-service", + ExpectError: true, + ExpectedRetryAttempts: float64(maximumRetry), + ExpectedRetryExhausted: 1.0, }, } for _, test := range tests { @@ -33,6 +45,8 @@ func TestRetryHandler_GetService(t *testing.T) { } else { assert.NoError(t, err) } + assert.Equal(t, test.ExpectedRetryAttempts, testutil.ToFloat64(metrics.GetServiceRetryAttempts)) + assert.Equal(t, test.ExpectedRetryExhausted, testutil.ToFloat64(metrics.GetServiceRetryExhausted)) } } @@ -40,24 +54,30 @@ func TestRetryHandler_PostService(t *testing.T) { mockStore := &mocks.Store{} mockStore.On("UpdateService", "service", "add", "192.0.0.1:8081").Return(nil) mockStore.On("UpdateService", "service", "invalid-operation", "xxx").Return(errors.New("new error")) - retryHandler := NewRetryHandler(5, mockStore) + retryHandler := NewRetryHandler(maximumRetry, mockStore, metrics) tests := []struct { - ExpectError bool - ServiceName string - Operation string - Host string + ExpectError bool + ServiceName string + Operation string + Host string + ExpectedRetryExhausted float64 + ExpectedRetryAttempts float64 }{ { - ServiceName: "service", - Operation: "add", - Host: "192.0.0.1:8081", - ExpectError: false, + ServiceName: "service", + Operation: "add", + Host: "192.0.0.1:8081", + ExpectError: false, + ExpectedRetryExhausted: 0.0, + ExpectedRetryAttempts: 0.0, }, { - ServiceName: "service", - Operation: "invalid-operation", - Host: "xxx", - ExpectError: true, + ServiceName: "service", + Operation: "invalid-operation", + Host: "xxx", + ExpectError: true, + ExpectedRetryExhausted: 1.0, + ExpectedRetryAttempts: float64(maximumRetry), }, } for _, test := range tests { @@ -67,5 +87,7 @@ func TestRetryHandler_PostService(t *testing.T) { } else { assert.NoError(t, err) } + assert.Equal(t, test.ExpectedRetryAttempts, testutil.ToFloat64(metrics.PostServiceRetryAttempts)) + assert.Equal(t, test.ExpectedRetryExhausted, testutil.ToFloat64(metrics.PostServiceRetryExhausted)) } }