Merge pull request #66 from guanw/jude-retry-metrics

add metrics to retry handler

main.go

@@ -43,7 +43,7 @@ func main() {
 			}
 			store := ctStore.NewStore(client)
 			// TODO move 5 to config/from flag
-			retryStore := ctStore.NewRetryHandler(5, store)
+			retryStore := ctStore.NewRetryHandler(5, store, ctStore.InitializeMetrics())
 			dnsServer := dns.NewServer(retryStore, dns.InitializeMetrics())
 			lis, err := net.Listen("tcp", fmt.Sprintf(":%s", cfg.GRPCPort))
 			if err != nil {

pkg/store/retry_handler.go

@@ -7,13 +7,15 @@ import (
 type retryHandler struct {
 	Store             Store
 	MaximumRetryTimes int
+	Metrics           *Metrics
 }
 
 // NewRetryHandler initializes RetryHandler
-func NewRetryHandler(maximumRetryTimes int, store Store) Store {
+func NewRetryHandler(maximumRetryTimes int, store Store, metrics *Metrics) Store {
 	return &retryHandler{
 		MaximumRetryTimes: maximumRetryTimes,
 		Store:             store,
+		Metrics:           metrics,
 	}
 }
 
@@ -25,7 +27,9 @@ func (r *retryHandler) GetService(serviceName string) ([]string, error) {
 		if res, err = r.Store.GetService(serviceName); err == nil {
 			return res, nil
 		}
+		r.Metrics.GetServiceRetryAttempts.Inc()
 	}
+	r.Metrics.GetServiceRetryExhausted.Inc()
 	return nil, errors.Wrap(err, "Failed to GetService with RetryHandler")
 }
 
@@ -36,6 +40,8 @@ func (r *retryHandler) UpdateService(serviceName, operation, Host string) error
 		if err = r.Store.UpdateService(serviceName, operation, Host); err == nil {
 			return nil
 		}
+		r.Metrics.PostServiceRetryAttempts.Inc()
 	}
+	r.Metrics.PostServiceRetryExhausted.Inc()
 	return errors.Wrap(err, "Failed to PostService with RetryHandler")
 }

pkg/store/retry_handler_metrics.go

@@ -0,0 +1,33 @@
+package store
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// Metrics defines all metrics for retry handler
+type Metrics struct {
+	GetServiceRetryAttempts  prometheus.Counter
+	GetServiceRetryExhausted prometheus.Counter
+
+	PostServiceRetryAttempts  prometheus.Counter
+	PostServiceRetryExhausted prometheus.Counter
+}
+
+// InitializeMetrics initialize retry metrics
+func InitializeMetrics() *Metrics {
+	return &Metrics{
+		GetServiceRetryAttempts: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "get_service_retry_attempts",
+		}),
+		GetServiceRetryExhausted: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "get_service_retry_exhausted",
+		}),
+		PostServiceRetryAttempts: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "update_service_retry_attempts",
+		}),
+		PostServiceRetryExhausted: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "update_service_retry_exhausted",
+		}),
+	}
+}

pkg/store/retry_handler_test.go

@@ -5,25 +5,37 @@ import (
 	"testing"
 
 	"github.com/guanw/ct-dns/pkg/store/mocks"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 )
 
+var (
+	metrics      = InitializeMetrics()
+	maximumRetry = 10
+)
+
 func TestRetryHandler_GetService(t *testing.T) {
 	mockStore := &mocks.Store{}
 	mockStore.On("GetService", "valid-service").Return([]string{"192.0.0.1:8081"}, nil)
 	mockStore.On("GetService", "error-service").Return(nil, errors.New("new error"))
-	retryHandler := NewRetryHandler(5, mockStore)
+	retryHandler := NewRetryHandler(maximumRetry, mockStore, metrics)
 	tests := []struct {
-		ExpectError bool
-		ServiceName string
+		ExpectError            bool
+		ServiceName            string
+		ExpectedRetryAttempts  float64
+		ExpectedRetryExhausted float64
 	}{
 		{
-			ServiceName: "valid-service",
-			ExpectError: false,
+			ServiceName:            "valid-service",
+			ExpectError:            false,
+			ExpectedRetryAttempts:  0.0,
+			ExpectedRetryExhausted: 0.0,
 		},
 		{
-			ServiceName: "error-service",
-			ExpectError: true,
+			ServiceName:            "error-service",
+			ExpectError:            true,
+			ExpectedRetryAttempts:  float64(maximumRetry),
+			ExpectedRetryExhausted: 1.0,
 		},
 	}
 	for _, test := range tests {
@@ -33,31 +45,39 @@ func TestRetryHandler_GetService(t *testing.T) {
 		} else {
 			assert.NoError(t, err)
 		}
+		assert.Equal(t, test.ExpectedRetryAttempts, testutil.ToFloat64(metrics.GetServiceRetryAttempts))
+		assert.Equal(t, test.ExpectedRetryExhausted, testutil.ToFloat64(metrics.GetServiceRetryExhausted))
 	}
 }
 
 func TestRetryHandler_PostService(t *testing.T) {
 	mockStore := &mocks.Store{}
 	mockStore.On("UpdateService", "service", "add", "192.0.0.1:8081").Return(nil)
 	mockStore.On("UpdateService", "service", "invalid-operation", "xxx").Return(errors.New("new error"))
-	retryHandler := NewRetryHandler(5, mockStore)
+	retryHandler := NewRetryHandler(maximumRetry, mockStore, metrics)
 	tests := []struct {
-		ExpectError bool
-		ServiceName string
-		Operation   string
-		Host        string
+		ExpectError            bool
+		ServiceName            string
+		Operation              string
+		Host                   string
+		ExpectedRetryExhausted float64
+		ExpectedRetryAttempts  float64
 	}{
 		{
-			ServiceName: "service",
-			Operation:   "add",
-			Host:        "192.0.0.1:8081",
-			ExpectError: false,
+			ServiceName:            "service",
+			Operation:              "add",
+			Host:                   "192.0.0.1:8081",
+			ExpectError:            false,
+			ExpectedRetryExhausted: 0.0,
+			ExpectedRetryAttempts:  0.0,
 		},
 		{
-			ServiceName: "service",
-			Operation:   "invalid-operation",
-			Host:        "xxx",
-			ExpectError: true,
+			ServiceName:            "service",
+			Operation:              "invalid-operation",
+			Host:                   "xxx",
+			ExpectError:            true,
+			ExpectedRetryExhausted: 1.0,
+			ExpectedRetryAttempts:  float64(maximumRetry),
 		},
 	}
 	for _, test := range tests {
@@ -67,5 +87,7 @@ func TestRetryHandler_PostService(t *testing.T) {
 		} else {
 			assert.NoError(t, err)
 		}
+		assert.Equal(t, test.ExpectedRetryAttempts, testutil.ToFloat64(metrics.PostServiceRetryAttempts))
+		assert.Equal(t, test.ExpectedRetryExhausted, testutil.ToFloat64(metrics.PostServiceRetryExhausted))
 	}
 }