adding prometheus metrics config, endpoint, setup, tests etc.; adding…

… a new metric

server/apns/apns.go

@@ -98,6 +98,7 @@ func (a *apns) HandleResponse(request connector.Request, responseIface interface
 	if errSend != nil {
 		logger.WithField("error", errSend.Error()).WithField("error_type", errSend).Error("error when trying to send APNS notification")
 		mTotalSendErrors.Add(1)
+		pSendErrors.Inc()
 		if *a.IntervalMetrics && metadata != nil {
 			addToLatenciesAndCountsMaps(currentTotalErrorsLatenciesKey, currentTotalErrorsKey, metadata.Latency)
 		}

server/apns/apns_prometheus.go

@@ -9,10 +9,16 @@ var (
 		Name: "apns_sent_messages",
 		Help: "Number of messages sent to APNS",
 	})
+
+	pSendErrors = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "apns_send_errors",
+		Help: "Number of errors when trying to send messages to APNS",
+	})
 )
 
 func init() {
 	prometheus.MustRegister(
 		pSentMessages,
+		pSendErrors,
 	)
 }

server/config.go

@@ -18,20 +18,21 @@ import (
 )
 
 const (
-	defaultHttpListen      = ":8080"
-	defaultHealthEndpoint  = "/admin/healthcheck"
-	defaultMetricsEndpoint = "/admin/metrics"
-	defaultKVSBackend      = "file"
-	defaultMSBackend       = "file"
-	defaultStoragePath     = "/var/lib/guble"
-	defaultNodePort        = "10000"
-	development            = "dev"
-	integration            = "int"
-	preproduction          = "pre"
-	production             = "prod"
-	memProfile             = "mem"
-	cpuProfile             = "cpu"
-	blockProfile           = "block"
+	defaultHttpListen         = ":8080"
+	defaultHealthEndpoint     = "/admin/healthcheck"
+	defaultMetricsEndpoint    = "/admin/metrics-old"
+	defaultPrometheusEndpoint = "/admin/metrics"
+	defaultKVSBackend         = "file"
+	defaultMSBackend          = "file"
+	defaultStoragePath        = "/var/lib/guble"
+	defaultNodePort           = "10000"
+	development               = "dev"
+	integration               = "int"
+	preproduction             = "pre"
+	production                = "prod"
+	memProfile                = "mem"
+	cpuProfile                = "cpu"
+	blockProfile              = "block"
 )
 
 var (
@@ -59,21 +60,22 @@ type (
 	}
 	// GubleConfig is used for configuring Guble server (including its modules / connectors).
 	GubleConfig struct {
-		Log             *string
-		EnvName         *string
-		HttpListen      *string
-		KVS             *string
-		MS              *string
-		StoragePath     *string
-		HealthEndpoint  *string
-		MetricsEndpoint *string
-		Profile         *string
-		Postgres        PostgresConfig
-		FCM             fcm.Config
-		APNS            apns.Config
-		SMS             sms.Config
-		WS		websocket.Config
-		Cluster         ClusterConfig
+		Log                *string
+		EnvName            *string
+		HttpListen         *string
+		KVS                *string
+		MS                 *string
+		StoragePath        *string
+		HealthEndpoint     *string
+		MetricsEndpoint    *string
+		PrometheusEndpoint *string
+		Profile            *string
+		Postgres           PostgresConfig
+		FCM                fcm.Config
+		APNS               apns.Config
+		SMS                sms.Config
+		WS                 websocket.Config
+		Cluster            ClusterConfig
 	}
 )
 
@@ -115,6 +117,10 @@ var (
 			Default(defaultMetricsEndpoint).
 			Envar("GUBLE_METRICS_ENDPOINT").
 			String(),
+		PrometheusEndpoint: kingpin.Flag("prometheus-endpoint", `The metrics Prometheus endpoint to be used by the HTTP server (value for disabling it: "")`).
+			Default(defaultPrometheusEndpoint).
+			Envar("GUBLE_PROMETHEUS_ENDPOINT").
+			String(),
 		Profile: kingpin.Flag("profile", `The profiler to be used (default: none): mem | cpu | block`).
 			Default("").
 			Envar("GUBLE_PROFILE").

server/config_test.go

@@ -39,6 +39,9 @@ func TestParsingOfEnvironmentVariables(t *testing.T) {
 	os.Setenv("GUBLE_METRICS_ENDPOINT", "metrics_endpoint")
 	defer os.Unsetenv("GUBLE_METRICS_ENDPOINT")
 
+	os.Setenv("GUBLE_PROMETHEUS_ENDPOINT", "prometheus_endpoint")
+	defer os.Unsetenv("GUBLE_PROMETHEUS_ENDPOINT")
+
 	os.Setenv("GUBLE_MS", "ms-backend")
 	defer os.Unsetenv("GUBLE_MS")
 
@@ -121,6 +124,7 @@ func TestParsingArgs(t *testing.T) {
 		"--ms", "ms-backend",
 		"--health-endpoint", "health_endpoint",
 		"--metrics-endpoint", "metrics_endpoint",
+		"--prometheus-endpoint", "prometheus_endpoint",
 		"--ws",
 		"--ws-prefix", "/wstream/",
 		"--fcm",
@@ -156,6 +160,7 @@ func assertArguments(a *assert.Assertions) {
 	a.Equal("health_endpoint", *Config.HealthEndpoint)
 
 	a.Equal("metrics_endpoint", *Config.MetricsEndpoint)
+	a.Equal("prometheus_endpoint", *Config.PrometheusEndpoint)
 
 	a.Equal(true, *Config.WS.Enabled)
 	a.Equal("/wstream/", *Config.WS.Prefix)

server/fcm_integration_test.go

@@ -134,7 +134,7 @@ func serviceSetUp(t *testing.T) (*service.Service, func()) {
 	*Config.MS = "file"
 	*Config.Cluster.NodeID = 0
 	*Config.StoragePath = dir
-	*Config.MetricsEndpoint = "/admin/metrics"
+	*Config.MetricsEndpoint = "/admin/metrics-old"
 	*Config.WS.Enabled = true
 	*Config.WS.Prefix = "/stream/"
 	*Config.FCM.Enabled = true

server/gobbler.go

@@ -282,7 +282,8 @@ func StartService() *service.Service {
 
 	srv := service.New(r, websrv).
 		HealthEndpoint(*Config.HealthEndpoint).
-		MetricsEndpoint(*Config.MetricsEndpoint)
+		MetricsEndpoint(*Config.MetricsEndpoint).
+		PrometheusEndpoint(*Config.PrometheusEndpoint)
 
 	srv.RegisterModules(0, 6, kvStore, messageStore)
 	srv.RegisterModules(4, 3, CreateModules(r)...)

server/service/service.go

@@ -3,6 +3,7 @@ package service
 import (
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/distribution/health"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/cosminrentea/gobbler/server/metrics"
 	"github.com/cosminrentea/gobbler/server/router"
@@ -21,13 +22,14 @@ const (
 
 // Service is the main struct for controlling a guble server
 type Service struct {
-	webserver       *webserver.WebServer
-	router          router.Router
-	modules         []module
-	healthEndpoint  string
-	healthFrequency time.Duration
-	healthThreshold int
-	metricsEndpoint string
+	webserver          *webserver.WebServer
+	router             router.Router
+	modules            []module
+	healthEndpoint     string
+	healthFrequency    time.Duration
+	healthThreshold    int
+	metricsEndpoint    string
+	prometheusEndpoint string
 }
 
 // New creates a new Service, using the given Router and WebServer.
@@ -80,6 +82,12 @@ func (s *Service) MetricsEndpoint(endpointPrefix string) *Service {
 	return s
 }
 
+// PrometheusEndpoint sets the endpoint used for Prometheus metrics. Parameter for disabling the endpoint is: "". Returns the updated service.
+func (s *Service) PrometheusEndpoint(endpointPrefix string) *Service {
+	s.prometheusEndpoint = endpointPrefix
+	return s
+}
+
 // Start checks the modules for the following interfaces and registers and/or starts:
 //   Startable:
 //   health.Checker:
@@ -98,6 +106,12 @@ func (s *Service) Start() error {
 	} else {
 		logger.Info("Metrics endpoint disabled")
 	}
+	if s.prometheusEndpoint != "" {
+		logger.WithField("prometheusEndpoint", s.prometheusEndpoint).Info("Prometheus metrics endpoint")
+		s.webserver.Handle(s.prometheusEndpoint, promhttp.Handler())
+	} else {
+		logger.Info("Prometheus metrics endpoint disabled")
+	}
 	for order, iface := range s.ModulesSortedByStartOrder() {
 		name := reflect.TypeOf(iface).String()
 		if s, ok := iface.(Startable); ok {

server/service/service_test.go

@@ -155,7 +155,34 @@ func TestMetricsEnabled(t *testing.T) {
 	url := fmt.Sprintf("http://%s/metrics_url", service.WebServer().GetAddr())
 	result, err := http.Get(url)
 
-	// then I get status 200 and JSON: {}
+	// then I get status 200 and a non-empty text
+	a.NoError(err)
+	body, err := ioutil.ReadAll(result.Body)
+	a.NoError(err)
+	a.True(len(body) > 0)
+}
+
+func TestPrometheusMetricsEnabled(t *testing.T) {
+	_, finish := testutil.NewMockCtrl(t)
+	defer finish()
+	defer testutil.ResetDefaultRegistryHealthCheck()
+	a := assert.New(t)
+
+	// given:
+	service, _, _, _ := aMockedServiceWithMockedRouterStandalone()
+	service = service.PrometheusEndpoint("/metrics_url")
+	a.Equal(2, len(service.ModulesSortedByStartOrder()))
+
+	// when starting the service
+	defer service.Stop()
+	service.Start()
+	time.Sleep(time.Millisecond * 10)
+
+	// and when I call the health URL
+	url := fmt.Sprintf("http://%s/metrics_url", service.WebServer().GetAddr())
+	result, err := http.Get(url)
+
+	// then I get status 200 and some non-empty text
 	a.NoError(err)
 	body, err := ioutil.ReadAll(result.Body)
 	a.NoError(err)