Merge pull request #32 from Financial-Times/deploy-on-push/k8s/promet…

…heus-metrics

Prometheus metrics

Gopkg.toml

@@ -57,6 +57,10 @@
   name = "k8s.io/kube-openapi"
   revision = "91cfa479c814065e420cee7ed227db0f63a5854e"
 
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  revision = "7858729281ec582767b20e0d696b6041d995d5e0"
+
 [prune]
   go-tests = true
   unused-packages = true

cachingController.go

@@ -16,8 +16,9 @@ const (
 func newMeasuredService(service service) measuredService {
 	cachedHealth := newCachedHealth()
 	bufferedHealths := newBufferedHealths()
+	bufferedMetrics := newBufferedHealths()
 	go cachedHealth.maintainLatest()
-	return measuredService{service, cachedHealth, bufferedHealths}
+	return measuredService{service, cachedHealth, bufferedHealths, bufferedMetrics}
 }
 
 func (c *healthCheckController) collectChecksFromCachesFor(categories map[string]category) ([]fthealth.CheckResult, map[string][]fthealth.CheckResult, error) {
@@ -121,6 +122,11 @@ func (c *healthCheckController) scheduleCheck(mService measuredService, refreshP
 	default:
 	}
 
+	select {
+	case mService.bufferedMetrics.buffer <- checkResult:
+	default:
+	}
+
 	go c.scheduleCheck(mService, refreshPeriod, time.NewTimer(refreshPeriod))
 }
 

helm/upp-aggregate-healthcheck/templates/service-monitor.yaml

@@ -0,0 +1,15 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    component: {{ .Values.service.name }}
+    prometheus: {{ .Values.metrics.prometheusInstance }}
+  name: {{ .Values.service.name }}
+spec:
+  endpoints:
+  - interval: {{ .Values.metrics.interval }}
+    path: {{ .Values.metrics.path }}
+  jobLabel: component
+  selector:
+    matchLabels:
+      app: {{ .Values.service.name }}

helm/upp-aggregate-healthcheck/values.yaml

@@ -27,4 +27,11 @@ categories:
     data:
       category.name: default
       category.refreshrate: "60"
-
+metrics:
+  # where we expose the metrics for prometheus
+  path: /metrics
+  # how often prometheus will call the metrics path
+  interval: 60s
+  # the name of the prometheus instance, as defined in
+  # content-k8s-prometheus\helm\content-k8s-prometheus\app-configs\monitoring-metrics_{env}.yaml
+  prometheusInstance: monitoring-metrics

main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/jawher/mow.cli"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const logPattern = log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile | log.LUTC
@@ -62,6 +63,9 @@ func main() {
 		graphiteFeeder := newGraphiteFeeder(*graphiteURL, *environment, controller)
 		go graphiteFeeder.feed()
 
+		prometheusFeeder := newPrometheusFeeder(*environment, controller)
+		go prometheusFeeder.feed()
+
 		listen(handler, *pathPrefix)
 	}
 
@@ -74,6 +78,7 @@ func main() {
 func listen(httpHandler *httpHandler, pathPrefix string) {
 	r := mux.NewRouter()
 	r.HandleFunc("/__gtg", httpHandler.handleGoodToGo)
+	r.Handle("/metrics", promhttp.Handler())
 	s := r.PathPrefix(pathPrefix).Subrouter()
 	s.HandleFunc("/add-ack", httpHandler.handleAddAck).Methods("POST")
 	s.HandleFunc("/enable-category", httpHandler.handleEnableCategory)

model.go

@@ -41,4 +41,5 @@ type measuredService struct {
 	service         service
 	cachedHealth    *cachedHealth
 	bufferedHealths *bufferedHealths
+	bufferedMetrics *bufferedHealths
 }

prometheusFeeder.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"strings"
+	"time"
+
+	prom "github.com/prometheus/client_golang/prometheus"
+)
+
+type prometheusFeeder struct {
+	environment string
+	ticker      *time.Ticker
+	controller  controller
+}
+
+func newPrometheusFeeder(environment string, controller controller) *prometheusFeeder {
+	ticker := time.NewTicker(60 * time.Second)
+	return &prometheusFeeder{
+		environment: environment,
+		ticker:      ticker,
+		controller:  controller,
+	}
+}
+
+func (p prometheusFeeder) feed() {
+	ignitePilotLight(p.environment)
+	serviceStatus := initServiceStatusMetrics()
+
+	for range p.ticker.C {
+		p.recordMetrics(serviceStatus)
+	}
+}
+
+func (p prometheusFeeder) recordMetrics(serviceStatus *prom.GaugeVec) {
+	for _, service := range p.controller.getMeasuredServices() {
+		select {
+		case checkResult := <-service.bufferedMetrics.buffer:
+			name := strings.Replace(checkResult.Name, ".", "-", -1)
+			checkStatus := inverseBoolToFloat64(checkResult.Ok)
+			serviceStatus.
+				With(prom.Labels{"environment": p.environment, "service": name}).
+				Set(checkStatus)
+		default:
+			continue
+		}
+	}
+}
+
+func initServiceStatusMetrics() *prom.GaugeVec {
+	serviceStatus := prom.NewGaugeVec(
+		prom.GaugeOpts{
+			Namespace: "upp",
+			Subsystem: "health",
+			Name:      "servicestatus",
+			Help:      "Status of the service: 0 - healthy; 1 - unhealthy",
+		},
+		[]string{
+			"environment",
+			"service",
+		})
+	prom.MustRegister(serviceStatus)
+	return serviceStatus
+}
+
+func ignitePilotLight(environment string) {
+	pilotLight := prom.NewGaugeVec(
+		prom.GaugeOpts{
+			Namespace: "upp",
+			Subsystem: "health",
+			Name:      "pilotlight",
+			Help:      "Pilot light for the service monitoring UPP service health",
+		},
+		[]string{
+			"environment",
+		})
+	prom.MustRegister(pilotLight)
+	pilotLight.With(prom.Labels{"environment": environment}).Set(1)
+}
+
+func inverseBoolToFloat64(b bool) float64 {
+	if b {
+		return 0
+	}
+	return 1
+}

prometheusFeeder_test.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"testing"
+
+	prom "github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
+)
+
+const ENV = "foobar"
+
+func TestNewPrometheusFeeder(t *testing.T) {
+	feeder := newPrometheusFeeder(ENV, &healthCheckController{})
+	assert.NotNil(t, feeder)
+	assert.Equal(t, ENV, feeder.environment, "Environment should be set correctly.")
+}
+
+func TestInitServiceStatusMetrics(t *testing.T) {
+	gauge := initServiceStatusMetrics()
+	assert.NotNil(t, gauge)
+	duplicateGauge := prom.NewGaugeVec(
+		prom.GaugeOpts{
+			Namespace: "upp",
+			Subsystem: "health",
+			Name:      "servicestatus",
+			Help:      "Status of the service: 0 - healthy; 1 - unhealthy",
+		},
+		[]string{
+			"environment",
+			"service",
+		})
+
+	err := prom.Register(duplicateGauge)
+	assert.NotNil(t, err, "Gauge should've been registered already.")
+	_, ok := err.(prom.AlreadyRegisteredError)
+	assert.True(t, ok, "Expecting an 'AlreadyRegisteredError'.")
+}
+func TestIgnitePilotLight(t *testing.T) {
+	ignitePilotLight(ENV)
+	duplicateGauge := prom.NewGaugeVec(
+		prom.GaugeOpts{
+			Namespace: "upp",
+			Subsystem: "health",
+			Name:      "pilotlight",
+			Help:      "Pilot light for the service monitoring UPP service health",
+		},
+		[]string{
+			"environment",
+		})
+
+	err := prom.Register(duplicateGauge)
+	assert.NotNil(t, err, "Gauge should've been registered already.")
+	_, ok := err.(prom.AlreadyRegisteredError)
+	assert.True(t, ok, "Expecting an 'AlreadyRegisteredError'.")
+}
+
+func TestInverseBoolToFloat64(t *testing.T) {
+	one := inverseBoolToFloat64(false)
+	assert.Equal(t, float64(1), one)
+
+	zero := inverseBoolToFloat64(true)
+	assert.Equal(t, float64(0), zero)
+}