Add prometheus metrics (#89)

cmds/webhok_config.go

@@ -10,7 +10,6 @@ import (
 	"github.com/appscode/kutil/tools/certstore"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
-	auth "k8s.io/api/authentication/v1"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/util/cert"
@@ -86,7 +85,7 @@ func NewCmdGetWebhookConfig() *cobra.Command {
 				APIVersion: "v1",
 				Clusters: map[string]*clientcmdapi.Cluster{
 					"guard-server": {
-						Server: fmt.Sprintf("https://%s/apis/%s/tokenreviews", addr, auth.SchemeGroupVersion),
+						Server: fmt.Sprintf("https://%s/tokenreviews", addr),
 						CertificateAuthorityData: caCert,
 					},
 				},

glide.yaml

@@ -16,7 +16,7 @@ import:
 - package: github.com/howeyc/gopass
 - package: github.com/jpillora/go-ogle-analytics
 - package: github.com/prometheus/client_golang
-  version: v0.8.0
+  version: c3324c1198cf3374996e9d3098edd46a6b55afc9
 - package: github.com/prometheus/common
   version: 3233b24a36715a2fe36d162260457857a56d672d
 - package: github.com/sirupsen/logrus

server/handler.go

@@ -16,7 +16,7 @@ import (
 )
 
 func (s Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	if req.TLS == nil || len(req.TLS.PeerCertificates) > 0 {
+	if req.TLS == nil || len(req.TLS.PeerCertificates) == 0 {
 		write(w, nil, WithCode(errors.New("Missing client certificate"), http.StatusBadRequest))
 		return
 	}

server/prometheus.go

@@ -0,0 +1,56 @@
+package server
+
+import (
+	v "github.com/appscode/go/version"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	version = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "version",
+		Help: "Version information about this binary",
+		ConstLabels: map[string]string{
+			"version": v.Version.Version,
+		},
+	})
+
+	inFlightGauge = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "tokenreviews_handler_requests_in_flight",
+		Help: "A gauge of requests currently being served by the tokenreviews handler.",
+	})
+
+	counter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "tokenreviews_hendler_requests_total",
+			Help: "A counter for requests to the tokenreviews handler.",
+		},
+		[]string{"code", "method"},
+	)
+
+	// duration is partitioned by the HTTP method and handler. It uses custom
+	// buckets based on the expected request duration.
+	duration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "request_duration_seconds",
+			Help:    "A histogram of latencies for requests.",
+			Buckets: []float64{.25, .5, 1, 2.5, 5, 10},
+		},
+		[]string{"handler", "method"},
+	)
+
+	// responseSize has no labels, making it a zero-dimensional
+	// ObserverVec.
+	responseSize = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "response_size_bytes",
+			Help:    "A histogram of response sizes for requests.",
+			Buckets: []float64{100, 200, 400, 1000},
+		},
+		[]string{"handler"},
+	)
+)
+
+func init() {
+	// Register all of the metrics in the standard registry.
+	prometheus.MustRegister(inFlightGauge, counter, duration, responseSize)
+}

server/server.go

@@ -3,7 +3,6 @@ package server
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	_ "net/http/pprof"
@@ -18,9 +17,9 @@ import (
 	"github.com/appscode/kutil/meta"
 	"github.com/appscode/kutil/tools/fsnotify"
 	"github.com/appscode/pat"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/pflag"
-	auth "k8s.io/api/authentication/v1"
 )
 
 type Server struct {
@@ -114,7 +113,17 @@ func (s Server) ListenAndServe() {
 	tlsConfig.BuildNameToCertificate()
 
 	m := pat.New()
-	m.Post(fmt.Sprintf("/apis/%s/tokenreviews", auth.SchemeGroupVersion), s)
+
+	// Instrument the handlers with all the metrics, injecting the "handler"
+	// label by currying.
+	handler := promhttp.InstrumentHandlerInFlight(inFlightGauge,
+		promhttp.InstrumentHandlerDuration(duration.MustCurryWith(prometheus.Labels{"handler": "tokenreviews"}),
+			promhttp.InstrumentHandlerCounter(counter,
+				promhttp.InstrumentHandlerResponseSize(responseSize.MustCurryWith(prometheus.Labels{"handler": "tokenreviews"}), s),
+			),
+		),
+	)
+	m.Post("/tokenreviews", handler)
 	m.Get("/metrics", promhttp.Handler())
 	m.Get("/healthz", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		w.WriteHeader(200)

server/utils.go

@@ -63,13 +63,12 @@ type httpStatusCode interface {
 }
 
 func printStackTrace(err error) {
-	c, ok := errors.Cause(err).(stackTracer)
-	if !ok {
-		panic("oops, err does not implement stackTracer")
-	}
+	log.Errorln(err)
 
-	st := c.StackTrace()
-	log.Errorf("%s\nStacktrace: %+v", err.Error(), st) // top two frames
+	if c, ok := errors.Cause(err).(stackTracer); ok {
+		st := c.StackTrace()
+		log.Debugf("Stacktrace: %+v", err.Error(), st) // top two frames
+	}
 }
 
 func GetSupportedOrg() []string {