-
Notifications
You must be signed in to change notification settings - Fork 2k
/
instrument_server.go
144 lines (123 loc) · 4.79 KB
/
instrument_server.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
// Copyright (c) The Thanos Authors.
// Licensed under the Apache License 2.0.
package http
import (
"fmt"
"net/http"
"strings"
"time"
"github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/uber/jaeger-client-go"
)
// InstrumentationMiddleware holds necessary metrics to instrument an http.Server
// and provides necessary behaviors.
type InstrumentationMiddleware interface {
// NewHandler wraps the given HTTP handler for instrumentation.
NewHandler(handlerName string, handler http.Handler) http.HandlerFunc
}
type nopInstrumentationMiddleware struct{}
func (ins nopInstrumentationMiddleware) NewHandler(handlerName string, handler http.Handler) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
handler.ServeHTTP(w, r)
}
}
// NewNopInstrumentationMiddleware provides a InstrumentationMiddleware which does nothing.
func NewNopInstrumentationMiddleware() InstrumentationMiddleware {
return nopInstrumentationMiddleware{}
}
type defaultInstrumentationMiddleware struct {
metrics *defaultMetrics
}
// NewInstrumentationMiddleware provides default InstrumentationMiddleware.
// Passing nil as buckets uses the default buckets.
func NewInstrumentationMiddleware(reg prometheus.Registerer, buckets []float64) InstrumentationMiddleware {
return &defaultInstrumentationMiddleware{
metrics: newDefaultMetrics(reg, buckets, []string{}),
}
}
// NewHandler wraps the given HTTP handler for instrumentation. It
// registers four metric collectors (if not already done) and reports HTTP
// metrics to the (newly or already) registered collectors: http_requests_total
// (CounterVec), http_request_duration_seconds (Histogram),
// http_request_size_bytes (Summary), http_response_size_bytes (Summary).
// Each has a constant label named "handler" with the provided handlerName as value.
func (ins *defaultInstrumentationMiddleware) NewHandler(handlerName string, handler http.Handler) http.HandlerFunc {
baseLabels := prometheus.Labels{"handler": handlerName}
return httpInstrumentationHandler(baseLabels, ins.metrics, handler)
}
func httpInstrumentationHandler(baseLabels prometheus.Labels, metrics *defaultMetrics, next http.Handler) http.HandlerFunc {
return promhttp.InstrumentHandlerRequestSize(
metrics.requestSize.MustCurryWith(baseLabels),
instrumentHandlerInFlight(
metrics.inflightHTTPRequests.MustCurryWith(baseLabels),
promhttp.InstrumentHandlerCounter(
metrics.requestsTotal.MustCurryWith(baseLabels),
promhttp.InstrumentHandlerResponseSize(
metrics.responseSize.MustCurryWith(baseLabels),
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
now := time.Now()
wd := &responseWriterDelegator{w: w}
next.ServeHTTP(wd, r)
requestLabels := prometheus.Labels{"code": wd.Status(), "method": strings.ToLower(r.Method)}
observer := metrics.requestDuration.MustCurryWith(baseLabels).With(requestLabels)
observer.Observe(time.Since(now).Seconds())
// If we find a tracingID we'll expose it as Exemplar.
span := opentracing.SpanFromContext(r.Context())
if span != nil {
spanCtx, ok := span.Context().(jaeger.SpanContext)
if ok && spanCtx.IsSampled() {
observer.(prometheus.ExemplarObserver).ObserveWithExemplar(
time.Since(now).Seconds(),
prometheus.Labels{
"traceID": spanCtx.TraceID().String(),
},
)
}
}
}),
),
),
),
)
}
// responseWriterDelegator implements http.ResponseWriter and extracts the statusCode.
type responseWriterDelegator struct {
w http.ResponseWriter
written bool
statusCode int
}
func (wd *responseWriterDelegator) Header() http.Header {
return wd.w.Header()
}
func (wd *responseWriterDelegator) Write(bytes []byte) (int, error) {
return wd.w.Write(bytes)
}
func (wd *responseWriterDelegator) WriteHeader(statusCode int) {
wd.written = true
wd.statusCode = statusCode
wd.w.WriteHeader(statusCode)
}
func (wd *responseWriterDelegator) StatusCode() int {
if !wd.written {
return http.StatusOK
}
return wd.statusCode
}
func (wd *responseWriterDelegator) Status() string {
return fmt.Sprintf("%d", wd.StatusCode())
}
// instrumentHandlerInFlight is responsible for counting the amount of
// in-flight HTTP requests (requests being processed by the handler) at a given
// moment in time.
// This is used instead of prometheus/client_golang/promhttp.InstrumentHandlerInFlight
// to be able to have the HTTP method as a label.
func instrumentHandlerInFlight(vec *prometheus.GaugeVec, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gauge := vec.With(prometheus.Labels{"method": r.Method})
gauge.Inc()
defer gauge.Dec()
next.ServeHTTP(w, r)
})
}