forked from fastly/fastly-exporter
-
Notifications
You must be signed in to change notification settings - Fork 0
/
monitor.go
211 lines (196 loc) · 11 KB
/
monitor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
)
// monitor polls the Fastly real-time stats API for the provided service ID on a
// regular cadence. All received metrics are processed and given to the
// Prometheus metrics struct. The service name label is regularly updated via
// the name resolver. The function exits when the context is canceled.
func monitor(ctx context.Context, client httpClient, token string, serviceID string, resolver serviceResolver, metrics prometheusMetrics, postprocess func(), logger log.Logger) error {
var ts uint64
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
var (
name, version = resolver.resolve(serviceID)
logger = log.With(logger, "service_name", name)
)
// rt.fastly.com blocks until it has data to return.
// It's safe to call in a (single-threaded!) hot loop.
u := fmt.Sprintf("https://rt.fastly.com/v1/channel/%s/ts/%d", serviceID, ts)
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return err // fatal for sure
}
req.Header.Set("User-Agent", "Fastly-Exporter ("+programVersion+")")
req.Header.Set("Fastly-Key", token)
req.Header.Set("Accept", "application/json")
resp, err := client.Do(req.WithContext(ctx))
if err != nil {
level.Error(logger).Log("during", "execute request", "err", err)
contextSleep(ctx, time.Second)
continue
}
var rt realtimeResponse
if err := json.NewDecoder(resp.Body).Decode(&rt); err != nil {
resp.Body.Close()
level.Error(logger).Log("during", "decode response", "err", err)
contextSleep(ctx, time.Second)
continue
}
resp.Body.Close()
rterr := rt.Error
if rterr == "" {
rterr = "<none>"
}
switch resp.StatusCode {
case http.StatusOK:
level.Debug(logger).Log("status_code", resp.StatusCode, "response_ts", rt.Timestamp, "err", rterr)
process(rt, serviceID, name, version, metrics)
postprocess()
case http.StatusUnauthorized, http.StatusForbidden:
level.Error(logger).Log("status_code", resp.StatusCode, "response_ts", rt.Timestamp, "err", rterr, "msg", "token may be invalid")
contextSleep(ctx, 15*time.Second)
default:
level.Error(logger).Log("status_code", resp.StatusCode, "response_ts", rt.Timestamp, "err", rterr)
contextSleep(ctx, 5*time.Second)
}
ts = rt.Timestamp
}
}
}
type realtimeResponse struct {
Timestamp uint64 `json:"Timestamp"`
Data []struct {
Datacenter map[string]datacenter `json:"datacenter"`
Aggregated datacenter `json:"aggregated"`
Recorded uint64 `json:"recorded"`
} `json:"Data"`
Error string `json:"error"`
}
type datacenter struct {
Requests uint64 `json:"requests"`
TLS uint64 `json:"tls"`
Shield uint64 `json:"shield"`
IPv6 uint64 `json:"ipv6"`
ImgOpto uint64 `json:"imgopto"`
ImgOptoShield uint64 `json:"imgopto_shield"`
ImgOptoTransform uint64 `json:"imgopto_transforms"`
OTFP uint64 `json:"otfp"`
OTFPShield uint64 `json:"otfp_shield"`
OTFPTransform uint64 `json:"otfp_transforms"`
OTFPManifest uint64 `json:"otfp_manifests"`
Video uint64 `json:"video"`
PCI uint64 `json:"pci"`
Logging uint64 `json:"logging"`
HTTP2 uint64 `json:"http2"`
RespHeaderBytes uint64 `json:"resp_header_bytes"`
HeaderSize uint64 `json:"header_size"`
RespBodyBytes uint64 `json:"resp_body_bytes"`
BodySize uint64 `json:"body_size"`
ReqHeaderBytes uint64 `json:"req_header_bytes"`
BackendReqHeaderBytes uint64 `json:"bereq_header_bytes"`
BilledHeaderBytes uint64 `json:"billed_header_bytes"`
BilledBodyBytes uint64 `json:"billed_body_bytes"`
WAFBlocked uint64 `json:"waf_blocked"`
WAFLogged uint64 `json:"waf_logged"`
WAFPassed uint64 `json:"waf_passed"`
AttackReqHeaderBytes uint64 `json:"attack_req_header_bytes"`
AttackReqBodyBytes uint64 `json:"attack_req_body_bytes"`
AttackRespSynthBytes uint64 `json:"attack_resp_synth_bytes"`
AttackLoggedReqHeaderBytes uint64 `json:"attack_logged_req_header_bytes"`
AttackLoggedReqBodyBytes uint64 `json:"attack_logged_req_body_bytes"`
AttackBlockedReqHeaderBytes uint64 `json:"attack_blocked_req_header_bytes"`
AttackBlockedReqBodyBytes uint64 `json:"attack_blocked_req_body_bytes"`
AttackPassedReqHeaderBytes uint64 `json:"attack_passed_req_header_bytes"`
AttackPassedReqBodyBytes uint64 `json:"attack_passed_req_body_bytes"`
ShieldRespHeaderBytes uint64 `json:"shield_resp_header_bytes"`
ShieldRespBodyBytes uint64 `json:"shield_resp_body_bytes"`
OTFPRespHeaderBytes uint64 `json:"otfp_resp_header_bytes"`
OTFPRespBodyBytes uint64 `json:"otfp_resp_body_bytes"`
OTFPShieldRespHeaderBytes uint64 `json:"otfp_shield_resp_header_bytes"`
OTFPShieldRespBodyBytes uint64 `json:"otfp_shield_resp_body_bytes"`
OTFPTransformRespHeaderBytes uint64 `json:"otfp_transform_resp_header_bytes"`
OTFPTransformRespBodyBytes uint64 `json:"otfp_transform_resp_body_bytes"`
OTFPShieldTime uint64 `json:"otfp_shield_time"`
OTFPTransformTime uint64 `json:"otfp_transform_time"`
OTFPDeliverTime uint64 `json:"otfp_deliver_time"`
ImgOptoRespHeaderBytes uint64 `json:"imgopto_resp_header_bytes"`
ImgOptoRespBodyBytes uint64 `json:"imgopto_resp_body_bytes"`
ImgOptoShieldRespHeaderBytes uint64 `json:"imgopto_shield_resp_header_bytes"`
ImgOptoShieldRespBodyBytes uint64 `json:"imgopto_shield_resp_body_bytes"`
ImgOptoTransformRespHeaderBytes uint64 `json:"imgopto_transform_resp_header_bytes"`
ImgOptoTransformRespBodyBytes uint64 `json:"imgopto_transform_resp_body_bytes"`
Status1xx uint64 `json:"status_1xx"`
Status2xx uint64 `json:"status_2xx"`
Status3xx uint64 `json:"status_3xx"`
Status4xx uint64 `json:"status_4xx"`
Status5xx uint64 `json:"status_5xx"`
Status200 uint64 `json:"status_200"`
Status204 uint64 `json:"status_204"`
Status301 uint64 `json:"status_301"`
Status302 uint64 `json:"status_302"`
Status304 uint64 `json:"status_304"`
Status400 uint64 `json:"status_400"`
Status401 uint64 `json:"status_401"`
Status403 uint64 `json:"status_403"`
Status404 uint64 `json:"status_404"`
Status416 uint64 `json:"status_416"`
Status500 uint64 `json:"status_500"`
Status501 uint64 `json:"status_501"`
Status502 uint64 `json:"status_502"`
Status503 uint64 `json:"status_503"`
Status504 uint64 `json:"status_504"`
Status505 uint64 `json:"status_505"`
Hits uint64 `json:"hits"`
Misses uint64 `json:"miss"`
Passes uint64 `json:"pass"`
Synths uint64 `json:"synth"`
Errors uint64 `json:"errors"`
Uncacheable uint64 `json:"uncacheable"`
HitsTime float64 `json:"hits_time"`
MissTime float64 `json:"miss_time"`
PassTime float64 `json:"pass_time"`
MissHistogram map[string]uint64 `json:"miss_histogram"`
TLSv10 uint64 `json:"tls_v10"`
TLSv11 uint64 `json:"tls_v11"`
TLSv12 uint64 `json:"tls_v12"`
TLSv13 uint64 `json:"tls_v13"`
ObjectSize1k uint64 `json:"object_size_1k"`
ObjectSize10k uint64 `json:"object_size_10k"`
ObjectSize100k uint64 `json:"object_size_100k"`
ObjectSize1m uint64 `json:"object_size_1m"`
ObjectSize10m uint64 `json:"object_size_10m"`
ObjectSize100m uint64 `json:"object_size_100m"`
ObjectSize1g uint64 `json:"object_size_1g"`
RecvSubTime uint64 `json:"recv_sub_time"`
RecvSubCount uint64 `json:"recv_sub_count"`
HashSubTime uint64 `json:"hash_sub_time"`
HashSubCount uint64 `json:"hash_sub_count"`
MissSubTime uint64 `json:"miss_sub_time"`
MissSubCount uint64 `json:"miss_sub_count"`
FetchSubTime uint64 `json:"fetch_sub_time"`
FetchSubCount uint64 `json:"fetch_sub_count"`
DeliverSubTime uint64 `json:"deliver_sub_time"`
DeliverSubCount uint64 `json:"deliver_sub_count"`
HitSubTime uint64 `json:"hit_sub_time"`
HitSubCount uint64 `json:"hit_sub_count"`
PrehashSubTime uint64 `json:"prehash_sub_time"`
PrehashSubCount uint64 `json:"prehash_sub_count"`
PredeliverSubTime uint64 `json:"predeliver_sub_time"`
PredeliverSubCount uint64 `json:"predeliver_sub_count"`
}
func contextSleep(ctx context.Context, d time.Duration) {
select {
case <-time.After(d):
case <-ctx.Done():
}
}