-
Notifications
You must be signed in to change notification settings - Fork 485
/
config_watcher.go
340 lines (286 loc) · 9.45 KB
/
config_watcher.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
package cluster
import (
"context"
"fmt"
"sync"
"time"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/grafana/agent/pkg/metrics/instance"
"github.com/grafana/agent/pkg/metrics/instance/configstore"
"github.com/grafana/agent/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
reshardDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "agent_metrics_scraping_service_reshard_duration",
Help: "How long it took for resharding to run.",
}, []string{"success"})
)
// configWatcher connects to a configstore and will apply configs to an
// instance.Manager.
type configWatcher struct {
log log.Logger
mut sync.Mutex
cfg Config
stopped bool
stop context.CancelFunc
store configstore.Store
im instance.Manager
owns OwnershipFunc
validate ValidationFunc
refreshCh chan struct{}
instanceMut sync.Mutex
instances map[string]struct{}
}
// OwnershipFunc should determine if a given keep is owned by the caller.
type OwnershipFunc = func(key string) (bool, error)
// ValidationFunc should validate a config.
type ValidationFunc = func(*instance.Config) error
// newConfigWatcher watches store for changes and checks for each config against
// owns. It will also poll the configstore at a configurable interval.
func newConfigWatcher(log log.Logger, cfg Config, store configstore.Store, im instance.Manager, owns OwnershipFunc, validate ValidationFunc) (*configWatcher, error) {
ctx, cancel := context.WithCancel(context.Background())
w := &configWatcher{
log: log,
stop: cancel,
store: store,
im: im,
owns: owns,
validate: validate,
refreshCh: make(chan struct{}, 1),
instances: make(map[string]struct{}),
}
if err := w.ApplyConfig(cfg); err != nil {
return nil, err
}
// Delay duration, this is to prevent a race condition, see method for details
delay := cfg.Lifecycler.HeartbeatPeriod * 3
go w.run(ctx, delay)
return w, nil
}
func (w *configWatcher) ApplyConfig(cfg Config) error {
w.mut.Lock()
defer w.mut.Unlock()
if util.CompareYAML(w.cfg, cfg) {
return nil
}
if w.stopped {
return fmt.Errorf("configWatcher already stopped")
}
w.cfg = cfg
return nil
}
func (w *configWatcher) run(ctx context.Context, delay time.Duration) {
defer level.Info(w.log).Log("msg", "config watcher run loop exiting")
// This is due to a race condition between the heartbeat and config ring in a very narrow set of circumstances
// https://gist.github.com/mattdurham/c15f27de17a6da97bf2e6a870991c7f2
time.Sleep(delay)
lastReshard := time.Now()
for {
select {
case <-ctx.Done():
return
case <-w.nextReshard(lastReshard):
level.Debug(w.log).Log("msg", "reshard timer ticked, scheduling refresh")
w.RequestRefresh()
lastReshard = time.Now()
case <-w.refreshCh:
err := w.refresh(ctx)
if err != nil {
level.Error(w.log).Log("msg", "refresh failed", "err", err)
}
case ev := <-w.store.Watch():
level.Debug(w.log).Log("msg", "handling event from config store")
if err := w.handleEvent(ev); err != nil {
level.Error(w.log).Log("msg", "failed to handle changed or deleted config", "key", ev.Key, "err", err)
}
}
}
}
// nextReshard returns a channel to that will fill a value when the reshard
// interval has elapsed.
func (w *configWatcher) nextReshard(lastReshard time.Time) <-chan time.Time {
w.mut.Lock()
nextReshard := lastReshard.Add(w.cfg.ReshardInterval)
w.mut.Unlock()
remaining := time.Until(nextReshard)
// NOTE(rfratto): clamping to 0 isn't necessary for time.After,
// but it makes the log message clearer to always use "0s" as
// "next reshard will be scheduled immediately."
if remaining < 0 {
remaining = 0
}
level.Debug(w.log).Log("msg", "waiting for next reshard interval", "last_reshard", lastReshard, "next_reshard", nextReshard, "remaining", remaining)
return time.After(remaining)
}
// RequestRefresh will queue a refresh. No more than one refresh can be queued at a time.
func (w *configWatcher) RequestRefresh() {
select {
case w.refreshCh <- struct{}{}:
level.Debug(w.log).Log("msg", "successfully scheduled a refresh")
default:
level.Debug(w.log).Log("msg", "ignoring request refresh: refresh already scheduled")
}
}
// refresh reloads all configs from the configstore. Deleted configs will be
// removed. refresh may not be called concurrently and must only be invoked from run.
// Call RequestRefresh to queue a call to refresh.
func (w *configWatcher) refresh(ctx context.Context) (err error) {
w.mut.Lock()
enabled := w.cfg.Enabled
refreshTimeout := w.cfg.ReshardTimeout
w.mut.Unlock()
if !enabled {
level.Debug(w.log).Log("msg", "refresh skipped because clustering is disabled")
return nil
}
level.Info(w.log).Log("msg", "starting refresh")
if refreshTimeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, refreshTimeout)
defer cancel()
}
start := time.Now()
defer func() {
success := "1"
if err != nil {
success = "0"
}
duration := time.Since(start)
level.Info(w.log).Log("msg", "refresh finished", "duration", duration, "success", success, "err", err)
reshardDuration.WithLabelValues(success).Observe(duration.Seconds())
}()
// This is used to determine if the context was already exceeded before calling the kv provider
if err = ctx.Err(); err != nil {
level.Error(w.log).Log("msg", "context deadline exceeded before calling store.all", "err", err)
return err
}
deadline, _ := ctx.Deadline()
level.Debug(w.log).Log("msg", "deadline before store.all", "deadline", deadline)
configs, err := w.store.All(ctx, func(key string) bool {
owns, err := w.owns(key)
if err != nil {
level.Error(w.log).Log("msg", "failed to check for ownership, instance will be deleted if it is running", "key", key, "err", err)
return false
}
return owns
})
level.Debug(w.log).Log("msg", "count of configs from store.all", "count", len(configs))
if err != nil {
return fmt.Errorf("failed to get configs from store: %w", err)
}
var (
keys = make(map[string]struct{})
firstError error
)
Outer:
for {
select {
case <-ctx.Done():
return ctx.Err()
case cfg, ok := <-configs:
// w.store.All will close configs when all of them have been read.
if !ok {
break Outer
}
if err := w.handleEvent(configstore.WatchEvent{Key: cfg.Name, Config: &cfg}); err != nil {
level.Error(w.log).Log("msg", "failed to process changed config", "key", cfg.Name, "err", err)
if firstError == nil {
firstError = err
}
}
keys[cfg.Name] = struct{}{}
}
}
// Any config we used to be running that disappeared from this most recent
// iteration should be deleted. We hold the lock just for the duration of
// populating deleted because handleEvent also grabs a hold on the lock.
var deleted []string
w.instanceMut.Lock()
for key := range w.instances {
if _, exist := keys[key]; exist {
continue
}
deleted = append(deleted, key)
}
w.instanceMut.Unlock()
// Send a deleted event for any key that has gone away.
for _, key := range deleted {
if err := w.handleEvent(configstore.WatchEvent{Key: key, Config: nil}); err != nil {
level.Error(w.log).Log("msg", "failed to process changed config", "key", key, "err", err)
}
}
return firstError
}
func (w *configWatcher) handleEvent(ev configstore.WatchEvent) error {
w.mut.Lock()
defer w.mut.Unlock()
if w.stopped {
return fmt.Errorf("configWatcher stopped")
}
w.instanceMut.Lock()
defer w.instanceMut.Unlock()
owned, err := w.owns(ev.Key)
if err != nil {
level.Error(w.log).Log("msg", "failed to see if config is owned. instance will be deleted if it is running", "err", err)
}
var (
_, isRunning = w.instances[ev.Key]
isDeleted = ev.Config == nil
)
switch {
// Two deletion scenarios:
// 1. A config we're running got moved to a new owner.
// 2. A config we're running got deleted
case (isRunning && !owned) || (isDeleted && isRunning):
if isDeleted {
level.Info(w.log).Log("msg", "untracking deleted config", "key", ev.Key)
} else {
level.Info(w.log).Log("msg", "untracking config that changed owners", "key", ev.Key)
}
err := w.im.DeleteConfig(ev.Key)
delete(w.instances, ev.Key)
if err != nil {
return fmt.Errorf("failed to delete: %w", err)
}
case !isDeleted && owned:
if err := w.validate(ev.Config); err != nil {
return fmt.Errorf(
"failed to validate config. %[1]s cannot run until the global settings are adjusted or the config is adjusted to operate within the global constraints. error: %[2]w",
ev.Key, err,
)
}
if _, exist := w.instances[ev.Key]; !exist {
level.Info(w.log).Log("msg", "tracking new config", "key", ev.Key)
}
if err := w.im.ApplyConfig(*ev.Config); err != nil {
return fmt.Errorf("failed to apply config: %w", err)
}
w.instances[ev.Key] = struct{}{}
}
return nil
}
// Stop stops the configWatcher. Cannot be called more than once.
func (w *configWatcher) Stop() error {
w.mut.Lock()
defer w.mut.Unlock()
if w.stopped {
return fmt.Errorf("already stopped")
}
w.stop()
w.stopped = true
// Shut down all the instances that this configWatcher managed. It *MUST*
// happen after w.stop() is called to prevent the run loop from applying any
// new configs.
w.instanceMut.Lock()
defer w.instanceMut.Unlock()
for key := range w.instances {
if err := w.im.DeleteConfig(key); err != nil {
level.Warn(w.log).Log("msg", "failed deleting config on shutdown", "key", key, "err", err)
}
}
w.instances = make(map[string]struct{})
return nil
}