-
Notifications
You must be signed in to change notification settings - Fork 684
/
accumulator.go
225 lines (197 loc) · 6.87 KB
/
accumulator.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
package kates
import (
"bytes"
"context"
"encoding/json"
"fmt"
"reflect"
"sort"
"strings"
"sync"
"k8s.io/apimachinery/pkg/api/meta"
)
// The Accumulator struct is used to efficiently maintain an in-memory copy of kubernetes resources
// present in a cluster in a form that is easy for business logic to process. It functions as a
// bridge between delta-based kubernetes watches on individual Kinds and the complete/consistent set
// of objects on which business logic needs to operate. In that sense it accumulates both multiple
// kinds of kubernetes resources into a single snapshot, as well as accumulating deltas on
// individual objects into relevant sets of objects.
//
// The Goals/Requirements below are based heavily on the needs of Ambassador as they have evolved
// over the years. A lot of this comes down to the fact that unlike the exemplary
// deployment/replicaset controller examples which typically operate on a single resource and render
// it into another (deployment -> N replicasets, replicaset -> N pods), Ambassador's controller
// logic has some additional requirements:
//
// 1. Complete knowledge of resources in a cluster. Because many thousands of Mappings are
// ultimately assembled into a single envoy configuration responsible for ingress into the
// cluster, the consequences of producing an envoy configuration when you e.g. know about only
// half of those Mappings is catastrophic (you are black-holing half your traffic).
//
// 2. Complete knowledge of multiple resources. Instead of having one self contained input like a
// deployment or a replicaset, Ambassador's business logic has many inputs, and the consequence
// of producing an envoy without knowledge of *all* of those inputs is equally catastrophic,
// e.g. it's no use knowing about all the Mappings if you don't know about any of the Hosts yet.
//
// Goals/Requirements:
//
// 1. Bootstrap of a single Kind: the Accumulator will ensure that all pre-existing resources of
// that Kind have been loaded into memory prior to triggering any notifications. This guarantees
// we will never trigger business logic on an egregiously incomplete view of the cluster
// (e.g. when 500 out of 1000 Mappings have been loaded) and makes it safe for the business
// logic to assume complete knowledge of the cluster.
//
// 2. When multiple Kinds are needed by a controller, the Accumulator will not notify the
// controller until all the Kinds have been fully bootstrapped.
//
// 3. Graceful load shedding: When the rate of change of resources is very fast, the API and
// implementation are structured so that individual object deltas get coalesced into a single
// snapshot update. This prevents excessively triggering business logic to process an entire
// snapshot for each individual object change that occurs.
type Accumulator struct {
client *Client
fields map[string]*field
mapsels map[string]mapsel
changed chan struct{}
mutex sync.Mutex
}
type field struct {
items []*Unstructured
prev []byte
}
type mapsel struct {
mapping *meta.RESTMapping
selector Selector
query Query
}
func newAccumulator(ctx context.Context, client *Client, queries ...Query) *Accumulator {
changed := make(chan struct{})
mapsels := make(map[string]mapsel)
rawUpdateCh := make(chan rawUpdate)
for _, q := range queries {
mapping, err := client.mappingFor(q.Kind)
if err != nil {
panic(err)
}
sel, err := ParseSelector(q.LabelSelector)
if err != nil {
panic(err)
}
mapsels[q.Name] = mapsel{mapping, sel, q}
client.watchRaw(ctx, q.Kind, rawUpdateCh, client.cliFor(mapping, q.Namespace), q.LabelSelector, q.Name)
}
acc := &Accumulator{client, make(map[string]*field), mapsels, changed, sync.Mutex{}}
// This coalesces reads from rawUpdateCh to notifications that changes are available to be
// processed. This loop along with the logic in storeField guarantees the 3
// Goals/Requirements listed in the documentation for the Accumulator struct, i.e. Ensuring
// all Kinds are bootstrapped before any notification occurs, as well as ensuring that we
// continue to coalesce updates in the background while business logic is executing in order
// to ensure graceful load shedding.
go func() {
canSend := false
for {
var rawUp rawUpdate
if canSend {
select {
case changed <- struct{}{}:
canSend = false
continue
case rawUp = <-rawUpdateCh:
case <-ctx.Done():
return
}
} else {
select {
case rawUp = <-rawUpdateCh:
case <-ctx.Done():
return
}
}
// Don't overwrite canSend if storeField returns false. We may not yet have
// had a chance to send a notification down the changed channel.
if acc.storeField(rawUp.correlation.(string), rawUp.items) {
canSend = true
}
}
}()
return acc
}
func (a *Accumulator) Changed() chan struct{} {
return a.changed
}
func (a *Accumulator) Update(target interface{}) bool {
a.mutex.Lock()
defer a.mutex.Unlock()
return a.update(reflect.ValueOf(target))
}
func (a *Accumulator) field(name string) *field {
f, ok := a.fields[name]
if !ok {
f = &field{}
a.fields[name] = f
}
return f
}
func (a *Accumulator) storeField(name string, items []*Unstructured) bool {
a.mutex.Lock()
defer a.mutex.Unlock()
a.field(name).items = items
return len(a.fields) >= len(a.mapsels)
}
func (a *Accumulator) updateField(target reflect.Value, name string, items []*Unstructured) bool {
field := a.field(name)
unKeySort(items)
jsonBytes, err := json.Marshal(items)
if err != nil {
panic(err)
}
fieldEntry, ok := target.Type().Elem().FieldByName(name)
if !ok {
panic(fmt.Sprintf("no such field: %q", name))
}
if !bytes.Equal(field.prev, jsonBytes) {
field.prev = jsonBytes
var val reflect.Value
if fieldEntry.Type.Kind() == reflect.Slice {
val = reflect.New(fieldEntry.Type)
err := json.Unmarshal(jsonBytes, val.Interface())
if err != nil {
panic(err)
}
} else if fieldEntry.Type.Kind() == reflect.Map {
val = reflect.MakeMap(fieldEntry.Type)
for _, item := range items {
innerVal := reflect.New(fieldEntry.Type.Elem())
err := convert(item, innerVal.Interface())
if err != nil {
panic(err)
}
val.SetMapIndex(reflect.ValueOf(item.GetName()), reflect.Indirect(innerVal))
}
} else {
panic(fmt.Sprintf("don't know how to unmarshal to: %v", fieldEntry.Type))
}
target.Elem().FieldByName(name).Set(reflect.Indirect(val))
return true
}
return false
}
func (a *Accumulator) update(target reflect.Value) bool {
updated := false
for name, field := range a.fields {
items := field.items[:]
ms := a.mapsels[name]
a.client.patchWatch(&items, ms.mapping, ms.selector)
if a.updateField(target, name, items) {
updated = true
}
}
return updated
}
func unKeySort(items []*Unstructured) {
sort.Slice(items, func(i, j int) bool {
ik := unKey(items[i])
jk := unKey(items[j])
return strings.Compare(ik, jk) < 0
})
}