-
Notifications
You must be signed in to change notification settings - Fork 2.8k
/
controller.go
341 lines (301 loc) · 11.3 KB
/
controller.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium
package agent
import (
"context"
"fmt"
"runtime/pprof"
"github.com/sirupsen/logrus"
"k8s.io/client-go/util/workqueue"
daemon_k8s "github.com/cilium/cilium/daemon/k8s"
"github.com/cilium/cilium/pkg/bgpv1/agent/signaler"
"github.com/cilium/cilium/pkg/hive"
"github.com/cilium/cilium/pkg/hive/cell"
"github.com/cilium/cilium/pkg/hive/job"
v2_api "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
v2alpha1api "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1"
"github.com/cilium/cilium/pkg/k8s/resource"
slimlabels "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/labels"
slimmetav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1"
"github.com/cilium/cilium/pkg/logging"
"github.com/cilium/cilium/pkg/logging/logfields"
"github.com/cilium/cilium/pkg/option"
)
var (
log = logging.DefaultLogger.WithField(logfields.LogSubsys, "bgp-control-plane")
)
var (
// ErrMultiplePolicies is a static error typed when the controller encounters
// multiple policies which apply to its host.
ErrMultiplePolicies = fmt.Errorf("more then one CiliumBGPPeeringPolicy applies to this node, please ensure only a single Policy matches this node's labels")
)
type policyLister interface {
List() ([]*v2alpha1api.CiliumBGPPeeringPolicy, error)
}
type policyListerFunc func() ([]*v2alpha1api.CiliumBGPPeeringPolicy, error)
func (plf policyListerFunc) List() ([]*v2alpha1api.CiliumBGPPeeringPolicy, error) {
return plf()
}
// Controller is the agent side BGP Control Plane controller.
//
// Controller listens for events and drives BGP related sub-systems
// to maintain a desired state.
type Controller struct {
// CiliumNodeResource provides a stream of events for changes to the local CiliumNode resource.
CiliumNodeResource daemon_k8s.LocalCiliumNodeResource
// LocalCiliumNode is the CiliumNode object for the local node.
LocalCiliumNode *v2_api.CiliumNode
// PolicyResource provides a store of cached policies and allows us to observe changes to the objects in its
// store.
PolicyResource resource.Resource[*v2alpha1api.CiliumBGPPeeringPolicy]
// PolicyLister is an interface which allows for the listing of all known policies
PolicyLister policyLister
// Sig informs the Controller that a Kubernetes
// event of interest has occurred.
//
// The signal itself provides no other information,
// when it occurs the Controller will query each
// informer for the latest API information required
// to drive it's control loop.
Sig *signaler.BGPCPSignaler
// BGPMgr is an implementation of the BGPRouterManager interface
// and provides a declarative API for configuring BGP peers.
BGPMgr BGPRouterManager
}
// ControllerParams contains all parameters needed to construct a Controller
type ControllerParams struct {
cell.In
Lifecycle hive.Lifecycle
Scope cell.Scope
JobRegistry job.Registry
Shutdowner hive.Shutdowner
Sig *signaler.BGPCPSignaler
RouteMgr BGPRouterManager
PolicyResource resource.Resource[*v2alpha1api.CiliumBGPPeeringPolicy]
DaemonConfig *option.DaemonConfig
LocalCiliumNodeResource daemon_k8s.LocalCiliumNodeResource
}
// NewController constructs a new BGP Control Plane Controller.
//
// When the constructor returns the Controller will be actively watching for
// events and configuring BGP related sub-systems.
//
// The constructor requires an implementation of BGPRouterManager to be provided.
// This implementation defines which BGP backend will be used (GoBGP, FRR, Bird, etc...)
// NOTE: only GoBGP currently implemented.
func NewController(params ControllerParams) (*Controller, error) {
// If the BGP control plane is disabled, just return nil. This way the hive dependency graph is always static
// regardless of config. The lifecycle has not been appended so no work will be done.
if !params.DaemonConfig.BGPControlPlaneEnabled() {
return nil, nil
}
c := &Controller{
Sig: params.Sig,
BGPMgr: params.RouteMgr,
PolicyResource: params.PolicyResource,
CiliumNodeResource: params.LocalCiliumNodeResource,
}
jobGroup := params.JobRegistry.NewGroup(
params.Scope,
job.WithLogger(log),
job.WithPprofLabels(pprof.Labels("cell", "bgp-cp")),
)
jobGroup.Add(
job.OneShot("bgp-policy-observer", func(ctx context.Context, health cell.HealthReporter) (err error) {
for ev := range c.PolicyResource.Events(ctx) {
switch ev.Kind {
case resource.Upsert, resource.Delete:
// Signal the reconciliation logic.
c.Sig.Event(struct{}{})
}
ev.Done(nil)
}
return nil
}),
job.OneShot("cilium-node-observer", func(ctx context.Context, health cell.HealthReporter) (err error) {
for ev := range c.CiliumNodeResource.Events(ctx) {
switch ev.Kind {
case resource.Upsert:
// Set the local CiliumNode.
c.LocalCiliumNode = ev.Object
// Signal the reconciliation logic.
c.Sig.Event(struct{}{})
}
ev.Done(nil)
}
return nil
}),
job.OneShot("bgp-controller", func(ctx context.Context, health cell.HealthReporter) (err error) {
// initialize PolicyLister used in the controller
policyStore, err := c.PolicyResource.Store(ctx)
if err != nil {
return fmt.Errorf("error creating CiliumBGPPeeringPolicy resource store: %w", err)
}
c.PolicyLister = policyListerFunc(func() ([]*v2alpha1api.CiliumBGPPeeringPolicy, error) {
return policyStore.List(), nil
})
// run the controller
c.Run(ctx)
return nil
}, job.WithRetry(3, workqueue.DefaultControllerRateLimiter()), job.WithShutdown()),
)
params.Lifecycle.Append(jobGroup)
return c, nil
}
// Run places the Controller into its control loop.
//
// When new events trigger a signal the control loop will be evaluated.
//
// A cancel of the provided ctx will kill the control loop along with the running
// informers.
func (c *Controller) Run(ctx context.Context) {
var (
l = log.WithFields(logrus.Fields{
"component": "Controller.Run",
})
)
// add an initial signal to kick things off
c.Sig.Event(struct{}{})
l.Info("Cilium BGP Control Plane Controller now running...")
for {
select {
case <-ctx.Done():
l.Info("Cilium BGP Control Plane Controller shut down")
return
case <-c.Sig.Sig:
l.Info("Cilium BGP Control Plane Controller woken for reconciliation")
if err := c.Reconcile(ctx); err != nil {
l.WithError(err).Error("Encountered error during reconciliation")
} else {
l.Debug("Successfully completed reconciliation")
}
}
}
}
// PolicySelection returns a CiliumBGPPeeringPolicy which applies to the provided
// *corev1.Node, enforced by a set of policy selection rules.
//
// Policy selection follows the following rules:
// - A policy matches a node if said policy's "nodeSelector" field matches
// the node's labels. If "nodeSelector" is omitted, it is unconditionally
// selected.
// - If (N > 1) policies match the provided *corev1.Node an error is returned.
// only a single policy may apply to a node to avoid ambiguity at this stage
// of development.
func PolicySelection(ctx context.Context, labels map[string]string, policies []*v2alpha1api.CiliumBGPPeeringPolicy) (*v2alpha1api.CiliumBGPPeeringPolicy, error) {
var (
l = log.WithFields(logrus.Fields{
"component": "PolicySelection",
})
// determine which policies match our node's labels.
selectedPolicy *v2alpha1api.CiliumBGPPeeringPolicy
slimLabels = slimlabels.Set(labels)
)
// range over policies and see if any match this node's labels.
//
// for now, only a single BGP policy can be applied to a node. if more than
// one policy applies to a node, we disconnect from all BGP peers and log
// an error.
for _, policy := range policies {
var selected bool
l.WithFields(logrus.Fields{
"policyName": policy.Name,
"nodeLabels": slimLabels,
"policyNodeSelector": policy.Spec.NodeSelector.String(),
}).Debug("Comparing BGP policy node selector with node's labels")
if policy.Spec.NodeSelector == nil {
selected = true
} else {
nodeSelector, err := slimmetav1.LabelSelectorAsSelector(policy.Spec.NodeSelector)
if err != nil {
l.WithError(err).Error("Failed to convert CiliumBGPPeeringPolicy's NodeSelector to a label.Selector interface")
continue
}
if nodeSelector.Matches(slimLabels) {
selected = true
}
}
if selected {
if selectedPolicy != nil {
return nil, ErrMultiplePolicies
}
selectedPolicy = policy
}
}
return selectedPolicy, nil
}
// Reconcile is the control loop for the Controller.
//
// Reconcile will be invoked when one or more event sources trigger a signal
// via the Controller's Signaler structure.
//
// On signal, Reconcile will obtain the state of the world necessary to drive
// the BGP control plane toward any new BGP peering policies.
//
// Reconcile will only allow a single CiliumBGPPeeringPolicy to apply to the
// node its running on.
func (c *Controller) Reconcile(ctx context.Context) error {
var (
l = log.WithFields(logrus.Fields{
"component": "Controller.Reconcile",
})
)
if c.LocalCiliumNode == nil {
return fmt.Errorf("attempted reconciliation with nil local CiliumNode")
}
if c.PolicyLister == nil {
return fmt.Errorf("attempted reconciliation with nil PolicyLister")
}
// retrieve all CiliumBGPPeeringPolicies
policies, err := c.PolicyLister.List()
if err != nil {
return fmt.Errorf("failed to list CiliumBGPPeeringPolicies")
}
l.WithField("count", len(policies)).Debug("Successfully listed CiliumBGPPeeringPolicies")
// perform policy selection based on node.
labels := c.LocalCiliumNode.Labels
policy, err := PolicySelection(ctx, labels, policies)
if err != nil {
l.WithError(err).Error("Policy selection failed")
c.FullWithdrawal(ctx)
return err
}
if policy == nil {
// no policy was discovered, tell router manager to withdrawal peers if
// they are configured.
l.Debug("No BGP peering policy applies to this node, any existing BGP sessions will be removed.")
c.FullWithdrawal(ctx)
return nil
}
// apply policy defaults to have consistent default config across sub-systems
policy = policy.DeepCopy() // deepcopy to not modify the policy object in store
policy.SetDefaults()
err = c.validatePolicy(policy)
if err != nil {
return fmt.Errorf("invalid BGP peering policy %s: %w", policy.Name, err)
}
// call bgp sub-systems required to apply this policy's BGP topology.
l.Debug("Asking configured BGPRouterManager to configure peering")
if err := c.BGPMgr.ConfigurePeers(ctx, policy, c.LocalCiliumNode); err != nil {
return fmt.Errorf("failed to configure BGP peers, cannot apply BGP peering policy: %w", err)
}
return nil
}
// FullWithdrawal will instruct the configured BGPRouterManager to withdraw all
// BGP servers and peers.
func (c *Controller) FullWithdrawal(ctx context.Context) {
_ = c.BGPMgr.ConfigurePeers(ctx, nil, nil) // cannot fail, no need for error handling
}
// validatePolicy validates the CiliumBGPPeeringPolicy.
// The validation is normally done by kube-apiserver (based on CRD validation markers),
// this validates only those constraints that cannot be enforced by them.
func (c *Controller) validatePolicy(policy *v2alpha1api.CiliumBGPPeeringPolicy) error {
for _, r := range policy.Spec.VirtualRouters {
for _, n := range r.Neighbors {
if err := n.Validate(); err != nil {
return err
}
}
}
return nil
}