-
Notifications
You must be signed in to change notification settings - Fork 0
/
version.go
323 lines (276 loc) · 10.5 KB
/
version.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium
// Package version keeps track of the Kubernetes version the client is
// connected to
package version
import (
"context"
"fmt"
"github.com/blang/semver/v4"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
k8sconfig "github.com/cilium/cilium/pkg/k8s/config"
"github.com/cilium/cilium/pkg/lock"
"github.com/cilium/cilium/pkg/logging"
"github.com/cilium/cilium/pkg/logging/logfields"
"github.com/cilium/cilium/pkg/versioncheck"
)
var log = logging.DefaultLogger.WithField(logfields.LogSubsys, "k8s")
// ServerCapabilities is a list of server capabilities derived based on
// version, the Kubernetes discovery API, or probing of individual API
// endpoints.
type ServerCapabilities struct {
// MinimalVersionMet is true when the minimal version of Kubernetes
// required to run Cilium has been met
MinimalVersionMet bool
// EndpointSlice is the ability of k8s server to support endpoint slices
EndpointSlice bool
// EndpointSliceV1 is the ability of k8s server to support endpoint slices
// v1. This version was introduced in K8s v1.21.0.
EndpointSliceV1 bool
// LeasesResourceLock is the ability of K8s server to support Lease type
// from coordination.k8s.io/v1 API for leader election purposes(currently only in operator).
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#lease-v1-coordination-k8s-io
//
// This capability was introduced in K8s version 1.14, prior to which
// we don't support HA mode for the cilium-operator.
LeasesResourceLock bool
// APIExtensionsV1CRD is set to true when the K8s server supports
// apiextensions/v1 CRDs. TODO: Add link to docs
//
// This capability was introduced in K8s version 1.16, prior to which
// apiextensions/v1beta1 CRDs were used exclusively.
APIExtensionsV1CRD bool
}
type cachedVersion struct {
mutex lock.RWMutex
capabilities ServerCapabilities
version semver.Version
}
const (
// MinimalVersionConstraint is the minimal version that Cilium supports to
// run kubernetes.
MinimalVersionConstraint = "1.16.0"
)
var (
cached = cachedVersion{}
discoveryAPIGroupV1beta1 = "discovery.k8s.io/v1beta1"
discoveryAPIGroupV1 = "discovery.k8s.io/v1"
coordinationV1APIGroup = "coordination.k8s.io/v1"
endpointSliceKind = "EndpointSlice"
leaseKind = "Lease"
// Constraint to check support for Lease type from coordination.k8s.io/v1.
// Support for Lease resource was introduced in K8s version 1.14.
isGEThanLeaseSupportConstraint = versioncheck.MustCompile(">=1.14.0")
// Constraint to check support for apiextensions/v1 CRD types. Support for
// v1 CRDs was introduced in K8s version 1.16.
isGEThanAPIExtensionsV1CRD = versioncheck.MustCompile(">=1.16.0")
// Constraint to check support for discovery/v1 types. Support for v1
// discovery was introduced in K8s version 1.21.
isGEThanAPIDiscoveryV1 = versioncheck.MustCompile(">=1.21.0")
// isGEThanMinimalVersionConstraint is the minimal version required to run
// Cilium
isGEThanMinimalVersionConstraint = versioncheck.MustCompile(">=" + MinimalVersionConstraint)
)
// Version returns the version of the Kubernetes apiserver
func Version() semver.Version {
cached.mutex.RLock()
c := cached.version
cached.mutex.RUnlock()
return c
}
// Capabilities returns the capabilities of the Kubernetes apiserver
func Capabilities() ServerCapabilities {
cached.mutex.RLock()
c := cached.capabilities
cached.mutex.RUnlock()
return c
}
func updateVersion(version semver.Version) {
cached.mutex.Lock()
defer cached.mutex.Unlock()
cached.version = version
cached.capabilities.MinimalVersionMet = isGEThanMinimalVersionConstraint(version)
cached.capabilities.APIExtensionsV1CRD = isGEThanAPIExtensionsV1CRD(version)
cached.capabilities.EndpointSliceV1 = isGEThanAPIDiscoveryV1(version)
}
func updateServerGroupsAndResources(apiResourceLists []*metav1.APIResourceList) {
cached.mutex.Lock()
defer cached.mutex.Unlock()
cached.capabilities.EndpointSlice = false
cached.capabilities.EndpointSliceV1 = false
cached.capabilities.LeasesResourceLock = false
for _, rscList := range apiResourceLists {
if rscList.GroupVersion == discoveryAPIGroupV1beta1 {
for _, rsc := range rscList.APIResources {
if rsc.Kind == endpointSliceKind {
cached.capabilities.EndpointSlice = true
break
}
}
}
if rscList.GroupVersion == discoveryAPIGroupV1 {
for _, rsc := range rscList.APIResources {
if rsc.Kind == endpointSliceKind {
cached.capabilities.EndpointSlice = true
cached.capabilities.EndpointSliceV1 = true
break
}
}
}
if rscList.GroupVersion == coordinationV1APIGroup {
for _, rsc := range rscList.APIResources {
if rsc.Kind == leaseKind {
cached.capabilities.LeasesResourceLock = true
break
}
}
}
}
}
// Force forces the use of a specific version
func Force(version string) error {
ver, err := versioncheck.Version(version)
if err != nil {
return err
}
updateVersion(ver)
return nil
}
func endpointSlicesFallbackDiscovery(client kubernetes.Interface) error {
// If a k8s version with discovery v1 is used, then do not even bother
// checking for v1beta1
cached.mutex.Lock()
if cached.capabilities.EndpointSliceV1 {
cached.capabilities.EndpointSlice = true
cached.mutex.Unlock()
return nil
}
cached.mutex.Unlock()
// Discovery of API groups requires the API services of the apiserver to be
// healthy. Such API services can depend on the readiness of regular pods
// which require Cilium to function correctly. By treating failure to
// discover API groups as fatal, a critial loop can be entered in which
// Cilium cannot start because the API groups can't be discovered.
//
// Here we acknowledge the lack of discovery ability as non Fatal and fall back to probing
// the API directly.
_, err := client.DiscoveryV1beta1().EndpointSlices("default").Get(context.TODO(), "kubernetes", metav1.GetOptions{})
if err == nil {
cached.mutex.Lock()
cached.capabilities.EndpointSlice = true
cached.mutex.Unlock()
return nil
}
if errors.IsNotFound(err) {
log.WithError(err).Info("Unable to retrieve EndpointSlices for default/kubernetes. Disabling EndpointSlices")
// StatusNotFound is a safe error, EndpointSlices are
// disabled and the agent can continue.
return nil
}
// Unknown error, we can't derive whether to enable or disable
// EndpointSlices and need to error out.
return fmt.Errorf("unable to validate EndpointSlices support: %s", err)
}
func leasesFallbackDiscovery(client kubernetes.Interface, conf k8sconfig.Configuration) error {
// K8sEnableLeasesFallbackDiscovery is used to fallback leases discovery to directly
// probing the API when we cannot discover API groups.
// We require to check for Leases capabilities in operator only, which uses Leases
// for leader election purposes in HA mode.
if !conf.K8sLeasesFallbackDiscoveryEnabled() {
log.Debugf("Skipping Leases support fallback discovery")
return nil
}
cached.mutex.RLock()
// Here we check if we are running a K8s version that has support for Leases.
if !isGEThanLeaseSupportConstraint(cached.version) {
cached.mutex.RUnlock()
return nil
}
cached.mutex.RUnlock()
// Similar to endpointSlicesFallbackDiscovery here we fallback to probing the Kubernetes
// API directly. `kube-controller-manager` creates a lease in the kube-system namespace
// and here we try and see if that Lease exists.
_, err := client.CoordinationV1().Leases("kube-system").Get(context.TODO(), "kube-controller-manager", metav1.GetOptions{})
if err == nil {
cached.mutex.Lock()
cached.capabilities.LeasesResourceLock = true
cached.mutex.Unlock()
return nil
}
if errors.IsNotFound(err) {
log.WithError(err).Info("Unable to retrieve Leases for kube-controller-manager. Disabling LeasesResourceLock")
// StatusNotFound is a safe error, Leases are
// disabled and the agent can continue
return nil
}
// Unknown error, we can't derive whether to enable or disable
// LeasesResourceLock and need to error out
return fmt.Errorf("unable to validate LeasesResourceLock support: %s", err)
}
func updateK8sServerVersion(client kubernetes.Interface) error {
var ver semver.Version
sv, err := client.Discovery().ServerVersion()
if err != nil {
return err
}
// Try GitVersion first. In case of error fallback to MajorMinor
if sv.GitVersion != "" {
// This is a string like "v1.9.0"
ver, err = versioncheck.Version(sv.GitVersion)
if err == nil {
updateVersion(ver)
return nil
}
}
if sv.Major != "" && sv.Minor != "" {
ver, err = versioncheck.Version(fmt.Sprintf("%s.%s", sv.Major, sv.Minor))
if err == nil {
updateVersion(ver)
return nil
}
}
return fmt.Errorf("cannot parse k8s server version from %+v: %s", sv, err)
}
// Update retrieves the version of the Kubernetes apiserver and derives the
// capabilities. This function must be called after connectivity to the
// apiserver has been established.
//
// Discovery of capabilities only works if the discovery API of the apiserver
// is functional. If it is not available, a warning is logged and the discovery
// falls back to probing individual API endpoints.
func Update(client kubernetes.Interface, conf k8sconfig.Configuration) error {
err := updateK8sServerVersion(client)
if err != nil {
return err
}
if conf.K8sAPIDiscoveryEnabled() {
// Discovery of API groups requires the API services of the
// apiserver to be healthy. Such API services can depend on the
// readiness of regular pods which require Cilium to function
// correctly. By treating failure to discover API groups as
// fatal, a critical loop can be entered in which Cilium cannot
// start because the API groups can't be discovered and th API
// groups will only become discoverable once Cilium is up.
_, apiResourceLists, err := client.Discovery().ServerGroupsAndResources()
if err != nil {
// It doesn't make sense to retry the retrieval of this
// information at a later point because the capabilities are
// primiarly used while the agent is starting up. Instead, fall
// back to probing API endpoints directly.
log.WithError(err).Warning("Unable to discover API groups and resources")
if err := endpointSlicesFallbackDiscovery(client); err != nil {
return err
}
return leasesFallbackDiscovery(client, conf)
}
updateServerGroupsAndResources(apiResourceLists)
} else {
if err := endpointSlicesFallbackDiscovery(client); err != nil {
return err
}
return leasesFallbackDiscovery(client, conf)
}
return nil
}