-
Notifications
You must be signed in to change notification settings - Fork 38.7k
/
common.go
364 lines (313 loc) · 13.8 KB
/
common.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"bytes"
"fmt"
"net"
"reflect"
"strings"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
netutil "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/version"
apimachineryversion "k8s.io/apimachinery/pkg/version"
componentversion "k8s.io/component-base/version"
"k8s.io/klog/v2"
netutils "k8s.io/utils/net"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1old "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
// MarshalKubeadmConfigObject marshals an Object registered in the kubeadm scheme. If the object is a InitConfiguration or ClusterConfiguration, some extra logic is run
func MarshalKubeadmConfigObject(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) {
switch internalcfg := obj.(type) {
case *kubeadmapi.InitConfiguration:
return MarshalInitConfigurationToBytes(internalcfg, gv)
default:
return kubeadmutil.MarshalToYamlForCodecs(obj, gv, kubeadmscheme.Codecs)
}
}
// validateSupportedVersion checks if the supplied GroupVersion is not on the lists of old unsupported or deprecated GVs.
// If it is, an error is returned.
func validateSupportedVersion(gv schema.GroupVersion, allowDeprecated, allowExperimental bool) error {
// The support matrix will look something like this now and in the future:
// v1.10 and earlier: v1alpha1
// v1.11: v1alpha1 read-only, writes only v1alpha2 config
// v1.12: v1alpha2 read-only, writes only v1alpha3 config. Errors if the user tries to use v1alpha1
// v1.13: v1alpha3 read-only, writes only v1beta1 config. Errors if the user tries to use v1alpha1 or v1alpha2
// v1.14: v1alpha3 convert only, writes only v1beta1 config. Errors if the user tries to use v1alpha1 or v1alpha2
// v1.15: v1beta1 read-only, writes only v1beta2 config. Errors if the user tries to use v1alpha1, v1alpha2 or v1alpha3
// v1.22: v1beta2 read-only, writes only v1beta3 config. Errors if the user tries to use v1beta1 and older
// v1.27: only v1beta3 config. Errors if the user tries to use v1beta2 and older
oldKnownAPIVersions := map[string]string{
"kubeadm.k8s.io/v1alpha1": "v1.11",
"kubeadm.k8s.io/v1alpha2": "v1.12",
"kubeadm.k8s.io/v1alpha3": "v1.14",
"kubeadm.k8s.io/v1beta1": "v1.15",
"kubeadm.k8s.io/v1beta2": "v1.22",
}
// v1.28: v1beta4 is released as experimental
experimentalAPIVersions := map[string]string{
// TODO: https://github.com/kubernetes/kubeadm/issues/2890
// remove this from experimental once v1beta4 is released
"kubeadm.k8s.io/v1beta4": "v1.28",
}
// Deprecated API versions are supported by us, but can only be used for migration.
deprecatedAPIVersions := map[string]struct{}{}
gvString := gv.String()
if useKubeadmVersion := oldKnownAPIVersions[gvString]; useKubeadmVersion != "" {
return errors.Errorf("your configuration file uses an old API spec: %q. Please use kubeadm %s instead and run 'kubeadm config migrate --old-config old.yaml --new-config new.yaml', which will write the new, similar spec using a newer API version.", gv.String(), useKubeadmVersion)
}
if _, present := deprecatedAPIVersions[gvString]; present && !allowDeprecated {
klog.Warningf("your configuration file uses a deprecated API spec: %q. Please use 'kubeadm config migrate --old-config old.yaml --new-config new.yaml', which will write the new, similar spec using a newer API version.", gv.String())
}
if _, present := experimentalAPIVersions[gvString]; present && !allowExperimental {
return errors.Errorf("experimental API spec: %q is not allowed. You can use the --%s flag if the command supports it.", gv, options.AllowExperimentalAPI)
}
return nil
}
// NormalizeKubernetesVersion resolves version labels, sets alternative
// image registry if requested for CI builds, and validates minimal
// version that kubeadm SetInitDynamicDefaultssupports.
func NormalizeKubernetesVersion(cfg *kubeadmapi.ClusterConfiguration) error {
isCIVersion := kubeadmutil.KubernetesIsCIVersion(cfg.KubernetesVersion)
// Requested version is automatic CI build, thus use KubernetesCI Image Repository for core images
if isCIVersion && cfg.ImageRepository == kubeadmapiv1.DefaultImageRepository {
cfg.CIImageRepository = constants.DefaultCIImageRepository
}
// Parse and validate the version argument and resolve possible CI version labels
ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion)
if err != nil {
return err
}
// Requested version is automatic CI build, thus mark CIKubernetesVersion as `ci/<resolved-version>`
if isCIVersion {
cfg.CIKubernetesVersion = fmt.Sprintf("%s%s", constants.CIKubernetesVersionPrefix, ver)
}
cfg.KubernetesVersion = ver
// Parse the given kubernetes version and make sure it's higher than the lowest supported
k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return errors.Wrapf(err, "couldn't parse Kubernetes version %q", cfg.KubernetesVersion)
}
// During the k8s release process, a kubeadm version in the main branch could be 1.23.0-pre,
// while the 1.22.0 version is not released yet. The MinimumControlPlaneVersion validation
// in such a case will not pass, since the value of MinimumControlPlaneVersion would be
// calculated as kubeadm version - 1 (1.22) and k8sVersion would still be at 1.21.x
// (fetched from the 'stable' marker). Handle this case by only showing a warning.
mcpVersion := constants.MinimumControlPlaneVersion
versionInfo := componentversion.Get()
if isKubeadmPrereleaseVersion(&versionInfo, k8sVersion, mcpVersion) {
klog.V(1).Infof("WARNING: tolerating control plane version %s as a pre-release version", cfg.KubernetesVersion)
return nil
}
// If not a pre-release version, handle the validation normally.
if k8sVersion.LessThan(mcpVersion) {
return errors.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s",
mcpVersion, cfg.KubernetesVersion)
}
return nil
}
// LowercaseSANs can be used to force all SANs to be lowercase so it passes IsDNS1123Subdomain
func LowercaseSANs(sans []string) {
for i, san := range sans {
lowercase := strings.ToLower(san)
if lowercase != san {
klog.V(1).Infof("lowercasing SAN %q to %q", san, lowercase)
sans[i] = lowercase
}
}
}
// VerifyAPIServerBindAddress can be used to verify if a bind address for the API Server is 0.0.0.0,
// in which case this address is not valid and should not be used.
func VerifyAPIServerBindAddress(address string) error {
ip := netutils.ParseIPSloppy(address)
if ip == nil {
return errors.Errorf("cannot parse IP address: %s", address)
}
// There are users with network setups where default routes are present, but network interfaces
// use only link-local addresses (e.g. as described in RFC5549).
// In many cases that matching global unicast IP address can be found on loopback interface,
// so kubeadm allows users to specify address=Loopback for handling supporting the scenario above.
// Nb. SetAPIEndpointDynamicDefaults will try to translate loopback to a valid address afterwards
if ip.IsLoopback() {
return nil
}
if !ip.IsGlobalUnicast() {
return errors.Errorf("cannot use %q as the bind address for the API Server", address)
}
return nil
}
// ChooseAPIServerBindAddress is a wrapper for netutil.ResolveBindAddress that also handles
// the case where no default routes were found and an IP for the API server could not be obtained.
func ChooseAPIServerBindAddress(bindAddress net.IP) (net.IP, error) {
ip, err := netutil.ResolveBindAddress(bindAddress)
if err != nil {
if netutil.IsNoRoutesError(err) {
klog.Warningf("WARNING: could not obtain a bind address for the API Server: %v; using: %s", err, constants.DefaultAPIServerBindAddress)
defaultIP := netutils.ParseIPSloppy(constants.DefaultAPIServerBindAddress)
if defaultIP == nil {
return nil, errors.Errorf("cannot parse default IP address: %s", constants.DefaultAPIServerBindAddress)
}
return defaultIP, nil
}
return nil, err
}
if bindAddress != nil && !bindAddress.IsUnspecified() && !reflect.DeepEqual(ip, bindAddress) {
klog.Warningf("WARNING: overriding requested API server bind address: requested %q, actual %q", bindAddress, ip)
}
return ip, nil
}
// validateKnownGVKs takes a list of GVKs and verifies if they are known in kubeadm or component config schemes
func validateKnownGVKs(gvks []schema.GroupVersionKind) error {
var unknown []schema.GroupVersionKind
schemes := []*runtime.Scheme{
kubeadmscheme.Scheme,
componentconfigs.Scheme,
}
for _, gvk := range gvks {
var scheme *runtime.Scheme
// Skip legacy known GVs so that they don't return errors.
// This makes the function return errors only for GVs that where never known.
if err := validateSupportedVersion(gvk.GroupVersion(), true, true); err != nil {
continue
}
for _, s := range schemes {
if _, err := s.New(gvk); err == nil {
scheme = s
break
}
}
if scheme == nil {
unknown = append(unknown, gvk)
}
}
if len(unknown) > 0 {
return errors.Errorf("unknown configuration APIs: %#v", unknown)
}
return nil
}
// MigrateOldConfig migrates an old configuration from a byte slice into a new one (returned again as a byte slice).
// Only kubeadm kinds are migrated.
func MigrateOldConfig(oldConfig []byte, allowExperimental bool) ([]byte, error) {
newConfig := [][]byte{}
gvkmap, err := kubeadmutil.SplitYAMLDocuments(oldConfig)
if err != nil {
return []byte{}, err
}
gvks := []schema.GroupVersionKind{}
for gvk := range gvkmap {
gvks = append(gvks, gvk)
}
if err := validateKnownGVKs(gvks); err != nil {
return []byte{}, err
}
gv := kubeadmapiv1old.SchemeGroupVersion
if allowExperimental {
gv = kubeadmapiv1.SchemeGroupVersion
}
// Migrate InitConfiguration and ClusterConfiguration if there are any in the config
if kubeadmutil.GroupVersionKindsHasInitConfiguration(gvks...) || kubeadmutil.GroupVersionKindsHasClusterConfiguration(gvks...) {
o, err := documentMapToInitConfiguration(gvkmap, true, allowExperimental, true)
if err != nil {
return []byte{}, err
}
b, err := MarshalKubeadmConfigObject(o, gv)
if err != nil {
return []byte{}, err
}
newConfig = append(newConfig, b)
}
// Migrate JoinConfiguration if there is any
if kubeadmutil.GroupVersionKindsHasJoinConfiguration(gvks...) {
o, err := documentMapToJoinConfiguration(gvkmap, true, allowExperimental, true)
if err != nil {
return []byte{}, err
}
b, err := MarshalKubeadmConfigObject(o, gv)
if err != nil {
return []byte{}, err
}
newConfig = append(newConfig, b)
}
// Migrate ResetConfiguration if there is any
if kubeadmutil.GroupVersionKindsHasResetConfiguration(gvks...) {
o, err := documentMapToResetConfiguration(gvkmap, true, allowExperimental, true)
if err != nil {
return []byte{}, err
}
b, err := MarshalKubeadmConfigObject(o, gv)
if err != nil {
return []byte{}, err
}
newConfig = append(newConfig, b)
}
return bytes.Join(newConfig, []byte(constants.YAMLDocumentSeparator)), nil
}
// ValidateConfig takes a byte slice containing a kubeadm configuration and performs conversion
// to internal types and validation.
func ValidateConfig(config []byte, allowExperimental bool) error {
gvkmap, err := kubeadmutil.SplitYAMLDocuments(config)
if err != nil {
return err
}
gvks := []schema.GroupVersionKind{}
for gvk := range gvkmap {
gvks = append(gvks, gvk)
}
if err := validateKnownGVKs(gvks); err != nil {
return err
}
// Validate InitConfiguration and ClusterConfiguration if there are any in the config
if kubeadmutil.GroupVersionKindsHasInitConfiguration(gvks...) || kubeadmutil.GroupVersionKindsHasClusterConfiguration(gvks...) {
if _, err := documentMapToInitConfiguration(gvkmap, true, allowExperimental, true); err != nil {
return err
}
}
// Validate JoinConfiguration if there is any
if kubeadmutil.GroupVersionKindsHasJoinConfiguration(gvks...) {
if _, err := documentMapToJoinConfiguration(gvkmap, true, allowExperimental, true); err != nil {
return err
}
}
// Validate ResetConfiguration if there is any
if kubeadmutil.GroupVersionKindsHasResetConfiguration(gvks...) {
if _, err := documentMapToResetConfiguration(gvkmap, true, allowExperimental, true); err != nil {
return err
}
}
return nil
}
// isKubeadmPrereleaseVersion returns true if the kubeadm version is a pre-release version and
// the minimum control plane version is N+2 MINOR version of the given k8sVersion.
func isKubeadmPrereleaseVersion(versionInfo *apimachineryversion.Info, k8sVersion, mcpVersion *version.Version) bool {
if len(versionInfo.Major) != 0 { // Make sure the component version is populated
kubeadmVersion := version.MustParseSemantic(versionInfo.String())
if len(kubeadmVersion.PreRelease()) != 0 { // Only handle this if the kubeadm binary is a pre-release
// After incrementing the k8s MINOR version by one, if this version is equal or greater than the
// MCP version, return true.
v := k8sVersion.WithMinor(k8sVersion.Minor() + 1)
if comp, _ := v.Compare(mcpVersion.String()); comp != -1 {
return true
}
}
}
return false
}