/
bootstraptoken.go
376 lines (331 loc) · 14.1 KB
/
bootstraptoken.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
/*
Copyright 2022 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package bootstraptoken
import (
"context"
"crypto/x509"
"fmt"
"sort"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
kubeclient "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
clientcertutil "k8s.io/client-go/util/cert"
bootstrapapi "k8s.io/cluster-bootstrap/token/api"
bootstraputil "k8s.io/cluster-bootstrap/token/util"
bootstrapsecretutil "k8s.io/cluster-bootstrap/util/secrets"
"k8s.io/klog/v2"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
"github.com/karmada-io/karmada/pkg/util/lifted/pubkeypin"
)
const (
// When a token is matched with 'BootstrapTokenPattern', the size of validated substrings returned by
// regexp functions which contains 'Submatch' in their names will be 3.
// Submatch 0 is the match of the entire expression, submatch 1 is
// the match of the first parenthesized subexpression, and so on.
// e.g.:
// result := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch("abcdef.1234567890123456")
// result == []string{"abcdef.1234567890123456","abcdef","1234567890123456"}
// len(result) == 3
validatedSubstringsSize = 3
// DefaultTokenDuration specifies the default amount of time that a bootstrap token will be valid
// Default behaviour is 24 hours
DefaultTokenDuration = 24 * time.Hour
)
var (
// DefaultUsages is the default usages of bootstrap token
DefaultUsages = bootstrapapi.KnownTokenUsages
// DefaultGroups is the default groups of bootstrap token
DefaultGroups = []string{"system:bootstrappers:karmada:default-cluster-token"}
)
// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster
type BootstrapToken struct {
// Token is used for establishing bidirectional trust between clusters and karmada-control-plane.
// Used for joining clusters to the karmada-control-plane.
Token *Token
// Description sets a human-friendly message why this token exists and what it's used
// for, so other administrators can know its purpose.
// +optional
Description string
// TTL defines the time to live for this token. Defaults to 24h.
// Expires and TTL are mutually exclusive.
// +optional
TTL *metav1.Duration
// Expires specifies the timestamp when this token expires. Defaults to being set
// dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive.
// +optional
Expires *metav1.Time
// Usages describes the ways in which this token can be used. Can by default be used
// for establishing bidirectional trust, but that can be changed here.
// +optional
Usages []string
// Groups specifies the extra groups that this token will authenticate as when/if
// used for authentication
// +optional
Groups []string
}
// Token is a token of the format abcdef.abcdef0123456789 that is used
// for both validation of the practically of the API server from a joining cluster's point
// of view and as an authentication method for the cluster in the bootstrap phase of
// "karmadactl join". This token is and should be short-lived
type Token struct {
ID string
Secret string
}
// GenerateRegisterCommand generate register command that will be printed
func GenerateRegisterCommand(kubeConfig, parentCommand, token string, karmadaContext string) (string, error) {
klog.V(1).Info("Print register command")
// load the kubeconfig file to get the CA certificate and endpoint
config, err := clientcmd.LoadFromFile(kubeConfig)
if err != nil {
return "", fmt.Errorf("failed to load kubeconfig, err: %w", err)
}
// load the cluster config with the given karmada-context
clusterConfig := GetClusterFromKubeConfig(config, karmadaContext)
if clusterConfig == nil {
return "", fmt.Errorf("failed to get default cluster config")
}
// load CA certificates from the kubeconfig (either from PEM data or by file path)
var caCerts []*x509.Certificate
if clusterConfig.CertificateAuthorityData != nil {
caCerts, err = clientcertutil.ParseCertsPEM(clusterConfig.CertificateAuthorityData)
if err != nil {
return "", fmt.Errorf("failed to parse CA certificate from kubeconfig, err: %w", err)
}
} else if clusterConfig.CertificateAuthority != "" {
caCerts, err = clientcertutil.CertsFromFile(clusterConfig.CertificateAuthority)
if err != nil {
return "", fmt.Errorf("failed to load CA certificate referenced by kubeconfig, err: %w", err)
}
} else {
return "", fmt.Errorf("no CA certificates found in kubeconfig")
}
// hash all the CA certs and include their public key pins as trusted values
publicKeyPins := make([]string, 0, len(caCerts))
for _, caCert := range caCerts {
publicKeyPins = append(publicKeyPins, pubkeypin.Hash(caCert))
}
return fmt.Sprintf("%s register %s --token %s --discovery-token-ca-cert-hash %s",
parentCommand, strings.Replace(clusterConfig.Server, "https://", "", -1),
token, strings.Join(publicKeyPins, ",")), nil
}
// GetClusterFromKubeConfig returns the Cluster of the specified KubeConfig, if karmada-context unset, it will use the current-context
func GetClusterFromKubeConfig(config *clientcmdapi.Config, karmadaContext string) *clientcmdapi.Cluster {
// If there is an unnamed cluster object, use it
if config.Clusters[""] != nil {
return config.Clusters[""]
}
if karmadaContext == "" {
karmadaContext = config.CurrentContext
}
if config.Contexts[karmadaContext] != nil {
return config.Clusters[config.Contexts[karmadaContext].Cluster]
}
return nil
}
// GenerateRandomBootstrapToken generate random bootstrap token
func GenerateRandomBootstrapToken(ttl *metav1.Duration, description string, groups, usages []string) (*BootstrapToken, error) {
tokenStr, err := bootstraputil.GenerateBootstrapToken()
if err != nil {
return nil, fmt.Errorf("couldn't generate random token, err: %w", err)
}
token, err := NewToken(tokenStr)
if err != nil {
return nil, err
}
bt := &BootstrapToken{
Token: token,
TTL: ttl,
Description: description,
Groups: groups,
Usages: usages,
}
return bt, nil
}
// NewToken converts the given Bootstrap Token as a string
// to the Token object used for serialization/deserialization
// and internal usage. It also automatically validates that the given token
// is of the right format
func NewToken(token string) (*Token, error) {
substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token)
if len(substrs) != validatedSubstringsSize {
return nil, fmt.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern)
}
return &Token{ID: substrs[1], Secret: substrs[2]}, nil
}
// ConvertBootstrapTokenToSecret converts the given BootstrapToken object to its Secret representation that
// may be submitted to the API Server in order to be stored.
func ConvertBootstrapTokenToSecret(bt *BootstrapToken) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: bootstraputil.BootstrapTokenSecretName(bt.Token.ID),
Namespace: metav1.NamespaceSystem,
},
Type: bootstrapapi.SecretTypeBootstrapToken,
Data: encodeTokenSecretData(bt, time.Now()),
}
}
// encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret
// now is passed in order to be able to used in unit testing
func encodeTokenSecretData(token *BootstrapToken, now time.Time) map[string][]byte {
data := map[string][]byte{
bootstrapapi.BootstrapTokenIDKey: []byte(token.Token.ID),
bootstrapapi.BootstrapTokenSecretKey: []byte(token.Token.Secret),
}
if len(token.Description) > 0 {
data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte(token.Description)
}
// If for some strange reason both token.TTL and token.Expires would be set
// (they are mutually exclusive in validation so this shouldn't be the case),
// token.Expires has higher priority, as can be seen in the logic here.
if token.Expires != nil {
// Format the expiration date accordingly
// TODO: This maybe should be a helper function in bootstraputil?
expirationString := token.Expires.Time.UTC().Format(time.RFC3339)
data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString)
} else if token.TTL != nil && token.TTL.Duration > 0 {
// Only if .Expires is unset, TTL might have an effect
// Get the current time, add the specified duration, and format it accordingly
expirationString := now.Add(token.TTL.Duration).UTC().Format(time.RFC3339)
data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString)
}
for _, usage := range token.Usages {
data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true")
}
if len(token.Groups) > 0 {
data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(strings.Join(token.Groups, ","))
}
return data
}
// NewTokenFromIDAndSecret is a wrapper around NewToken
// that allows the caller to specify the ID and Secret separately
func NewTokenFromIDAndSecret(id, secret string) (*Token, error) {
return NewToken(bootstraputil.TokenFromIDAndSecret(id, secret))
}
// GetBootstrapTokenFromSecret returns a BootstrapToken object from the given Secret
func GetBootstrapTokenFromSecret(secret *corev1.Secret) (*BootstrapToken, error) {
// Get the Token ID field from the Secret data
tokenID := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenIDKey)
if len(tokenID) == 0 {
return nil, fmt.Errorf("bootstrap Token Secret has no token-id data: %s", secret.Name)
}
// Enforce the right naming convention
if secret.Name != bootstraputil.BootstrapTokenSecretName(tokenID) {
return nil, fmt.Errorf("bootstrap token name is not of the form '%s(token-id)'. Actual: %q. Expected: %q",
bootstrapapi.BootstrapTokenSecretPrefix, secret.Name, bootstraputil.BootstrapTokenSecretName(tokenID))
}
tokenSecret := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenSecretKey)
if len(tokenSecret) == 0 {
return nil, fmt.Errorf("bootstrap Token Secret has no token-secret data: %s", secret.Name)
}
// Create the Token object based on the ID and Secret
bts, err := NewTokenFromIDAndSecret(tokenID, tokenSecret)
if err != nil {
return nil, fmt.Errorf("bootstrap Token Secret is invalid and couldn't be parsed, err: %w", err)
}
// Get the description (if any) from the Secret
description := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenDescriptionKey)
// Expiration time is optional, if not specified this implies the token
// never expires.
secretExpiration := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExpirationKey)
var expires *metav1.Time
if len(secretExpiration) > 0 {
expTime, err := time.Parse(time.RFC3339, secretExpiration)
if err != nil {
return nil, fmt.Errorf("can't parse expiration time of bootstrap token %q, err: %w", secret.Name, err)
}
expires = &metav1.Time{Time: expTime}
}
// Build an usages string slice from the Secret data
var usages []string
for k, v := range secret.Data {
// Skip all fields that don't include this prefix
if !strings.HasPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix) {
continue
}
// Skip those that don't have this usage set to true
if string(v) != "true" {
continue
}
usages = append(usages, strings.TrimPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix))
}
// Only sort the slice if defined
if usages != nil {
sort.Strings(usages)
}
// Get the extra groups information from the Secret
// It's done this way to make .Groups be nil in case there is no items, rather than an
// empty slice or an empty slice with a "" string only
var groups []string
groupsString := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExtraGroupsKey)
g := strings.Split(groupsString, ",")
if len(g) > 0 && len(g[0]) > 0 {
groups = g
}
return &BootstrapToken{
Token: bts,
Description: description,
Expires: expires,
Usages: usages,
Groups: groups,
}, nil
}
// CreateNewToken tries to create a token and fails if one with the same ID already exists
func CreateNewToken(client kubeclient.Interface, token *BootstrapToken) error {
return UpdateOrCreateToken(client, true, token)
}
// UpdateOrCreateToken attempts to update a token with the given ID, or create if it does not already exist.
func UpdateOrCreateToken(client kubeclient.Interface, failIfExists bool, token *BootstrapToken) error {
secretName := bootstraputil.BootstrapTokenSecretName(token.Token.ID)
secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), secretName, metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
if secret != nil && err == nil && failIfExists {
return fmt.Errorf("a token with id %q already exists", token.Token.ID)
}
updatedOrNewSecret := ConvertBootstrapTokenToSecret(token)
// Try to create or update the token with an exponential backoff
err = TryRunCommand(func() error {
if err := cmdutil.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil {
return fmt.Errorf("failed to create or update bootstrap token with name %s, err: %w", secretName, err)
}
return nil
}, 5)
if err != nil {
return err
}
return nil
}
// TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned
func TryRunCommand(f func() error, failureThreshold int) error {
backoff := wait.Backoff{
Duration: 5 * time.Second,
Factor: 2, // double the timeout for every failure
Steps: failureThreshold,
}
return wait.ExponentialBackoff(backoff, func() (bool, error) {
err := f()
if err != nil {
// Retry until the timeout
return false, nil
}
// The last f() call was a success, return cleanly
return true, nil
})
}