-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support multiple bootstrapkubeconfigs in registration agent.
Signed-off-by: xuezhaojun <zxue@redhat.com>
- Loading branch information
1 parent
0882f6d
commit 649f468
Showing
14 changed files
with
990 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 106 additions & 0 deletions
106
pkg/registration/spoke/bootstrapkubeconfigsmanager/bootstrapkubeconfig.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| package bootstrapkubeconfigsmanager | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| ) | ||
|
|
||
| type boostrapKubeConfigStatus string | ||
|
|
||
| const ( | ||
| boostrapKubeConfigStatusInValid boostrapKubeConfigStatus = "InValid" | ||
| boostrapKubeConfigStatusValid boostrapKubeConfigStatus = "Valid" | ||
| ) | ||
|
|
||
| // bootstrapKubeConfig represents a bootstrap kubeconfig that agent can use to bootstrap a managed cluster. | ||
| type boostrapKubeConfig interface { | ||
| // Name returns the name of the bootstrap kubeconfig. It helps to identify the bootstrap kubeconfig. | ||
| Name() string | ||
|
|
||
| // KubeConfigData returns the kubeconfig data of the bootstrap kubeconfig. | ||
| KubeConfigData() ([]byte, error) | ||
|
|
||
| // Status returns the status of the bootstrap kubeconfig. | ||
| // A bootstrap kubeconfig has two status: Valid and InValid. | ||
| Status() (boostrapKubeConfigStatus, error) | ||
|
|
||
| // Fail means at the time t, the bootstrap kubeconfig failed to connect to the hub cluster. | ||
| Fail(t time.Time) error | ||
| } | ||
|
|
||
| var _ boostrapKubeConfig = &boostrapKubeConfigSecretImpl{} | ||
|
|
||
| const ( | ||
| // BootstrapKubeconfigFailedTimeAnnotationKey represents the time when the bootstrap kubeconfig failed | ||
| BootstrapKubeconfigFailedTimeAnnotationKey = "agent.open-cluster-management.io/bootstrap-kubeconfig-failed-time" | ||
| ) | ||
|
|
||
| type boostrapKubeConfigSecretImpl struct { | ||
| secretName string | ||
| secretNamespace string | ||
| skipFailedBootstrapKubeconfigSeconds int32 // if a bootstrap kubeconfig failed, in 3 mins, it can't be used in rebootstrap. | ||
| kubeClient kubernetes.Interface | ||
| } | ||
|
|
||
| func (b *boostrapKubeConfigSecretImpl) Name() string { | ||
| return b.secretName | ||
| } | ||
|
|
||
| func (b *boostrapKubeConfigSecretImpl) KubeConfigData() ([]byte, error) { | ||
| secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{}) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err) | ||
| } | ||
| if secret.Data == nil { | ||
| return nil, fmt.Errorf("the bootstrap kubeconfig secret %s has no data", b.secretName) | ||
| } | ||
| kubeconfigData, ok := secret.Data["kubeconfig"] | ||
| if !ok { | ||
| return nil, fmt.Errorf("the bootstrap kubeconfig secret %s has no kubeconfig data", b.secretName) | ||
| } | ||
| return kubeconfigData, nil | ||
| } | ||
|
|
||
| func (b *boostrapKubeConfigSecretImpl) Status() (boostrapKubeConfigStatus, error) { | ||
| secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{}) | ||
| if err != nil { | ||
| return boostrapKubeConfigStatusInValid, fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err) | ||
| } | ||
|
|
||
| if secret.Annotations == nil { | ||
| return boostrapKubeConfigStatusValid, nil | ||
| } | ||
|
|
||
| now := time.Now() | ||
| if failedTime, ok := secret.Annotations[BootstrapKubeconfigFailedTimeAnnotationKey]; ok { | ||
| failedTimeParsed, err := time.Parse(time.RFC3339, failedTime) | ||
| if err != nil { | ||
| return boostrapKubeConfigStatusInValid, fmt.Errorf("failed to parse the failed time %s of the secret %s: %v", failedTime, secret.Name, err) | ||
| } | ||
| if now.Sub(failedTimeParsed).Seconds() < float64(b.skipFailedBootstrapKubeconfigSeconds) { | ||
| return boostrapKubeConfigStatusInValid, nil | ||
| } | ||
| } | ||
| return boostrapKubeConfigStatusValid, nil | ||
| } | ||
|
|
||
| func (b *boostrapKubeConfigSecretImpl) Fail(t time.Time) error { | ||
| secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{}) | ||
| if err != nil { | ||
| return fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err) | ||
| } | ||
| secretCopy := secret.DeepCopy() | ||
| if secretCopy.Annotations == nil { | ||
| secretCopy.Annotations = make(map[string]string) | ||
| } | ||
| secretCopy.Annotations[BootstrapKubeconfigFailedTimeAnnotationKey] = t.Format(time.RFC3339) | ||
| _, err = b.kubeClient.CoreV1().Secrets(b.secretNamespace).Update(context.Background(), secretCopy, metav1.UpdateOptions{}) | ||
| if err != nil { | ||
| return fmt.Errorf("update the secret %s failed: %v", b.secretName, err) | ||
| } | ||
| return nil | ||
| } |
202 changes: 202 additions & 0 deletions
202
pkg/registration/spoke/bootstrapkubeconfigsmanager/bootstrapkubeconfig_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| package bootstrapkubeconfigsmanager | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| "k8s.io/client-go/kubernetes/fake" | ||
| k8stesting "k8s.io/client-go/testing" | ||
| ) | ||
|
|
||
| func TestBootstrapKubeConfigSecretImpl(t *testing.T) { | ||
| testcases := []struct { | ||
| name string | ||
| mockClient func(failedTime time.Time) *fake.Clientset | ||
| validate func(c boostrapKubeConfig) | ||
| }{ | ||
| { | ||
| name: "Valid BootstrapKubeConfig", | ||
| mockClient: func(_ time.Time) *fake.Clientset { | ||
| // Create a mock secret | ||
| mockSecret := &corev1.Secret{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "mock-secret", | ||
| Namespace: "test-namespace", | ||
| }, | ||
| Data: map[string][]byte{ | ||
| "kubeconfig": []byte("mock-kubeconfig"), | ||
| }, | ||
| } | ||
|
|
||
| // Create a mock kubeClient | ||
| mockKubeClient := &fake.Clientset{} | ||
| mockKubeClient.AddReactor("get", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { | ||
| return true, mockSecret, nil | ||
| }) | ||
| return mockKubeClient | ||
| }, | ||
| validate: func(c boostrapKubeConfig) { | ||
| if c.Name() != "mock-secret" { | ||
| t.Errorf("Expected name %v, but got %v", "mock-secret", c.Name()) | ||
| } | ||
|
|
||
| kubeConfigData, err := c.KubeConfigData() | ||
| if err != nil { | ||
| t.Errorf("Unexpected error: %v", err) | ||
| } | ||
|
|
||
| if string(kubeConfigData) != "mock-kubeconfig" { | ||
| t.Errorf("Expected kubeconfig data %v, but got %v", "mock-kubeconfig", string(kubeConfigData)) | ||
| } | ||
|
|
||
| status, err := c.Status() | ||
| if err != nil { | ||
| t.Errorf("Unexpected error: %v", err) | ||
| } | ||
|
|
||
| if status != boostrapKubeConfigStatusValid { | ||
| t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusValid, status) | ||
| } | ||
| }, | ||
| }, | ||
| { | ||
| name: "Once failed but now valid BootstrapKubeConfig", | ||
| mockClient: func(_ time.Time) *fake.Clientset { | ||
| // Create a mock secret | ||
| mockSecret := &corev1.Secret{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "mock-secret", | ||
| Namespace: "test-namespace", | ||
| Annotations: map[string]string{ | ||
| BootstrapKubeconfigFailedTimeAnnotationKey: "2021-08-01T00:00:00Z", | ||
| }, | ||
| }, | ||
| Data: map[string][]byte{ | ||
| "kubeconfig": []byte("mock-kubeconfig"), | ||
| }, | ||
| } | ||
|
|
||
| // Create a mock kubeClient | ||
| mockKubeClient := &fake.Clientset{} | ||
| mockKubeClient.AddReactor("get", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { | ||
| return true, mockSecret, nil | ||
| }) | ||
| return mockKubeClient | ||
| }, | ||
| validate: func(c boostrapKubeConfig) { | ||
| status, err := c.Status() | ||
| if err != nil { | ||
| t.Errorf("Unexpected error: %v", err) | ||
| } | ||
|
|
||
| if status != boostrapKubeConfigStatusValid { | ||
| t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusValid, status) | ||
| } | ||
| }, | ||
| }, | ||
| { | ||
| name: "Recently failed and invalid BootstrapKubeConfig", | ||
| mockClient: func(failedTime time.Time) *fake.Clientset { | ||
| // Create a mock secret | ||
| mockSecret := &corev1.Secret{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "mock-secret", | ||
| Namespace: "test-namespace", | ||
| Annotations: map[string]string{ | ||
| BootstrapKubeconfigFailedTimeAnnotationKey: failedTime.Format(time.RFC3339), | ||
| }, | ||
| }, | ||
| Data: map[string][]byte{ | ||
| "kubeconfig": []byte("mock-kubeconfig"), | ||
| }, | ||
| } | ||
|
|
||
| // Create a mock kubeClient | ||
| mockKubeClient := &fake.Clientset{} | ||
| mockKubeClient.AddReactor("get", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { | ||
| return true, mockSecret, nil | ||
| }) | ||
| return mockKubeClient | ||
| }, | ||
| validate: func(c boostrapKubeConfig) { | ||
| status, err := c.Status() | ||
| if err != nil { | ||
| t.Errorf("Unexpected error: %v", err) | ||
| } | ||
|
|
||
| if status != boostrapKubeConfigStatusInValid { | ||
| t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusInValid, status) | ||
| } | ||
| }, | ||
| }, | ||
| { | ||
| name: "Fail a BootstrapKubeConfig", | ||
| mockClient: func(_ time.Time) *fake.Clientset { | ||
| // Create a mock secret | ||
| mockSecret := &corev1.Secret{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "mock-secret", | ||
| Namespace: "test-namespace", | ||
| }, | ||
| Data: map[string][]byte{ | ||
| "kubeconfig": []byte("mock-kubeconfig"), | ||
| }, | ||
| } | ||
|
|
||
| // Create a mock kubeClient | ||
| mockKubeClient := &fake.Clientset{} | ||
| mockKubeClient.AddReactor("get", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { | ||
| return true, mockSecret, nil | ||
| }) | ||
| mockKubeClient.AddReactor("update", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { | ||
| // Get annotation from action | ||
| annotations := action.(k8stesting.UpdateAction).GetObject().(*corev1.Secret).Annotations | ||
| mockSecret.Annotations = annotations | ||
| return true, nil, nil | ||
| }) | ||
| return mockKubeClient | ||
| }, | ||
| validate: func(c boostrapKubeConfig) { | ||
| status, err := c.Status() | ||
| if err != nil { | ||
| t.Errorf("Unexpected error: %v", err) | ||
| } | ||
|
|
||
| if status != boostrapKubeConfigStatusValid { | ||
| t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusValid, status) | ||
| } | ||
|
|
||
| // Fail the BootstrapKubeConfig | ||
| err = c.Fail(time.Now()) | ||
| if err != nil { | ||
| t.Errorf("Unexpected error: %v", err) | ||
| } | ||
|
|
||
| status, err = c.Status() | ||
| if err != nil { | ||
| t.Errorf("Unexpected error: %v", err) | ||
| } | ||
|
|
||
| if status != boostrapKubeConfigStatusInValid { | ||
| t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusInValid, status) | ||
| } | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range testcases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| mockClient := tc.mockClient(time.Now()) | ||
| bootstrapKubeConfig := &boostrapKubeConfigSecretImpl{ | ||
| kubeClient: mockClient, | ||
| secretNamespace: "test-namespace", | ||
| secretName: "mock-secret", | ||
| skipFailedBootstrapKubeconfigSeconds: 60, | ||
| } | ||
| tc.validate(bootstrapKubeConfig) | ||
| }) | ||
| } | ||
| } |
Oops, something went wrong.