Skip to content

Commit

Permalink
Support multiple bootstrapkubeconfigs in registration agent.
Browse files Browse the repository at this point in the history
Signed-off-by: xuezhaojun <zxue@redhat.com>
  • Loading branch information
xuezhaojun committed Apr 28, 2024
1 parent 0882f6d commit eeeee2a
Show file tree
Hide file tree
Showing 20 changed files with 1,547 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,51 @@ spec:
description: RegistrationConfiguration contains the configuration
of registration
properties:
bootstrapKubeConfigs:
description: "BootstrapKubeConfigs defines the ordered list of
bootstrap kubeconfigs. The order decides which bootstrap kubeconfig
to use first when rebootstrap. \n When the agent loses the connection
to the current hub over HubConnectionTimeoutSeconds, or the
managedcluster CR is set `hubAcceptsClient=false` on the hub,
the controller marks the related bootstrap kubeconfig as \"failed\".
\n A failed bootstrapkubeconfig won't be used for the duration
specified by SkipFailedBootstrapKubeConfigSeconds. But if the
user updates the content of a failed bootstrapkubeconfig, the
\"failed\" mark will be cleared."
properties:
localSecretsConfig:
description: LocalSecretsConfig include a list of secrets
that contains the kubeconfigs for ordered bootstrap kubeconifigs.
The secrets must be in the same namespace where the agent
controller runs.
properties:
hubConnectionTimeoutSeconds:
default: 600
description: HubConnectionTimeoutSeconds is used to set
the timeout of connecting to the hub cluster. When agent
loses the connection to the hub over the timeout seconds,
the agent do a rebootstrap. By default is 10 mins.
format: int32
minimum: 180
type: integer
secretNames:
description: SecretNames is a list of secret names. The
secrets are in the same namespace where the agent controller
runs.
items:
type: string
type: array
type: object
type:
default: None
description: Type specifies the type of priority bootstrap
kubeconfigs. By default, it is set to None, representing
no priority bootstrap kubeconfigs are set.
enum:
- None
- LocalSecrets
type: string
type: object
clientCertExpirationSeconds:
description: clientCertExpirationSeconds represents the seconds
of a client certificate to expire. If it is not set or 0, the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ metadata:
categories: Integration & Delivery,OpenShift Optional
certified: "false"
containerImage: quay.io/open-cluster-management/registration-operator:latest
createdAt: "2024-04-10T15:46:14Z"
createdAt: "2024-04-25T09:39:17Z"
description: Manages the installation and upgrade of the Klusterlet.
operators.operatorframework.io/builder: operator-sdk-v1.32.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,12 @@ spec:
{{if gt .AgentKubeAPIBurst 0}}
- "--kube-api-burst={{ .AgentKubeAPIBurst }}"
{{end}}
{{range .BootStrapKubeConfigSecrets}}
- "--bootstrap-kubeconfig-secrets={{ . }}"
{{end}}
{{if gt .HubConnectionTimeoutSeconds 0}}
- "--hub-connection-timeout-seconds={{ .HubConnectionTimeoutSeconds }}"
{{end}}
env:
- name: POD_NAME
valueFrom:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,12 @@ spec:
{{if gt .RegistrationKubeAPIBurst 0}}
- "--kube-api-burst={{ .RegistrationKubeAPIBurst }}"
{{end}}
{{range .BootStrapKubeConfigSecrets}}
- "--bootstrap-kubeconfig-secrets={{ . }}"
{{end}}
{{if gt .HubConnectionTimeoutSeconds 0}}
- "--hub-connection-timeout-seconds={{ .HubConnectionTimeoutSeconds }}"
{{end}}
env:
- name: POD_NAME
valueFrom:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ type klusterletConfig struct {
ExternalServerURL string
HubKubeConfigSecret string
BootStrapKubeConfigSecret string
BootStrapKubeConfigSecrets []string
HubConnectionTimeoutSeconds int32
OperatorNamespace string
Replica int32
ClientCertExpirationSeconds int32
Expand Down Expand Up @@ -264,6 +266,13 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
annotationsArray = append(annotationsArray, fmt.Sprintf("%s=%s", k, v))
}
config.ClusterAnnotationsString = strings.Join(annotationsArray, ",")

// get bootstrap kubeconfig secrets from the klusterlet configuration
bootstrapKubeConfigs := klusterlet.Spec.RegistrationConfiguration.BootstrapKubeConfigs
if bootstrapKubeConfigs.Type == operatorapiv1.LocalSecrets {
config.BootStrapKubeConfigSecrets = bootstrapKubeConfigs.LocalSecrets.SecretNames
config.HubConnectionTimeoutSeconds = bootstrapKubeConfigs.LocalSecrets.HubConnectionTimeoutSeconds
}
}
config.RegistrationFeatureGates, registrationFeatureMsgs = helpers.ConvertToFeatureGateFlags("Registration",
registrationFeatureGates, ocmfeature.DefaultSpokeRegistrationFeatureGates)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package bootstrapkubeconfigsmanager

import (
"context"
"fmt"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

type boostrapKubeConfigStatus string

const (
boostrapKubeConfigStatusInValid boostrapKubeConfigStatus = "InValid"
boostrapKubeConfigStatusValid boostrapKubeConfigStatus = "Valid"
)

// bootstrapKubeConfig represents a bootstrap kubeconfig that agent can use to bootstrap a managed cluster.
type boostrapKubeConfig interface {
// Name returns the name of the bootstrap kubeconfig. It helps to identify the bootstrap kubeconfig.
Name() string

// KubeConfigData returns the kubeconfig data of the bootstrap kubeconfig.
// KubeConfigData includes the kubeconfig and the credentials to connect to the hub cluster.
KubeConfigData() (map[string][]byte, error)

// Status returns the status of the bootstrap kubeconfig.
// A bootstrap kubeconfig has two status: Valid and InValid.
Status() (boostrapKubeConfigStatus, error)

// Fail means at the time t, the bootstrap kubeconfig failed to connect to the hub cluster.
Fail(t time.Time) error
}

var _ boostrapKubeConfig = &boostrapKubeConfigSecretImpl{}

const (
// BootstrapKubeconfigFailedTimeAnnotationKey represents the time when the bootstrap kubeconfig failed
BootstrapKubeconfigFailedTimeAnnotationKey = "agent.open-cluster-management.io/bootstrap-kubeconfig-failed-time"
)

type boostrapKubeConfigSecretImpl struct {
secretName string
secretNamespace string
skipFailedBootstrapKubeconfigSeconds int32 // if a bootstrap kubeconfig failed, in 3 mins, it can't be used in rebootstrap.
kubeClient kubernetes.Interface
}

func (b *boostrapKubeConfigSecretImpl) Name() string {
return b.secretName
}

func (b *boostrapKubeConfigSecretImpl) KubeConfigData() (map[string][]byte, error) {
secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err)
}
return secret.Data, nil
}

func (b *boostrapKubeConfigSecretImpl) Status() (boostrapKubeConfigStatus, error) {
secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{})
if err != nil {
return boostrapKubeConfigStatusInValid, fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err)
}

if secret.Annotations == nil {
return boostrapKubeConfigStatusValid, nil
}

now := time.Now()
if failedTime, ok := secret.Annotations[BootstrapKubeconfigFailedTimeAnnotationKey]; ok {
failedTimeParsed, err := time.Parse(time.RFC3339, failedTime)
if err != nil {
return boostrapKubeConfigStatusInValid, fmt.Errorf("failed to parse the failed time %s of the secret %s: %v", failedTime, secret.Name, err)
}
if now.Sub(failedTimeParsed).Seconds() < float64(b.skipFailedBootstrapKubeconfigSeconds) {
return boostrapKubeConfigStatusInValid, nil
}
}
return boostrapKubeConfigStatusValid, nil
}

func (b *boostrapKubeConfigSecretImpl) Fail(t time.Time) error {
secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err)
}
secretCopy := secret.DeepCopy()
if secretCopy.Annotations == nil {
secretCopy.Annotations = make(map[string]string)
}
secretCopy.Annotations[BootstrapKubeconfigFailedTimeAnnotationKey] = t.Format(time.RFC3339)
_, err = b.kubeClient.CoreV1().Secrets(b.secretNamespace).Update(context.Background(), secretCopy, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("update the secret %s failed: %v", b.secretName, err)
}
return nil
}

0 comments on commit eeeee2a

Please sign in to comment.