From 02efd3b711ba66a333edfde8a7ef460c1aed22dc Mon Sep 17 00:00:00 2001 From: Mulham Raee Date: Wed, 8 Mar 2023 11:18:42 +0100 Subject: [PATCH] check kas loadbalancer health --- api/v1beta1/hostedcluster_conditions.go | 2 + .../hostedcontrolplane_controller.go | 87 +++++++++++++++++-- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/api/v1beta1/hostedcluster_conditions.go b/api/v1beta1/hostedcluster_conditions.go index a1dab483215..795e7139cc5 100644 --- a/api/v1beta1/hostedcluster_conditions.go +++ b/api/v1beta1/hostedcluster_conditions.go @@ -174,6 +174,8 @@ const ( ExternalDNSHostNotReachableReason = "ExternalDNSHostNotReachable" + ExternalKASEndpointNotReachableReason = "ExternalKASEndpointNotReachable" + ReconciliationPausedConditionReason = "ReconciliationPaused" ReconciliationInvalidPausedUntilConditionReason = "InvalidPausedUntilValue" ) diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 501a8acdcc3..95c86694ab0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -3,9 +3,11 @@ package hostedcontrolplane import ( "context" crand "crypto/rand" + "crypto/tls" "errors" "fmt" "math/big" + "net/http" "os" "sort" "strings" @@ -79,6 +81,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/workqueue" "k8s.io/utils/pointer" @@ -575,6 +578,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R kubeConfigAvailable := hostedControlPlane.Status.KubeConfig != nil etcdCondition := meta.FindStatusCondition(hostedControlPlane.Status.Conditions, string(hyperv1.EtcdAvailable)) kubeAPIServerCondition := meta.FindStatusCondition(hostedControlPlane.Status.Conditions, string(hyperv1.KubeAPIServerAvailable)) + healthCheckErr := r.healthCheckKASExternalEndpoint(ctx, hostedControlPlane) status := metav1.ConditionFalse var reason, message string @@ -594,6 +598,9 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R case kubeAPIServerCondition != nil && kubeAPIServerCondition.Status == metav1.ConditionFalse: reason = kubeAPIServerCondition.Reason message = kubeAPIServerCondition.Message + case healthCheckErr != nil: + reason = hyperv1.ExternalKASEndpointNotReachableReason + message = healthCheckErr.Error() default: reason = hyperv1.AsExpectedReason message = "" @@ -660,6 +667,73 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R return result, nil } +func (r *HostedControlPlaneReconciler) healthCheckKASExternalEndpoint(ctx context.Context, hcp *hyperv1.HostedControlPlane) error { + serviceStrategy := util.ServicePublishingStrategyByTypeForHCP(hcp, hyperv1.APIServer) + if serviceStrategy == nil { + return fmt.Errorf("APIServer service strategy not specified") + } + + if util.IsPublicHCP(hcp) { + var svc *corev1.Service + if serviceStrategy.Type == hyperv1.Route { + svc = manifests.RouterPublicService(hcp.Namespace) + } else { + svc = manifests.KubeAPIServerService(hcp.Namespace) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil { + return fmt.Errorf("failed to get kube apiserver service: %w", err) + } + + if len(svc.Status.LoadBalancer.Ingress) == 0 || svc.Status.LoadBalancer.Ingress[0].Hostname == "" { + return fmt.Errorf("APIServer load balancer is not provisioned") + } + + hostName := svc.Status.LoadBalancer.Ingress[0].Hostname + port := util.InternalAPIPortWithDefault(hcp, config.DefaultAPIServerPort) + healthEndpoint := fmt.Sprintf("https://%s:%d/healthz", hostName, port) + + httpClient := http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + + resp, err := httpClient.Get(healthEndpoint) + if err != nil { + return err + } + + // req, err := http.NewRequest(http.MethodGet, healthEndpoint, nil) + // if err != nil { + // return err + // } + + // guestConfig, err := r.GetGuestClusterConfig(ctx, hcp) + // if err != nil { + // return err + // } + + // transport, err := rest.TransportFor(guestConfig) + // if err != nil { + // return err + // } + + // resp, err := transport.RoundTrip(req) + // if err != nil { + // return err + // } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("APIServer external endpoint %s is not healthy", healthEndpoint) + } + } + + // private cluster, what to do? + return nil +} + func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(hc *hyperv1.HostedControlPlane) error { for _, svc := range hc.Spec.Services { if svc.Type == hyperv1.Route && !r.ManagementClusterCapabilities.Has(capabilities.CapabilityRoute) { @@ -3903,22 +3977,21 @@ func (r *HostedControlPlaneReconciler) validateAWSKMSConfig(ctx context.Context, meta.SetStatusCondition(&hcp.Status.Conditions, condition) } -func (r *HostedControlPlaneReconciler) GetGuestClusterClient(ctx context.Context, hcp *hyperv1.HostedControlPlane) (*kubernetes.Clientset, error) { +func (r *HostedControlPlaneReconciler) GetGuestClusterConfig(ctx context.Context, hcp *hyperv1.HostedControlPlane) (*rest.Config, error) { kubeconfigSecret := manifests.KASExternalKubeconfigSecret(hcp.Namespace, hcp.Spec.KubeConfig) if err := r.Get(ctx, client.ObjectKeyFromObject(kubeconfigSecret), kubeconfigSecret); err != nil { return nil, err } kubeconfig := kubeconfigSecret.Data[DefaultAdminKubeconfigKey] - restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig) - if err != nil { - return nil, err - } + return clientcmd.RESTConfigFromKubeConfig(kubeconfig) +} - clientset, err := kubernetes.NewForConfig(restConfig) +func (r *HostedControlPlaneReconciler) GetGuestClusterClient(ctx context.Context, hcp *hyperv1.HostedControlPlane) (*kubernetes.Clientset, error) { + restConfig, err := r.GetGuestClusterConfig(ctx, hcp) if err != nil { return nil, err } - return clientset, nil + return kubernetes.NewForConfig(restConfig) }