check kas loadbalancer health

api/v1beta1/hostedcluster_conditions.go

@@ -174,6 +174,8 @@ const (
 
 	ExternalDNSHostNotReachableReason = "ExternalDNSHostNotReachable"
 
+	ExternalKASEndpointNotReachableReason = "ExternalKASEndpointNotReachable"
+
 	ReconciliationPausedConditionReason             = "ReconciliationPaused"
 	ReconciliationInvalidPausedUntilConditionReason = "InvalidPausedUntilValue"
 )

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -3,9 +3,11 @@ package hostedcontrolplane
 import (
 	"context"
 	crand "crypto/rand"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"math/big"
+	"net/http"
 	"os"
 	"sort"
 	"strings"
@@ -79,6 +81,7 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/utils/pointer"
@@ -575,6 +578,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		kubeConfigAvailable := hostedControlPlane.Status.KubeConfig != nil
 		etcdCondition := meta.FindStatusCondition(hostedControlPlane.Status.Conditions, string(hyperv1.EtcdAvailable))
 		kubeAPIServerCondition := meta.FindStatusCondition(hostedControlPlane.Status.Conditions, string(hyperv1.KubeAPIServerAvailable))
+		healthCheckErr := r.healthCheckKASExternalEndpoint(ctx, hostedControlPlane)
 
 		status := metav1.ConditionFalse
 		var reason, message string
@@ -594,6 +598,9 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		case kubeAPIServerCondition != nil && kubeAPIServerCondition.Status == metav1.ConditionFalse:
 			reason = kubeAPIServerCondition.Reason
 			message = kubeAPIServerCondition.Message
+		case healthCheckErr != nil:
+			reason = hyperv1.ExternalKASEndpointNotReachableReason
+			message = healthCheckErr.Error()
 		default:
 			reason = hyperv1.AsExpectedReason
 			message = ""
@@ -660,6 +667,73 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	return result, nil
 }
 
+func (r *HostedControlPlaneReconciler) healthCheckKASExternalEndpoint(ctx context.Context, hcp *hyperv1.HostedControlPlane) error {
+	serviceStrategy := util.ServicePublishingStrategyByTypeForHCP(hcp, hyperv1.APIServer)
+	if serviceStrategy == nil {
+		return fmt.Errorf("APIServer service strategy not specified")
+	}
+
+	if util.IsPublicHCP(hcp) {
+		var svc *corev1.Service
+		if serviceStrategy.Type == hyperv1.Route {
+			svc = manifests.RouterPublicService(hcp.Namespace)
+		} else {
+			svc = manifests.KubeAPIServerService(hcp.Namespace)
+		}
+		if err := r.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil {
+			return fmt.Errorf("failed to get kube apiserver service: %w", err)
+		}
+
+		if len(svc.Status.LoadBalancer.Ingress) == 0 || svc.Status.LoadBalancer.Ingress[0].Hostname == "" {
+			return fmt.Errorf("APIServer load balancer is not provisioned")
+		}
+
+		hostName := svc.Status.LoadBalancer.Ingress[0].Hostname
+		port := util.InternalAPIPortWithDefault(hcp, config.DefaultAPIServerPort)
+		healthEndpoint := fmt.Sprintf("https://%s:%d/healthz", hostName, port)
+
+		httpClient := http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: true,
+				},
+			},
+		}
+
+		resp, err := httpClient.Get(healthEndpoint)
+		if err != nil {
+			return err
+		}
+
+		// req, err := http.NewRequest(http.MethodGet, healthEndpoint, nil)
+		// if err != nil {
+		// 	return err
+		// }
+
+		// guestConfig, err := r.GetGuestClusterConfig(ctx, hcp)
+		// if err != nil {
+		// 	return err
+		// }
+
+		// transport, err := rest.TransportFor(guestConfig)
+		// if err != nil {
+		// 	return err
+		// }
+
+		// resp, err := transport.RoundTrip(req)
+		// if err != nil {
+		// 	return err
+		// }
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("APIServer external endpoint %s is not healthy", healthEndpoint)
+		}
+	}
+
+	// private cluster, what to do?
+	return nil
+}
+
 func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(hc *hyperv1.HostedControlPlane) error {
 	for _, svc := range hc.Spec.Services {
 		if svc.Type == hyperv1.Route && !r.ManagementClusterCapabilities.Has(capabilities.CapabilityRoute) {
@@ -3903,22 +3977,21 @@ func (r *HostedControlPlaneReconciler) validateAWSKMSConfig(ctx context.Context,
 	meta.SetStatusCondition(&hcp.Status.Conditions, condition)
 }
 
-func (r *HostedControlPlaneReconciler) GetGuestClusterClient(ctx context.Context, hcp *hyperv1.HostedControlPlane) (*kubernetes.Clientset, error) {
+func (r *HostedControlPlaneReconciler) GetGuestClusterConfig(ctx context.Context, hcp *hyperv1.HostedControlPlane) (*rest.Config, error) {
 	kubeconfigSecret := manifests.KASExternalKubeconfigSecret(hcp.Namespace, hcp.Spec.KubeConfig)
 	if err := r.Get(ctx, client.ObjectKeyFromObject(kubeconfigSecret), kubeconfigSecret); err != nil {
 		return nil, err
 	}
 
 	kubeconfig := kubeconfigSecret.Data[DefaultAdminKubeconfigKey]
-	restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
-	if err != nil {
-		return nil, err
-	}
+	return clientcmd.RESTConfigFromKubeConfig(kubeconfig)
+}
 
-	clientset, err := kubernetes.NewForConfig(restConfig)
+func (r *HostedControlPlaneReconciler) GetGuestClusterClient(ctx context.Context, hcp *hyperv1.HostedControlPlane) (*kubernetes.Clientset, error) {
+	restConfig, err := r.GetGuestClusterConfig(ctx, hcp)
 	if err != nil {
 		return nil, err
 	}
 
-	return clientset, nil
+	return kubernetes.NewForConfig(restConfig)
 }