add konnectivity proxy sidecar to ingress-operator to ensure it can p…

…roperly perform in cluster canary healthchecks

Currently the ingress operator fails to properly perform canary health checks in the guest cluster if it does not have direct network access to the ingress subdomain in the guest cluster. This is not a guarentee to have since the management
cluster and guest cluster can run in a split network environment. This pr introduces the socks proxy which will allow the ingress operator to proxy these canary healthcheck https requests through konnectivity and ultimately into the guest cluster. This will allow the healthchecks to properly be executed in all environments and prevent Degragaded status reports on the ingress resource which can lead to customer concerns/tickets. Fixes: #1130

control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go

@@ -2,9 +2,9 @@ package ingressoperator
 
 import (
 	"fmt"
-
 	hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/konnectivity"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
 	"github.com/openshift/hypershift/support/config"
 	"github.com/openshift/hypershift/support/util"
@@ -19,6 +19,7 @@ import (
 
 const (
 	ingressOperatorContainerName = "ingress-operator"
+	socks5ProxyContainerName     = "socks-proxy"
 	ingressOperatorMetricsPort   = 60000
 )
 
@@ -29,6 +30,7 @@ type Params struct {
 	ReleaseVersion          string
 	TokenMinterImage        string
 	AvailabilityProberImage string
+	Socks5ProxyImage        string
 	Platform                hyperv1.PlatformType
 	DeploymentConfig        config.DeploymentConfig
 }
@@ -39,6 +41,7 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, images map[strin
 		HAProxyRouterImage:      images["haproxy-router"],
 		ReleaseVersion:          version,
 		TokenMinterImage:        images["token-minter"],
+		Socks5ProxyImage:        images["socks5-proxy"],
 		AvailabilityProberImage: images[util.AvailabilityProberImageName],
 		Platform:                platform,
 	}
@@ -118,6 +121,18 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, apiPort *int32)
 			{Name: "IMAGE", Value: params.HAProxyRouterImage},
 			{Name: "CANARY_IMAGE", Value: params.IngressOperatorImage},
 			{Name: "KUBECONFIG", Value: "/etc/kubernetes/kubeconfig"},
+			{
+				Name:  "HTTP_PROXY",
+				Value: fmt.Sprintf("socks5://127.0.0.1:%d", konnectivity.KonnectivityServerLocalPort),
+			},
+			{
+				Name:  "HTTPS_PROXY",
+				Value: fmt.Sprintf("socks5://127.0.0.1:%d", konnectivity.KonnectivityServerLocalPort),
+			},
+			{
+				Name:  "NO_PROXY",
+				Value: fmt.Sprintf(manifests.KubeAPIServerService("").Name),
+			},
 		},
 		Name:            ingressOperatorContainerName,
 		Image:           params.IngressOperatorImage,
@@ -131,9 +146,11 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, apiPort *int32)
 			{Name: "ingress-operator-kubeconfig", MountPath: "/etc/kubernetes"},
 		},
 	}}
-
+	dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, ingressOperatorSocks5ProxyContainer(params.Socks5ProxyImage))
 	dep.Spec.Template.Spec.Volumes = []corev1.Volume{
 		{Name: "ingress-operator-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: manifests.IngressOperatorKubeconfig("").Name}}},
+		{Name: "admin-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "service-network-admin-kubeconfig"}}},
+		{Name: "konnectivity-proxy-cert", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: manifests.KonnectivityClientSecret("").Name}}},
 	}
 
 	if params.Platform == hyperv1.AWSPlatform {
@@ -162,9 +179,7 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, apiPort *int32)
 			},
 		})
 		dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes,
-			corev1.Volume{Name: "serviceaccount-token", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
-			corev1.Volume{Name: "admin-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "service-network-admin-kubeconfig"}}},
-		)
+			corev1.Volume{Name: "serviceaccount-token", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}})
 	}
 
 	util.AvailabilityProber(
@@ -181,3 +196,26 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, apiPort *int32)
 
 	params.DeploymentConfig.ApplyTo(dep)
 }
+
+func ingressOperatorSocks5ProxyContainer(socks5ProxyImage string) corev1.Container {
+	return corev1.Container{
+		Name:    socks5ProxyContainerName,
+		Image:   socks5ProxyImage,
+		Command: []string{"/usr/bin/konnectivity-socks5-proxy"},
+		Args:    []string{"run"},
+		Env: []corev1.EnvVar{{
+			Name:  "KUBECONFIG",
+			Value: "/etc/kubernetes/kubeconfig",
+		}},
+		Resources: corev1.ResourceRequirements{
+			Requests: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("10m"),
+				corev1.ResourceMemory: resource.MustParse("10Mi"),
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: "admin-kubeconfig", MountPath: "/etc/kubernetes"},
+			{Name: "konnectivity-proxy-cert", MountPath: "/etc/konnectivity-proxy-tls"},
+		},
+	}
+}