Merge pull request #1131 from relyt0925/ingress-operator-konnectivity…

…-sidecar add konnectivity proxy sidecar to ingress-operator to ensure it can properly perform in cluster canary healthchecks
openshift · Mar 10, 2022 · 9fd6e1d · 9fd6e1d
2 parents c5b0827 + 28546dc
commit 9fd6e1d
Showing 1 changed file with 48 additions and 6 deletions.
diff --git a/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go b/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go
@@ -2,9 +2,9 @@ package ingressoperator
 
 import (
 	"fmt"
-
 	hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/konnectivity"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
 	"github.com/openshift/hypershift/support/config"
 	"github.com/openshift/hypershift/support/util"
@@ -19,6 +19,7 @@ import (
 
 const (
 	ingressOperatorContainerName = "ingress-operator"
+	socks5ProxyContainerName     = "socks-proxy"
 	ingressOperatorMetricsPort   = 60000
 )
 
@@ -29,6 +30,7 @@ type Params struct {
 	ReleaseVersion          string
 	TokenMinterImage        string
 	AvailabilityProberImage string
+	Socks5ProxyImage        string
 	Platform                hyperv1.PlatformType
 	DeploymentConfig        config.DeploymentConfig
 }
@@ -39,6 +41,7 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, images map[strin
 		HAProxyRouterImage:      images["haproxy-router"],
 		ReleaseVersion:          version,
 		TokenMinterImage:        images["token-minter"],
+		Socks5ProxyImage:        images["socks5-proxy"],
 		AvailabilityProberImage: images[util.AvailabilityProberImageName],
 		Platform:                platform,
 	}
@@ -118,6 +121,22 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, apiPort *int32)
 			{Name: "IMAGE", Value: params.HAProxyRouterImage},
 			{Name: "CANARY_IMAGE", Value: params.IngressOperatorImage},
 			{Name: "KUBECONFIG", Value: "/etc/kubernetes/kubeconfig"},
+			{
+				Name:  "HTTP_PROXY",
+				Value: fmt.Sprintf("socks5://127.0.0.1:%d", konnectivity.KonnectivityServerLocalPort),
+			},
+			{
+				Name:  "HTTPS_PROXY",
+				Value: fmt.Sprintf("socks5://127.0.0.1:%d", konnectivity.KonnectivityServerLocalPort),
+			},
+			// cloud provider APIs need to be included since the ingress operator reaches out to them to provision
+			// DNS domains. The API list can be found below:
+			// AWS: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints
+			// AZURE: https://docs.microsoft.com/en-us/rest/api/azure/#how-to-call-azure-rest-apis-with-curl
+			{
+				Name:  "NO_PROXY",
+				Value: fmt.Sprintf(".amazonaws.com,.microsoftonline.com,.azure.com,%s", manifests.KubeAPIServerService("").Name),
+			},
 		},
 		Name:            ingressOperatorContainerName,
 		Image:           params.IngressOperatorImage,
@@ -131,9 +150,11 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, apiPort *int32)
 			{Name: "ingress-operator-kubeconfig", MountPath: "/etc/kubernetes"},
 		},
 	}}
-
+	dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, ingressOperatorSocks5ProxyContainer(params.Socks5ProxyImage))
 	dep.Spec.Template.Spec.Volumes = []corev1.Volume{
-		{Name: "ingress-operator-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: manifests.IngressOperatorKubeconfig("").Name}}},
+		{Name: "ingress-operator-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: manifests.IngressOperatorKubeconfig("").Name, DefaultMode: utilpointer.Int32Ptr(416)}}},
+		{Name: "admin-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "service-network-admin-kubeconfig", DefaultMode: utilpointer.Int32Ptr(416)}}},
+		{Name: "konnectivity-proxy-cert", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: manifests.KonnectivityClientSecret("").Name, DefaultMode: utilpointer.Int32Ptr(416)}}},
 	}
 
 	if params.Platform == hyperv1.AWSPlatform {
@@ -162,9 +183,7 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, apiPort *int32)
 			},
 		})
 		dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes,
-			corev1.Volume{Name: "serviceaccount-token", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
-			corev1.Volume{Name: "admin-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "service-network-admin-kubeconfig"}}},
-		)
+			corev1.Volume{Name: "serviceaccount-token", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}})
 	}
 
 	util.AvailabilityProber(
@@ -181,3 +200,26 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, apiPort *int32)
 
 	params.DeploymentConfig.ApplyTo(dep)
 }
+
+func ingressOperatorSocks5ProxyContainer(socks5ProxyImage string) corev1.Container {
+	return corev1.Container{
+		Name:    socks5ProxyContainerName,
+		Image:   socks5ProxyImage,
+		Command: []string{"/usr/bin/konnectivity-socks5-proxy"},
+		Args:    []string{"run"},
+		Env: []corev1.EnvVar{{
+			Name:  "KUBECONFIG",
+			Value: "/etc/kubernetes/kubeconfig",
+		}},
+		Resources: corev1.ResourceRequirements{
+			Requests: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("10m"),
+				corev1.ResourceMemory: resource.MustParse("10Mi"),
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: "admin-kubeconfig", MountPath: "/etc/kubernetes"},
+			{Name: "konnectivity-proxy-cert", MountPath: "/etc/konnectivity-proxy-tls"},
+		},
+	}
+}