Ensuring kubeconfig upon TCP change

e2e/utils_test.go

@@ -0,0 +1,19 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func GetKindIPAddress() string {
+	ep := &corev1.Endpoints{}
+	Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "kubernetes", Namespace: "default"}, ep)).ToNot(HaveOccurred())
+
+	return ep.Subsets[0].Addresses[0].IP
+}

e2e/worker_kubeadm_join_test.go

@@ -36,10 +36,6 @@ var _ = Describe("starting a kind worker with kubeadm", func() {
 	var kubeconfigFile *os.File
 
 	JustBeforeEach(func() {
-		// Retrieving the kind instance IP from the `kubernetes` service in the `default` namespace
-		ep := &corev1.Endpoints{}
-		Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "kubernetes", Namespace: "default"}, ep)).ToNot(HaveOccurred())
-
 		tcp = kamajiv1alpha1.TenantControlPlane{
 			TypeMeta: metav1.TypeMeta{},
 			ObjectMeta: metav1.ObjectMeta{
@@ -59,7 +55,7 @@ var _ = Describe("starting a kind worker with kubeadm", func() {
 					},
 				},
 				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
-					Address: ep.Subsets[0].Addresses[0].IP,
+					Address: GetKindIPAddress(),
 					Port:    31443,
 				},
 				Kubernetes: kamajiv1alpha1.KubernetesSpec{

e2e/worker_tcp_change_port_test.go

@@ -0,0 +1,163 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("validating kubeconfig", func() {
+	ctx := context.Background()
+
+	var tcp *kamajiv1alpha1.TenantControlPlane
+
+	var kubeconfigFile *os.File
+
+	JustBeforeEach(func() {
+		tcp = &kamajiv1alpha1.TenantControlPlane{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "kubeconfig",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+				ControlPlane: kamajiv1alpha1.ControlPlane{
+					Deployment: kamajiv1alpha1.DeploymentSpec{
+						Replicas: 1,
+					},
+					Ingress: kamajiv1alpha1.IngressSpec{
+						Enabled: false,
+					},
+					Service: kamajiv1alpha1.ServiceSpec{
+						ServiceType: "NodePort",
+					},
+				},
+				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+					Address: GetKindIPAddress(),
+					Port:    31443,
+				},
+				Kubernetes: kamajiv1alpha1.KubernetesSpec{
+					Version: "v1.23.6",
+					Kubelet: kamajiv1alpha1.KubeletSpec{
+						CGroupFS: "cgroupfs",
+					},
+					AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+						"LimitRanger",
+						"ResourceQuota",
+					},
+				},
+				Addons: kamajiv1alpha1.AddonsSpec{},
+			},
+		}
+		Expect(k8sClient.Create(ctx, tcp)).NotTo(HaveOccurred())
+
+		var err error
+
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(ctx, tcp)).Should(Succeed())
+		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
+	})
+
+	It("return kubernetes version", func() {
+		for _, port := range []int32{31444, 31445, 31446} {
+			Eventually(func() string {
+				By(fmt.Sprintf("ensuring TCP port is set to %d", port), func() {
+					Eventually(func() (err error) {
+						if err = k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return err
+						}
+
+						tcp.Spec.NetworkProfile.Port = port
+
+						return k8sClient.Update(ctx, tcp)
+					}, time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+				})
+
+				By("ensuring port change is defined in the TCP status", func() {
+					Eventually(func() int32 {
+						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return 0
+						}
+
+						return tcp.Status.Kubernetes.Service.Port
+					}, time.Minute, 5*time.Second).Should(Equal(port))
+				})
+
+				By("ensuring downloading the updated kubeconfig", func() {
+					Eventually(func() (err error) {
+						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return err
+						}
+
+						secret := &corev1.Secret{}
+
+						if err = k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.Status.KubeConfig.Admin.SecretName}, secret); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve kubeconfig secret name:", err.Error())
+
+							return err
+						}
+
+						_, err = kubeconfigFile.Write(secret.Data["admin.conf"])
+
+						return err
+					}, time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+				})
+
+				var version version.Info
+
+				By("retrieving TCP version using the kubeconfig", func() {
+					config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot generate REST configuration:", err.Error())
+
+						return
+					}
+
+					clientset, err := kubernetes.NewForConfig(config)
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot generate clientset:", err.Error())
+
+						return
+					}
+
+					serverVersion, err := clientset.ServerVersion()
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve server version:", err.Error())
+
+						return
+					}
+
+					version = *serverVersion
+				})
+
+				return version.GitVersion
+			}, 5*time.Minute, 5*time.Second).Should(Equal(tcp.Spec.Kubernetes.Version))
+		}
+	})
+})

internal/kubeadm/types.go

@@ -18,7 +18,7 @@ type Configuration struct {
 }
 
 func (c *Configuration) Checksum() string {
-	initConfiguration, _ := json.Marshal(c.InitConfiguration)
+	initConfiguration, _ := utilities.EncondeToYaml(&c.InitConfiguration)
 	kubeconfig, _ := json.Marshal(c.Kubeconfig)
 	parameters, _ := json.Marshal(c.Parameters)
 

internal/resources/kubeconfig.go

@@ -5,8 +5,6 @@ package resources
 
 import (
 	"context"
-	"crypto/md5"
-	"encoding/hex"
 	"fmt"
 
 	"github.com/go-logr/logr"
@@ -108,19 +106,16 @@ func (r *KubeconfigResource) CreateOrUpdate(ctx context.Context, tenantControlPl
 	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
+func (r *KubeconfigResource) checksum(apiServerCertificatesSecret *corev1.Secret, kubeadmChecksum string) string {
+	return utilities.CalculateConfigMapChecksum(map[string]string{
+		"ca-cert-checksum": string(apiServerCertificatesSecret.Data[kubeadmconstants.CACertName]),
+		"ca-key-checksum":  string(apiServerCertificatesSecret.Data[kubeadmconstants.CAKeyName]),
+		"kubeadmconfig":    kubeadmChecksum,
+	})
+}
+
 func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	return func() error {
-		status, err := r.getKubeconfigStatus(tenantControlPlane)
-		if err != nil {
-			return err
-		}
-
-		if status.Checksum == r.resource.GetAnnotations()["checksum"] {
-			if kubeadm.IsKubeconfigValid(r.resource.Data[r.KubeConfigFileName]) {
-				return nil
-			}
-		}
-
 		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
@@ -136,6 +131,19 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
 			return err
 		}
 
+		checksum := r.checksum(apiServerCertificatesSecret, config.Checksum())
+
+		status, err := r.getKubeconfigStatus(tenantControlPlane)
+		if err != nil {
+			return err
+		}
+
+		if status.Checksum == checksum {
+			if kubeadm.IsKubeconfigValid(r.resource.Data[r.KubeConfigFileName]) {
+				return nil
+			}
+		}
+
 		kubeconfig, err := kubeadm.CreateKubeconfig(
 			r.KubeConfigFileName,
 			kubeadm.CertificatePrivateKeyPair{
@@ -159,10 +167,8 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
 			},
 		))
 
-		hash := md5.Sum(kubeconfig)
-
 		r.resource.SetAnnotations(map[string]string{
-			"checksum": hex.EncodeToString(hash[:]),
+			"checksum": checksum,
 		})
 
 		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
@@ -171,8 +177,6 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
 
 func (r *KubeconfigResource) customizeConfig(config *kubeadm.Configuration) error {
 	switch r.KubeConfigFileName {
-	case kubeadmconstants.AdminKubeConfigFileName:
-		return r.ingressAsAdvertiseAddress(config)
 	case kubeadmconstants.ControllerManagerKubeConfigFileName:
 		return r.localhostAsAdvertiseAddress(config)
 	case kubeadmconstants.SchedulerKubeConfigFileName:
@@ -182,12 +186,6 @@ func (r *KubeconfigResource) customizeConfig(config *kubeadm.Configuration) erro
 	}
 }
 
-func (r *KubeconfigResource) ingressAsAdvertiseAddress(config *kubeadm.Configuration) error {
-	config.InitConfiguration.LocalAPIEndpoint.BindPort = defaultIngressPort
-
-	return nil
-}
-
 func (r *KubeconfigResource) localhostAsAdvertiseAddress(config *kubeadm.Configuration) error {
 	config.InitConfiguration.LocalAPIEndpoint.AdvertiseAddress = localhost