From a3373175333bbfd4a6fd545ce703bca9584650b2 Mon Sep 17 00:00:00 2001
From: Tomofumi Hayashi <tohayash@redhat.com>
Date: Wed, 14 Feb 2024 00:21:04 +0900
Subject: [PATCH] Reload bootstrap kubeconfig if cert mgr failed to load valid
 certs

When user recreate whole cluster certs, multus thick plugin's
previous cert is no longer valid. In such case, we need to prevent
to use cert manager's old certs and restart it from bootstrap
kubeconfig. This fix reloads client config from bootstrap
kubeconfig if cert mgr's cert is failed to load pod.
---
 pkg/k8sclient/kubeconfig.go | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/pkg/k8sclient/kubeconfig.go b/pkg/k8sclient/kubeconfig.go
index 0856b1d3a..79ab61ae5 100644
--- a/pkg/k8sclient/kubeconfig.go
+++ b/pkg/k8sclient/kubeconfig.go
@@ -22,10 +22,12 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strings"
 	"time"
 
 	certificatesv1 "k8s.io/api/certificates/v1"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -87,8 +89,28 @@ func PerNodeK8sClient(nodeName, bootstrapKubeconfigFile string, certDuration tim
 	// https://github.com/kubernetes/kubernetes/blob/068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8/cmd/kubelet/app/server.go#L953-L963
 	newClientsetFn := func(current *tls.Certificate) (kubernetes.Interface, error) {
 		cfg := bootstrapKubeconfig
-		if current != nil {
-			cfg = config
+
+		// validate the kubeconfig
+		tempClient, err := kubernetes.NewForConfig(cfg)
+		if err != nil {
+			logging.Errorf("failed to read kubeconfig from cert manager: %v", err)
+		} else {
+			_, err := tempClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
+			// tls unknown authority error is unrecoverable error with retry
+			if err != nil {
+				if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
+					logging.Verbosef("cert mgr gets invalid config. rebuild from bootstrap kubeconfig")
+					// reload and use bootstrapKubeconfig again
+					newBootstrapKubeconfig, _ := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
+					cfg = newBootstrapKubeconfig
+				} else {
+					logging.Errorf("failed to list pods with new certs: %v", err)
+				}
+			}
+
+			if current != nil {
+				cfg = config
+			}
 		}
 		return kubernetes.NewForConfig(cfg)
 	}