diff --git a/pkg/k8sclient/kubeconfig.go b/pkg/k8sclient/kubeconfig.go index 0856b1d3a..79ab61ae5 100644 --- a/pkg/k8sclient/kubeconfig.go +++ b/pkg/k8sclient/kubeconfig.go @@ -22,10 +22,12 @@ import ( "fmt" "os" "path" + "strings" "time" certificatesv1 "k8s.io/api/certificates/v1" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -87,8 +89,28 @@ func PerNodeK8sClient(nodeName, bootstrapKubeconfigFile string, certDuration tim // https://github.com/kubernetes/kubernetes/blob/068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8/cmd/kubelet/app/server.go#L953-L963 newClientsetFn := func(current *tls.Certificate) (kubernetes.Interface, error) { cfg := bootstrapKubeconfig - if current != nil { - cfg = config + + // validate the kubeconfig + tempClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + logging.Errorf("failed to read kubeconfig from cert manager: %v", err) + } else { + _, err := tempClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{}) + // tls unknown authority error is unrecoverable error with retry + if err != nil { + if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") { + logging.Verbosef("cert mgr gets invalid config. rebuild from bootstrap kubeconfig") + // reload and use bootstrapKubeconfig again + newBootstrapKubeconfig, _ := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile) + cfg = newBootstrapKubeconfig + } else { + logging.Errorf("failed to list pods with new certs: %v", err) + } + } + + if current != nil { + cfg = config + } } return kubernetes.NewForConfig(cfg) }