fix(health check): check cluster status for multiple times (#2311)

in health check to go get rid of socket connection failure #2310

pkg/controller/controller_utils.go

@@ -21,10 +21,13 @@ package controller
 import (
 	"context"
 	"fmt"
+	"math/rand"
+	"time"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	"tkestack.io/tke/pkg/util/log"
@@ -109,3 +112,21 @@ func CatchPanic(funcName string, addon string) {
 		runtime.HandleError(fmt.Errorf("recover from %s.%s(), err is %v", addon, funcName, err))
 	}
 }
+
+// CheckClusterHealthStatus uses client to probe cluster health status for mutiple time
+func CheckClusterHealthStatus(client kubernetes.Interface) (*version.Info, error) {
+	totalProbeCount := 3
+	var version *version.Info
+	var err error
+
+	for i := 0; i < totalProbeCount; i++ {
+		version, err = client.Discovery().ServerVersion()
+		if err == nil {
+			break
+		}
+		// sleep 20ms-100ms when failed
+		randomNumber := rand.Intn(80) + 20
+		time.Sleep(time.Millisecond * time.Duration(randomNumber))
+	}
+	return version, err
+}

pkg/platform/controller/cluster/cluster_controller.go

@@ -615,7 +615,7 @@ func (c *Controller) checkHealth(ctx context.Context, cluster *typesv1.Cluster)
 		healthCheckCondition.Reason = failedHealthCheckReason
 		healthCheckCondition.Message = err.Error()
 	} else {
-		version, err := client.Discovery().ServerVersion()
+		version, err := controllerutil.CheckClusterHealthStatus(client)
 		if err != nil {
 			cluster.Status.Phase = platformv1.ClusterFailed