Merge pull request #437 from calmkart/fix-main-process-block

FIX: fix the bud -> when check pod pending delete it, main process blocked && format project

cmd/check-reaper/main.go

@@ -20,7 +20,7 @@ var ReapCheckerPods map[string]v1.Pod
 var MaxPodsThreshold = 4
 var Namespace string
 
-func init(){
+func init() {
 	Namespace = os.Getenv("SINGLE_NAMESPACE")
 	if len(Namespace) == 0 {
 		log.Infoln("Single namespace not specified, running check reaper across all namespaces")

cmd/deployment-check/deployment_test.go

@@ -23,7 +23,7 @@ func TestCreateContainerConfig(t *testing.T) {
 
 		if len(containerConfig.ImagePullPolicy) == 0 {
 			t.Fatalf("nil image pull policy: %s", containerConfig.ImagePullPolicy)
-		}	
+		}
 
 		if len(containerConfig.Ports) == 0 {
 			t.Fatalf("no ports given for container: found %d ports\n", len(containerConfig.Ports))
@@ -51,7 +51,7 @@ func TestCreateDeploymentConfig(t *testing.T) {
 		if len(deploymentConfig.ObjectMeta.Namespace) == 0 {
 			t.Fatalf("nil deployment object meta namespace: %s\n", deploymentConfig.ObjectMeta.Namespace)
 		}
-		
+
 		reps := int32(1)
 		if *deploymentConfig.Spec.Replicas < reps {
 			t.Fatalf("deployment config was created with less than 1 replica: %d", deploymentConfig.Spec.Replicas)

cmd/kuberhealthy/main_test.go

@@ -4,8 +4,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func init(){
+func init() {
 	// tests always run with debug logging
 	log.SetLevel(log.DebugLevel)
 }
-

cmd/pod-restarts-check/main.go

@@ -24,8 +24,8 @@ import (
 	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	checkclient "github.com/Comcast/kuberhealthy/v2/pkg/checks/external/checkclient"
 	"github.com/Comcast/kuberhealthy/v2/pkg/kubeClient"

cmd/pod-restarts-check/main_test.go

@@ -13,15 +13,15 @@ func pod(podName string, containerName string, restartCount int32) *v1.Pod {
 	var pod = v1.Pod{
 		ObjectMeta: v12.ObjectMeta{
 			Namespace: "test-namespace",
-			Name: podName,
+			Name:      podName,
 		},
 		Spec: v1.PodSpec{},
 		Status: v1.PodStatus{
 			Reason: "Ready",
 			ContainerStatuses: []v1.ContainerStatus{
 				{Name: containerName,
-				Ready: true,
-				RestartCount: restartCount},
+					Ready:        true,
+					RestartCount: restartCount},
 			},
 		},
 	}
@@ -45,8 +45,8 @@ func TestAddPodRestartCount(t *testing.T) {
 
 	var testCase = struct {
 		description string
-		pod v1.Pod
-		expected map[string]map[string]int32
+		pod         v1.Pod
+		expected    map[string]map[string]int32
 	}{"Pod: test-pod, container: container-name, restart count: 5",
 		*pod("test-pod", "container-name", 5),
 		expectedRestartObservations}
@@ -70,7 +70,7 @@ func TestCheckBadPodRestarts(t *testing.T) {
 
 	var testCases = []struct {
 		description string
-		pod v1.Pod
+		pod         v1.Pod
 	}{
 		{
 			"Bad pod with more than max restarts",
@@ -109,7 +109,7 @@ func TestRemoveBadPodRestarts(t *testing.T) {
 
 	var testCases = []struct {
 		description string
-		pod v1.Pod
+		pod         v1.Pod
 	}{
 		{"Remove deleted pod from BadPodRestarts", *pod("removed-bad-pod", "removed-bad-container", 3)},
 		{"Don't remove pod from BadPodRestarts", *pod("bad-test-pod", "bad-test-container", 8)},

pkg/checks/external/main.go

@@ -6,14 +6,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
+
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/watch"
@@ -25,7 +26,6 @@ import (
 	typedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
 
 	"github.com/Comcast/kuberhealthy/v2/pkg/checks/external/util"
-
 	"github.com/Comcast/kuberhealthy/v2/pkg/health"
 	"github.com/Comcast/kuberhealthy/v2/pkg/khcheckcrd"
 	"github.com/Comcast/kuberhealthy/v2/pkg/khstatecrd"
@@ -64,6 +64,9 @@ const defaultTimeout = time.Minute * 15
 // constant for the error when a pod is deleted expectedly during a check run
 var ErrPodRemovedExpectedly = errors.New("pod deleted expectedly")
 
+// constant for the error when a pod is deleted before the check pod running
+var ErrPodDeletedBeforeRunning = errors.New("the khcheck check pod is deleted, waiting for start failed")
+
 // DefaultName is used when no check name is supplied
 var DefaultName = "external-check"
 
@@ -955,6 +958,17 @@ func (ext *Checker) waitForPodStart() chan error {
 
 			ext.log("starting pod running watcher")
 
+			pods, err := podClient.List(metav1.ListOptions{
+				LabelSelector: kuberhealthyRunIDLabel + "=" + ext.currentCheckUUID,
+			})
+			if err != nil {
+				outChan <- err
+				return
+			}
+			if pods.Size() == 0 {
+				outChan <- ErrPodDeletedBeforeRunning
+				return
+			}
 			// start watching
 			watcher, err := podClient.Watch(metav1.ListOptions{
 				LabelSelector: kuberhealthyRunIDLabel + "=" + ext.currentCheckUUID,
@@ -969,6 +983,13 @@ func (ext *Checker) waitForPodStart() chan error {
 
 				ext.log("got an event while waiting for pod to start running")
 
+				if e.Type == watch.Deleted {
+					ext.log("the khcheck check pod is deleted, waiting for start failed!")
+					outChan <- ErrPodDeletedBeforeRunning
+					watcher.Stop()
+					return
+				}
+
 				// try to cast the incoming object to a pod and skip the event if we cant
 				p, ok := e.Object.(*apiv1.Pod)
 				if !ok {

pkg/health/checkDetails.go

@@ -17,7 +17,7 @@ import "time"
 type CheckDetails struct {
 	OK               bool
 	Errors           []string
-	RunDuration		 string
+	RunDuration      string
 	Namespace        string
 	LastRun          time.Time // the time the check last was last run
 	AuthoritativePod string    // the pod that last ran the check

pkg/khcheckcrd/functions_test.go

@@ -56,7 +56,7 @@ func TestCreate(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	checkDetails := NewKuberhealthyCheck(testCheckName, defaultNamespace, NewCheckConfig(time.Second,v1.PodSpec{}))
+	checkDetails := NewKuberhealthyCheck(testCheckName, defaultNamespace, NewCheckConfig(time.Second, v1.PodSpec{}))
 	result, err := client.Create(&checkDetails, resource, defaultNamespace)
 	if err != nil {
 		t.Fatal(err)