Calculate health for services (#1707)

A service is considered healthy if its cluster IP is accessible. Furthermore, if the service is load-balanced, then balancer needs to have an ingress defined.

Signed-off-by: Jan Schlicht <jan@d2iq.com>

Co-authored-by: Aleksey Dukhovniy <adukhovniy@mesosphere.io>

pkg/kubernetes/status/health.go

@@ -156,12 +156,44 @@ func IsHealthy(obj runtime.Object) (healthy bool, msg string, err error) {
 		}
 		return false, fmt.Sprintf("namespace %s is not active: %s", obj.Name, obj.Status.Phase), nil
 
+	case *corev1.Service:
+		return isServiceHealthy(obj)
+
 	// unless we build logic for what a healthy object is, assume it's healthy when created.
 	default:
 		return true, fmt.Sprintf("unknown type %s is marked healthy by default", reflect.TypeOf(obj)), nil
 	}
 }
 
+// Service health depends on the ingress type.
+// To be considered healthy, a service needs to be accessible by its cluster IP.
+// If the service is load-balanced, the balancer need to have an ingress defined.
+// Adapted from https://github.com/helm/helm/blob/v3.3.4/pkg/kube/wait.go#L185.
+func isServiceHealthy(obj *corev1.Service) (healthy bool, msg string, err error) {
+	// ExternalName services are external to cluster. KUDO shouldn't be checking to see if they're 'ready' (i.e. have an IP set).
+	if obj.Spec.Type == corev1.ServiceTypeExternalName {
+		return true, fmt.Sprintf("external name service %s/%s is marked healthy", obj.Namespace, obj.Name), nil
+	}
+
+	if obj.Spec.ClusterIP == "" {
+		return false, fmt.Sprintf("service %s/%s does not have cluster IP address", obj.Namespace, obj.Name), nil
+	}
+
+	// Check if the service has a LoadBalancer and that balancer has an Ingress defined.
+	if obj.Spec.Type == corev1.ServiceTypeLoadBalancer {
+		if len(obj.Spec.ExternalIPs) > 0 {
+			return true, fmt.Sprintf("service %s/%s has external IP addresses (%v), marked healthy", obj.Namespace, obj.Name, obj.Spec.ExternalIPs), nil
+		}
+
+		if obj.Status.LoadBalancer.Ingress == nil {
+			return false, fmt.Sprintf("service %s/%s does not have load balancer ingress IP address", obj.Namespace, obj.Name), nil
+		}
+	}
+
+	// If none of the above conditions are met, we can assume that the service is healthy.
+	return true, fmt.Sprintf("service %s/%s is marked healthy", obj.Namespace, obj.Name), nil
+}
+
 func toUnstructured(obj runtime.Object) (*unstructured.Unstructured, error) {
 	unstructMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
 	if err != nil {

pkg/kubernetes/status/health_test.go

@@ -0,0 +1,202 @@
+package status
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	kudoapi "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
+)
+
+func TestIsHealthy(t *testing.T) {
+	// this table skips resources covered by 'polymorphichelpers' and older APIs
+	// for which the same code is used.
+	tests := []struct {
+		name    string
+		input   runtime.Object
+		healthy bool
+		msg     string
+	}{
+		{
+			name:    "unknown object type",
+			input:   &corev1.ConfigMap{},
+			healthy: true,
+			msg:     "unknown type *v1.ConfigMap is marked healthy by default",
+		},
+		{
+			name: "unhealthy CRD",
+			input: &apiextv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "foo",
+				},
+				Status: apiextv1.CustomResourceDefinitionStatus{
+					Conditions: []apiextv1.CustomResourceDefinitionCondition{
+						{
+							Type:   apiextv1.Established,
+							Status: apiextv1.ConditionFalse,
+						},
+					},
+				},
+			},
+			healthy: false,
+			msg:     "CRD foo is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC  }] )",
+		},
+		{
+			name: "healthy CRD",
+			input: &apiextv1.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "foo",
+				},
+				Status: apiextv1.CustomResourceDefinitionStatus{
+					Conditions: []apiextv1.CustomResourceDefinitionCondition{
+						{
+							Type:   apiextv1.Established,
+							Status: apiextv1.ConditionTrue,
+						},
+					},
+				},
+			},
+			healthy: true,
+			msg:     "CRD foo is now healthy",
+		},
+		{
+			name: "unhealthy Job",
+			input: &batchv1.Job{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "foo",
+				},
+				Status: batchv1.JobStatus{
+					Succeeded: 0,
+				},
+			},
+			healthy: false,
+			msg:     "job \"foo\" still running or failed",
+		},
+		{
+			name: "healthy Job",
+			input: &batchv1.Job{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "foo",
+				},
+				Status: batchv1.JobStatus{
+					Succeeded: 1,
+				},
+			},
+			healthy: true,
+			msg:     "job \"foo\" is marked healthy",
+		},
+		{
+			name: "unhealthy Instance",
+			input: &kudoapi.Instance{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "foo",
+					Namespace: "default",
+				},
+				Spec: kudoapi.InstanceSpec{
+					PlanExecution: kudoapi.PlanExecution{
+						PlanName: "deploy",
+						Status:   kudoapi.ExecutionInProgress,
+					},
+				},
+			},
+			healthy: false,
+			msg:     "instance default/foo active plan is in state IN_PROGRESS",
+		},
+		{
+			name: "healthy Instance",
+			input: &kudoapi.Instance{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "foo",
+					Namespace: "default",
+				},
+				Spec: kudoapi.InstanceSpec{
+					PlanExecution: kudoapi.PlanExecution{
+						PlanName: "",
+					},
+				},
+			},
+			healthy: true,
+			msg:     "instance default/foo is marked healthy",
+		},
+		{
+			name: "unhealthy Pod",
+			input: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "foo",
+					Namespace: "default",
+				},
+				Status: corev1.PodStatus{
+					Phase: corev1.PodUnknown,
+					Conditions: []corev1.PodCondition{
+						{
+							Type: corev1.PodReady,
+						},
+					},
+				},
+			},
+			healthy: false,
+			msg:     "pod default/foo is not running yet: Unknown",
+		},
+		{
+			name: "healthy Pod",
+			input: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "foo",
+					Namespace: "default",
+				},
+				Status: corev1.PodStatus{
+					Phase: corev1.PodRunning,
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+				},
+			},
+			healthy: true,
+			msg:     "",
+		},
+		{
+			name: "unhealthy Namespace",
+			input: &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "default",
+				},
+				Status: corev1.NamespaceStatus{
+					Phase: corev1.NamespaceTerminating,
+				},
+			},
+			healthy: false,
+			msg:     "namespace default is not active: Terminating",
+		},
+		{
+			name: "healthy Namespace",
+			input: &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "default",
+				},
+				Status: corev1.NamespaceStatus{
+					Phase: corev1.NamespaceActive,
+				},
+			},
+			healthy: true,
+			msg:     "",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			healthy, msg, _ := IsHealthy(test.input)
+
+			assert.Equal(t, test.healthy, healthy)
+			assert.Equal(t, test.msg, msg)
+		})
+	}
+}