Move health checks to kubernetes package and extract logging (#1677)

* Refactored health tool

Signed-off-by: Andreas Neumann <aneumann@mesosphere.com>

Co-authored-by: Ken Sipe <kensipe@gmail.com>

pkg/engine/task/task_apply.go

@@ -20,8 +20,8 @@ import (
 
 	kudoapi "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
 	"github.com/kudobuilder/kudo/pkg/engine"
-	"github.com/kudobuilder/kudo/pkg/engine/health"
 	"github.com/kudobuilder/kudo/pkg/engine/resource"
+	"github.com/kudobuilder/kudo/pkg/kubernetes/status"
 	"github.com/kudobuilder/kudo/pkg/util/kudo"
 )
 
@@ -238,15 +238,39 @@ func strategicThreeWayMergePatch(r runtime.Object, original, modified, current [
 
 func isHealthy(ro []runtime.Object) error {
 	for _, r := range ro {
-		err := health.IsHealthy(r)
+		err := isResourceHealthy(r)
 		if err != nil {
-			key, _ := client.ObjectKeyFromObject(r)
+			key, _ := client.ObjectKeyFromObject(r) // err not possible as all runtime.Objects have metadata
 			return fmt.Errorf("object %s/%s is NOT healthy: %w", key.Namespace, key.Name, err)
 		}
 	}
 	return nil
 }
 
+func isResourceHealthy(obj runtime.Object) error {
+	healthy, msg, err := status.IsHealthy(obj)
+	if err != nil {
+		return err
+	}
+	if healthy {
+		if msg != "" {
+			log.Printf("HealthUtil: %s", msg)
+		}
+		return nil
+	}
+	isTerminal, terminalMsg, err := status.IsTerminallyFailed(obj)
+	if err != nil {
+		return err
+	}
+	if isTerminal {
+		log.Printf("HealthUtil: %s", terminalMsg)
+		return fmt.Errorf("%wHealthUtil: %s", engine.ErrFatalExecution, terminalMsg)
+	}
+
+	log.Printf("HealthUtil: %s", msg)
+	return errors.New(msg)
+}
+
 // copy from k8s.io/kubectl@v0.16.6/pkg/util/apply.go, with adjustments
 // GetOriginalConfiguration retrieves the original configuration of the object
 // from the annotation.

pkg/engine/task/task_delete.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/kudobuilder/kudo/pkg/engine/health"
+	"github.com/kudobuilder/kudo/pkg/kubernetes/status"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -54,11 +54,20 @@ func (dt DeleteTask) Run(ctx Context) (bool, error) {
 		return false, err
 	}
 
-	// 6. - Check health: always true for Delete task -
-	err = health.IsDeleted(ctx.Client, ctx.Discovery, objs)
-	if err != nil {
-		log.Printf("TaskExecution: %v", err)
-		return false, nil
+	// 6. - Check health - wait for object deletion
+	return allObjsDeleted(ctx, objs)
+}
+
+func allObjsDeleted(ctx Context, objs []runtime.Object) (bool, error) {
+	for _, obj := range objs {
+		objDeleted, _, err := status.IsDeleted(ctx.Client, ctx.Discovery, obj)
+		if err != nil {
+			log.Printf("TaskExecution: wait for object deletion: %v", err)
+			return false, nil
+		}
+		if !objDeleted {
+			return false, nil
+		}
 	}
 	return true, nil
 }

pkg/engine/task/task_kudo_operator.go

@@ -15,7 +15,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	kudoapi "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
-	"github.com/kudobuilder/kudo/pkg/engine/health"
 	"github.com/kudobuilder/kudo/pkg/engine/renderer"
 	parser "github.com/kudobuilder/kudo/pkg/kudoctl/cmd/params"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/packages"
@@ -63,7 +62,7 @@ func (kt KudoOperatorTask) Run(ctx Context) (bool, error) {
 	}
 
 	// 4. - Check the Instance health -
-	if err := health.IsHealthy(instance); err != nil {
+	if err := isResourceHealthy(instance); err != nil {
 		return false, nil
 	}