[Deployment] Add PollRolloutStatus and poll deployment on updates

modules/common/deployment/deployment.go

@@ -18,30 +18,59 @@ package deployment
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
 // NewDeployment returns an initialized Deployment.
+//
+// Pass nil for pollingInterval and pollingTimeout to initialize with default.
 func NewDeployment(
 	deployment *appsv1.Deployment,
 	timeout time.Duration,
 ) *Deployment {
 	return &Deployment{
-		deployment: deployment,
-		timeout:    timeout,
+		deployment:          deployment,
+		timeout:             timeout,
+		rolloutPollInterval: ptr.To(DefaultPollInterval),
+		rolloutPollTimeout:  ptr.To(DefaultPollTimeout),
 	}
 }
 
+// SetRolloutPollInterval -
+func (d *Deployment) SetRolloutPollInterval(interval time.Duration) {
+	d.rolloutPollInterval = ptr.To(interval)
+}
+
+// GetRolloutPollInterval -
+func (d *Deployment) GetRolloutPollInterval() *time.Duration {
+	return d.rolloutPollInterval
+}
+
+// SetRolloutPollTimeout -
+func (d *Deployment) SetRolloutPollTimeout(timeout time.Duration) {
+	d.rolloutPollTimeout = ptr.To(timeout)
+}
+
+// GetRolloutPollTimeout -
+func (d *Deployment) GetRolloutPollTimeout() *time.Duration {
+	return d.rolloutPollTimeout
+}
+
 // CreateOrPatch - creates or patches a deployment, reconciles after Xs if object won't exist.
 func (d *Deployment) CreateOrPatch(
 	ctx context.Context,
@@ -80,19 +109,116 @@ func (d *Deployment) CreateOrPatch(
 		}
 		return ctrl.Result{}, err
 	}
-	if op != controllerutil.OperationResultNone {
-		h.GetLogger().Info(fmt.Sprintf("Deployment %s - %s", deployment.Name, op))
-	}
-
 	// update the deployment object of the deployment type
-	d.deployment, err = GetDeploymentWithName(ctx, h, deployment.GetName(), deployment.GetNamespace())
-	if err != nil {
-		return ctrl.Result{}, err
+	d.deployment = deployment
+
+	h.GetLogger().Info(fmt.Sprintf("Deployment %s %s", deployment.Name, op))
+	// Only poll on Deployment updates, not on initial create.
+	if op != controllerutil.OperationResultCreated {
+		// only poll if replicas > 0
+		if d.deployment.Spec.Replicas != nil && *d.deployment.Spec.Replicas > 0 {
+			// Ignore context.DeadlineExceeded when PollUntilContextTimeout reached
+			// the poll timeout. d.rolloutStatus as information on the
+			// replica rollout, the consumer can evaluate the rolloutStatus and
+			// retry/reconcile until RolloutComplete, or ProgressDeadlineExceeded.
+			if err := d.PollRolloutStatus(ctx, h); err != nil && !errors.Is(err, context.DeadlineExceeded) &&
+				!strings.Contains(err.Error(), "would exceed context deadline") {
+				return ctrl.Result{}, fmt.Errorf("poll rollout error: %w", err)
+			}
+		}
 	}
 
 	return ctrl.Result{}, nil
 }
 
+// PollRolloutStatus - will poll the deployment rollout to verify its status for Complet, Failed or polling until timeout.
+//
+// - Complete - all replicas updated using RolloutComplete()
+//
+// - Failed   - rollout of new config failed and the new pod is stuck in ProgressDeadlineExceeded using ProgressDeadlineExceeded()
+func (d *Deployment) PollRolloutStatus(
+	ctx context.Context,
+	h *helper.Helper,
+) error {
+	if d.rolloutPollInterval == nil {
+		d.rolloutPollInterval = ptr.To(DefaultPollInterval)
+	}
+	if d.rolloutPollTimeout == nil {
+		d.rolloutPollTimeout = ptr.To(DefaultPollTimeout)
+	}
+
+	err := wait.PollUntilContextTimeout(ctx, *d.rolloutPollInterval, *d.rolloutPollTimeout, true, func(ctx context.Context) (bool, error) {
+		// Fetch deployment object
+		depl, err := GetDeploymentWithName(ctx, h, d.deployment.Name, d.deployment.Namespace)
+		if err != nil {
+			return false, err
+		}
+		d.deployment = depl
+
+		// Check if rollout is complete
+		if Complete(d.deployment.Status, d.deployment.Generation) {
+			d.rolloutStatus = ptr.To(DeploymentPollCompleted)
+			d.rolloutMessage = fmt.Sprintf(DeploymentPollCompletedMessage, d.deployment.Name)
+			h.GetLogger().Info(d.rolloutMessage)
+			// If rollout is complete, return true to stop polling
+			return true, nil
+		}
+
+		// check if we already reached the ProgressDeadlineExceeded
+		if ok, msg := ProgressDeadlineExceeded(d.deployment.Status); ok {
+			d.rolloutStatus = ptr.To(DeploymentPollProgressDeadlineExceeded)
+			d.rolloutMessage = fmt.Sprintf(DeploymentPollProgressDeadlineExceededMessage, d.deployment.Name, msg)
+			// If rollout reached ProgressDeadlineExceeded, return true to stop polling
+			return true, nil
+		}
+
+		// If not yet complete, continue waiting
+		d.rolloutStatus = ptr.To(DeploymentPollProgressing)
+		d.rolloutMessage = fmt.Sprintf(DeploymentPollProgressingMessage, d.deployment.Name,
+			d.deployment.Status.UpdatedReplicas, d.deployment.Status.Replicas)
+		h.GetLogger().Info(*d.rolloutStatus)
+
+		return false, nil
+	})
+
+	return err
+}
+
+// RolloutComplete -
+func (d *Deployment) RolloutComplete() bool {
+	return d.GetRolloutStatus() != nil && *d.GetRolloutStatus() == DeploymentPollCompleted
+}
+
+// Complete -
+func Complete(status appsv1.DeploymentStatus, generation int64) bool {
+	return status.UpdatedReplicas == status.Replicas &&
+		status.Replicas == status.AvailableReplicas &&
+		status.ObservedGeneration == generation
+}
+
+// ProgressDeadlineExceeded -
+func ProgressDeadlineExceeded(status appsv1.DeploymentStatus) (bool, string) {
+	for _, condition := range status.Conditions {
+		if condition.Type == appsv1.DeploymentProgressing &&
+			condition.Status == corev1.ConditionFalse &&
+			condition.Reason == DeploymentPollProgressDeadlineExceeded {
+			return true, condition.Message
+		}
+	}
+
+	return false, ""
+}
+
+// GetRolloutStatus - get rollout status of the deployment.
+func (d *Deployment) GetRolloutStatus() *string {
+	return d.rolloutStatus
+}
+
+// GetRolloutMessage - get rollout message of the deployment.
+func (d *Deployment) GetRolloutMessage() string {
+	return d.rolloutMessage
+}
+
 // Delete - delete a deployment.
 func (d *Deployment) Delete(
 	ctx context.Context,

modules/common/deployment/types.go

@@ -22,8 +22,31 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 )
 
+const (
+	// DeploymentPollCompleted -
+	DeploymentPollCompleted = "Completed"
+	// DeploymentPollCompletedMessage -
+	DeploymentPollCompletedMessage = "%s Completed"
+	// DeploymentPollProgressing -
+	DeploymentPollProgressing = "Progressing"
+	// DeploymentPollProgressingMessage -
+	DeploymentPollProgressingMessage = "%s - %d/%d replicas updated"
+	// DeploymentPollProgressDeadlineExceeded -
+	DeploymentPollProgressDeadlineExceeded = "ProgressDeadlineExceeded"
+	// DeploymentPollProgressDeadlineExceededMessage -
+	DeploymentPollProgressDeadlineExceededMessage = "%s ProgressDeadlineExceeded - %s"
+	// DefaultPollInterval -
+	DefaultPollInterval = 2 * time.Second
+	// DefaultPollTimeout -
+	DefaultPollTimeout = 20 * time.Second
+)
+
 // Deployment -
 type Deployment struct {
-	deployment *appsv1.Deployment
-	timeout    time.Duration
+	deployment          *appsv1.Deployment
+	timeout             time.Duration
+	rolloutStatus       *string
+	rolloutMessage      string
+	rolloutPollInterval *time.Duration
+	rolloutPollTimeout  *time.Duration
 }