Add a wait for API pods when image version changed

closes: #969

CHANGES/969.feature

@@ -0,0 +1,2 @@
+Modified the reconciliation for `pulpcore-content` to wait for `API` pods get
+into a READY state before updating the `Deployment` in case of image version change.

controllers/repo_manager/content.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"os"
 	"strconv"
+	"time"
 
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
@@ -71,6 +72,12 @@ func (r *RepoManagerReconciler) pulpContentController(ctx context.Context, pulp
 	deployment := &appsv1.Deployment{}
 	r.Get(ctx, types.NamespacedName{Name: pulp.Name + "-content", Namespace: pulp.Namespace}, deployment)
 	expected := deploymentForPulpContent(funcResources)
+	// before doing the reconciliation, in case of image version change
+	// we should wait for all API pods get upgraded
+	if controllers.CheckImageVersionModified(pulp, deployment) {
+		log.Info("A new image version has been provided! Waiting for API pods to upgrade first ...")
+		controllers.WaitAPIPods(r, pulp, deployment, time.Second*60)
+	}
 	if requeue, err := reconcileObject(funcResources, expected, deployment, conditionType); err != nil || requeue {
 		return ctrl.Result{Requeue: requeue}, err
 	}

controllers/utils.go

@@ -4,13 +4,15 @@ import (
 	"bytes"
 	"context"
 	"reflect"
+	"regexp"
 	"strings"
 	"time"
 
 	repomanagerpulpprojectorgv1beta2 "github.com/pulp/pulp-operator/apis/repo-manager.pulpproject.org/v1beta2"
 
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -272,3 +274,28 @@ func CheckEmptyDir(pulp *repomanagerpulpprojectorgv1beta2.Pulp, resource string)
 		logger.Warn("CONFIGURING " + strings.ToUpper(resource) + " POD VOLUME AS EMPTYDIR. THIS SHOULD NOT BE USED IN PRODUCTION CLUSTERS.")
 	}
 }
+
+// CheckImageVersionModified verifies if the container image tag defined in
+// Pulp CR matches the one in the Deployment
+func CheckImageVersionModified(pulp *repomanagerpulpprojectorgv1beta2.Pulp, deployment *appsv1.Deployment) bool {
+	r := regexp.MustCompile(`(?P<ImageName>.*?):(?P<Tag>.*)`)
+	currentImageVersion := r.FindStringSubmatch(deployment.Spec.Template.Spec.Containers[0].Image)
+	return pulp.Spec.ImageVersion != currentImageVersion[2]
+}
+
+// WaitAPIPods waits until all API pods are in a READY state
+func WaitAPIPods[T any](resource T, pulp *repomanagerpulpprojectorgv1beta2.Pulp, deployment *appsv1.Deployment, timeout time.Duration) {
+
+	// we need to add a litte "stand by" to give time for the operator get the updated status from database/cluster
+	time.Sleep(time.Millisecond * 500)
+	clientConcrete := reflect.ValueOf(resource)
+	restClient := reflect.Indirect(clientConcrete).FieldByName("Client").Elem().Interface().(client.Client)
+	for i := 0; i < int(timeout.Seconds()); i++ {
+		apiDeployment := &appsv1.Deployment{}
+		restClient.Get(context.TODO(), types.NamespacedName{Name: pulp.Name + "-api", Namespace: pulp.Namespace}, apiDeployment)
+		if apiDeployment.Status.ReadyReplicas == apiDeployment.Status.Replicas {
+			return
+		}
+		time.Sleep(time.Second)
+	}
+}