Failed to reconcile persistence should update

ReconcileSuccess status.Condidition and publish events - correct logging on storage capacity
rabbitmq · Feb 17, 2021 · 3afb8db · 3afb8db
1 parent 8ab7f0c
commit 3afb8db
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 17 deletions.
diff --git a/controllers/rabbitmqcluster_controller.go b/controllers/rabbitmqcluster_controller.go
@@ -161,7 +161,11 @@ func (r *RabbitmqClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 			return ctrl.Result{}, err
 		}
 
-		if err = r.reconcilePVC(ctx, builder, rabbitmqCluster, resource); err != nil  {
+		if err = r.reconcilePVC(ctx, builder, rabbitmqCluster, resource); err != nil {
+			rabbitmqCluster.Status.SetCondition(status.ReconcileSuccess, corev1.ConditionFalse, "FailedReconcilePVC", err.Error())
+			if statusErr := r.Status().Update(ctx, rabbitmqCluster); statusErr != nil {
+				logger.Error(statusErr, "Failed to update ReconcileSuccess condition state")
+			}
 			return ctrl.Result{}, err
 		}
 

diff --git a/controllers/reconcile_persistence.go b/controllers/reconcile_persistence.go
@@ -37,14 +37,12 @@ func (r *RabbitmqClusterReconciler) reconcilePVC(ctx context.Context, builder re
 		}
 
 		resize, err := r.needsPVCResize(current, sts)
-
 		if err != nil {
 			return err
 		}
 
 		if resize {
 			if err := r.expandPVC(ctx, cluster, current, sts); err != nil {
-				logger.Error(err, "Failed to expand PersistentVolumeClaims", "statefulSet", cluster.ChildResourceName("server"))
 				return err
 			}
 		}
@@ -65,7 +63,7 @@ func (r *RabbitmqClusterReconciler) expandPVC(ctx context.Context, rmq *rabbitmq
 		return err
 	}
 
-	logger.Info(fmt.Sprintf("updating storage capacity from %v to %v", currentCapacity, desiredCapacity))
+	logger.Info(fmt.Sprintf("updating storage capacity from %s to %s", currentCapacity.String(), desiredCapacity.String()))
 
 	if err := r.deleteSts(ctx, rmq); err != nil {
 		return err
@@ -87,13 +85,17 @@ func (r *RabbitmqClusterReconciler) updatePVC(ctx context.Context, rmq *rabbitmq
 		PVC := corev1.PersistentVolumeClaim{}
 
 		if err := r.Client.Get(ctx, types.NamespacedName{Namespace: rmq.Namespace, Name: PVCName}, &PVC); err != nil {
-			logger.Error(err, "failed to get PersistentVolumeClaim")
-			return err
+			msg := "failed to get PersistentVolumeClaim"
+			logger.Error(err, msg, "PersistentVolumeClaim", PVCName)
+			r.Recorder.Event(rmq, corev1.EventTypeWarning, "FailedReconcilePersistence", fmt.Sprintf("%s %s", msg, PVCName))
+			return fmt.Errorf("%s %s: %v", msg, PVCName, err)
 		}
 		PVC.Spec.Resources.Requests[corev1.ResourceStorage] = desiredCapacity
 		if err := r.Client.Update(ctx, &PVC, &client.UpdateOptions{}); err != nil {
-			logger.Error(err, "failed to update PersistentVolumeClaim")
-			return err
+			msg := "failed to update PersistentVolumeClaim"
+			logger.Error(err, msg, "PersistentVolumeClaim", PVCName)
+			r.Recorder.Event(rmq, corev1.EventTypeWarning, "FailedReconcilePersistence", fmt.Sprintf("%s %s", msg, PVCName))
+			return fmt.Errorf("%s %s: %v", msg, PVCName, err)
 		}
 		logger.Info("successfully expanded", "PVC", PVCName)
 	}
@@ -141,8 +143,10 @@ func (r *RabbitmqClusterReconciler) deleteSts(ctx context.Context, rmq *rabbitmq
 		return err
 	}
 	if err := r.Delete(ctx, current, deleteOptions); err != nil {
-		logger.Error(err, "failed to delete statefulSet", "statefulSet", stsName)
-		return err
+		msg := "failed to delete statefulSet"
+		logger.Error(err, msg, "statefulSet", stsName)
+		r.Recorder.Event(rmq, corev1.EventTypeWarning, "FailedReconcilePersistence", fmt.Sprintf("%s %s", msg, stsName))
+		return fmt.Errorf("%s %s: %v", msg, stsName, err)
 	}
 
 	if err := retryWithInterval(logger, "delete statefulSet", 10, 3*time.Second, func() bool {
@@ -152,8 +156,10 @@ func (r *RabbitmqClusterReconciler) deleteSts(ctx context.Context, rmq *rabbitmq
 		}
 		return false
 	}); err != nil {
-		logger.Error(err, "statefulSet not deleting after 50 seconds", "statefulSet", stsName)
-		return err
+		msg := "statefulSet not deleting after 30 seconds"
+		logger.Error(err, msg, "statefulSet", stsName)
+		r.Recorder.Event(rmq, corev1.EventTypeWarning, "FailedReconcilePersistence", fmt.Sprintf("%s %s", msg, stsName))
+		return fmt.Errorf("%s %s: %v", msg, stsName, err)
 	}
 	logger.Info("statefulSet deleted", "statefulSet", stsName)
 	return nil

diff --git a/system_tests/system_tests.go b/system_tests/system_tests.go
@@ -240,10 +240,6 @@ CONSOLE_LOG=new`
 			password string
 		)
 
-		AfterEach(func() {
-			Expect(rmqClusterClient.Delete(context.TODO(), cluster)).To(Succeed())
-		})
-
 		BeforeEach(func() {
 			cluster = newRabbitmqCluster(namespace, "persistence-rabbit")
 			Expect(createRabbitmqCluster(ctx, rmqClusterClient, cluster)).To(Succeed())
@@ -259,6 +255,10 @@ CONSOLE_LOG=new`
 			assertHttpReady(hostname, port)
 		})
 
+		AfterEach(func() {
+			Expect(rmqClusterClient.Delete(context.TODO(), cluster)).To(Succeed())
+		})
+
 		It("persists messages", func() {
 			By("publishing a message", func() {
 				Expect(publishToQueue(hostname, port, username, password)).To(Succeed())
@@ -296,7 +296,7 @@ CONSOLE_LOG=new`
 		})
 
 		BeforeEach(func() {
-			// volume expansion is supported in kinD which is use in github action
+			// volume expansion is not supported in kinD which is use in github action
 			if os.Getenv("SUPPORT_VOLUME_EXPANSION") == "false" {
 				Skip("SUPPORT_VOLUME_EXPANSION is set to false; skipping volume expansion test")
 			}