Separate dispatching and running timeouts

Author: tardieu

internal/controller/appwrapper_controller.go

@@ -155,7 +155,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	case mcadv1beta1.Requeuing:
 		// delete wrapped resources
 		if r.deleteResources(ctx, appWrapper) != 0 {
-			if isSlowDeletion(appWrapper) {
+			if isSlowRequeuing(appWrapper) {
 				// give up requeuing and fail instead
 				return r.updateStatus(ctx, appWrapper, mcadv1beta1.Failed)
 			} else {
@@ -167,7 +167,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 	case mcadv1beta1.Dispatching:
 		// dispatching is taking too long?
-		if isSlowCreation(appWrapper) {
+		if isSlowDispatching(appWrapper) {
 			// set requeuing or failed status
 			return r.requeueOrFail(ctx, appWrapper)
 		}
@@ -190,7 +190,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
-		if counts.Failed > 0 || isSlowCreation(appWrapper) && (counts.Other > 0 || counts.Running < int(appWrapper.Spec.MinPods)) {
+		if counts.Failed > 0 || isSlowRunning(appWrapper) && (counts.Other > 0 || counts.Running < int(appWrapper.Spec.MinPods)) {
 			// set requeuing or failed status
 			return r.requeueOrFail(ctx, appWrapper)
 		}

internal/controller/resource_manager.go

@@ -117,12 +117,17 @@ func (r *AppWrapperReconciler) monitorPods(ctx context.Context, appWrapper *mcad
 	return counts, nil
 }
 
-// Is dispatch too slow?
-func isSlowCreation(appWrapper *mcadv1beta1.AppWrapper) bool {
-	return metav1.Now().After(appWrapper.Status.LastDispatchTime.Add(creationTimeout))
+// Is requeuing too slow?
+func isSlowRequeuing(appWrapper *mcadv1beta1.AppWrapper) bool {
+	return metav1.Now().After(appWrapper.Status.LastRequeuingTime.Add(requeuingTimeout))
 }
 
-// Is requeuing too slow?
-func isSlowDeletion(appWrapper *mcadv1beta1.AppWrapper) bool {
-	return metav1.Now().After(appWrapper.Status.LastRequeuingTime.Add(deletionTimeout))
+// Is dispatching too slow?
+func isSlowDispatching(appWrapper *mcadv1beta1.AppWrapper) bool {
+	return metav1.Now().After(appWrapper.Status.LastDispatchTime.Add(dispatchingTimeout))
+}
+
+// Is running too slow?
+func isSlowRunning(appWrapper *mcadv1beta1.AppWrapper) bool {
+	return metav1.Now().After(appWrapper.Status.LastDispatchTime.Add(runningTimeout))
 }

internal/controller/timings.go

@@ -30,9 +30,10 @@ const (
 
 	cacheConflictTimeout = 5 * time.Minute // when to give up on a cache conflict
 
-	// Timeouts for the creation and deletion of wrapped resources and pods
-	creationTimeout = 5 * time.Minute // minimum wait before aborting an incomplete resource/pod creation
-	deletionTimeout = 5 * time.Minute // minimum wait before aborting an incomplete resource deletion
+	// Timeouts
+	requeuingTimeout   = 2 * time.Minute // minimum wait before aborting Requeuing
+	dispatchingTimeout = 2 * time.Minute // minimum wait before aborting Dispatching
+	runningTimeout     = 5 * time.Minute // minimum wait before aborting Running
 
 	// RequeueAfter delays
 	// This is the maximum delay before the next reconciliation event but reconciliation