diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go
index f5bda0b530..b3e0620443 100644
--- a/pkg/controller/core/workload_controller.go
+++ b/pkg/controller/core/workload_controller.go
@@ -255,6 +255,16 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	return ctrl.Result{}, nil
 }
 
+func (r *WorkloadReconciler) isScaledDown(wl *kueue.Workload) bool {
+	podSetSize := len(wl.Spec.PodSets)
+	for i := 1; i < podSetSize; i++ {
+		if ptr.Deref(wl.Status.Admission.PodSetAssignments[i].Count, 0) > wl.Spec.PodSets[i].Count {
+			return true
+		}
+	}
+	return false
+}
+
 func (r *WorkloadReconciler) downSizeJobIfNecessary(wl *kueue.Workload, ctx context.Context) error {
 	statusUpdate := false
 	podSetSize := len(wl.Spec.PodSets)
@@ -528,6 +538,7 @@ func (r *WorkloadReconciler) Delete(e event.DeleteEvent) bool {
 
 func (r *WorkloadReconciler) Update(e event.UpdateEvent) bool {
 	oldWl, isWorkload := e.ObjectOld.(*kueue.Workload)
+
 	if !isWorkload {
 		// this event will be handled by the LimitRange/RuntimeClass handle
 		return true
@@ -620,7 +631,7 @@ func (r *WorkloadReconciler) Update(e event.UpdateEvent) bool {
 				}
 			})
 		}
-	case prevStatus == admitted && status == admitted && !equality.Semantic.DeepEqual(oldWl.Status.ReclaimablePods, wl.Status.ReclaimablePods):
+	case prevStatus == admitted && status == admitted && !equality.Semantic.DeepEqual(oldWl.Status.ReclaimablePods, wl.Status.ReclaimablePods) || r.isScaledDown(oldWl):
 		// trigger the move of associated inadmissibleWorkloads, if there are any.
 		r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, wl, func() {
 			// Update the workload from cache while holding the queues lock