diff --git a/pkg/webhooks/workload_webhook.go b/pkg/webhooks/workload_webhook.go
index e473e77af8..8e584b8868 100644
--- a/pkg/webhooks/workload_webhook.go
+++ b/pkg/webhooks/workload_webhook.go
@@ -84,7 +84,7 @@ func (w *WorkloadWebhook) ValidateCreate(ctx context.Context, obj runtime.Object
 	wl := obj.(*kueue.Workload)
 	log := ctrl.LoggerFrom(ctx).WithName("workload-webhook")
 	log.V(5).Info("Validating create", "workload", klog.KObj(wl))
-	return nil, ValidateWorkload(wl).ToAggregate()
+	return nil, ValidateWorkload(wl, nil).ToAggregate()
 }
 
 // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type
@@ -101,7 +101,7 @@ func (w *WorkloadWebhook) ValidateDelete(ctx context.Context, obj runtime.Object
 	return nil, nil
 }
 
-func ValidateWorkload(obj *kueue.Workload) field.ErrorList {
+func ValidateWorkload(obj, oldObj *kueue.Workload) field.ErrorList {
 	var allErrs field.ErrorList
 	specPath := field.NewPath("spec")
 
@@ -109,6 +109,9 @@ func ValidateWorkload(obj *kueue.Workload) field.ErrorList {
 	for i := range obj.Spec.PodSets {
 		ps := &obj.Spec.PodSets[i]
 		allErrs = append(allErrs, validatePodSet(ps, specPath.Child("podSets").Index(i))...)
+		if oldObj != nil {
+			allErrs = append(allErrs, apivalidation.ValidateImmutableField(ps.Count, oldObj.Spec.PodSets[i].Count, specPath.Child("podSets").Index(i))...)
+		}
 		if ps.MinCount != nil {
 			variableCountPosets++
 		}
@@ -219,7 +222,7 @@ func validateReclaimablePods(obj *kueue.Workload, basePath *field.Path) field.Er
 func ValidateWorkloadUpdate(newObj, oldObj *kueue.Workload) field.ErrorList {
 	var allErrs field.ErrorList
 	specPath := field.NewPath("spec")
-	allErrs = append(allErrs, ValidateWorkload(newObj)...)
+	allErrs = append(allErrs, ValidateWorkload(newObj, oldObj)...)
 	allErrs = append(allErrs, apivalidation.ValidateImmutableField(newObj.Spec.PriorityClassSource, oldObj.Spec.PriorityClassSource, specPath.Child("priorityClassSource"))...)
 	allErrs = append(allErrs, apivalidation.ValidateImmutableField(newObj.Spec.PriorityClassName, oldObj.Spec.PriorityClassName, specPath.Child("priorityClassName"))...)
 	if workload.HasQuotaReservation(newObj) && workload.HasQuotaReservation(oldObj) {
diff --git a/test/integration/webhook/workload_test.go b/test/integration/webhook/workload_test.go
index 043707dbff..0d889dfc04 100644
--- a/test/integration/webhook/workload_test.go
+++ b/test/integration/webhook/workload_test.go
@@ -154,8 +154,12 @@ var _ = ginkgo.Describe("Workload validating webhook", func() {
 		})
 
 		ginkgo.It("Should forbid the change of spec.podSet", func() {
+			ginkgo.By("Creating a pod template")
+			podtemplate := testing.MakePodTemplate(workloadName, ns.Name).Labels(map[string]string{constants.WorkloadNameLabel: workloadName}).Obj()
+			gomega.Expect(k8sClient.Create(ctx, podtemplate)).Should(gomega.Succeed())
+
 			ginkgo.By("Creating a new Workload")
-			workload := testing.MakeWorkload(workloadName, ns.Name).Obj()
+			workload := testing.MakeWorkload(workloadName, ns.Name).SetPodSetName(podtemplate.Name).SetPodTemplateName(podtemplate.Name).Obj()
 			gomega.Expect(k8sClient.Create(ctx, workload)).Should(gomega.Succeed())
 
 			ginkgo.By("Updating podSet")