diff --git a/apis/kueue/v1beta1/workload_types.go b/apis/kueue/v1beta1/workload_types.go index 3f98fc970c..b358752a16 100644 --- a/apis/kueue/v1beta1/workload_types.go +++ b/apis/kueue/v1beta1/workload_types.go @@ -113,6 +113,8 @@ type PodSet struct { // the keys in the nodeLabels from the ResourceFlavors considered for this // Workload are used to filter the ResourceFlavors that can be assigned to // this podSet. + // + // +optional Template *corev1.PodTemplateSpec `json:"template"` // count is the number of pods for the spec. @@ -133,6 +135,8 @@ type PodSet struct { MinCount *int32 `json:"minCount,omitempty"` // PodTemplateName is the name of the PodTemplate the Workload is associated with. + // + // +optional PodTemplateName *string `json:"podTemplateName,omitempty"` } diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index b71b2465f0..02b3ea810f 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -8186,7 +8186,6 @@ spec: required: - count - name - - template type: object maxItems: 8 minItems: 1 diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 38fd5a8c9e..17ff8d67c7 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -37,4 +37,5 @@ const ( WorkloadPriorityClassSource = "kueue.x-k8s.io/workloadpriorityclass" PodPriorityClassSource = "scheduling.k8s.io/priorityclass" + WorkloadNameSource = "kueue.x-k8s.io/workload-name" ) diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 08f930d0e0..330f4628ea 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -19,13 +19,16 @@ package core import ( "context" "fmt" + "strings" "time" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" nodev1 "k8s.io/api/node/v1" + "k8s.io/apimachinery/pkg/api/equality" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/workqueue" @@ -135,17 +138,17 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } - for i, ps := range wl.Spec.PodSets { - if wl.Spec.PodSets[i].PodTemplateName != nil { - continue - } - var pt corev1.PodTemplate - if err := r.client.Get(ctx, client.ObjectKey{Name: fmt.Sprintf("%s-%s", wl.Name, ps.Name), Namespace: wl.Namespace}, &pt); err != nil { - continue - } - wl.Spec.PodSets[i].PodTemplateName = &pt.Name + wlCopy := wl.DeepCopy() + + if err := r.updatePodTemplateName(ctx, &wl); err != nil { + log.V(2).Error(err, "updatePodTemplateName") + return ctrl.Result{}, err + } + + if !equality.Semantic.DeepEqual(wl.Spec.PodSets, wlCopy.Spec.PodSets) { if err := r.client.Update(ctx, &wl); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) + log.V(2).Error(err, "client.Update", "wl", wl) + return ctrl.Result{}, err } } @@ -191,6 +194,27 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } +func (r *WorkloadReconciler) updatePodTemplateName(ctx context.Context, wl *kueue.Workload) error { + labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{constants.WorkloadNameSource: wl.Name}} + var pts corev1.PodTemplateList + if err := r.client.List(ctx, &pts, &client.ListOptions{LabelSelector: labels.Set(labelSelector.MatchLabels).AsSelector(), Namespace: wl.Namespace}); err != nil { + return err + } + + for i, ps := range wl.Spec.PodSets { + if wl.Spec.PodSets[i].PodTemplateName != nil { + continue + } + for _, pt := range pts.Items { + if strings.Contains(pt.Name, ps.Name) { + wl.Spec.PodSets[i].PodTemplateName = &pt.Name + } + } + } + + return nil +} + func (r *WorkloadReconciler) reconcileCheckBasedEviction(ctx context.Context, wl *kueue.Workload) (bool, error) { if apimeta.IsStatusConditionTrue(wl.Status.Conditions, kueue.WorkloadEvicted) || !workload.HasRetryOrRejectedChecks(wl) { return false, nil @@ -289,8 +313,12 @@ func (r *WorkloadReconciler) Create(e event.CreateEvent) bool { } pts := r.extractPodTemplates(ctx, *wl) + if len(pts) == 0 { + return false + } wlCopy := wl.DeepCopy() + r.adjustResources(ctx, wlCopy, pts) if !workload.HasQuotaReservation(wl) { if !r.queues.AddOrUpdateWorkload(wlCopy, pts) { @@ -378,6 +406,8 @@ func (r *WorkloadReconciler) Update(e event.UpdateEvent) bool { wlCopy := wl.DeepCopy() + r.adjustResources(ctx, wlCopy, pts) + switch { case status == finished: // The workload could have been in the queues if we missed an event. @@ -500,26 +530,28 @@ func workloadStatus(w *kueue.Workload) string { // As a result, the pod's Overhead is not always correct. E.g. if we set a non-existent runtime class name to // `pod.Spec.RuntimeClassName` and we also set the `pod.Spec.Overhead`, in real world, the pod creation will be // rejected due to the mismatch with RuntimeClass. However, in the future we assume that they are correct. -func (h *podTemplateHandler) handlePodOverhead(ctx context.Context, pt *corev1.PodTemplate) { +func (r *WorkloadReconciler) handlePodOverhead(ctx context.Context, pts []corev1.PodTemplate) { log := ctrl.LoggerFrom(ctx) - podSpec := &pt.Template.Spec - if podSpec.RuntimeClassName != nil && len(podSpec.Overhead) == 0 { - var runtimeClass nodev1.RuntimeClass - if err := h.client.Get(ctx, types.NamespacedName{Name: *podSpec.RuntimeClassName}, &runtimeClass); err != nil { - log.Error(err, "Could not get RuntimeClass") - } - if runtimeClass.Overhead != nil { - podSpec.Overhead = runtimeClass.Overhead.PodFixed + for pi := range pts { + podSpec := &pts[pi].Template.Spec + if podSpec.RuntimeClassName != nil && len(podSpec.Overhead) == 0 { + var runtimeClass nodev1.RuntimeClass + if err := r.client.Get(ctx, types.NamespacedName{Name: *podSpec.RuntimeClassName}, &runtimeClass); err != nil { + log.Error(err, "Could not get RuntimeClass") + } + if runtimeClass.Overhead != nil { + podSpec.Overhead = runtimeClass.Overhead.PodFixed + } } } } -func (h *podTemplateHandler) handlePodLimitRange(ctx context.Context, pt *corev1.PodTemplate) { +func (r *WorkloadReconciler) handlePodLimitRange(ctx context.Context, wl *kueue.Workload, pts []corev1.PodTemplate) { log := ctrl.LoggerFrom(ctx) // get the list of limit ranges var list corev1.LimitRangeList - if err := h.client.List(ctx, &list, &client.ListOptions{Namespace: pt.Namespace}, client.MatchingFields{indexer.LimitRangeHasContainerType: "true"}); err != nil { + if err := r.client.List(ctx, &list, &client.ListOptions{Namespace: wl.Namespace}, client.MatchingFields{indexer.LimitRangeHasContainerType: "true"}); err != nil { log.Error(err, "Could not list LimitRanges") return } @@ -533,54 +565,51 @@ func (h *podTemplateHandler) handlePodLimitRange(ctx context.Context, pt *corev1 return } - pod := &pt.Template.Spec - for ci := range pod.InitContainers { - res := &pod.InitContainers[ci].Resources - res.Limits = resource.MergeResourceListKeepFirst(res.Limits, containerLimits.Default) - res.Requests = resource.MergeResourceListKeepFirst(res.Requests, containerLimits.DefaultRequest) - } - for ci := range pod.Containers { - res := &pod.Containers[ci].Resources - res.Limits = resource.MergeResourceListKeepFirst(res.Limits, containerLimits.Default) - res.Requests = resource.MergeResourceListKeepFirst(res.Requests, containerLimits.DefaultRequest) + for pi := range pts { + pod := &pts[pi].Template.Spec + for ci := range pod.InitContainers { + res := &pod.InitContainers[ci].Resources + res.Limits = resource.MergeResourceListKeepFirst(res.Limits, containerLimits.Default) + res.Requests = resource.MergeResourceListKeepFirst(res.Requests, containerLimits.DefaultRequest) + } + for ci := range pod.Containers { + res := &pod.Containers[ci].Resources + res.Limits = resource.MergeResourceListKeepFirst(res.Limits, containerLimits.Default) + res.Requests = resource.MergeResourceListKeepFirst(res.Requests, containerLimits.DefaultRequest) + } } } -func (h *podTemplateHandler) handleLimitsToRequests(pt *corev1.PodTemplate) { - pod := &pt.Template.Spec - for ci := range pod.InitContainers { - res := &pod.InitContainers[ci].Resources - res.Requests = resource.MergeResourceListKeepFirst(res.Requests, res.Limits) - } - for ci := range pod.Containers { - res := &pod.Containers[ci].Resources - res.Requests = resource.MergeResourceListKeepFirst(res.Requests, res.Limits) +func (r *WorkloadReconciler) handleLimitsToRequests(pts []corev1.PodTemplate) { + for pi := range pts { + pod := &pts[pi].Template.Spec + for ci := range pod.InitContainers { + res := &pod.InitContainers[ci].Resources + res.Requests = resource.MergeResourceListKeepFirst(res.Requests, res.Limits) + } + for ci := range pod.Containers { + res := &pod.Containers[ci].Resources + res.Requests = resource.MergeResourceListKeepFirst(res.Requests, res.Limits) + } } } -func (h *podTemplateHandler) adjustResources(ctx context.Context, obj client.Object) { - pt := obj.(*corev1.PodTemplate) - if pt.Name == "" { - return - } - h.handlePodOverhead(ctx, pt) - h.handlePodLimitRange(ctx, pt) - h.handleLimitsToRequests(pt) - if err := h.client.Update(ctx, pt); err != nil { - return - } +func (r *WorkloadReconciler) adjustResources(ctx context.Context, wl *kueue.Workload, pts []corev1.PodTemplate) { + r.handlePodOverhead(ctx, pts) + r.handlePodLimitRange(ctx, wl, pts) + r.handleLimitsToRequests(pts) } func (r *WorkloadReconciler) extractPodTemplates(ctx context.Context, wl kueue.Workload) []corev1.PodTemplate { - podTemplates := make([]corev1.PodTemplate, 0, len(wl.Spec.PodSets)) - for _, ps := range wl.Spec.PodSets { - var pt corev1.PodTemplate - if err := r.client.Get(ctx, client.ObjectKey{Name: fmt.Sprintf("%s-%s", wl.Name, ps.Name), Namespace: wl.Namespace}, &pt); err != nil { - continue - } - podTemplates = append(podTemplates, pt) + log := ctrl.LoggerFrom(ctx) + labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{constants.WorkloadNameSource: wl.Name}} + var pts corev1.PodTemplateList + if err := r.client.List(ctx, &pts, &client.ListOptions{LabelSelector: labels.Set(labelSelector.MatchLabels).AsSelector(), Namespace: wl.Namespace}); err != nil { + log.V(2).Error(err, "Unable to find list of podtemplates") + return nil } - return podTemplates + log.V(2).Info("extractPodTemplates", "pt", pts.Items) + return pts.Items } type resourceUpdatesHandler struct { @@ -620,7 +649,7 @@ func (h *resourceUpdatesHandler) handle(ctx context.Context, obj client.Object, case *nodev1.RuntimeClass: log := ctrl.LoggerFrom(ctx).WithValues("runtimeClass", klog.KObj(v)) ctx = ctrl.LoggerInto(ctx, log) - h.queueReconcileForPending(ctx, q, client.MatchingFields{indexer.WorkloadRuntimeClassKey: v.Name}) + h.queueReconcileForPending(ctx, q) default: panic(v) } @@ -634,12 +663,14 @@ func (h *resourceUpdatesHandler) queueReconcileForPending(ctx context.Context, _ if err != nil { log.Error(err, "Could not list pending workloads") } - log.V(4).Info("Updating pending workload requests", "count", len(lst.Items)) + log.V(2).Info("Updating pending workload requests", "count", len(lst.Items), "opts", opts) for _, w := range lst.Items { wlCopy := w.DeepCopy() log := log.WithValues("workload", klog.KObj(wlCopy)) log.V(5).Info("Queue reconcile for") - if !h.r.queues.AddOrUpdateWorkload(wlCopy, h.r.extractPodTemplates(ctx, w)) { + pts := h.r.extractPodTemplates(ctx, w) + h.r.adjustResources(ctx, wlCopy, pts) + if !h.r.queues.AddOrUpdateWorkload(wlCopy, pts) { log.V(2).Info("Queue for workload didn't exist") } } @@ -658,7 +689,6 @@ func (h *podTemplateHandler) Create(ctx context.Context, e event.CreateEvent, q ctx = ctrl.LoggerInto(ctx, log) log.V(5).Info("Create event") h.handle(ctx, e.Object, q) - h.adjustResources(ctx, e.Object) } func (h *podTemplateHandler) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { @@ -666,7 +696,6 @@ func (h *podTemplateHandler) Update(ctx context.Context, e event.UpdateEvent, q ctx = ctrl.LoggerInto(ctx, log) log.V(5).Info("Update event") h.handle(ctx, e.ObjectNew, q) - h.adjustResources(ctx, e.ObjectNew) } func (h *podTemplateHandler) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { @@ -674,7 +703,6 @@ func (h *podTemplateHandler) Delete(ctx context.Context, e event.DeleteEvent, q ctx = ctrl.LoggerInto(ctx, log) log.V(5).Info("Delete event") h.handle(ctx, e.Object, q) - h.adjustResources(ctx, e.Object) } func (h *podTemplateHandler) Generic(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { diff --git a/pkg/controller/jobframework/podtemplate_names.go b/pkg/controller/jobframework/podtemplate_names.go new file mode 100644 index 0000000000..6df0fd3ff2 --- /dev/null +++ b/pkg/controller/jobframework/podtemplate_names.go @@ -0,0 +1,38 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jobframework + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" +) + +func GetPodTemplateName(workloadName, podSetName string) string { + fullName := fmt.Sprintf("%s-%s", workloadName, podSetName) + return limitObjectName(fullName) +} + +func limitObjectName(fullName string) string { + if len(fullName) <= maxPrefixLength { + return fullName + } + h := sha1.New() + h.Write([]byte(fullName)) + hashBytes := hex.EncodeToString(h.Sum(nil)) + return fmt.Sprintf("%s-%s", fullName[:maxPrefixLength], hashBytes[:hashLength]) +} diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index e1362ae5b0..705ef17ba0 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -731,9 +731,11 @@ func newPodTemplate(ps kueue.PodSet, job GenericJob, object client.Object) *core return &corev1.PodTemplate{ Template: *ps.Template.DeepCopy(), ObjectMeta: metav1.ObjectMeta{ - //Name: fmt.Sprintf("%s-%s", GetWorkloadNameForOwnerWithGVK(object.GetName(), job.GVK()), ps.Name), - Name: ps.Name, + Name: GetPodTemplateName(object.GetName(), ps.Name), Namespace: object.GetNamespace(), + Labels: map[string]string{ + constants.WorkloadNameSource: GetWorkloadNameForOwnerWithGVK(object.GetName(), job.GVK()), + }, }, } } diff --git a/pkg/util/testing/wrappers.go b/pkg/util/testing/wrappers.go index 060d51c41d..5cc4ef4883 100644 --- a/pkg/util/testing/wrappers.go +++ b/pkg/util/testing/wrappers.go @@ -617,6 +617,18 @@ func (w *PodTemplateWrapper) Request(r corev1.ResourceName, q string) *PodTempla return w } +func (w *PodTemplateWrapper) Limit(r corev1.ResourceName, q string) *PodTemplateWrapper { + res := &w.Template.Spec.Containers[0].Resources + if res.Limits == nil { + res.Limits = corev1.ResourceList{ + r: resource.MustParse(q), + } + } else { + res.Limits[r] = resource.MustParse(q) + } + return w +} + func (w *PodTemplateWrapper) Toleration(t corev1.Toleration) *PodTemplateWrapper { w.Template.Spec.Tolerations = append(w.Template.Spec.Tolerations, t) return w diff --git a/pkg/util/testingjobs/rayjob/wrappers.go b/pkg/util/testingjobs/rayjob/wrappers.go index 19ae78d414..034a007cb6 100644 --- a/pkg/util/testingjobs/rayjob/wrappers.go +++ b/pkg/util/testingjobs/rayjob/wrappers.go @@ -46,7 +46,8 @@ func MakeJob(name, ns string) *JobWrapper { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "head-container", + Name: "head-container", + Image: "pause", }, }, }, @@ -63,7 +64,8 @@ func MakeJob(name, ns string) *JobWrapper { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "worker-container", + Name: "worker-container", + Image: "pause", }, }, }, diff --git a/test/integration/controller/core/clusterqueue_controller_test.go b/test/integration/controller/core/clusterqueue_controller_test.go index 8c9b1d5dbc..c19589d91e 100644 --- a/test/integration/controller/core/clusterqueue_controller_test.go +++ b/test/integration/controller/core/clusterqueue_controller_test.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/util/testing" @@ -145,19 +146,46 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin }) ginkgo.It("Should update status and report metrics when workloads are assigned and finish", func() { + podTemplates := []*corev1.PodTemplate{ + testing.MakePodTemplate("one", ns.Name). + Request(corev1.ResourceCPU, "2").Obj(), + //Request(corev1.ResourceCPU, "2").Request(resourceGPU, "2").Obj(), + testing.MakePodTemplate("two", ns.Name). + Request(corev1.ResourceCPU, "3").Obj(), + //Request(corev1.ResourceCPU, "3").Request(resourceGPU, "3").Obj(), + testing.MakePodTemplate("three", ns.Name). + Request(corev1.ResourceCPU, "1").Obj(), + //Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(), + testing.MakePodTemplate("four", ns.Name). + Request(corev1.ResourceCPU, "1").Obj(), + //Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(), + testing.MakePodTemplate("five", ns.Name). + Request(corev1.ResourceCPU, "1").Obj(), + //Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(), + testing.MakePodTemplate("six", ns.Name). + Request(corev1.ResourceCPU, "1").Obj(), + //Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(), + } + workloads := []*kueue.Workload{ testing.MakeWorkload("one", ns.Name).Queue(localQueue.Name). - Request(corev1.ResourceCPU, "2").Request(resourceGPU, "2").Obj(), + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName("one").Obj()}...). + Obj(), testing.MakeWorkload("two", ns.Name).Queue(localQueue.Name). - Request(corev1.ResourceCPU, "3").Request(resourceGPU, "3").Obj(), + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName("two").Obj()}...). + Obj(), testing.MakeWorkload("three", ns.Name).Queue(localQueue.Name). - Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(), + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName("three").Obj()}...). + Obj(), testing.MakeWorkload("four", ns.Name).Queue(localQueue.Name). - Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(), + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName("four").Obj()}...). + Obj(), testing.MakeWorkload("five", ns.Name).Queue("other"). - Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(), + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName("five").Obj()}...). + Obj(), testing.MakeWorkload("six", ns.Name).Queue(localQueue.Name). - Request(corev1.ResourceCPU, "1").Request(resourceGPU, "1").Obj(), + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName("six").Obj()}...). + Obj(), } ginkgo.By("Checking that the resource metrics are published", func() { @@ -177,6 +205,12 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin util.ExpectCQResourceUsage(clusterQueue, flavorModelB, string(resourceGPU), 0) }) + ginkgo.By("Creating podtemplates") + for _, pt := range podTemplates { + pt.SetLabels(map[string]string{constants.WorkloadNameSource: pt.Name}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + } + ginkgo.By("Creating workloads") for _, w := range workloads { gomega.Expect(k8sClient.Create(ctx, w)).To(gomega.Succeed()) @@ -327,14 +361,29 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin gomega.Expect(k8sClient.Create(ctx, modelBFlavor)).To(gomega.Succeed()) }) + pt1 := testing.MakePodTemplate("driver", ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + + pt2 := testing.MakePodTemplate("workers", ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + + ginkgo.By("Creating the podtemplates", func() { + gomega.Expect(k8sClient.Create(ctx, pt1)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Create(ctx, pt2)).To(gomega.Succeed()) + }) + wl := testing.MakeWorkload("one", ns.Name). Queue(localQueue.Name). PodSets( *testing.MakePodSet("driver", 2). - Request(corev1.ResourceCPU, "1"). + SetPodTemplateName(pt1.Name). Obj(), *testing.MakePodSet("workers", 5). - Request(resourceGPU, "1"). + SetPodTemplateName(pt2.Name). Obj(), ). Obj() @@ -744,11 +793,32 @@ var _ = ginkgo.Describe("ClusterQueue controller with queue visibility is enable ginkgo.It("Should update of the pending workloads when a new workload is scheduled", func() { const lowPrio, midLowerPrio, midHigherPrio, highPrio = 0, 10, 20, 100 + podTemplateFirstBatch := []*corev1.PodTemplate{ + testing.MakePodTemplate("one", ns.Name). + Request(corev1.ResourceCPU, "2").Obj(), + testing.MakePodTemplate("two", ns.Name). + Request(corev1.ResourceCPU, "3").Obj(), + } + workloadsFirstBatch := []*kueue.Workload{ - testing.MakeWorkload("one", ns.Name).Queue(localQueue.Name).Priority(highPrio). - Request(corev1.ResourceCPU, "2").Request(resourceGPU, "2").Obj(), - testing.MakeWorkload("two", ns.Name).Queue(localQueue.Name).Priority(midHigherPrio). - Request(corev1.ResourceCPU, "3").Request(resourceGPU, "3").Obj(), + testing.MakeWorkload("one", ns.Name). + PodSets([]kueue.PodSet{ + *testing.MakePodSet("main", 1). + SetPodTemplateName("one"). + Obj(), + }...). + Queue(localQueue.Name). + Priority(highPrio). + Obj(), + testing.MakeWorkload("two", ns.Name). + PodSets([]kueue.PodSet{ + *testing.MakePodSet("main", 1). + SetPodTemplateName("two"). + Obj(), + }...). + Queue(localQueue.Name). + Priority(midHigherPrio). + Obj(), } ginkgo.By("Verify pending workload status before adding workloads") @@ -758,6 +828,12 @@ var _ = ginkgo.Describe("ClusterQueue controller with queue visibility is enable return updatedCq.Status.PendingWorkloadsStatus }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(&kueue.ClusterQueuePendingWorkloadsStatus{}, ignoreLastChangeTime)) + ginkgo.By("Creating pod template") + for _, pt := range podTemplateFirstBatch { + pt.SetLabels(map[string]string{constants.WorkloadNameSource: pt.Name}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + } + ginkgo.By("Creating workloads") for _, w := range workloadsFirstBatch { gomega.Expect(k8sClient.Create(ctx, w)).To(gomega.Succeed()) @@ -782,12 +858,38 @@ var _ = ginkgo.Describe("ClusterQueue controller with queue visibility is enable }, ignoreLastChangeTime)) ginkgo.By("Creating new workloads to so that the number of workloads exceeds the MaxCount") + podTemplateSecondBatch := []*corev1.PodTemplate{ + testing.MakePodTemplate("three", ns.Name). + Request(corev1.ResourceCPU, "2").Obj(), + testing.MakePodTemplate("four", ns.Name). + Request(corev1.ResourceCPU, "3").Obj(), + } + workloadsSecondBatch := []*kueue.Workload{ - testing.MakeWorkload("three", ns.Name).Queue(localQueue.Name).Priority(midLowerPrio). - Request(corev1.ResourceCPU, "2").Request(resourceGPU, "2").Obj(), - testing.MakeWorkload("four", ns.Name).Queue(localQueue.Name).Priority(lowPrio). - Request(corev1.ResourceCPU, "3").Request(resourceGPU, "3").Obj(), + testing.MakeWorkload("three", ns.Name). + PodSets([]kueue.PodSet{ + *testing.MakePodSet("main", 1). + SetPodTemplateName("one"). + Obj(), + }...). + Queue(localQueue.Name). + Priority(midLowerPrio). + Obj(), + testing.MakeWorkload("four", ns.Name). + PodSets([]kueue.PodSet{ + *testing.MakePodSet("main", 1). + SetPodTemplateName("one"). + Obj(), + }...). + Queue(localQueue.Name). + Priority(lowPrio). + Obj(), } + for _, pt := range podTemplateSecondBatch { + pt.SetLabels(map[string]string{constants.WorkloadNameSource: pt.Name}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + } + for _, w := range workloadsSecondBatch { gomega.Expect(k8sClient.Create(ctx, w)).To(gomega.Succeed()) } diff --git a/test/integration/controller/core/localqueue_controller_test.go b/test/integration/controller/core/localqueue_controller_test.go index 793275a829..dfe38201f7 100644 --- a/test/integration/controller/core/localqueue_controller_test.go +++ b/test/integration/controller/core/localqueue_controller_test.go @@ -25,6 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" @@ -165,18 +166,26 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai ginkgo.By("Creating a clusterQueue") gomega.Expect(k8sClient.Create(ctx, clusterQueue)).To(gomega.Succeed()) + podtemplates := []*corev1.PodTemplate{ + testing.MakePodTemplate("one", ns.Name). + //Request(resourceGPU, "2"). + Obj(), + testing.MakePodTemplate("two", ns.Name). + //Request(resourceGPU, "3"). + Obj(), + testing.MakePodTemplate("three", ns.Name). + //Request(resourceGPU, "1"). + Obj(), + } workloads := []*kueue.Workload{ testing.MakeWorkload("one", ns.Name). Queue(queue.Name). - Request(resourceGPU, "2"). Obj(), testing.MakeWorkload("two", ns.Name). Queue(queue.Name). - Request(resourceGPU, "3"). Obj(), testing.MakeWorkload("three", ns.Name). Queue(queue.Name). - Request(resourceGPU, "1"). Obj(), } admissions := []*kueue.Admission{ @@ -188,6 +197,12 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai Assignment(resourceGPU, flavorModelD, "1").Obj(), } + ginkgo.By("Creating podtemplates") + for _, pt := range podtemplates { + pt.SetLabels(map[string]string{constants.WorkloadNameSource: pt.Name}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + } + ginkgo.By("Creating workloads") for _, w := range workloads { gomega.Expect(k8sClient.Create(ctx, w)).To(gomega.Succeed()) diff --git a/test/integration/controller/core/workload_controller_test.go b/test/integration/controller/core/workload_controller_test.go index ee3b82d491..a9c818eb1a 100644 --- a/test/integration/controller/core/workload_controller_test.go +++ b/test/integration/controller/core/workload_controller_test.go @@ -78,7 +78,12 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) }) ginkgo.It("Should update status when workloads are created", func() { - wl = testing.MakeWorkload("one", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt := testing.MakePodTemplate("one", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + wl = testing.MakeWorkload("one", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Obj() message = fmt.Sprintf("LocalQueue %s doesn't exist", "") gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) gomega.Eventually(func() int { @@ -94,7 +99,13 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) }) ginkgo.It("Should update status when workloads are created", func() { - wl = testing.MakeWorkload("two", ns.Name).Queue("non-created-queue").Request(corev1.ResourceCPU, "1").Obj() + pt := testing.MakePodTemplate("two", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "two"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + wl = testing.MakeWorkload("two", ns.Name). + Queue("non-created-queue"). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Obj() message = fmt.Sprintf("LocalQueue %s doesn't exist", "non-created-queue") gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) gomega.Eventually(func() int { @@ -114,7 +125,13 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) }) ginkgo.It("Should update status when workloads are created", func() { - wl = testing.MakeWorkload("three", ns.Name).Queue(localQueue.Name).Request(corev1.ResourceCPU, "1").Obj() + pt := testing.MakePodTemplate("three", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "three"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + wl = testing.MakeWorkload("three", ns.Name). + Queue(localQueue.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Obj() message = fmt.Sprintf("ClusterQueue %s doesn't exist", "fooclusterqueue") gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) gomega.Eventually(func() []metav1.Condition { @@ -142,32 +159,22 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn util.ExpectClusterQueueToBeDeleted(ctx, k8sClient, clusterQueue, true) }) ginkgo.It("Should update status when workloads are created", func() { - pt := testing.MakePodTemplate(fmt.Sprintf("%s-%s", "four", kueue.DefaultPodSetName), ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt := testing.MakePodTemplate(kueue.DefaultPodSetName, ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "four"}) gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) - wl = testing.MakeWorkload("four", ns.Name).Queue(localQueue.Name).Obj() - gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) - gomega.Eventually(func() bool { - gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedQueueWorkload)).To(gomega.Succeed()) - return updatedQueueWorkload.Spec.PodSets[0].PodTemplateName != nil - }, util.Timeout, util.Interval).Should(gomega.BeTrue()) - }) - }) - ginkgo.When("check pod template name when podtemplate does not exist", func() { - ginkgo.BeforeEach(func() { - localQueue = testing.MakeLocalQueue("queue", ns.Name).ClusterQueue("fooclusterqueue").Obj() - gomega.Expect(k8sClient.Create(ctx, localQueue)).To(gomega.Succeed()) - }) - ginkgo.AfterEach(func() { - gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) - }) - ginkgo.It("Should update status when workloads are created", func() { - wl = testing.MakeWorkload("five", ns.Name).Queue(localQueue.Name).Obj() + wl = testing.MakeWorkload("four", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Queue(localQueue.Name). + Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) + gomega.Eventually(func() bool { gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedQueueWorkload)).To(gomega.Succeed()) return updatedQueueWorkload.Spec.PodSets[0].PodTemplateName != nil - }, util.Timeout, util.Interval).Should(gomega.BeFalse()) + }, util.Timeout, util.Interval).Should(gomega.BeTrue()) }) }) @@ -214,7 +221,16 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn }) ginkgo.It("the workload should get the AdditionalChecks added", func() { - wl := testing.MakeWorkload("wl", ns.Name).Queue("queue").Obj() + pt := testing.MakePodTemplate(kueue.DefaultPodSetName, ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "wl"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + + wl := testing.MakeWorkload("wl", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Queue("queue"). + Obj() wlKey := client.ObjectKeyFromObject(wl) createdWl := kueue.Workload{} ginkgo.By("creating the workload, the check conditions should be added", func() { @@ -282,7 +298,13 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn }) ginkgo.It("case of WorkloadPriorityClass", func() { ginkgo.By("creating workload") - wl = testing.MakeWorkload("wl", ns.Name).Queue("lq").Request(corev1.ResourceCPU, "1"). + pt := testing.MakePodTemplate(kueue.DefaultPodSetName, ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "wl"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + wl = testing.MakeWorkload("wl", ns.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Queue("lq"). PriorityClass("workload-priority-class").PriorityClassSource(constants.WorkloadPriorityClassSource).Priority(200).Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) gomega.Eventually(func() []metav1.Condition { diff --git a/test/integration/scheduler/preemption_test.go b/test/integration/scheduler/preemption_test.go index a7673b893b..ca5c594477 100644 --- a/test/integration/scheduler/preemption_test.go +++ b/test/integration/scheduler/preemption_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/test/util" ) @@ -82,27 +83,55 @@ var _ = ginkgo.Describe("Preemption", func() { }) ginkgo.It("Should preempt Workloads with lower priority when there is not enough quota", func() { + ginkgo.By("Creating pod templates with different priorities") + lowPt1 := testing.MakePodTemplate("main-1", ns.Name). + Request(corev1.ResourceCPU, "1"). + Priority(lowPriority). + Obj() + lowPt1.SetLabels(map[string]string{constants.WorkloadNameSource: "low-wl-1"}) + lowPt2 := testing.MakePodTemplate("main-2", ns.Name). + Request(corev1.ResourceCPU, "1"). + Priority(lowPriority). + Obj() + lowPt2.SetLabels(map[string]string{constants.WorkloadNameSource: "low-wl-2"}) + midPt := testing.MakePodTemplate("main-mid", ns.Name). + Request(corev1.ResourceCPU, "1"). + Priority(midPriority). + Obj() + midPt.SetLabels(map[string]string{constants.WorkloadNameSource: "mid-wl"}) + highPt := testing.MakePodTemplate("main-high", ns.Name). + Request(corev1.ResourceCPU, "1"). + Priority(highPriority). + Obj() + highPt.SetLabels(map[string]string{constants.WorkloadNameSource: "high-wl-1"}) + + gomega.Expect(k8sClient.Create(ctx, lowPt1)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Create(ctx, lowPt2)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Create(ctx, midPt)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Create(ctx, highPt)).To(gomega.Succeed()) + ginkgo.By("Creating initial Workloads with different priorities") lowWl1 := testing.MakeWorkload("low-wl-1", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(lowPt1.Name).Obj()). Priority(lowPriority). - Request(corev1.ResourceCPU, "1"). + Queue(q.Name). Obj() lowWl2 := testing.MakeWorkload("low-wl-2", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(lowPt2.Name).Obj()). Priority(lowPriority). - Request(corev1.ResourceCPU, "1"). + Queue(q.Name). Obj() midWl := testing.MakeWorkload("mid-wl", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(midPt.Name).Obj()). Priority(midPriority). - Request(corev1.ResourceCPU, "1"). + Queue(q.Name). Obj() highWl1 := testing.MakeWorkload("high-wl-1", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(highPt.Name).Obj()). Priority(highPriority). - Request(corev1.ResourceCPU, "1"). + Queue(q.Name). Obj() + gomega.Expect(k8sClient.Create(ctx, lowWl1)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, lowWl2)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, midWl)).To(gomega.Succeed()) @@ -110,21 +139,35 @@ var _ = ginkgo.Describe("Preemption", func() { util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, cq.Name, lowWl1, lowWl2, midWl, highWl1) - ginkgo.By("Creating a low priority Workload") + ginkgo.By("Creating a low priority pod template") + lowPt3 := testing.MakePodTemplate("low-main-3", ns.Name). + Request(corev1.ResourceCPU, "1"). + Priority(lowPriority). + Obj() + lowPt3.SetLabels(map[string]string{constants.WorkloadNameSource: "low-wl-3"}) + gomega.Expect(k8sClient.Create(ctx, lowPt3)).To(gomega.Succeed()) + lowWl3 := testing.MakeWorkload("low-wl-3", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(lowPt3.Name).Obj()). Priority(lowPriority). - Request(corev1.ResourceCPU, "1"). + Queue(q.Name). Obj() gomega.Expect(k8sClient.Create(ctx, lowWl3)).To(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, lowWl3) ginkgo.By("Creating a high priority Workload") + highPt2 := testing.MakePodTemplate("high-pt-2", ns.Name). + Request(corev1.ResourceCPU, "2"). + Priority(highPriority). + Obj() + highPt2.SetLabels(map[string]string{constants.WorkloadNameSource: "high-wl-2"}) + gomega.Expect(k8sClient.Create(ctx, highPt2)).To(gomega.Succeed()) + highWl2 := testing.MakeWorkload("high-wl-2", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(highPt2.Name).Obj()). Priority(highPriority). - Request(corev1.ResourceCPU, "2"). + Queue(q.Name). Obj() gomega.Expect(k8sClient.Create(ctx, highWl2)).To(gomega.Succeed()) @@ -135,38 +178,65 @@ var _ = ginkgo.Describe("Preemption", func() { }) ginkgo.It("Should preempt newer Workloads with the same priority when there is not enough quota", func() { + ginkgo.By("Creating pod templates") + pt1 := testing.MakePodTemplate("main-1", ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-1"}) + pt2 := testing.MakePodTemplate("main-2", ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-2"}) + pt3 := testing.MakePodTemplate("main-3", ns.Name). + Request(corev1.ResourceCPU, "3"). + Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-3"}) + ginkgo.By("Creating initial Workloads") wl1 := testing.MakeWorkload("wl-1", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()). Priority(lowPriority). - Request(corev1.ResourceCPU, "1"). + Queue(q.Name). Obj() wl2 := testing.MakeWorkload("wl-2", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()). Priority(lowPriority). - Request(corev1.ResourceCPU, "1"). + Queue(q.Name). Obj() wl3 := testing.MakeWorkload("wl-3", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt3.Name).Obj()). Priority(lowPriority). - Request(corev1.ResourceCPU, "3"). + Queue(q.Name). Obj() + + gomega.Expect(k8sClient.Create(ctx, pt1)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Create(ctx, pt2)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, wl1)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, wl2)).To(gomega.Succeed()) util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, cq.Name, wl1, wl2) + gomega.Expect(k8sClient.Create(ctx, pt3)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, wl3)).To(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, wl3) ginkgo.By("Waiting one second, to ensure that the new workload has a later creation time") time.Sleep(time.Second) + ginkgo.By("Creating a new PodTemplate") + pt4 := testing.MakePodTemplate("main-4", ns.Name). + Request(corev1.ResourceCPU, "1"). + Priority(lowPriority). + Obj() + pt4.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-4"}) + ginkgo.By("Creating a new Workload") wl4 := testing.MakeWorkload("wl-4", ns.Name). - Queue(q.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt4.Name).Obj()). Priority(lowPriority). - Request(corev1.ResourceCPU, "1"). + Queue(q.Name). Obj() + + gomega.Expect(k8sClient.Create(ctx, pt4)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, wl4)).To(gomega.Succeed()) util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, cq.Name, wl1, wl2, wl4) @@ -233,23 +303,42 @@ var _ = ginkgo.Describe("Preemption", func() { ginkgo.It("Should preempt Workloads in the cohort borrowing quota, when the ClusterQueue is using less than nominal quota", func() { ginkgo.By("Creating workloads in beta-cq that borrow quota") + alphaLowPt := testing.MakePodTemplate("alpha-low", ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + alphaLowPt.SetLabels(map[string]string{constants.WorkloadNameSource: "alpha-low"}) + gomega.Expect(k8sClient.Create(ctx, alphaLowPt)).To(gomega.Succeed()) + alphaLowWl := testing.MakeWorkload("alpha-low", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(alphaLowPt.Name).Obj()). Queue(alphaQ.Name). Priority(lowPriority). - Request(corev1.ResourceCPU, "1"). Obj() gomega.Expect(k8sClient.Create(ctx, alphaLowWl)).To(gomega.Succeed()) + betaMidPt := testing.MakePodTemplate("beta-mid", ns.Name). + Request(corev1.ResourceCPU, "1"). + Obj() + betaMidPt.SetLabels(map[string]string{constants.WorkloadNameSource: "beta-mid"}) + gomega.Expect(k8sClient.Create(ctx, betaMidPt)).To(gomega.Succeed()) + betaMidWl := testing.MakeWorkload("beta-mid", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(betaMidPt.Name).Obj()). Queue(betaQ.Name). Priority(midPriority). - Request(corev1.ResourceCPU, "1"). Obj() gomega.Expect(k8sClient.Create(ctx, betaMidWl)).To(gomega.Succeed()) + + betaHighPt := testing.MakePodTemplate("beta-high", ns.Name). + Request(corev1.ResourceCPU, "4"). + Obj() + betaHighPt.SetLabels(map[string]string{constants.WorkloadNameSource: "beta-high"}) + gomega.Expect(k8sClient.Create(ctx, betaHighPt)).To(gomega.Succeed()) + betaHighWl := testing.MakeWorkload("beta-high", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(betaHighPt.Name).Obj()). Queue(betaQ.Name). Priority(highPriority). - Request(corev1.ResourceCPU, "4"). Obj() gomega.Expect(k8sClient.Create(ctx, betaHighWl)).To(gomega.Succeed()) @@ -257,10 +346,16 @@ var _ = ginkgo.Describe("Preemption", func() { util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, betaCQ.Name, betaMidWl, betaHighWl) ginkgo.By("Creating workload in alpha-cq to preempt workloads in both ClusterQueues") + alphaMidPt := testing.MakePodTemplate("alpha-mid", ns.Name). + Request(corev1.ResourceCPU, "2"). + Obj() + alphaMidPt.SetLabels(map[string]string{constants.WorkloadNameSource: "alpha-mid"}) + gomega.Expect(k8sClient.Create(ctx, alphaMidPt)).To(gomega.Succeed()) + alphaMidWl := testing.MakeWorkload("alpha-mid", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(alphaMidPt.Name).Obj()). Queue(alphaQ.Name). Priority(midPriority). - Request(corev1.ResourceCPU, "2"). Obj() gomega.Expect(k8sClient.Create(ctx, alphaMidWl)).To(gomega.Succeed()) @@ -272,17 +367,29 @@ var _ = ginkgo.Describe("Preemption", func() { ginkgo.It("Should not preempt Workloads in the cohort, if the ClusterQueue requires borrowing", func() { ginkgo.By("Creating workloads in beta-cq that borrow quota") + alphaHighPt1 := testing.MakePodTemplate("alpha-high-1", ns.Name). + Request(corev1.ResourceCPU, "2"). + Obj() + alphaHighPt1.SetLabels(map[string]string{constants.WorkloadNameSource: "alpha-high-1"}) + gomega.Expect(k8sClient.Create(ctx, alphaHighPt1)).To(gomega.Succeed()) alphaHighWl1 := testing.MakeWorkload("alpha-high-1", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(alphaHighPt1.Name).Obj()). Queue(alphaQ.Name). Priority(highPriority). - Request(corev1.ResourceCPU, "2"). Obj() gomega.Expect(k8sClient.Create(ctx, alphaHighWl1)).To(gomega.Succeed()) + + betaLowPt1 := testing.MakePodTemplate("beta-low", ns.Name). + Request(corev1.ResourceCPU, "4"). + Obj() + betaLowPt1.SetLabels(map[string]string{constants.WorkloadNameSource: "beta-low"}) + gomega.Expect(k8sClient.Create(ctx, betaLowPt1)).To(gomega.Succeed()) + betaLowWl := testing.MakeWorkload("beta-low", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(betaLowPt1.Name).Obj()). Queue(betaQ.Name). Priority(lowPriority). - Request(corev1.ResourceCPU, "4"). Obj() gomega.Expect(k8sClient.Create(ctx, betaLowWl)).To(gomega.Succeed()) @@ -290,10 +397,16 @@ var _ = ginkgo.Describe("Preemption", func() { util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, betaCQ.Name, betaLowWl) ginkgo.By("Creating high priority workload in alpha-cq that doesn't fit without borrowing") + alphaHighPt2 := testing.MakePodTemplate("alpha-high-2", ns.Name). + Request(corev1.ResourceCPU, "2"). + Obj() + alphaHighPt2.SetLabels(map[string]string{constants.WorkloadNameSource: "alpha-high-2"}) + gomega.Expect(k8sClient.Create(ctx, alphaHighPt2)).To(gomega.Succeed()) + alphaHighWl2 := testing.MakeWorkload("alpha-high-2", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(alphaHighPt2.Name).Obj()). Queue(alphaQ.Name). Priority(highPriority). - Request(corev1.ResourceCPU, "2"). Obj() gomega.Expect(k8sClient.Create(ctx, alphaHighWl2)).To(gomega.Succeed()) @@ -305,33 +418,57 @@ var _ = ginkgo.Describe("Preemption", func() { ginkgo.It("Should preempt all necessary workloads in concurrent scheduling", func() { ginkgo.By("Creating workloads in beta-cq that borrow quota") + betaMidPt := testing.MakePodTemplate("beta-mid", ns.Name). + Request(corev1.ResourceCPU, "3"). + Obj() + betaMidPt.SetLabels(map[string]string{constants.WorkloadNameSource: "beta-mid"}) + gomega.Expect(k8sClient.Create(ctx, betaMidPt)).To(gomega.Succeed()) betaMidWl := testing.MakeWorkload("beta-mid", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(betaMidPt.Name).Obj()). Queue(betaQ.Name). Priority(midPriority). - Request(corev1.ResourceCPU, "3"). Obj() gomega.Expect(k8sClient.Create(ctx, betaMidWl)).To(gomega.Succeed()) + + betaHighPt := testing.MakePodTemplate("beta-high", ns.Name). + Request(corev1.ResourceCPU, "3"). + Obj() + betaHighPt.SetLabels(map[string]string{constants.WorkloadNameSource: "beta-high"}) + gomega.Expect(k8sClient.Create(ctx, betaHighPt)).To(gomega.Succeed()) + betaHighWl := testing.MakeWorkload("beta-high", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(betaHighPt.Name).Obj()). Queue(betaQ.Name). Priority(highPriority). - Request(corev1.ResourceCPU, "3"). Obj() gomega.Expect(k8sClient.Create(ctx, betaHighWl)).To(gomega.Succeed()) util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, betaCQ.Name, betaMidWl, betaHighWl) ginkgo.By("Creating workload in alpha-cq and gamma-cq that need to preempt") + alphaMidPt := testing.MakePodTemplate("alpha-mid", ns.Name). + Request(corev1.ResourceCPU, "2"). + Obj() + alphaMidPt.SetLabels(map[string]string{constants.WorkloadNameSource: "alpha-mid"}) + gomega.Expect(k8sClient.Create(ctx, alphaMidPt)).To(gomega.Succeed()) + alphaMidWl := testing.MakeWorkload("alpha-mid", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(alphaMidPt.Name).Obj()). Queue(alphaQ.Name). Priority(midPriority). + Obj() + + gammaMidPt := testing.MakePodTemplate("gamma-mid", ns.Name). Request(corev1.ResourceCPU, "2"). Obj() + gammaMidPt.SetLabels(map[string]string{constants.WorkloadNameSource: "gamma-mid"}) + gomega.Expect(k8sClient.Create(ctx, gammaMidPt)).To(gomega.Succeed()) gammaMidWl := testing.MakeWorkload("gamma-mid", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(gammaMidPt.Name).Obj()). Queue(gammaQ.Name). Priority(midPriority). - Request(corev1.ResourceCPU, "2"). Obj() gomega.Expect(k8sClient.Create(ctx, alphaMidWl)).To(gomega.Succeed()) diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 29ccfdbc82..32e7b179e5 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/pkg/workload" @@ -155,7 +156,12 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.It("Should admit workloads as they fit in their ClusterQueue", func() { ginkgo.By("checking the first prod workload gets admitted") - prodWl1 := testing.MakeWorkload("prod-wl1", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2").Obj() + prodPt1 := testing.MakePodTemplate("prod-pt1", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + prodPt1.SetLabels(map[string]string{constants.WorkloadNameSource: "prod-wl1"}) + gomega.Expect(k8sClient.Create(ctx, prodPt1)).Should(gomega.Succeed()) + prodWl1 := testing.MakeWorkload("prod-wl1", ns.Name). + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName(prodPt1.Name).Obj()}...). + Queue(prodQueue.Name).Obj() gomega.Expect(k8sClient.Create(ctx, prodWl1)).Should(gomega.Succeed()) prodWl1Admission := testing.MakeAdmission(prodClusterQ.Name).Assignment(corev1.ResourceCPU, "on-demand", "2").Obj() util.ExpectWorkloadToBeAdmittedAs(ctx, k8sClient, prodWl1, prodWl1Admission) @@ -164,13 +170,24 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedWorkloadsTotalMetric(prodClusterQ, 1) ginkgo.By("checking a second no-fit workload does not get admitted") - prodWl2 := testing.MakeWorkload("prod-wl2", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "5").Obj() + prodPt2 := testing.MakePodTemplate("prod-pt2", ns.Name).Request(corev1.ResourceCPU, "5").Obj() + prodPt2.SetLabels(map[string]string{constants.WorkloadNameSource: "prod-wl2"}) + gomega.Expect(k8sClient.Create(ctx, prodPt2)).Should(gomega.Succeed()) + prodWl2 := testing.MakeWorkload("prod-wl2", ns.Name). + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName(prodPt2.Name).Obj()}...). + Queue(prodQueue.Name).Obj() gomega.Expect(k8sClient.Create(ctx, prodWl2)).Should(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, prodWl2) util.ExpectPendingWorkloadsMetric(prodClusterQ, 0, 1) ginkgo.By("checking a dev workload gets admitted") - devWl := testing.MakeWorkload("dev-wl", ns.Name).Queue(devQueue.Name).Request(corev1.ResourceCPU, "5").Obj() + devPt := testing.MakePodTemplate("dev-pt", ns.Name).Request(corev1.ResourceCPU, "5").Obj() + devPt.SetLabels(map[string]string{constants.WorkloadNameSource: "dev-wl"}) + gomega.Expect(k8sClient.Create(ctx, devPt)).Should(gomega.Succeed()) + + devWl := testing.MakeWorkload("dev-wl", ns.Name). + PodSets([]kueue.PodSet{*testing.MakePodSet("main", 1).SetPodTemplateName(devPt.Name).Obj()}...). + Queue(devQueue.Name).Obj() gomega.Expect(k8sClient.Create(ctx, devWl)).Should(gomega.Succeed()) spotUntaintedFlavorAdmission := testing.MakeAdmission(devClusterQ.Name).Assignment(corev1.ResourceCPU, "spot-untainted", "5").Obj() util.ExpectWorkloadToBeAdmittedAs(ctx, k8sClient, devWl, spotUntaintedFlavorAdmission) @@ -188,10 +205,14 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Should admit workloads as number of pods allows it", func() { + pt1 := testing.MakePodTemplate("pt1", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + wl1 := testing.MakeWorkload("wl1", ns.Name). Queue(podsCountQueue.Name). PodSets(*testing.MakePodSet("main", 3). - Request(corev1.ResourceCPU, "2"). + SetPodTemplateName(pt1.Name). Obj()). Obj() @@ -208,17 +229,25 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedWorkloadsTotalMetric(podsCountClusterQ, 1) }) + pt2 := testing.MakePodTemplate("pt2", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + wl2 := testing.MakeWorkload("wl2", ns.Name). Queue(podsCountQueue.Name). PodSets(*testing.MakePodSet("main", 3). - Request(corev1.ResourceCPU, "2"). + SetPodTemplateName(pt2.Name). Obj()). Obj() + pt3 := testing.MakePodTemplate("pt3", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "wl3"}) + gomega.Expect(k8sClient.Create(ctx, pt3)).Should(gomega.Succeed()) + wl3 := testing.MakeWorkload("wl3", ns.Name). Queue(podsCountQueue.Name). PodSets(*testing.MakePodSet("main", 2). - Request(corev1.ResourceCPU, "2"). + SetPodTemplateName(pt3.Name). Obj()). Obj() @@ -248,9 +277,14 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Should admit workloads as the number of pods (only) allows it", func() { + pt1 := testing.MakePodTemplate("pt1", ns.Name).Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + wl1 := testing.MakeWorkload("wl1", ns.Name). Queue(podsCountOnlyQueue.Name). PodSets(*testing.MakePodSet("main", 3). + SetPodTemplateName(pt1.Name). Obj()). Obj() @@ -266,15 +300,25 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedWorkloadsTotalMetric(podsCountOnlyClusterQ, 1) }) + pt2 := testing.MakePodTemplate("pt2", ns.Name).Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + wl2 := testing.MakeWorkload("wl2", ns.Name). Queue(podsCountOnlyQueue.Name). PodSets(*testing.MakePodSet("main", 3). + SetPodTemplateName(pt2.Name). Obj()). Obj() + pt3 := testing.MakePodTemplate("pt3", ns.Name).Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "wl3"}) + gomega.Expect(k8sClient.Create(ctx, pt3)).Should(gomega.Succeed()) + wl3 := testing.MakeWorkload("wl3", ns.Name). Queue(podsCountOnlyQueue.Name). PodSets(*testing.MakePodSet("main", 2). + SetPodTemplateName(pt3.Name). Obj()). Obj() @@ -308,15 +352,36 @@ var _ = ginkgo.Describe("Scheduler", func() { const lowPrio, midPrio, highPrio = 0, 10, 100 - wlLow := testing.MakeWorkload("wl-low-priority", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "2").Priority(lowPrio).Obj() + ptLow := testing.MakePodTemplate("pt-low-priority", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + ptLow.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-low-priority"}) + gomega.Expect(k8sClient.Create(ctx, ptLow)).Should(gomega.Succeed()) + ptMid1 := testing.MakePodTemplate("pt-mid-priority-1", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + ptMid1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-mid-priority-1"}) + gomega.Expect(k8sClient.Create(ctx, ptMid1)).Should(gomega.Succeed()) + ptMid2 := testing.MakePodTemplate("pt-mid-priority-2", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + ptMid2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-mid-priority-2"}) + gomega.Expect(k8sClient.Create(ctx, ptMid2)).Should(gomega.Succeed()) + ptHigh1 := testing.MakePodTemplate("pt-high-priority-1", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + ptHigh1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-high-priority-1"}) + gomega.Expect(k8sClient.Create(ctx, ptHigh1)).Should(gomega.Succeed()) + ptHigh2 := testing.MakePodTemplate("pt-high-priority-2", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + ptHigh2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-high-priority-2"}) + gomega.Expect(k8sClient.Create(ctx, ptHigh2)).Should(gomega.Succeed()) + + wlLow := testing.MakeWorkload("wl-low-priority", ns.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(ptLow.Name).Obj()). + Queue(queue.Name).Priority(lowPrio).Obj() gomega.Expect(k8sClient.Create(ctx, wlLow)).Should(gomega.Succeed()) - wlMid1 := testing.MakeWorkload("wl-mid-priority-1", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "2").Priority(midPrio).Obj() + wlMid1 := testing.MakeWorkload("wl-mid-priority-1", ns.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(ptMid1.Name).Obj()). + Queue(queue.Name).Priority(midPrio).Obj() gomega.Expect(k8sClient.Create(ctx, wlMid1)).Should(gomega.Succeed()) - wlMid2 := testing.MakeWorkload("wl-mid-priority-2", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "2").Priority(midPrio).Obj() + wlMid2 := testing.MakeWorkload("wl-mid-priority-2", ns.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(ptMid2.Name).Obj()). + Queue(queue.Name).Priority(midPrio).Obj() gomega.Expect(k8sClient.Create(ctx, wlMid2)).Should(gomega.Succeed()) - wlHigh1 := testing.MakeWorkload("wl-high-priority-1", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "2").Priority(highPrio).Obj() + wlHigh1 := testing.MakeWorkload("wl-high-priority-1", ns.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(ptHigh1.Name).Obj()). + Queue(queue.Name).Priority(highPrio).Obj() gomega.Expect(k8sClient.Create(ctx, wlHigh1)).Should(gomega.Succeed()) - wlHigh2 := testing.MakeWorkload("wl-high-priority-2", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "2").Priority(highPrio).Obj() + wlHigh2 := testing.MakeWorkload("wl-high-priority-2", ns.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(ptHigh2.Name).Obj()). + Queue(queue.Name).Priority(highPrio).Obj() gomega.Expect(k8sClient.Create(ctx, wlHigh2)).Should(gomega.Succeed()) util.ExpectPendingWorkloadsMetric(prodClusterQ, 0, 0) @@ -344,7 +409,11 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Should admit two small workloads after a big one finishes", func() { - bigWl := testing.MakeWorkload("big-wl", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "5").Obj() + bigPt := testing.MakePodTemplate("big-pt", ns.Name).Request(corev1.ResourceCPU, "5").Obj() + bigPt.SetLabels(map[string]string{constants.WorkloadNameSource: "big-wl"}) + gomega.Expect(k8sClient.Create(ctx, bigPt)).Should(gomega.Succeed()) + + bigWl := testing.MakeWorkload("big-wl", ns.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(bigPt.Name).Obj()).Queue(prodQueue.Name).Obj() ginkgo.By("Creating big workload") gomega.Expect(k8sClient.Create(ctx, bigWl)).Should(gomega.Succeed()) @@ -353,8 +422,20 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedActiveWorkloadsMetric(prodClusterQ, 1) util.ExpectAdmittedWorkloadsTotalMetric(prodClusterQ, 1) - smallWl1 := testing.MakeWorkload("small-wl-1", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2.5").Obj() - smallWl2 := testing.MakeWorkload("small-wl-2", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2.5").Obj() + smallPt1 := testing.MakePodTemplate("small-pt-1", ns.Name).Request(corev1.ResourceCPU, "2.5").Obj() + smallPt1.SetLabels(map[string]string{constants.WorkloadNameSource: "small-wl-1"}) + smallPt2 := testing.MakePodTemplate("small-pt-2", ns.Name).Request(corev1.ResourceCPU, "2.5").Obj() + smallPt2.SetLabels(map[string]string{constants.WorkloadNameSource: "small-wl-2"}) + ginkgo.By("Creating two small podtemplates") + gomega.Expect(k8sClient.Create(ctx, smallPt1)).Should(gomega.Succeed()) + gomega.Expect(k8sClient.Create(ctx, smallPt2)).Should(gomega.Succeed()) + + smallWl1 := testing.MakeWorkload("small-wl-1", ns.Name).Queue(prodQueue.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(smallPt1.Name).Obj()). + Obj() + smallWl2 := testing.MakeWorkload("small-wl-2", ns.Name).Queue(prodQueue.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(smallPt2.Name).Obj()). + Obj() ginkgo.By("Creating two small workloads") gomega.Expect(k8sClient.Create(ctx, smallWl1)).Should(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, smallWl2)).Should(gomega.Succeed()) @@ -373,10 +454,13 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Should admit workloads when resources are dynamically reclaimed", func() { + firstPt := testing.MakePodTemplate("first-pt", ns.Name).Request(corev1.ResourceCPU, "3").Obj() + firstPt.SetLabels(map[string]string{constants.WorkloadNameSource: "first-wl"}) firstWl := testing.MakeWorkload("first-wl", ns.Name).Queue(prodQueue.Name). - PodSets(*testing.MakePodSet("main", 1).Request(corev1.ResourceCPU, "3").Obj()). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(firstPt.Name).Obj()). Obj() ginkgo.By("Creating first workload", func() { + gomega.Expect(k8sClient.Create(ctx, firstPt)).Should(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, firstWl)).Should(gomega.Succeed()) util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, prodClusterQ.Name, firstWl) @@ -384,8 +468,13 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedActiveWorkloadsMetric(prodClusterQ, 1) }) - secondWl := testing.MakeWorkload("second-wl", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "3").Obj() + secondPt := testing.MakePodTemplate("second-pt", ns.Name).Request(corev1.ResourceCPU, "3").Obj() + secondPt.SetLabels(map[string]string{constants.WorkloadNameSource: "second-wl"}) + secondWl := testing.MakeWorkload("second-wl", ns.Name).Queue(prodQueue.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(secondPt.Name).Obj()). + Obj() ginkgo.By("Creating the second workload", func() { + gomega.Expect(k8sClient.Create(ctx, secondPt)).Should(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, secondWl)).Should(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, secondWl) @@ -403,8 +492,13 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Should admit workloads with 0 count for podSets due to reclaim", func() { + firstPt := testing.MakePodTemplate("first-pt", ns.Name).Request(corev1.ResourceCPU, "3").Obj() + firstPt.SetLabels(map[string]string{constants.WorkloadNameSource: "first-wl"}) + ginkgo.By("Creating first podtemplate", func() { + gomega.Expect(k8sClient.Create(ctx, firstPt)).Should(gomega.Succeed()) + }) firstWl := testing.MakeWorkload("first-wl", ns.Name).Queue(prodQueue.Name). - PodSets(*testing.MakePodSet("main", 1).Request(corev1.ResourceCPU, "3").Obj()). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(firstPt.Name).Obj()). Obj() ginkgo.By("Creating first workload", func() { gomega.Expect(k8sClient.Create(ctx, firstWl)).Should(gomega.Succeed()) @@ -414,14 +508,22 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedActiveWorkloadsMetric(prodClusterQ, 1) }) + secondPt := testing.MakePodTemplate("second-pt", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + secondPt.SetLabels(map[string]string{constants.WorkloadNameSource: "second-wl"}) + thirdPt := testing.MakePodTemplate("third-pt", ns.Name).Request(corev1.ResourceCPU, "3").Obj() + thirdPt.SetLabels(map[string]string{constants.WorkloadNameSource: "second-wl"}) + ginkgo.By("Creating two podtemplates", func() { + gomega.Expect(k8sClient.Create(ctx, secondPt)).Should(gomega.Succeed()) + gomega.Expect(k8sClient.Create(ctx, thirdPt)).Should(gomega.Succeed()) + }) + secondWl := testing.MakeWorkload("second-wl", ns.Name).Queue(prodQueue.Name). - Request(corev1.ResourceCPU, "3"). PodSets( *testing.MakePodSet("main", 2). - Request(corev1.ResourceCPU, "1"). + SetPodTemplateName(secondPt.Name). Obj(), *testing.MakePodSet("secondary", 2). - Request(corev1.ResourceCPU, "3"). + SetPodTemplateName(thirdPt.Name). Obj(), ). Obj() @@ -443,8 +545,13 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Reclaimed resources are not accounted during admission", func() { + pt := testing.MakePodTemplate("first-pt", ns.Name).Request(corev1.ResourceCPU, "3").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "first-wl"}) + ginkgo.By("Creating the podtemplate", func() { + gomega.Expect(k8sClient.Create(ctx, pt)).Should(gomega.Succeed()) + }) wl := testing.MakeWorkload("first-wl", ns.Name).Queue(prodQueue.Name). - PodSets(*testing.MakePodSet("main", 2).Request(corev1.ResourceCPU, "3").Obj()). + PodSets(*testing.MakePodSet("main", 2).SetPodTemplateName(pt.Name).Obj()). Obj() ginkgo.By("Creating the workload", func() { gomega.Expect(k8sClient.Create(ctx, wl)).Should(gomega.Succeed()) @@ -497,8 +604,13 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Should re-enqueue by the delete event of workload belonging to the same ClusterQueue", func() { + ginkgo.By("Creating the podtemplate") + pt1 := testing.MakePodTemplate("on-demand-pt1", ns.Name).Request(corev1.ResourceCPU, "4").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + ginkgo.By("First big workload starts") - wl1 := testing.MakeWorkload("on-demand-wl1", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "4").Obj() + wl1 := testing.MakeWorkload("on-demand-wl1", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) expectWl1Admission := testing.MakeAdmission(cq.Name).Assignment(corev1.ResourceCPU, "on-demand", "4").Obj() util.ExpectWorkloadToBeAdmittedAs(ctx, k8sClient, wl1, expectWl1Admission) @@ -506,16 +618,26 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedActiveWorkloadsMetric(cq, 1) util.ExpectAdmittedWorkloadsTotalMetric(cq, 1) + ginkgo.By("Creating second big podtemplate") + pt2 := testing.MakePodTemplate("on-demand-pt2", ns.Name).Request(corev1.ResourceCPU, "4").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + ginkgo.By("Second big workload is pending") - wl2 := testing.MakeWorkload("on-demand-wl2", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "4").Obj() + wl2 := testing.MakeWorkload("on-demand-wl2", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).Should(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, wl2) util.ExpectPendingWorkloadsMetric(cq, 0, 1) util.ExpectAdmittedActiveWorkloadsMetric(cq, 1) util.ExpectAdmittedWorkloadsTotalMetric(cq, 1) + ginkgo.By("Creating third big podtemplate") + pt3 := testing.MakePodTemplate("on-demand-pt3", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl3"}) + gomega.Expect(k8sClient.Create(ctx, pt3)).Should(gomega.Succeed()) + ginkgo.By("Third small workload starts") - wl3 := testing.MakeWorkload("on-demand-wl3", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "1").Obj() + wl3 := testing.MakeWorkload("on-demand-wl3", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt3.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl3)).Should(gomega.Succeed()) expectWl3Admission := testing.MakeAdmission(cq.Name).Assignment(corev1.ResourceCPU, "on-demand", "1").Obj() util.ExpectWorkloadToBeAdmittedAs(ctx, k8sClient, wl3, expectWl3Admission) @@ -545,8 +667,13 @@ var _ = ginkgo.Describe("Scheduler", func() { fooQ := testing.MakeLocalQueue("foo-queue", ns.Name).ClusterQueue(fooCQ.Name).Obj() gomega.Expect(k8sClient.Create(ctx, fooQ)).Should(gomega.Succeed()) + ginkgo.By("Creating the podtemplate") + pt1 := testing.MakePodTemplate("on-demand-pt1", ns.Name).Request(corev1.ResourceCPU, "8").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + ginkgo.By("First big workload starts") - wl1 := testing.MakeWorkload("on-demand-wl1", ns.Name).Queue(fooQ.Name).Request(corev1.ResourceCPU, "8").Obj() + wl1 := testing.MakeWorkload("on-demand-wl1", ns.Name).Queue(fooQ.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) expectAdmission := testing.MakeAdmission(fooCQ.Name).Assignment(corev1.ResourceCPU, "on-demand", "8").Obj() util.ExpectWorkloadToBeAdmittedAs(ctx, k8sClient, wl1, expectAdmission) @@ -554,16 +681,26 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedActiveWorkloadsMetric(fooCQ, 1) util.ExpectAdmittedWorkloadsTotalMetric(fooCQ, 1) + ginkgo.By("Creating second big podtemplate") + pt2 := testing.MakePodTemplate("on-demand-pt2", ns.Name).Request(corev1.ResourceCPU, "8").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + ginkgo.By("Second big workload is pending") - wl2 := testing.MakeWorkload("on-demand-wl2", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "8").Obj() + wl2 := testing.MakeWorkload("on-demand-wl2", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).Should(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, wl2) util.ExpectPendingWorkloadsMetric(cq, 0, 1) util.ExpectAdmittedActiveWorkloadsMetric(cq, 0) util.ExpectAdmittedWorkloadsTotalMetric(cq, 0) + ginkgo.By("Creating third big podtemplate") + pt3 := testing.MakePodTemplate("on-demand-pt3", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl3"}) + gomega.Expect(k8sClient.Create(ctx, pt3)).Should(gomega.Succeed()) + ginkgo.By("Third small workload starts") - wl3 := testing.MakeWorkload("on-demand-wl3", ns.Name).Queue(fooQ.Name).Request(corev1.ResourceCPU, "2").Obj() + wl3 := testing.MakeWorkload("on-demand-wl3", ns.Name).Queue(fooQ.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt3.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl3)).Should(gomega.Succeed()) expectAdmission = testing.MakeAdmission(fooCQ.Name).Assignment(corev1.ResourceCPU, "on-demand", "2").Obj() util.ExpectWorkloadToBeAdmittedAs(ctx, k8sClient, wl3, expectAdmission) @@ -604,7 +741,16 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectResourceFlavorToBeDeleted(ctx, k8sClient, onDemandFlavor, true) }) ginkgo.It("Should re-enqueue by the update event of ClusterQueue", func() { - wl := testing.MakeWorkload("on-demand-wl", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "6").Obj() + pt := testing.MakePodTemplate("on-demand-pt", ns.Name).Request(corev1.ResourceCPU, "6").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl"}) + gomega.Expect(k8sClient.Create(ctx, pt)).Should(gomega.Succeed()) + + wl := testing.MakeWorkload("on-demand-wl", ns.Name). + PodSets(*testing.MakePodSet("main", 1). + SetPodTemplateName(pt.Name). + Obj()). + Queue(queue.Name). + Obj() gomega.Expect(k8sClient.Create(ctx, wl)).Should(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, wl) util.ExpectPendingWorkloadsMetric(cq, 0, 1) @@ -681,9 +827,15 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.It("Should admit workloads from the selected namespaces", func() { ginkgo.By("checking the workloads don't get admitted at first") - wl1 := testing.MakeWorkload("wl1", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "1").Obj() + pt1 := testing.MakePodTemplate("pt1", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + pt2 := testing.MakePodTemplate("pt2", nsFoo.Name).Request(corev1.ResourceCPU, "1").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + wl1 := testing.MakeWorkload("wl1", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) - wl2 := testing.MakeWorkload("wl2", nsFoo.Name).Queue(queueFoo.Name).Request(corev1.ResourceCPU, "1").Obj() + wl2 := testing.MakeWorkload("wl2", nsFoo.Name).Queue(queueFoo.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).Should(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, wl1, wl2) util.ExpectPendingWorkloadsMetric(cq, 0, 2) @@ -724,7 +876,10 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.It("Should be inactive until the flavor is created", func() { ginkgo.By("Creating one workload") util.ExpectClusterQueueStatusMetric(fooCQ, metrics.CQStatusPending) - wl := testing.MakeWorkload("workload", ns.Name).Queue(fooQ.Name).Request(corev1.ResourceCPU, "1").Obj() + pt := testing.MakePodTemplate("pt", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "workload"}) + gomega.Expect(k8sClient.Create(ctx, pt)).Should(gomega.Succeed()) + wl := testing.MakeWorkload("workload", ns.Name).Queue(fooQ.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl)).Should(gomega.Succeed()) util.ExpectWorkloadsToBeFrozen(ctx, k8sClient, fooCQ.Name, wl) util.ExpectPendingWorkloadsMetric(fooCQ, 0, 1) @@ -777,7 +932,11 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.It("Should schedule workloads on tolerated flavors", func() { ginkgo.By("checking a workload without toleration starts on the non-tainted flavor") - wl1 := testing.MakeWorkload("on-demand-wl1", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "5").Obj() + pt1 := testing.MakePodTemplate("on-demand-pt1", ns.Name).Request(corev1.ResourceCPU, "5").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + + wl1 := testing.MakeWorkload("on-demand-wl1", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) expectAdmission := testing.MakeAdmission(cq.Name).Assignment(corev1.ResourceCPU, "on-demand", "5").Obj() @@ -787,7 +946,11 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedWorkloadsTotalMetric(cq, 1) ginkgo.By("checking a second workload without toleration doesn't start") - wl2 := testing.MakeWorkload("on-demand-wl2", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "5").Obj() + pt2 := testing.MakePodTemplate("on-demand-pt2", ns.Name).Request(corev1.ResourceCPU, "5").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + + wl2 := testing.MakeWorkload("on-demand-wl2", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).Should(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, wl2) util.ExpectPendingWorkloadsMetric(cq, 0, 1) @@ -795,7 +958,11 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectAdmittedWorkloadsTotalMetric(cq, 1) ginkgo.By("checking a third workload with toleration starts") - wl3 := testing.MakeWorkload("on-demand-wl3", ns.Name).Queue(queue.Name).Toleration(spotToleration).Request(corev1.ResourceCPU, "5").Obj() + pt3 := testing.MakePodTemplate("on-demand-pt3", ns.Name).Request(corev1.ResourceCPU, "5").Toleration(spotToleration).Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "on-demand-wl3"}) + gomega.Expect(k8sClient.Create(ctx, pt3)).Should(gomega.Succeed()) + + wl3 := testing.MakeWorkload("on-demand-wl3", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt3.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl3)).Should(gomega.Succeed()) expectAdmission = testing.MakeAdmission(cq.Name).Assignment(corev1.ResourceCPU, "spot-tainted", "5").Obj() @@ -836,7 +1003,12 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.It("Should admit workloads with affinity to specific flavor", func() { ginkgo.By("checking a workload without affinity gets admitted on the first flavor") - wl1 := testing.MakeWorkload("no-affinity-workload", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "1").Obj() + pt1 := testing.MakePodTemplate("no-affinity-pt", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "no-affinity-workload"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + + wl1 := testing.MakeWorkload("no-affinity-workload", ns.Name).Queue(queue.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) expectAdmission := testing.MakeAdmission(cq.Name).Assignment(corev1.ResourceCPU, "spot-untainted", "1").Obj() util.ExpectWorkloadToBeAdmittedAs(ctx, k8sClient, wl1, expectAdmission) @@ -845,11 +1017,16 @@ var _ = ginkgo.Describe("Scheduler", func() { util.ExpectPendingWorkloadsMetric(cq, 0, 0) ginkgo.By("checking a second workload with affinity to on-demand gets admitted") - wl2 := testing.MakeWorkload("affinity-wl", ns.Name).Queue(queue.Name). + pt2 := testing.MakePodTemplate("affinity-pt", ns.Name).Request(corev1.ResourceCPU, "1"). NodeSelector(map[string]string{instanceKey: onDemandFlavor.Name, "foo": "bar"}). - Request(corev1.ResourceCPU, "1").Obj() + Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "affinity-wl"}) + gomega.Expect(len(pt2.Template.Spec.NodeSelector)).Should(gomega.Equal(2)) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + + wl2 := testing.MakeWorkload("affinity-wl", ns.Name).Queue(queue.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).Should(gomega.Succeed()) - gomega.Expect(len(wl2.Spec.PodSets[0].Template.Spec.NodeSelector)).Should(gomega.Equal(2)) expectAdmission = testing.MakeAdmission(cq.Name).Assignment(corev1.ResourceCPU, "on-demand", "1").Obj() util.ExpectWorkloadToBeAdmittedAs(ctx, k8sClient, wl2, expectAdmission) util.ExpectPendingWorkloadsMetric(cq, 0, 0) @@ -863,6 +1040,7 @@ var _ = ginkgo.Describe("Scheduler", func() { cq *kueue.ClusterQueue q *kueue.LocalQueue w *kueue.Workload + pt *corev1.PodTemplate ) ginkgo.BeforeEach(func() { @@ -870,7 +1048,12 @@ var _ = ginkgo.Describe("Scheduler", func() { ResourceGroup(*testing.MakeFlavorQuotas("on-demand").Resource(corev1.ResourceCPU, "5").Obj()). Obj() q = testing.MakeLocalQueue("queue", ns.Name).ClusterQueue(cq.Name).Obj() - w = testing.MakeWorkload("workload", ns.Name).Queue(q.Name).Request(corev1.ResourceCPU, "2").Obj() + pt = testing.MakePodTemplate("pt", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "workload"}) + w = testing.MakeWorkload("workload", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Queue(q.Name). + Obj() }) ginkgo.AfterEach(func() { @@ -882,12 +1065,14 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.It("Should admit workload when creating ResourceFlavor->LocalQueue->Workload->ClusterQueue", func() { gomega.Expect(k8sClient.Create(ctx, onDemandFlavor)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, q)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Create(ctx, pt)).Should(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, w)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, cq)).To(gomega.Succeed()) util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, cq.Name, w) }) ginkgo.It("Should admit workload when creating Workload->ResourceFlavor->LocalQueue->ClusterQueue", func() { + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, w)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, onDemandFlavor)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, q)).To(gomega.Succeed()) @@ -896,6 +1081,7 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Should admit workload when creating Workload->ResourceFlavor->ClusterQueue->LocalQueue", func() { + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, w)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, onDemandFlavor)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, cq)).To(gomega.Succeed()) @@ -904,6 +1090,7 @@ var _ = ginkgo.Describe("Scheduler", func() { }) ginkgo.It("Should admit workload when creating Workload->ClusterQueue->LocalQueue->ResourceFlavor", func() { + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, w)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, cq)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, q)).To(gomega.Succeed()) @@ -946,8 +1133,13 @@ var _ = ginkgo.Describe("Scheduler", func() { gomega.Expect(k8sClient.Create(ctx, queue)).Should(gomega.Succeed()) ginkgo.By("checking a no-fit workload does not get admitted") + pt := testing.MakePodTemplate("pt", ns.Name).Request(corev1.ResourceCPU, "10").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "wl"}) + gomega.Expect(k8sClient.Create(ctx, pt)).Should(gomega.Succeed()) + wl := testing.MakeWorkload("wl", ns.Name).Queue(queue.Name). - Request(corev1.ResourceCPU, "10").Toleration(spotToleration).Obj() + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Toleration(spotToleration).Obj() gomega.Expect(k8sClient.Create(ctx, wl)).Should(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, wl) util.ExpectPendingWorkloadsMetric(prodCQ, 0, 1) @@ -991,8 +1183,16 @@ var _ = ginkgo.Describe("Scheduler", func() { podQueue := testing.MakeLocalQueue("dev-queue", ns.Name).ClusterQueue(devCQ.Name).Obj() gomega.Expect(k8sClient.Create(ctx, podQueue)).Should(gomega.Succeed()) - wl1 := testing.MakeWorkload("wl-1", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "11").Obj() - wl2 := testing.MakeWorkload("wl-2", ns.Name).Queue(podQueue.Name).Request(corev1.ResourceCPU, "11").Obj() + pt1 := testing.MakePodTemplate("pt-1", ns.Name).Request(corev1.ResourceCPU, "11").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + + pt2 := testing.MakePodTemplate("pt-2", ns.Name).Request(corev1.ResourceCPU, "11").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl-2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + + wl1 := testing.MakeWorkload("wl-1", ns.Name).Queue(prodQueue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Obj() + wl2 := testing.MakeWorkload("wl-2", ns.Name).Queue(podQueue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Obj() ginkgo.By("Creating two workloads") gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) @@ -1048,15 +1248,31 @@ var _ = ginkgo.Describe("Scheduler", func() { gomega.Expect(k8sClient.Create(ctx, devQueue)).To(gomega.Succeed()) ginkgo.By("Creating two workloads for prod ClusterQueue") - pWl1 := testing.MakeWorkload("p-wl-1", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "1").Obj() - pWl2 := testing.MakeWorkload("p-wl-2", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "1").Obj() + pt1 := testing.MakePodTemplate("pt-1", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "p-wl-1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + + pt2 := testing.MakePodTemplate("pt-2", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "p-wl-2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + + pWl1 := testing.MakeWorkload("p-wl-1", ns.Name).Queue(prodQueue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Obj() + pWl2 := testing.MakeWorkload("p-wl-2", ns.Name).Queue(prodQueue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, pWl1)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, pWl2)).To(gomega.Succeed()) util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, prodCQ.Name, pWl1, pWl2) ginkgo.By("Creating a workload for each ClusterQueue") - dWl1 := testing.MakeWorkload("d-wl-1", ns.Name).Queue(devQueue.Name).Request(corev1.ResourceCPU, "1").Obj() - pWl3 := testing.MakeWorkload("p-wl-3", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2").Obj() + dPt1 := testing.MakePodTemplate("d-pt-1", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + dPt1.SetLabels(map[string]string{constants.WorkloadNameSource: "d-wl-1"}) + gomega.Expect(k8sClient.Create(ctx, dPt1)).Should(gomega.Succeed()) + + pt3 := testing.MakePodTemplate("p-pt-3", ns.Name).Request(corev1.ResourceCPU, "2").Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "p-wl-3"}) + gomega.Expect(k8sClient.Create(ctx, pt3)).Should(gomega.Succeed()) + + dWl1 := testing.MakeWorkload("d-wl-1", ns.Name).Queue(devQueue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(dPt1.Name).Obj()).Obj() + pWl3 := testing.MakeWorkload("p-wl-3", ns.Name).Queue(prodQueue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt3.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, dWl1)).To(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, pWl3)).To(gomega.Succeed()) util.ExpectWorkloadsToBePending(ctx, k8sClient, dWl1, pWl3) @@ -1118,16 +1334,27 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.It("Should schedule workloads by their priority strictly", func() { strictFIFOQueue := testing.MakeLocalQueue("strict-fifo-q", matchingNS.Name).ClusterQueue(strictFIFOClusterQ.Name).Obj() + ginkgo.By("Creating podtemplates") + pt1 := testing.MakePodTemplate("pt1", matchingNS.Name).Request(corev1.ResourceCPU, "2").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + pt2 := testing.MakePodTemplate("pt2", matchingNS.Name).Request(corev1.ResourceCPU, "5").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + pt3 := testing.MakePodTemplate("pt3", matchingNS.Name).Request(corev1.ResourceCPU, "1").Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "wl3"}) + gomega.Expect(k8sClient.Create(ctx, pt3)).Should(gomega.Succeed()) + ginkgo.By("Creating workloads") wl1 := testing.MakeWorkload("wl1", matchingNS.Name).Queue(strictFIFOQueue. - Name).Request(corev1.ResourceCPU, "2").Priority(100).Obj() + Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Priority(100).Obj() gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) wl2 := testing.MakeWorkload("wl2", matchingNS.Name).Queue(strictFIFOQueue. - Name).Request(corev1.ResourceCPU, "5").Priority(10).Obj() + Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Priority(10).Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).Should(gomega.Succeed()) // wl3 can't be scheduled before wl2 even though there is enough quota. wl3 := testing.MakeWorkload("wl3", matchingNS.Name).Queue(strictFIFOQueue. - Name).Request(corev1.ResourceCPU, "1").Priority(1).Obj() + Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt3.Name).Obj()).Priority(1).Obj() gomega.Expect(k8sClient.Create(ctx, wl3)).Should(gomega.Succeed()) gomega.Expect(k8sClient.Create(ctx, strictFIFOQueue)).Should(gomega.Succeed()) @@ -1152,16 +1379,27 @@ var _ = ginkgo.Describe("Scheduler", func() { matchingQueue := testing.MakeLocalQueue("matching-queue", matchingNS.Name).ClusterQueue(strictFIFOClusterQ.Name).Obj() gomega.Expect(k8sClient.Create(ctx, matchingQueue)).Should(gomega.Succeed()) + ginkgo.By("Creating podtemplates") + pt1 := testing.MakePodTemplate("pt1", matchingNS.Name).Request(corev1.ResourceCPU, "2").Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "wl1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + pt2 := testing.MakePodTemplate("pt2", ns.Name).Request(corev1.ResourceCPU, "5").Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "wl2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + pt3 := testing.MakePodTemplate("pt3", matchingNS.Name).Request(corev1.ResourceCPU, "1").Obj() + pt3.SetLabels(map[string]string{constants.WorkloadNameSource: "wl3"}) + gomega.Expect(k8sClient.Create(ctx, pt3)).Should(gomega.Succeed()) + ginkgo.By("Creating workloads") wl1 := testing.MakeWorkload("wl1", matchingNS.Name).Queue(matchingQueue. - Name).Request(corev1.ResourceCPU, "2").Priority(100).Obj() + Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Priority(100).Obj() gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) wl2 := testing.MakeWorkload("wl2", ns.Name).Queue(notMatchingQueue. - Name).Request(corev1.ResourceCPU, "5").Priority(10).Obj() + Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Priority(10).Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).Should(gomega.Succeed()) // wl2 can't block wl3 from getting scheduled. wl3 := testing.MakeWorkload("wl3", matchingNS.Name).Queue(matchingQueue. - Name).Request(corev1.ResourceCPU, "1").Priority(1).Obj() + Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt3.Name).Obj()).Priority(1).Obj() gomega.Expect(k8sClient.Create(ctx, wl3)).Should(gomega.Succeed()) util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, strictFIFOClusterQ.Name, wl1, wl3) @@ -1189,7 +1427,10 @@ var _ = ginkgo.Describe("Scheduler", func() { gomega.Expect(k8sClient.Create(ctx, queue)).Should(gomega.Succeed()) ginkgo.By("New created workloads should be admitted") - wl1 := testing.MakeWorkload("workload1", ns.Name).Queue(queue.Name).Obj() + pt1 := testing.MakePodTemplate("pt1", ns.Name).Obj() + pt1.SetLabels(map[string]string{constants.WorkloadNameSource: "workload1"}) + gomega.Expect(k8sClient.Create(ctx, pt1)).Should(gomega.Succeed()) + wl1 := testing.MakeWorkload("workload1", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt1.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl1)).Should(gomega.Succeed()) defer func() { gomega.Expect(util.DeleteWorkload(ctx, k8sClient, wl1)).To(gomega.Succeed()) @@ -1207,7 +1448,10 @@ var _ = ginkgo.Describe("Scheduler", func() { }, util.ConsistentDuration, util.Interval).Should(gomega.Equal([]string{kueue.ResourceInUseFinalizerName})) ginkgo.By("New created workloads should be frozen") - wl2 := testing.MakeWorkload("workload2", ns.Name).Queue(queue.Name).Obj() + pt2 := testing.MakePodTemplate("pt2", ns.Name).Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "workload2"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).Should(gomega.Succeed()) + wl2 := testing.MakeWorkload("workload2", ns.Name).Queue(queue.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()).Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).Should(gomega.Succeed()) defer func() { gomega.Expect(util.DeleteWorkload(ctx, k8sClient, wl2)).To(gomega.Succeed()) @@ -1267,16 +1511,20 @@ var _ = ginkgo.Describe("Scheduler", func() { lr := lrBuilder.Obj() gomega.Expect(k8sClient.Create(ctx, lr)).To(gomega.Succeed()) - wlBuilder := testing.MakeWorkload("workload", ns.Name).Queue(queue.Name) + pt := testing.MakePodTemplate("pt", ns.Name) if tp.reqCPU != "" { - wlBuilder.Request(corev1.ResourceCPU, tp.reqCPU) + pt.Request(corev1.ResourceCPU, tp.reqCPU) } if tp.limitCPU != "" { - wlBuilder.Limit(corev1.ResourceCPU, tp.limitCPU) + pt.Limit(corev1.ResourceCPU, tp.limitCPU) } - wl := wlBuilder.Obj() + ptObj := pt.Obj() + ptObj.SetLabels(map[string]string{constants.WorkloadNameSource: "workload"}) + gomega.Expect(k8sClient.Create(ctx, ptObj)).Should(gomega.Succeed()) + + wl := testing.MakeWorkload("workload", ns.Name).PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(ptObj.Name).Obj()).Queue(queue.Name).Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) if tp.shouldBeAdmited { @@ -1294,7 +1542,9 @@ var _ = ginkgo.Describe("Scheduler", func() { } return cond.Message }, util.Timeout, util.Interval).Should(gomega.ContainSubstring(tp.wantedStatus)) + } + gomega.Expect(util.DeletePodTemplate(ctx, k8sClient, ptObj)).To(gomega.Succeed()) gomega.Expect(util.DeleteWorkload(ctx, k8sClient, wl)).To(gomega.Succeed()) gomega.Expect(k8sClient.Delete(ctx, lr)).To(gomega.Succeed()) }, diff --git a/test/integration/scheduler/workload_controller_test.go b/test/integration/scheduler/workload_controller_test.go index 2cd6e818cd..76c7983b46 100644 --- a/test/integration/scheduler/workload_controller_test.go +++ b/test/integration/scheduler/workload_controller_test.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" @@ -92,11 +93,17 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.It("Should accumulate RuntimeClass's overhead", func() { ginkgo.By("Create and wait for workload admission", func() { - wl = testing.MakeWorkload("one", ns.Name). - Queue(localQueue.Name). + pt := testing.MakePodTemplate("one", ns.Name). Request(corev1.ResourceCPU, "1"). RuntimeClass("kata"). Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + + wl = testing.MakeWorkload("one", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Queue(localQueue.Name). + Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) gomega.Eventually(func() bool { @@ -147,11 +154,17 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.It("Should not accumulate RuntimeClass's overhead", func() { ginkgo.By("Create and wait for workload admission", func() { - wl = testing.MakeWorkload("one", ns.Name). - Queue(localQueue.Name). + pt := testing.MakePodTemplate("one", ns.Name). Request(corev1.ResourceCPU, "1"). RuntimeClass("kata"). Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + + wl = testing.MakeWorkload("one", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Queue(localQueue.Name). + Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) gomega.Eventually(func() bool { @@ -203,7 +216,12 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.It("Should use the range defined default requests, if provided", func() { ginkgo.By("Create and wait for workload admission", func() { + pt := testing.MakePodTemplate("one", ns.Name).Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + wl = testing.MakeWorkload("one", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). Queue(localQueue.Name). Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) @@ -244,9 +262,13 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { }) ginkgo.It("Should not use the range defined requests, if provided by the workload", func() { ginkgo.By("Create and wait for workload admission", func() { + pt := testing.MakePodTemplate("one", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + wl = testing.MakeWorkload("one", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). Queue(localQueue.Name). - Request(corev1.ResourceCPU, "1"). Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) @@ -305,9 +327,13 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.It("The limits should be used as request values", func() { ginkgo.By("Create and wait for workload admission", func() { + pt := testing.MakePodTemplate("one", ns.Name).Request(corev1.ResourceCPU, "1").Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + wl = testing.MakeWorkload("one", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). Queue(localQueue.Name). - Request(corev1.ResourceCPU, "1"). Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) @@ -369,11 +395,17 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.It("Should sync the resource requests with the new overhead", func() { ginkgo.By("Create and wait for the first workload admission", func() { - wl = testing.MakeWorkload("one", ns.Name). - Queue(localQueue.Name). + pt := testing.MakePodTemplate("one", ns.Name). Request(corev1.ResourceCPU, "1"). RuntimeClass("kata"). Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + + wl = testing.MakeWorkload("one", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). + Queue(localQueue.Name). + Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) gomega.Eventually(func() bool { @@ -387,11 +419,17 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { var wl2 *kueue.Workload ginkgo.By("Create a second workload, should stay pending", func() { - wl2 = testing.MakeWorkload("two", ns.Name). - Queue(localQueue.Name). + pt2 := testing.MakePodTemplate("two", ns.Name). Request(corev1.ResourceCPU, "1"). RuntimeClass("kata"). Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "two"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).To(gomega.Succeed()) + + wl2 = testing.MakeWorkload("two", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()). + Queue(localQueue.Name). + Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).To(gomega.Succeed()) gomega.Consistently(func() bool { @@ -463,7 +501,12 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.It("Should sync the resource requests with the limit", func() { ginkgo.By("Create and wait for the first workload admission", func() { + pt := testing.MakePodTemplate("one", ns.Name).Obj() + pt.SetLabels(map[string]string{constants.WorkloadNameSource: "one"}) + gomega.Expect(k8sClient.Create(ctx, pt)).To(gomega.Succeed()) + wl = testing.MakeWorkload("one", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt.Name).Obj()). Queue(localQueue.Name). Obj() gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) @@ -479,7 +522,12 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { var wl2 *kueue.Workload ginkgo.By("Create a second workload, should stay pending", func() { + pt2 := testing.MakePodTemplate("two", ns.Name).Obj() + pt2.SetLabels(map[string]string{constants.WorkloadNameSource: "two"}) + gomega.Expect(k8sClient.Create(ctx, pt2)).To(gomega.Succeed()) + wl2 = testing.MakeWorkload("two", ns.Name). + PodSets(*testing.MakePodSet("main", 1).SetPodTemplateName(pt2.Name).Obj()). Queue(localQueue.Name). Obj() gomega.Expect(k8sClient.Create(ctx, wl2)).To(gomega.Succeed()) diff --git a/test/util/util.go b/test/util/util.go index 28906cffec..555d29f604 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -45,6 +45,15 @@ func DeleteAdmissionCheck(ctx context.Context, c client.Client, ac *kueue.Admiss return nil } +func DeletePodTemplate(ctx context.Context, c client.Client, pt *corev1.PodTemplate) error { + if pt != nil { + if err := c.Delete(ctx, pt); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + func DeleteWorkload(ctx context.Context, c client.Client, wl *kueue.Workload) error { if wl != nil { if err := c.Delete(ctx, wl); err != nil && !apierrors.IsNotFound(err) {