clusterQueue is a reference to a clusterQueue that backs this localQueue.
+stopPolicy
+StopPolicy
+ |
+
+ stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive,
+no new reservation being made.
+Depending on its value, its associated workloads will:
+
+- None - Workloads are admitted
+- HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation.
+- Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation.
+
+ |
+
@@ -1792,6 +1806,8 @@ words, it's the used quota that is over the nominalQuota.
- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta1-ClusterQueueSpec)
+- [LocalQueueSpec](#kueue-x-k8s-io-v1beta1-LocalQueueSpec)
+
diff --git a/test/integration/controller/core/workload_controller_test.go b/test/integration/controller/core/workload_controller_test.go
index f8ede73b63..7643c8fd87 100644
--- a/test/integration/controller/core/workload_controller_test.go
+++ b/test/integration/controller/core/workload_controller_test.go
@@ -90,7 +90,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn
gomega.BeComparableTo(metav1.Condition{
Type: kueue.WorkloadQuotaReserved,
Status: metav1.ConditionFalse,
- Reason: "Inadmissible",
+ Reason: kueue.WorkloadInadmissible,
Message: message,
}, util.IgnoreConditionTimestampsAndObservedGeneration),
)
@@ -113,7 +113,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn
gomega.BeComparableTo(metav1.Condition{
Type: kueue.WorkloadQuotaReserved,
Status: metav1.ConditionFalse,
- Reason: "Inadmissible",
+ Reason: kueue.WorkloadInadmissible,
Message: message,
}, util.IgnoreConditionTimestampsAndObservedGeneration),
)
@@ -140,7 +140,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn
gomega.BeComparableTo(metav1.Condition{
Type: kueue.WorkloadQuotaReserved,
Status: metav1.ConditionFalse,
- Reason: "Inadmissible",
+ Reason: kueue.WorkloadInadmissible,
Message: message,
}, util.IgnoreConditionTimestampsAndObservedGeneration),
)
diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go
index 10119107b1..0f26191842 100644
--- a/test/integration/scheduler/scheduler_test.go
+++ b/test/integration/scheduler/scheduler_test.go
@@ -96,6 +96,7 @@ var _ = ginkgo.Describe("Scheduler", func() {
preemptionQueue *kueue.LocalQueue
admissionCheckQueue *kueue.LocalQueue
cqsStopPolicy *kueue.StopPolicy
+ lqsStopPolicy *kueue.StopPolicy
)
ginkgo.JustBeforeEach(func() {
@@ -103,6 +104,7 @@ var _ = ginkgo.Describe("Scheduler", func() {
gomega.Expect(k8sClient.Create(ctx, spotTaintedFlavor)).To(gomega.Succeed())
gomega.Expect(k8sClient.Create(ctx, spotUntaintedFlavor)).To(gomega.Succeed())
cqsStopPolicy := ptr.Deref(cqsStopPolicy, kueue.None)
+ lqsStopPolicy := ptr.Deref(lqsStopPolicy, kueue.None)
admissionCheck1 = testing.MakeAdmissionCheck("check1").ControllerName("ctrl").Obj()
gomega.Expect(k8sClient.Create(ctx, admissionCheck1)).Should(gomega.Succeed())
@@ -172,22 +174,40 @@ var _ = ginkgo.Describe("Scheduler", func() {
Obj()
gomega.Expect(k8sClient.Create(ctx, admissionCheckClusterQ)).Should(gomega.Succeed())
- prodQueue = testing.MakeLocalQueue("prod-queue", ns.Name).ClusterQueue(prodClusterQ.Name).Obj()
+ prodQueue = testing.MakeLocalQueue("prod-queue", ns.Name).
+ StopPolicy(lqsStopPolicy).
+ ClusterQueue(prodClusterQ.Name).
+ Obj()
gomega.Expect(k8sClient.Create(ctx, prodQueue)).Should(gomega.Succeed())
- devQueue = testing.MakeLocalQueue("dev-queue", ns.Name).ClusterQueue(devClusterQ.Name).Obj()
+ devQueue = testing.MakeLocalQueue("dev-queue", ns.Name).
+ ClusterQueue(devClusterQ.Name).
+ StopPolicy(lqsStopPolicy).
+ Obj()
gomega.Expect(k8sClient.Create(ctx, devQueue)).Should(gomega.Succeed())
- podsCountQueue = testing.MakeLocalQueue("pods-count-queue", ns.Name).ClusterQueue(podsCountClusterQ.Name).Obj()
+ podsCountQueue = testing.MakeLocalQueue("pods-count-queue", ns.Name).
+ ClusterQueue(podsCountClusterQ.Name).
+ StopPolicy(lqsStopPolicy).
+ Obj()
gomega.Expect(k8sClient.Create(ctx, podsCountQueue)).Should(gomega.Succeed())
- podsCountOnlyQueue = testing.MakeLocalQueue("pods-count-only-queue", ns.Name).ClusterQueue(podsCountOnlyClusterQ.Name).Obj()
+ podsCountOnlyQueue = testing.MakeLocalQueue("pods-count-only-queue", ns.Name).
+ ClusterQueue(podsCountOnlyClusterQ.Name).
+ StopPolicy(lqsStopPolicy).
+ Obj()
gomega.Expect(k8sClient.Create(ctx, podsCountOnlyQueue)).Should(gomega.Succeed())
- preemptionQueue = testing.MakeLocalQueue("preemption-queue", ns.Name).ClusterQueue(preemptionClusterQ.Name).Obj()
+ preemptionQueue = testing.MakeLocalQueue("preemption-queue", ns.Name).
+ ClusterQueue(preemptionClusterQ.Name).
+ StopPolicy(lqsStopPolicy).
+ Obj()
gomega.Expect(k8sClient.Create(ctx, preemptionQueue)).Should(gomega.Succeed())
- admissionCheckQueue = testing.MakeLocalQueue("admission-check-queue", ns.Name).ClusterQueue(admissionCheckClusterQ.Name).Obj()
+ admissionCheckQueue = testing.MakeLocalQueue("admission-check-queue", ns.Name).
+ ClusterQueue(admissionCheckClusterQ.Name).
+ StopPolicy(lqsStopPolicy).
+ Obj()
gomega.Expect(k8sClient.Create(ctx, admissionCheckQueue)).Should(gomega.Succeed())
})
@@ -430,12 +450,14 @@ var _ = ginkgo.Describe("Scheduler", func() {
})
})
- ginkgo.When("Hold at startup", func() {
+ ginkgo.When("Hold ClusterQueue at startup", func() {
ginkgo.BeforeEach(func() {
cqsStopPolicy = ptr.To(kueue.Hold)
+ lqsStopPolicy = nil
})
ginkgo.AfterEach(func() {
cqsStopPolicy = nil
+ lqsStopPolicy = nil
})
ginkgo.It("Should admit workloads according to their priorities", func() {
const lowPrio, midPrio, highPrio = 0, 10, 100
@@ -454,7 +476,57 @@ var _ = ginkgo.Describe("Scheduler", func() {
util.ExpectPendingWorkloadsMetric(prodClusterQ, 0, 5)
util.ExpectReservingActiveWorkloadsMetric(prodClusterQ, 0)
- util.UnholdQueue(ctx, k8sClient, prodClusterQ)
+ util.UnholdClusterQueue(ctx, k8sClient, prodClusterQ)
+
+ ginkgo.By("checking the workloads with lower priority do not get admitted")
+ util.ExpectWorkloadsToBePending(ctx, k8sClient, wlLow, wlMid1, wlMid2)
+
+ ginkgo.By("checking the workloads with high priority get admitted")
+ util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, prodClusterQ.Name, wlHigh1, wlHigh2)
+
+ util.ExpectPendingWorkloadsMetric(prodClusterQ, 0, 3)
+ util.ExpectReservingActiveWorkloadsMetric(prodClusterQ, 2)
+ util.ExpectQuotaReservedWorkloadsTotalMetric(prodClusterQ, 2)
+ util.ExpectAdmittedWorkloadsTotalMetric(prodClusterQ, 2)
+
+ ginkgo.By("after the high priority workloads finish, only the mid priority workloads should be admitted")
+ util.FinishWorkloads(ctx, k8sClient, wlHigh1, wlHigh2)
+
+ util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, prodClusterQ.Name, wlMid1, wlMid2)
+ util.ExpectPendingWorkloadsMetric(prodClusterQ, 0, 1)
+ util.ExpectReservingActiveWorkloadsMetric(prodClusterQ, 2)
+ util.ExpectQuotaReservedWorkloadsTotalMetric(prodClusterQ, 4)
+ util.ExpectAdmittedWorkloadsTotalMetric(prodClusterQ, 4)
+ })
+ })
+
+ ginkgo.When("Hold LocalQueue at startup", func() {
+ ginkgo.BeforeEach(func() {
+ cqsStopPolicy = nil
+ lqsStopPolicy = ptr.To(kueue.Hold)
+ })
+ ginkgo.AfterEach(func() {
+ cqsStopPolicy = nil
+ lqsStopPolicy = nil
+ })
+ ginkgo.It("Should admit workloads according to their priorities", func() {
+ const lowPrio, midPrio, highPrio = 0, 10, 100
+
+ wlLow := testing.MakeWorkload("wl-low-priority", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2").Priority(lowPrio).Obj()
+ gomega.Expect(k8sClient.Create(ctx, wlLow)).Should(gomega.Succeed())
+ wlMid1 := testing.MakeWorkload("wl-mid-priority-1", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2").Priority(midPrio).Obj()
+ gomega.Expect(k8sClient.Create(ctx, wlMid1)).Should(gomega.Succeed())
+ wlMid2 := testing.MakeWorkload("wl-mid-priority-2", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2").Priority(midPrio).Obj()
+ gomega.Expect(k8sClient.Create(ctx, wlMid2)).Should(gomega.Succeed())
+ wlHigh1 := testing.MakeWorkload("wl-high-priority-1", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2").Priority(highPrio).Obj()
+ gomega.Expect(k8sClient.Create(ctx, wlHigh1)).Should(gomega.Succeed())
+ wlHigh2 := testing.MakeWorkload("wl-high-priority-2", ns.Name).Queue(prodQueue.Name).Request(corev1.ResourceCPU, "2").Priority(highPrio).Obj()
+ gomega.Expect(k8sClient.Create(ctx, wlHigh2)).Should(gomega.Succeed())
+
+ util.ExpectPendingWorkloadsMetric(prodClusterQ, 0, 0)
+ util.ExpectReservingActiveWorkloadsMetric(prodClusterQ, 0)
+
+ util.UnholdLocalQueue(ctx, k8sClient, prodQueue)
ginkgo.By("checking the workloads with lower priority do not get admitted")
util.ExpectWorkloadsToBePending(ctx, k8sClient, wlLow, wlMid1, wlMid2)
diff --git a/test/util/util.go b/test/util/util.go
index d33ff2b723..eb7241d65e 100644
--- a/test/util/util.go
+++ b/test/util/util.go
@@ -211,7 +211,7 @@ func DeleteRuntimeClass(ctx context.Context, c client.Client, runtimeClass *node
return nil
}
-func UnholdQueue(ctx context.Context, k8sClient client.Client, cq *kueue.ClusterQueue) {
+func UnholdClusterQueue(ctx context.Context, k8sClient client.Client, cq *kueue.ClusterQueue) {
gomega.EventuallyWithOffset(1, func(g gomega.Gomega) {
var cqCopy kueue.ClusterQueue
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cq), &cqCopy)).To(gomega.Succeed())
@@ -223,6 +223,18 @@ func UnholdQueue(ctx context.Context, k8sClient client.Client, cq *kueue.Cluster
}, Timeout, Interval).Should(gomega.Succeed())
}
+func UnholdLocalQueue(ctx context.Context, k8sClient client.Client, lq *kueue.LocalQueue) {
+ gomega.EventuallyWithOffset(1, func(g gomega.Gomega) {
+ var lqCopy kueue.LocalQueue
+ g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lq), &lqCopy)).To(gomega.Succeed())
+ if ptr.Deref(lqCopy.Spec.StopPolicy, kueue.None) == kueue.None {
+ return
+ }
+ lqCopy.Spec.StopPolicy = ptr.To(kueue.None)
+ g.Expect(k8sClient.Update(ctx, &lqCopy)).To(gomega.Succeed())
+ }, Timeout, Interval).Should(gomega.Succeed())
+}
+
func FinishWorkloads(ctx context.Context, k8sClient client.Client, workloads ...*kueue.Workload) {
for _, w := range workloads {
gomega.EventuallyWithOffset(1, func() error {