From fedb72eada86c286033ef608435b425c18a40eb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Nerl=C3=B8e?= Date: Fri, 24 Oct 2025 12:56:38 +0200 Subject: [PATCH 1/2] feat(supervisor): add Kubernetes worker pod priority class support Add KUBERNETES_POD_PRIORITY_CLASS_NAME environment variable to allow configuring a priority class for task run pods in Kubernetes environments. This addresses pod preemption issues in resource-constrained clusters, particularly in GKE Autopilot where low-priority pods can be preempted by system pods during scale-up operations. Fixes: Pod preemption causing SIGTERM failures during cluster scale-up Related: GKE Autopilot resource contention --- apps/supervisor/src/env.ts | 1 + apps/supervisor/src/workloadManager/kubernetes.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 453e001187..2ffd0ea4b2 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -91,6 +91,7 @@ const Env = z.object({ KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods + KUBERNETES_POD_PRIORITY_CLASS_NAME: z.string().optional(), // Priority class name for task run pods // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index 2042bf8b6a..1df6287eca 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -286,6 +286,11 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, } : {}), + ...(env.KUBERNETES_POD_PRIORITY_CLASS_NAME + ? { + priorityClassName: env.KUBERNETES_POD_PRIORITY_CLASS_NAME, + } + : {}), }; } From 9045e31a9dcf9d6bc17246e83aa6be218640acea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Nerl=C3=B8e?= Date: Fri, 24 Oct 2025 13:04:58 +0200 Subject: [PATCH 2/2] fix: renamed env variable --- apps/supervisor/src/env.ts | 2 +- apps/supervisor/src/workloadManager/kubernetes.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 2ffd0ea4b2..bd7c03cdb0 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -81,6 +81,7 @@ const Env = z.object({ KUBERNETES_FORCE_ENABLED: BoolEnv.default(false), KUBERNETES_NAMESPACE: z.string().default("default"), KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"), + KUBERNETES_WORKER_PRIORITY_CLASS_NAME: z.string().optional(), KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"), KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"), @@ -91,7 +92,6 @@ const Env = z.object({ KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods - KUBERNETES_POD_PRIORITY_CLASS_NAME: z.string().optional(), // Priority class name for task run pods // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index 1df6287eca..dd8d701e97 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -286,9 +286,9 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, } : {}), - ...(env.KUBERNETES_POD_PRIORITY_CLASS_NAME + ...(env.KUBERNETES_WORKER_PRIORITY_CLASS_NAME ? { - priorityClassName: env.KUBERNETES_POD_PRIORITY_CLASS_NAME, + priorityClassName: env.KUBERNETES_WORKER_PRIORITY_CLASS_NAME, } : {}), };