From 76ab96dbd65720ed0836c2b3ecb66c8fd4c93abb Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Mon, 10 Feb 2025 12:17:22 +0000 Subject: [PATCH] Fix process_submitted_jobs instance lock Fixes: https://github.com/dstackai/dstack/issues/2274 --- .../server/background/tasks/process_submitted_jobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py b/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py index b86ecdb6c..0f1e63962 100644 --- a/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py +++ b/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py @@ -207,12 +207,14 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel): detaching_instances_ids = await get_instances_ids_with_detaching_volumes(session) # Refetch after lock res = await session.execute( - select(InstanceModel).where( + select(InstanceModel) + .where( InstanceModel.id.not_in(detaching_instances_ids), InstanceModel.id.in_(instances_ids), InstanceModel.deleted == False, InstanceModel.total_blocks > InstanceModel.busy_blocks, ) + .execution_options(populate_existing=True) ) pool_instances = list(res.unique().scalars().all()) instance = await _assign_job_to_pool_instance(