From 90d8068862b562e33a8a311d3e53dd2d9683988c Mon Sep 17 00:00:00 2001 From: Elijah DeLee Date: Wed, 2 Feb 2022 19:27:27 -0500 Subject: [PATCH] jobs_running not accurate for control nodes We can either NOT use "idle instances" for control nodes, or we need to update the jobs_running property on the Instance model to count jobs where the node is the controller_node. I didn't do that because it may be an expensive query, and it would be hard to make it match with jobs_running on the InstanceGroup which filters on tasks assigned to the instance group. This change chooses to stop considering "idle" control nodes an option, since we can't acurrately identify them. The way things are without any change, is we are continuing to over consume capacity on control nodes because this method sees all control nodes as "idle" at the beginning of the task manager run, and then only counts jobs started in that run in the in-memory tracking. So jobs which last over a number of task manager runs build up consuming capacity, which is accurately reported via Instance.consumed_capacity --- awx/main/scheduler/task_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/scheduler/task_manager.py b/awx/main/scheduler/task_manager.py index 5a77965c2423..8765c2871b52 100644 --- a/awx/main/scheduler/task_manager.py +++ b/awx/main/scheduler/task_manager.py @@ -502,7 +502,7 @@ def process_pending_tasks(self, pending_tasks): control_impact = settings.AWX_CONTROL_NODE_TASK_IMPACT control_instance = InstanceGroup.fit_task_to_most_remaining_capacity_instance( task, self.graph[settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME]['instances'], impact=control_impact, capacity_type='control' - ) or InstanceGroup.find_largest_idle_instance(self.graph[settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME]['instances'], capacity_type='control') + ) if not control_instance: self.task_needs_capacity(task, tasks_to_update_job_explanation) logger.debug(f"Skipping task {task.log_format} in pending, not enough capacity left on controlplane to control new tasks")