From 532b4db68c318472dc3405ae11f10da64f18b4dc Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 23 Jul 2020 14:03:30 +0200 Subject: [PATCH 1/4] Improve check in the loop of async policy --- reframe/frontend/executors/policies.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py index 83240978bd..3de1cb4781 100644 --- a/reframe/frontend/executors/policies.py +++ b/reframe/frontend/executors/policies.py @@ -416,7 +416,8 @@ def exit(self): pollrate = PollRateFunction(0.2, 60) num_polls = 0 t_start = datetime.now() - while (self._running_tasks or self._waiting_tasks): + while (self._running_tasks or self._waiting_tasks or + self._completed_tasks or sum(self._ready_tasks.values(), []): getlogger().debug('running tasks: %s' % len(self._running_tasks)) num_polls += len(self._running_tasks) try: From 783c5b8c68dba17972ceb2e4305e024a67d5e337 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 23 Jul 2020 14:08:56 +0200 Subject: [PATCH 2/4] Improve check in the loop of async policy --- reframe/frontend/executors/policies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py index 3de1cb4781..0fa7054023 100644 --- a/reframe/frontend/executors/policies.py +++ b/reframe/frontend/executors/policies.py @@ -417,7 +417,7 @@ def exit(self): num_polls = 0 t_start = datetime.now() while (self._running_tasks or self._waiting_tasks or - self._completed_tasks or sum(self._ready_tasks.values(), []): + self._completed_tasks or sum(self._ready_tasks.values(), [])): getlogger().debug('running tasks: %s' % len(self._running_tasks)) num_polls += len(self._running_tasks) try: From 63e58d02535876df44747d6cd4a751445a8cc0b4 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 23 Jul 2020 17:51:51 +0200 Subject: [PATCH 3/4] add values_len function --- reframe/frontend/executors/policies.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py index 0fa7054023..70efe3c054 100644 --- a/reframe/frontend/executors/policies.py +++ b/reframe/frontend/executors/policies.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import contextlib +import functools import itertools import math import sys @@ -17,6 +18,10 @@ TaskEventListener, ABORT_REASONS) +def values_len(d): + return functools.reduce(lambda l, r: l + len(r), d.values(), 0) + + def _cleanup_all(tasks, *args, **kwargs): for task in tasks: if task.ref_count == 0: @@ -417,7 +422,7 @@ def exit(self): num_polls = 0 t_start = datetime.now() while (self._running_tasks or self._waiting_tasks or - self._completed_tasks or sum(self._ready_tasks.values(), [])): + self._completed_tasks or values_len(self._ready_tasks)): getlogger().debug('running tasks: %s' % len(self._running_tasks)) num_polls += len(self._running_tasks) try: From 8f992387fc7111356b2015bdd3650bcfa45dc2be Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Fri, 24 Jul 2020 10:20:15 +0200 Subject: [PATCH 4/4] rename values_len to dictlist_len --- reframe/frontend/executors/policies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py index 70efe3c054..83e5680a3c 100644 --- a/reframe/frontend/executors/policies.py +++ b/reframe/frontend/executors/policies.py @@ -18,7 +18,7 @@ TaskEventListener, ABORT_REASONS) -def values_len(d): +def dictlist_len(d): return functools.reduce(lambda l, r: l + len(r), d.values(), 0) @@ -422,7 +422,7 @@ def exit(self): num_polls = 0 t_start = datetime.now() while (self._running_tasks or self._waiting_tasks or - self._completed_tasks or values_len(self._ready_tasks)): + self._completed_tasks or dictlist_len(self._ready_tasks)): getlogger().debug('running tasks: %s' % len(self._running_tasks)) num_polls += len(self._running_tasks) try: