From bba0f8143cdf3f0875531ba0048d65a71ea41df5 Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Fri, 23 Oct 2020 08:49:52 -0400 Subject: [PATCH] callback: tweak threading based on api client When using >=2 threads with the offline client, the playbook execution can sometimes, but not always, lock up. The issue doesn't appear to reproduce when using a single thread so do that for now. Otherwise, default to 4 threads as a more conservative default to prevent hitting urllib's connectionpool max. Change-Id: I2b0c57f784be36685619d80549fa626c385b0e04 --- ara/plugins/callback/ara_default.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ara/plugins/callback/ara_default.py b/ara/plugins/callback/ara_default.py index af2fc059..27df55b2 100644 --- a/ara/plugins/callback/ara_default.py +++ b/ara/plugins/callback/ara_default.py @@ -175,14 +175,13 @@ class CallbackModule(CallbackBase): def __init__(self): super(CallbackModule, self).__init__() self.log = logging.getLogger("ara.plugins.callback.default") + # These are configured in self.set_options self.client = None - # TODO: Consider un-hardcoding this and plumbing pool_maxsize to requests.adapters.HTTPAdapter. - # In the meantime default to 10 because that's the value of requests.adapters.DEFAULT_POOLSIZE. - # Otherwise we can hit "urllib3.connectionpool: Connection pool is full" - self.thread_count = 10 - self.global_threads = ThreadPoolExecutor(max_workers=self.thread_count) + self.thread_count = None + self.global_threads = None # Need individual threads for tasks to ensure all results are saved before moving on to next task self.task_threads = None + self.ignored_facts = [] self.ignored_arguments = [] self.ignored_files = [] @@ -221,6 +220,14 @@ def set_options(self, task_keys=None, var_options=None, direct=None): verify=False if insecure else True, ) + # TODO: Consider un-hardcoding this and plumbing pool_maxsize to requests.adapters.HTTPAdapter. + # In the meantime default to 4 so we don't go above requests.adapters.DEFAULT_POOLSIZE. + # Otherwise we can hit "urllib3.connectionpool: Connection pool is full" + # TODO: Using >= 2 threads with the offline client can result in execution getting locked up + self.thread_count = 1 if client == "offline" else 4 + self.global_threads = ThreadPoolExecutor(max_workers=self.thread_count) + self.log.debug("working with %s thread(s)" % self.thread_count) + def v2_playbook_on_start(self, playbook): self.log.debug("v2_playbook_on_start") @@ -382,6 +389,7 @@ def _end_task(self): ended=datetime.datetime.now().isoformat(), ) # Flush threads before moving on to next task to make sure all results are saved + self.log.debug("waiting for task threads...") self.task_threads.shutdown(wait=True) self.task_threads = None self.task = None @@ -409,6 +417,7 @@ def _end_playbook(self, stats): status=status, ended=datetime.datetime.now().isoformat(), ) + self.log.debug("waiting for global threads...") self.global_threads.shutdown(wait=True) def _set_playbook_name(self, name):