From e698f246e901c9d4dfbc3f80e8f896a20e707689 Mon Sep 17 00:00:00 2001 From: Nemanja Martinovic Date: Mon, 15 Aug 2022 13:12:22 -0400 Subject: [PATCH 1/4] Reduce no of retries for no connection to 1 --- adsws/api/discoverer/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adsws/api/discoverer/views.py b/adsws/api/discoverer/views.py index 7b9e094..da8b346 100644 --- a/adsws/api/discoverer/views.py +++ b/adsws/api/discoverer/views.py @@ -40,7 +40,7 @@ def __init__(self, endpoint, service_uri, deploy_path, route): # requests sessions) # http://docs.python-requests.org/en/latest/api/?highlight=max_retries#requests.adapters.HTTPAdapter # - http_adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=1000, max_retries=3, pool_block=False) + http_adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=1000, max_retries=1, pool_block=False) self.session.mount('http://', http_adapter) @staticmethod From 5f321f266b37c77e1fa29b03120f6ad9f515e3b9 Mon Sep 17 00:00:00 2001 From: Nemanja Martinovic Date: Fri, 2 Sep 2022 16:22:52 -0400 Subject: [PATCH 2/4] Switch request properties to be loaded via env vars --- .github/workflows/python_actions.yml | 1 + adsws/api/discoverer/views.py | 2 +- config.py | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_actions.yml b/.github/workflows/python_actions.yml index 143eeab..b2ec238 100644 --- a/.github/workflows/python_actions.yml +++ b/.github/workflows/python_actions.yml @@ -64,6 +64,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade setuptools pip + git config --global url."https://".insteadOf git:// pip install -U -r requirements.txt pip install -U -r dev-requirements.txt diff --git a/adsws/api/discoverer/views.py b/adsws/api/discoverer/views.py index da8b346..e57ea2a 100644 --- a/adsws/api/discoverer/views.py +++ b/adsws/api/discoverer/views.py @@ -40,7 +40,7 @@ def __init__(self, endpoint, service_uri, deploy_path, route): # requests sessions) # http://docs.python-requests.org/en/latest/api/?highlight=max_retries#requests.adapters.HTTPAdapter # - http_adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=1000, max_retries=1, pool_block=False) + http_adapter = requests.adapters.HTTPAdapter(pool_connections=current_app.config.get("REQUESTS_POOL_CONNECTIONS", 20), pool_maxsize=current_app.config.get("REQUESTS_POOL_MAXSIZE", 1000), max_retries=current_app.config.get("REQUESTS_POOL_RETRIES", 1), pool_block=False) self.session.mount('http://', http_adapter) @staticmethod diff --git a/config.py b/config.py index c783c72..aecf7a5 100644 --- a/config.py +++ b/config.py @@ -17,3 +17,7 @@ # Flask session config (http://flask.pocoo.org/docs/0.12/config/) PERMANENT_SESSION_LIFETIME = 3600*24*365.25 # 1 year in seconds SESSION_REFRESH_EACH_REQUEST = True + +REQUESTS_POOL_CONNECTIONS=20 +REQUESTS_POOL_MAXSIZE=1000 +REQUESTS_POOL_RETRIES=1 From bc62ea3be7bd49fe77277f49f8186b8c6335d3d0 Mon Sep 17 00:00:00 2001 From: Nemanja Martinovic Date: Sat, 3 Sep 2022 16:33:01 -0400 Subject: [PATCH 3/4] Correction for unit tests --- adsws/api/discoverer/views.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/adsws/api/discoverer/views.py b/adsws/api/discoverer/views.py index e57ea2a..124da26 100644 --- a/adsws/api/discoverer/views.py +++ b/adsws/api/discoverer/views.py @@ -18,9 +18,15 @@ def __init__(self, endpoint, service_uri, deploy_path, route): self.cs = None try: self.default_request_timeout = current_app.config.get("DEFAULT_REQUEST_TIMEOUT", 60) + self.pool_connections = current_app.config.get("REQUESTS_POOL_CONNECTIONS", 20) + self.pool_maxsize = current_app.config.get("REQUESTS_POOL_MAXSIZE", 1000) + self.max_retries = current_app.config.get("REQUESTS_MAX_RETRIES", 1) except RuntimeError: # Unit testing fails: "RuntimeError: Working outside of application context." self.default_request_timeout = 60 + self.pool_connections = 20 + self.pool_maxsize = 1000 + self.max_retries = 1 if service_uri.startswith('consul://'): self.cs = ConsulService( service_uri, @@ -40,7 +46,10 @@ def __init__(self, endpoint, service_uri, deploy_path, route): # requests sessions) # http://docs.python-requests.org/en/latest/api/?highlight=max_retries#requests.adapters.HTTPAdapter # - http_adapter = requests.adapters.HTTPAdapter(pool_connections=current_app.config.get("REQUESTS_POOL_CONNECTIONS", 20), pool_maxsize=current_app.config.get("REQUESTS_POOL_MAXSIZE", 1000), max_retries=current_app.config.get("REQUESTS_POOL_RETRIES", 1), pool_block=False) + http_adapter = requests.adapters.HTTPAdapter(pool_connections=self.pool_connections, \ + pool_maxsize=self.pool_maxsize, \ + max_retries=self.max_retries, \ + pool_block=False) self.session.mount('http://', http_adapter) @staticmethod From 0ae2c811387bf22a84ba57817eb1cf3e61e6c2ef Mon Sep 17 00:00:00 2001 From: Nemanja Martinovic Date: Sat, 3 Sep 2022 22:04:04 -0400 Subject: [PATCH 4/4] Correct env naming in config --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index aecf7a5..8d3bd47 100644 --- a/config.py +++ b/config.py @@ -20,4 +20,4 @@ REQUESTS_POOL_CONNECTIONS=20 REQUESTS_POOL_MAXSIZE=1000 -REQUESTS_POOL_RETRIES=1 +REQUESTS_MAX_RETRIES=1