Merge branch 'requests_timeouts_and_retries' of https://github.com/dk…

…oston/python-cloudflare into dkoston-requests_timeouts_and_retries

Pull Request 173 from dkoston - requests_timeouts_and_retries

CloudFlare/cloudflare.py

@@ -38,8 +38,14 @@ def __init__(self, config):
 
             self.raw = config['raw']
             self.use_sessions = config['use_sessions']
+            self.global_request_timeout = config['global_request_timeout'] if 'global_request_timeout' in config else None
+            self.max_request_retries = config['max_request_retries'] if 'max_request_retries' in config else None
             self.profile = config['profile']
-            self.network = CFnetwork(use_sessions=self.use_sessions)
+            self.network = CFnetwork(
+                use_sessions=self.use_sessions,
+                global_request_timeout=self.global_request_timeout,
+                max_request_retries=self.max_request_retries
+            )
             self.user_agent = user_agent()
 
             self.logger = CFlogger(config['debug']).getLogger() if 'debug' in config and config['debug'] else None
@@ -909,7 +915,7 @@ def api_from_openapi(self, url):
 
         return api_decode_from_openapi(self._base.api_from_openapi(url))
 
-    def __init__(self, email=None, key=None, token=None, certtoken=None, debug=False, raw=False, use_sessions=True, profile=None, base_url=None):
+    def __init__(self, email=None, key=None, token=None, certtoken=None, debug=False, raw=False, use_sessions=True, profile=None, base_url=None, global_request_timeout=5, max_request_retries=5):
         """ Cloudflare v4 API"""
 
         self._base = None
@@ -938,6 +944,10 @@ def __init__(self, email=None, key=None, token=None, certtoken=None, debug=False
             config['profile'] = profile
         if base_url is not None:
             config['base_url'] = base_url
+        if global_request_timeout is not None:
+            config['global_request_timeout'] = global_request_timeout
+        if max_request_retries is not None:
+            config['max_request_retries'] = max_request_retries
 
         # we do not need to handle item.call values - they pass straight thru
 

CloudFlare/network.py

@@ -1,60 +1,133 @@
 """ Network for Cloudflare API"""
 from __future__ import absolute_import
 
+from urllib.parse import urlparse
+
 import requests
+from requests.adapters import HTTPAdapter
 
 from .exceptions import CloudFlareAPIError
 
-class CFnetwork():
-    """ Network for Cloudflare API"""
 
-    def __init__(self, use_sessions=True):
-        """ Network for Cloudflare API"""
+class CFnetwork:
+    """Network for Cloudflare API"""
+
+    def __init__(
+        self, max_request_retries, use_sessions=True, global_request_timeout=5,
+    ):
+        """Network for Cloudflare API"""
 
         self.use_sessions = use_sessions
+        self.global_request_timeout = global_request_timeout
+        self.max_request_retries = max_request_retries
         self.session = None
 
     def __call__(self, method, url, headers=None, params=None, data=None, files=None):
-        """ Network for Cloudflare API"""
+        """Network for Cloudflare API"""
 
         if self.use_sessions:
             if self.session is None:
-                self.session = requests.Session()
+                s = requests.Session()
+                if self.max_request_retries is not None:
+                    hostname = urlparse(url).netloc
+                    s.mount(
+                        f"https://{hostname}",
+                        HTTPAdapter(max_retries=self.max_request_retries),
+                    )
+                self.session = s
         else:
             self.session = requests
 
         method = method.upper()
 
         if method == 'GET':
-            r = self.session.get(url, headers=headers, params=params, data=data)
+            r = self.session.get(
+                url,
+                headers=headers,
+                params=params,
+                data=data,
+                timeout=self.global_request_timeout,
+            )
         elif method == 'POST':
             if isinstance(data, str):
-                r = self.session.post(url, headers=headers, params=params, data=data, files=files)
+                r = self.session.post(
+                    url,
+                    headers=headers,
+                    params=params,
+                    data=data,
+                    files=files,
+                    timeout=self.global_request_timeout,
+                )
             else:
-                r = self.session.post(url, headers=headers, params=params, json=data, files=files)
+                r = self.session.post(
+                    url,
+                    headers=headers,
+                    params=params,
+                    json=data,
+                    files=files,
+                    timeout=self.global_request_timeout,
+                )
         elif method == 'PUT':
             if isinstance(data, str):
-                r = self.session.put(url, headers=headers, params=params, data=data)
+                r = self.session.put(
+                    url,
+                    headers=headers,
+                    params=params,
+                    data=data,
+                    timeout=self.global_request_timeout,
+                )
             else:
-                r = self.session.put(url, headers=headers, params=params, json=data)
+                r = self.session.put(
+                    url,
+                    headers=headers,
+                    params=params,
+                    json=data,
+                    timeout=self.global_request_timeout,
+                )
         elif method == 'DELETE':
             if isinstance(data, str):
-                r = self.session.delete(url, headers=headers, params=params, data=data)
+                r = self.session.delete(
+                    url,
+                    headers=headers,
+                    params=params,
+                    data=data,
+                    timeout=self.global_request_timeout,
+                )
             else:
-                r = self.session.delete(url, headers=headers, params=params, json=data)
+                r = self.session.delete(
+                    url,
+                    headers=headers,
+                    params=params,
+                    json=data,
+                    timeout=self.global_request_timeout,
+                )
         elif method == 'PATCH':
             if isinstance(data, str):
-                r = self.session.request('PATCH', url, headers=headers, params=params, data=data)
+                r = self.session.request(
+                    'PATCH',
+                    url,
+                    headers=headers,
+                    params=params,
+                    data=data,
+                    timeout=self.global_request_timeout,
+                )
             else:
-                r = self.session.request('PATCH', url, headers=headers, params=params, json=data)
+                r = self.session.request(
+                    'PATCH',
+                    url,
+                    headers=headers,
+                    params=params,
+                    json=data,
+                    timeout=self.global_request_timeout,
+                )
         else:
             # should never happen
             raise CloudFlareAPIError(0, 'method not supported')
 
         return r
 
     def __del__(self):
-        """ Network for Cloudflare API"""
+        """Network for Cloudflare API"""
 
         if self.use_sessions and self.session:
             self.session.close()

Makefile

@@ -3,6 +3,7 @@ PYTHON = python
 # PANDOC = pandoc
 PYLINT = pylint
 TWINE = twine
+PYTEST = pytest
 
 EMAIL = "mahtin@mahtin.com"
 NAME = "cloudflare"
@@ -36,7 +37,7 @@ install: build
 	sudo rm -rf ${NAME}.egg-info
 
 test: all
-#	 to be done
+	$(PYTEST) -vv
 
 cli4test: all
 	$(PYTHON) -m cli4 /ips > /dev/null

README.md

@@ -180,7 +180,12 @@ When you create a **CloudFlare** class you can pass some combination of these fo
 
 This parameter controls how the data is returned from a successful call (see notes below).
 
- * `raw - An optional Raw flag (True/False) - defaults to False
+ * `raw` - An optional Raw flag (True/False) - defaults to False
+
+Timeouts (10s) and Retries (5) are configured by default. Should you wish to override them, use these settings:
+* `global_request_timeout` - How long before each API call to Cloudflare should time out (in seconds)
+* `max_requests_retries` - How many times to retry an API call when DNS lookups, socket connections, or connect timeouts occur.
+> NOTE: `max_request_retries` is only available when `use_sessions` is not disabled.
 
 The following paramaters are for debug and/or development usage
 

requirements.txt

@@ -3,3 +3,4 @@ future
 pyyaml
 jsonlines
 beautifulsoup4
+pytest

setup.cfg

@@ -9,3 +9,10 @@
 #
 #[upload]
 #
+[options.extras_require]
+test =
+    pytest
+
+[tool:pytest]
+testpaths =
+    tests

tests/test_cloudflare.py

@@ -0,0 +1,19 @@
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+import CloudFlare
+
+class TestCloudflare:
+    def test_creating_default_client(self):
+        cf = CloudFlare.CloudFlare()
+        assert isinstance(cf, CloudFlare.CloudFlare)
+
+
+    def test_with_global_request_timeout(self):
+        cf = CloudFlare.CloudFlare({'global_request_timeout': 10})
+        assert isinstance(cf, CloudFlare.CloudFlare)
+
+    def test_with_max_request_retries(self):
+        cf = CloudFlare.CloudFlare({'max_request_retries': 2})
+        assert isinstance(cf, CloudFlare.CloudFlare)