SHould fix #381

pyowm/commons/http_client.py

@@ -3,6 +3,8 @@
 
 import json
 import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 from pyowm.commons import exceptions
 from pyowm.commons.enums import ImageTypeEnum
@@ -119,6 +121,22 @@ def __init__(self, api_key, config, root_uri, admits_subdomains=True):
         assert isinstance(admits_subdomains, bool)
         self.admits_subdomains = admits_subdomains
 
+        if self.config['connection']['max_retries'] is not None:
+            # this adapter tells how to perform retries
+            self.session_adapter = HTTPAdapter(
+                max_retries=Retry(
+                    total=self.config['connection']['max_retries'],
+                    status_forcelist=[429, 500, 502, 503, 504],
+                    method_whitelist=["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
+                )
+            )
+            # this is the adapted requests client
+            self.http = requests.Session()
+            self.http.mount("https://", self.session_adapter)
+            self.http.mount("http://", self.session_adapter)
+        else:
+            self.http = requests
+
     def get_json(self, path, params=None, headers=None):
         builder = HttpRequestBuilder(self.root_uri, self.api_key, self.config, has_subdomains=self.admits_subdomains)\
             .with_path(path)\
@@ -128,7 +146,7 @@ def get_json(self, path, params=None, headers=None):
             .with_headers(headers if headers is not None else dict())
         url, params, headers, proxies = builder.build()
         try:
-            resp = requests.get(url, params=params, headers=headers, proxies=proxies,
+            resp = self.http.get(url, params=params, headers=headers, proxies=proxies,
                                 timeout=self.config['connection']['timeout_secs'],
                                 verify=self.config['connection']['verify_ssl_certs'])
         except requests.exceptions.SSLError as e:
@@ -159,7 +177,7 @@ def get_png(self, path, params=None, headers=None):
             .with_header('Accept', ImageTypeEnum.PNG.mime_type)
         url, params, headers, proxies = builder.build()
         try:
-            resp = requests.get(url, stream=True, params=params, headers=headers, proxies=proxies,
+            resp = self.http.get(url, stream=True, params=params, headers=headers, proxies=proxies,
                                 timeout=self.config['connection']['timeout_secs'],
                                 verify=self.config['connection']['verify_ssl_certs'])
         except requests.exceptions.SSLError as e:
@@ -191,7 +209,7 @@ def get_geotiff(self, path, params=None, headers=None):
             .with_header('Accept', ImageTypeEnum.GEOTIFF.mime_type)
         url, params, headers, proxies = builder.build()
         try:
-            resp = requests.get(url, stream=True, params=params, headers=headers, proxies=proxies,
+            resp = self.http.get(url, stream=True, params=params, headers=headers, proxies=proxies,
                                 timeout=self.config['connection']['timeout_secs'],
                                 verify=self.config['connection']['verify_ssl_certs'])
         except requests.exceptions.SSLError as e:
@@ -216,7 +234,7 @@ def post(self, path, params=None, data=None, headers=None):
             .with_headers(headers if headers is not None else dict())
         url, params, headers, proxies = builder.build()
         try:
-            resp = requests.post(url, params=params, json=data, headers=headers, proxies=proxies,
+            resp = self.http.post(url, params=params, json=data, headers=headers, proxies=proxies,
                                  timeout=self.config['connection']['timeout_secs'],
                                  verify=self.config['connection']['verify_ssl_certs'])
         except requests.exceptions.SSLError as e:
@@ -242,7 +260,7 @@ def put(self, path, params=None, data=None, headers=None):
             .with_headers(headers if headers is not None else dict())
         url, params, headers, proxies = builder.build()
         try:
-            resp = requests.put(url, params=params, json=data, headers=headers, proxies=proxies,
+            resp = self.http.put(url, params=params, json=data, headers=headers, proxies=proxies,
                                 timeout=self.config['connection']['timeout_secs'],
                                 verify=self.config['connection']['verify_ssl_certs'])
         except requests.exceptions.SSLError as e:
@@ -268,7 +286,7 @@ def delete(self, path, params=None, data=None, headers=None):
             .with_headers(headers if headers is not None else dict())
         url, params, headers, proxies = builder.build()
         try:
-            resp = requests.delete(url, params=params, json=data, headers=headers, proxies=proxies,
+            resp = self.http.delete(url, params=params, json=data, headers=headers, proxies=proxies,
                                    timeout=self.config['connection']['timeout_secs'],
                                    verify=self.config['connection']['verify_ssl_certs'])
         except requests.exceptions.SSLError as e:

pyowm/config.py

@@ -10,7 +10,8 @@
         'use_ssl': True,
         'verify_ssl_certs': True,
         'use_proxy': False,
-        'timeout_secs': 5
+        'timeout_secs': 5,
+        'max_retries': None
     },
     'proxies': {
         'http': 'http://user:pass@host:port',

sphinx/v3/pyowm-configuration-description.md

@@ -16,9 +16,10 @@ The config dict is formatted as follows:
     "language": <str>,
     "connection": {
         "use_ssl": <bool>
-        "verify_ssl_certs": <bool>>,
+        "verify_ssl_certs": <bool>,
         "use_proxy": <bool>,
-        "timeout_secs": <int>
+        "timeout_secs": <int>,
+        "max_retries": <int>|<None>
     },
     "proxies": {
         "http": <str>,
@@ -36,6 +37,7 @@ Here are the keys:
     * `verify_ssl_certs`: speaks by itself..
     * `use_proxy`: whether to use a proxy server or not (useful if you're eg. in a corporate network). HTTP and SOCKS5 proxies are allowed
     * `timeout_secs`: after how many seconds the API calls should be timeouted
+    * `max_retries`: how many times PyOWM should retry to call the API if it responds with an error or timeouts. Defaults to `None`, which means: call forever.
   * `proxies` (this sub-dict is ignored if `use_proxy == False`)
     * `http`: the HTTP URL of the proxy server
     * `https`: the HTTPS/SOCKS5 URL of the proxy server

tests/integration/utils/test_config.json

@@ -5,6 +5,7 @@
     "connection": {
         "use_ssl": true,
         "verify_ssl_certs": true,
-        "timeout_secs": 1
+        "timeout_secs": 1,
+        "max_retries": null
     }
 }

tests/integration/utils/test_default_config.py

@@ -35,6 +35,9 @@ def test_default_config_is_complete(self):
         self.assertTrue('timeout_secs' in connection)
         self.assertEqual(5, connection['timeout_secs'])
 
+        self.assertTrue('max_retries' in connection)
+        self.assertEqual(None, connection['max_retries'])
+
         # proxies is a sub-dict, check its keys
         self.assertTrue('proxies' in DEFAULT_CONFIG)
         proxies = DEFAULT_CONFIG['proxies']

tests/unit/commons/test_http_client.py

@@ -132,6 +132,7 @@ def monkey_patched_get_timeouting(uri, params=None, headers=None, proxies=None,
         requests.get = monkey_patched_get_timeouting
         config = DEFAULT_CONFIG.copy()
         config['connection']['timeout_secs'] = timeout
+        config['connection']['max_retries'] = None
         try:
             status, data = HttpClient('apikey', config, 'anyurl.com').get_json('/resource')
             self.fail()