Fix #35: Add HTTP connection retries.

docs/config.rst

@@ -92,10 +92,13 @@ Configuration Values
 
    The cache time-to-live in seconds.
 
+.. confval:: internetarchive/retries
+
+   The maximum number of retries each HTTP connection should attempt.
+
 .. confval:: internetarchive/timeout
 
-   The request timeout in seconds for HTTP requests to the Internet
-   Archive.
+   The timeout in seconds for HTTP requests to the Internet Archive.
 
 
 .. _sortorder:

mopidy_internetarchive/__init__.py

@@ -57,6 +57,7 @@ def get_config_schema(self):
         schema['exclude_mediatypes'] = config.List(optional=True)
         schema['cache_size'] = config.Integer(minimum=1, optional=True)
         schema['cache_ttl'] = config.Integer(minimum=0, optional=True)
+        schema['retries'] = config.Integer(minimum=0)
         schema['timeout'] = config.Integer(minimum=0, optional=True)
         return schema
 

mopidy_internetarchive/backend.py

@@ -22,7 +22,8 @@ def __init__(self, config, audio):
         super(InternetArchiveBackend, self).__init__()
         self.client = InternetArchiveClient(
             config[Extension.ext_name]['base_url'],
-            config[Extension.ext_name]['timeout']
+            retries=config[Extension.ext_name]['retries'],
+            timeout=config[Extension.ext_name]['timeout']
         )
         self.library = InternetArchiveLibraryProvider(config, self)
         self.playback = InternetArchivePlaybackProvider(audio, self)

mopidy_internetarchive/client.py

@@ -1,65 +1,79 @@
 from __future__ import unicode_literals
 
 import collections
+import logging
 import requests
 import uritools
 
 BASE_URL = 'http://archive.org/'
 
+logger = logging.getLogger(__name__)
+
 
 class InternetArchiveClient(object):
 
     pykka_traversable = True
 
-    def __init__(self, base_url=BASE_URL, timeout=None):
-        self.search_url = uritools.urijoin(base_url, '/advancedsearch.php')
-        self.metadata_url = uritools.urijoin(base_url, '/metadata/')
-        self.download_url = uritools.urijoin(base_url, '/download/')
-        self.bookmarks_url = uritools.urijoin(base_url, '/bookmarks/')
-        self.session = requests.Session()
+    def __init__(self, base_url=BASE_URL, retries=0, timeout=None):
+        self.base_url = base_url
+        self.retries = retries
         self.timeout = timeout
+        self.session = requests.Session()
 
     def search(self, query, fields=None, sort=None, rows=None, start=None):
-        response = self.session.get(self.search_url, params={
+        response = self._get('/advancedsearch.php', params={
             'q': query,
             'fl[]': fields,
             'sort[]': sort,
             'rows': rows,
             'start': start,
             'output': 'json'
-        }, timeout=self.timeout)
-        if not response.content:
+        })
+        if response.content:
+            return self.SearchResult(response.json())
+        else:
             raise self.SearchError(response.url)
-        return self.SearchResult(response.json())
-
-    def metadata(self, path):
-        url = uritools.urijoin(self.metadata_url, path.lstrip('/'))
-        response = self.session.get(url, timeout=self.timeout)
-        data = response.json()
-
-        if not data:
-            raise LookupError('Internet Archive item %s not found' % path)
-        elif 'error' in data:
-            raise LookupError(data['error'])
-        elif 'result' in data:
-            return data['result']
+
+    def metadata(self, identifier):
+        obj = self._get('/metadata/' + identifier).json()
+        if not obj:
+            raise LookupError('Internet Archive item %s not found' % identifier)  # noqa
+        elif 'error' in obj:
+            raise LookupError(obj['error'])
+        elif 'result' in obj:
+            return obj['result']
         else:
-            return data
+            return obj
 
     def bookmarks(self, username):
-        url = uritools.urijoin(self.bookmarks_url, username + '?output=json')
-        response = self.session.get(url, timeout=self.timeout)
+        response = self._get('/bookmarks/' + username, params={
+            'output': 'json'
+        })
         # requests for non-existant users yield text/xml response
-        if response.headers['Content-Type'] != 'application/json':
+        if response.headers.get('Content-Type') != 'application/json':
             raise LookupError('Internet Archive user %s not found' % username)
         return response.json()
 
     def geturl(self, identifier, filename=None):
         if filename:
-            ref = identifier + '/' + uritools.uriencode(filename)
+            path = identifier + '/' + uritools.uriencode(filename)
         else:
-            ref = identifier + '/'
-        return uritools.urijoin(self.download_url, ref)
+            path = identifier + '/'
+        return uritools.urijoin(self.base_url, '/download/' + path)
+
+    def _get(self, path, params=None):
+        url = uritools.urijoin(self.base_url, path)
+        retries = self.retries
+        timeout = self.timeout
+
+        while True:
+            try:
+                return self.session.get(url, params=params, timeout=timeout)
+            except requests.exceptions.ConnectionError as e:
+                if not retries:
+                    raise e
+                logger.warn('Error connecting to the Internet Archive: %s', e)
+                retries -= 1
 
     class SearchResult(collections.Sequence):
 

mopidy_internetarchive/ext.conf

@@ -53,5 +53,8 @@ cache_size = 128
 # cache time-to-live in seconds
 cache_ttl = 86400
 
-# request timeout in seconds
+# maximum number of HTTP connection retries
+retries = 3
+
+# HTTP request timeout in seconds
 timeout = 10

scripts/iaclient.py

@@ -17,13 +17,18 @@
 parser.add_argument('-i', '--indent', type=int, default=2)
 parser.add_argument('-q', '--query', action='store_true')
 parser.add_argument('-r', '--rows', type=int)
+parser.add_argument('-R', '--retries', type=int, default=0)
 parser.add_argument('-S', '--sort', nargs='+')
 parser.add_argument('-t', '--timeout', type=float)
 args = parser.parse_args()
 
 logging.basicConfig(level=logging.DEBUG if args.debug else logging.WARN)
 
-client = InternetArchiveClient(args.base_url, timeout=args.timeout)
+client = InternetArchiveClient(
+    args.base_url,
+    retries=args.retries,
+    timeout=args.timeout
+)
 
 if args.query:
     query = args.arg.decode(args.encoding)

tests/test_extension.py

@@ -30,4 +30,5 @@ def test_get_config_schema(self):
         self.assertIn('exclude_mediatypes', schema)
         self.assertIn('cache_size', schema)
         self.assertIn('cache_ttl', schema)
+        self.assertIn('retries', schema)
         self.assertIn('timeout', schema)

tests/test_library.py

@@ -27,6 +27,7 @@ class LibraryTest(unittest.TestCase):
             'exclude_mediatypes': (),
             'cache_size': 1,
             'cache_ttl': None,
+            'retries': 0,
             'timeout': None
         }
     }