Merge remote-tracking branch 'origin/develop' into feature/ltv_valid_…

…archives

subliminal/core.py

@@ -11,10 +11,12 @@
 
 from babelfish import Language, LanguageReverseError
 from guessit import guessit
+from six.moves.xmlrpc_client import ProtocolError
 from rarfile import BadRarFile, NotRarFile, RarCannotExec, RarFile
 from zipfile import BadZipfile
 import requests
 
+from .exceptions import ServiceUnavailable
 from .extensions import provider_manager, refiner_manager
 from .score import compute_score as default_compute_score
 from .subtitle import SUBTITLE_EXTENSIONS, get_subtitle_path
@@ -80,6 +82,13 @@ def __delitem__(self, name):
             self.initialized_providers[name].terminate()
         except (requests.Timeout, socket.timeout):
             logger.error('Provider %r timed out, improperly terminated', name)
+        except (ServiceUnavailable, ProtocolError):  # OpenSubtitles raises xmlrpclib.ProtocolError when unavailable
+            logger.error('Provider %r unavailable, improperly terminated', name)
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code in range(500, 600):
+                logger.error('Provider %r unavailable, improperly terminated', name)
+            else:
+                logger.exception('Provider %r http error %r, improperly terminated', name, e.response.status_code)
         except:
             logger.exception('Provider %r terminated unexpectedly', name)
 
@@ -119,6 +128,13 @@ def list_subtitles_provider(self, provider, video, languages):
             return self[provider].list_subtitles(video, provider_languages)
         except (requests.Timeout, socket.timeout):
             logger.error('Provider %r timed out', provider)
+        except (ServiceUnavailable, ProtocolError):  # OpenSubtitles raises xmlrpclib.ProtocolError when unavailable
+            logger.error('Provider %r unavailable', provider)
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code in range(500, 600):
+                logger.error('Provider %r unavailable', provider)
+            else:
+                logger.exception('Provider %r http error %r', provider, e.response.status_code)
         except:
             logger.exception('Unexpected error in provider %r', provider)
 
@@ -174,6 +190,17 @@ def download_subtitle(self, subtitle):
             logger.error('Provider %r timed out, discarding it', subtitle.provider_name)
             self.discarded_providers.add(subtitle.provider_name)
             return False
+        except (ServiceUnavailable, ProtocolError):  # OpenSubtitles raises xmlrpclib.ProtocolError when unavailable
+            logger.error('Provider %r unavailable, discarding it', subtitle.provider_name)
+            self.discarded_providers.add(subtitle.provider_name)
+            return False
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code in range(500, 600):
+                logger.error('Provider %r unavailable, improperly terminated', subtitle.provider_name)
+            else:
+                logger.exception('Provider %r http error %r, improperly terminated', subtitle.provider_name,
+                                 e.response.status_code)
+            return False
         except (BadRarFile, BadZipfile):
             logger.error('Bad archive for %r', subtitle)
             return False

subliminal/exceptions.py

@@ -19,8 +19,8 @@ class AuthenticationError(ProviderError):
     pass
 
 
-class TooManyRequests(ProviderError):
-    """Exception raised by providers when too many requests are made."""
+class ServiceUnavailable(ProviderError):
+    """Exception raised when status is '503 Service Unavailable'."""
     pass
 
 

subliminal/providers/addic7ed.py

@@ -9,7 +9,7 @@
 from . import ParserBeautifulSoup, Provider
 from .. import __short_version__
 from ..cache import SHOW_EXPIRATION_TIME, region
-from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, TooManyRequests
+from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded
 from ..score import get_equivalent_release_groups
 from ..subtitle import Subtitle, fix_line_ending, guess_matches
 from ..utils import sanitize, sanitize_release_group
@@ -19,6 +19,9 @@
 
 language_converters.register('addic7ed = subliminal.converters.addic7ed:Addic7edConverter')
 
+# Series cell matching regex
+show_cells_re = re.compile(b'<td class="version">.*?</td>', re.DOTALL)
+
 #: Series header parsing regex
 series_year_re = re.compile(r'^(?P<series>[ \w\'.:(),&!?-]+?)(?: \((?P<year>\d{4})\))?$')
 
@@ -137,7 +140,16 @@ def _get_show_ids(self):
         logger.info('Getting show ids')
         r = self.session.get(self.server_url + 'shows.php', timeout=10)
         r.raise_for_status()
-        soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+        # LXML parser seems to fail when parsing Addic7ed.com HTML markup.
+        # Last known version to work properly is 3.6.4 (next version, 3.7.0, fails)
+        # Assuming the site's markup is bad, and stripping it down to only contain what's needed.
+        show_cells = re.findall(show_cells_re, r.content)
+        if show_cells:
+            soup = ParserBeautifulSoup(b''.join(show_cells), ['lxml', 'html.parser'])
+        else:
+            # If RegEx fails, fall back to original r.content and use 'html.parser'
+            soup = ParserBeautifulSoup(r.content, ['html.parser'])
 
         # populate the show ids
         show_ids = {}
@@ -169,8 +181,6 @@ def _search_show_id(self, series, year=None):
         logger.info('Searching show ids with %r', params)
         r = self.session.get(self.server_url + 'search.php', params=params, timeout=10)
         r.raise_for_status()
-        if r.status_code == 304:
-            raise TooManyRequests()
         soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
 
         # get the suggestion
@@ -237,8 +247,13 @@ def query(self, series, season, year=None, country=None):
         logger.info('Getting the page of show id %d, season %d', show_id, season)
         r = self.session.get(self.server_url + 'show/%d' % show_id, params={'season': season}, timeout=10)
         r.raise_for_status()
-        if r.status_code == 304:
-            raise TooManyRequests()
+
+        if not r.content:
+            # Provider returns a status of 304 Not Modified with an empty content
+            # raise_for_status won't raise exception for that status code
+            logger.debug('No data returned from provider')
+            return []
+
         soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
 
         # loop over subtitle rows
@@ -289,6 +304,12 @@ def download_subtitle(self, subtitle):
                              timeout=10)
         r.raise_for_status()
 
+        if not r.content:
+            # Provider returns a status of 304 Not Modified with an empty content
+            # raise_for_status won't raise exception for that status code
+            logger.debug('Unable to download subtitle. No data returned from provider')
+            return
+
         # detect download limit exceeded
         if r.headers['Content-Type'] == 'text/html':
             raise DownloadLimitExceeded

subliminal/providers/legendastv.py

@@ -18,7 +18,7 @@
 from . import ParserBeautifulSoup, Provider
 from .. import __short_version__
 from ..cache import SHOW_EXPIRATION_TIME, region
-from ..exceptions import AuthenticationError, ConfigurationError, ProviderError
+from ..exceptions import AuthenticationError, ConfigurationError, ProviderError, ServiceUnavailable
 from ..subtitle import SUBTITLE_EXTENSIONS, Subtitle, fix_line_ending, guess_matches, sanitize
 from ..video import Episode, Movie
 
@@ -184,7 +184,7 @@ def initialize(self):
             logger.info('Logging in')
             data = {'_method': 'POST', 'data[User][username]': self.username, 'data[User][password]': self.password}
             r = self.session.post(self.server_url + 'login', data, allow_redirects=False, timeout=10)
-            r.raise_for_status()
+            raise_for_status(r)
 
             soup = ParserBeautifulSoup(r.content, ['html.parser'])
             if soup.find('div', {'class': 'alert-error'}, string=re.compile(u'Usuário ou senha inválidos')):
@@ -198,7 +198,7 @@ def terminate(self):
         if self.logged_in:
             logger.info('Logging out')
             r = self.session.get(self.server_url + 'users/logout', allow_redirects=False, timeout=10)
-            r.raise_for_status()
+            raise_for_status(r)
             logger.debug('Logged out')
             self.logged_in = False
 
@@ -263,7 +263,7 @@ def search_titles(self, title, season, title_year):
                 logger.info('Searching movie title %r', sanitized_title)
 
             r = self.session.get(self.server_url + 'legenda/sugestao/{}'.format(sanitized_title), timeout=10)
-            r.raise_for_status()
+            raise_for_status(r)
             results = json.loads(r.text)
 
             # loop over results
@@ -334,7 +334,7 @@ def get_archives(self, title_id, language_code, title_type, season, episode):
             url = self.server_url + 'legenda/busca/-/{language}/-/{page}/{title}'.format(
                 language=language_code, page=page, title=title_id)
             r = self.session.get(url)
-            r.raise_for_status()
+            raise_for_status(r)
 
             # parse the results
             soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
@@ -403,7 +403,7 @@ def download_archive(self, archive):
         """
         logger.info('Downloading archive %s', archive.id)
         r = self.session.get(self.server_url + 'downloadarquivo/{}'.format(archive.id))
-        r.raise_for_status()
+        raise_for_status(r)
 
         # open the archive
         archive_stream = io.BytesIO(r.content)
@@ -499,3 +499,11 @@ def download_subtitle(self, subtitle):
 
         # extract subtitle's content
         subtitle.content = fix_line_ending(subtitle.archive.content.read(subtitle.name))
+
+
+def raise_for_status(r):
+    # When site is under maintaince and http status code 200.
+    if 'Em breve estaremos de volta' in r.text:
+        raise ServiceUnavailable
+    else:
+        r.raise_for_status()

subliminal/providers/napiprojekt.py

@@ -86,16 +86,16 @@ def query(self, language, hash):
             'f': hash,
             't': get_subhash(hash)}
         logger.info('Searching subtitle %r', params)
-        response = self.session.get(self.server_url, params=params, timeout=10)
-        response.raise_for_status()
+        r = self.session.get(self.server_url, params=params, timeout=10)
+        r.raise_for_status()
 
         # handle subtitles not found and errors
-        if response.content[:4] == b'NPc0':
+        if r.content[:4] == b'NPc0':
             logger.debug('No subtitles found')
             return None
 
         subtitle = self.subtitle_class(language, hash)
-        subtitle.content = response.content
+        subtitle.content = r.content
         logger.debug('Found subtitle %r', subtitle)
 
         return subtitle

subliminal/providers/opensubtitles.py

@@ -11,7 +11,8 @@
 
 from . import Provider, TimeoutSafeTransport
 from .. import __short_version__
-from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError
+from ..exceptions import (AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError,
+                          ServiceUnavailable)
 from ..subtitle import Subtitle, fix_line_ending, guess_matches
 from ..utils import sanitize
 from ..video import Episode, Movie
@@ -261,11 +262,6 @@ class DisabledUserAgent(OpenSubtitlesError, AuthenticationError):
     pass
 
 
-class ServiceUnavailable(OpenSubtitlesError):
-    """Exception raised when status is '503 Service Unavailable'."""
-    pass
-
-
 def checked(response):
     """Check a response status before returning it.
 

subliminal/providers/podnapisi.py

@@ -114,7 +114,9 @@ def query(self, language, keyword, season=None, episode=None, year=None):
         pids = set()
         while True:
             # query the server
-            xml = etree.fromstring(self.session.get(self.server_url + 'search/old', params=params, timeout=10).content)
+            r = self.session.get(self.server_url + 'search/old', params=params, timeout=10)
+            r.raise_for_status()
+            xml = etree.fromstring(r.content)
 
             # exit if no results
             if not int(xml.find('pagination/results').text):

tests/cassettes/legendastv/test_under_maintenance.yaml

@@ -0,0 +1,41 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [Subliminal/2.1]
+    method: GET
+    uri: http://legendas.tv/legenda/sugestao/man%20of%20steel
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAA3VU207cMBB971cMqSrvqptkUUHQXEAg8YZohWhfEEImmU0Mjh3syS6r0o+p+tAP
+        4cfqrLPLRW2kyJ45Y885nrGzrVIXtGwRamrkwbusH0ByVeVBS8fnwcE7gKxGXvYTN22QOBQ1NxYp
+        DzqahfvBAJEgiQenWKEquYWL71nsXR62hREtQZ8sDwgfKL7lc+69wxYAc27guuL3kPvh8REur9IB
+        7D1R29l6dMmuXf6jotCdIjYB9u0o3N+fft7dCbfZ1fifC8jw4u4rr3AucPEiaDTrVEFCq9EYfgw+
+        T6TijoY7n65BRVFhkBOeSOytEfPE2Th1YdHqBHNgb2SxFcjtUhUOJdNhukngAGt694jVRK1NGOQv
+        skld8J5U1BpNutASDmEIjGNrJYPE285cLBZsDB+BRZXWlcSQKy6XJAobFbqJXaZby9JX0uxLZRXS
+        IMseLy94dcYbfBZ4Ob1KwUYtNy7gTJcYCWXR0DHOtMFRxSdgN4f5czwa5lnsNxiqL4W6g9rgLN+o
+        mGmXcKDMW+HJFtYezngj5DI/1zdOerIznU7cL4hLUUz2nLW3thgYlDmztJRoa0Rivr18HdxWbNW+
+        8bp/sxtdLgdCpZjDamEeuNPVJnn/afWl0PNKmM/uestyZUMnWMw8FHp6/4moUVQ19aTbhxQabiqh
+        EtjedSbwjjSsJz34EC5ESXWy66O9sT2dfkg3FyJr1yx7TaFTXamkcJVA44Iy0VTg2igPhlYQkRzu
+        X0TzWOpKR62qAuDS3dbTZ8gtjdvnHAdf5NOvrdeuE0u80RawcUxVR6ie/jz91hGcNHBjcI6ALsJg
+        H1MizLUkHkEyfrPLfSfc3dik7l+FAc9iVwNfIF8XV6jVI/QX3eL2o5UEAAA=
+    headers:
+      accept-ranges: [bytes]
+      age: ['0']
+      connection: [keep-alive]
+      content-encoding: [gzip]
+      content-length: ['671']
+      content-type: [text/html]
+      date: ['Thu, 22 Jun 2017 21:09:05 GMT']
+      vary: [Accept-Encoding]
+      via: [1.1 varnish]
+      x-backend: [default_director]
+      x-cache: [MISS]
+      x-cacheable: ['NO:Not Cacheable']
+      x-varnish: ['1338282906']
+    status: {code: 200, message: OK}
+version: 1

tests/test_legendastv.py

@@ -4,7 +4,7 @@
 from babelfish import Language, language_converters
 import pytest
 from vcr import VCR
-from subliminal.exceptions import ConfigurationError, AuthenticationError
+from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable
 from subliminal.providers.legendastv import LegendasTVSubtitle, LegendasTVProvider, LegendasTVArchive
 
 USERNAME = 'python-subliminal'
@@ -318,3 +318,18 @@ def test_download_subtitle(movies):
         provider.download_subtitle(subtitles[0])
     assert subtitles[0].content is not None
     assert subtitles[0].is_valid() is True
+
+
+@pytest.mark.integration
+@vcr.use_cassette
+def test_under_maintenance(movies):
+    """Tests when is under maintenance and http status code 200."""
+    video = movies['man_of_steel']
+    languages = {Language('eng')}
+    with LegendasTVProvider() as provider:
+        try:
+            provider.list_subtitles(video, languages)
+        except ServiceUnavailable:
+            pass
+        else:
+            pytest.fail()