diff --git a/subliminal/core.py b/subliminal/core.py index 0716fbb7a..30923ee4d 100644 --- a/subliminal/core.py +++ b/subliminal/core.py @@ -11,10 +11,12 @@ from babelfish import Language, LanguageReverseError from guessit import guessit +from six.moves.xmlrpc_client import ProtocolError from rarfile import BadRarFile, NotRarFile, RarCannotExec, RarFile from zipfile import BadZipfile import requests +from .exceptions import ServiceUnavailable from .extensions import provider_manager, refiner_manager from .score import compute_score as default_compute_score from .subtitle import SUBTITLE_EXTENSIONS, get_subtitle_path @@ -80,6 +82,13 @@ def __delitem__(self, name): self.initialized_providers[name].terminate() except (requests.Timeout, socket.timeout): logger.error('Provider %r timed out, improperly terminated', name) + except (ServiceUnavailable, ProtocolError): # OpenSubtitles raises xmlrpclib.ProtocolError when unavailable + logger.error('Provider %r unavailable, improperly terminated', name) + except requests.exceptions.HTTPError as e: + if e.response.status_code in range(500, 600): + logger.error('Provider %r unavailable, improperly terminated', name) + else: + logger.exception('Provider %r http error %r, improperly terminated', name, e.response.status_code) except: logger.exception('Provider %r terminated unexpectedly', name) @@ -119,6 +128,13 @@ def list_subtitles_provider(self, provider, video, languages): return self[provider].list_subtitles(video, provider_languages) except (requests.Timeout, socket.timeout): logger.error('Provider %r timed out', provider) + except (ServiceUnavailable, ProtocolError): # OpenSubtitles raises xmlrpclib.ProtocolError when unavailable + logger.error('Provider %r unavailable', provider) + except requests.exceptions.HTTPError as e: + if e.response.status_code in range(500, 600): + logger.error('Provider %r unavailable', provider) + else: + logger.exception('Provider %r http error %r', provider, e.response.status_code) except: logger.exception('Unexpected error in provider %r', provider) @@ -174,6 +190,17 @@ def download_subtitle(self, subtitle): logger.error('Provider %r timed out, discarding it', subtitle.provider_name) self.discarded_providers.add(subtitle.provider_name) return False + except (ServiceUnavailable, ProtocolError): # OpenSubtitles raises xmlrpclib.ProtocolError when unavailable + logger.error('Provider %r unavailable, discarding it', subtitle.provider_name) + self.discarded_providers.add(subtitle.provider_name) + return False + except requests.exceptions.HTTPError as e: + if e.response.status_code in range(500, 600): + logger.error('Provider %r unavailable, improperly terminated', subtitle.provider_name) + else: + logger.exception('Provider %r http error %r, improperly terminated', subtitle.provider_name, + e.response.status_code) + return False except (BadRarFile, BadZipfile): logger.error('Bad archive for %r', subtitle) return False diff --git a/subliminal/exceptions.py b/subliminal/exceptions.py index 5f5c7a773..14d4f6412 100644 --- a/subliminal/exceptions.py +++ b/subliminal/exceptions.py @@ -19,8 +19,8 @@ class AuthenticationError(ProviderError): pass -class TooManyRequests(ProviderError): - """Exception raised by providers when too many requests are made.""" +class ServiceUnavailable(ProviderError): + """Exception raised when status is '503 Service Unavailable'.""" pass diff --git a/subliminal/providers/addic7ed.py b/subliminal/providers/addic7ed.py index e8c2791b4..2e1f1b678 100644 --- a/subliminal/providers/addic7ed.py +++ b/subliminal/providers/addic7ed.py @@ -9,7 +9,7 @@ from . import ParserBeautifulSoup, Provider from .. import __short_version__ from ..cache import SHOW_EXPIRATION_TIME, region -from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, TooManyRequests +from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded from ..score import get_equivalent_release_groups from ..subtitle import Subtitle, fix_line_ending, guess_matches from ..utils import sanitize, sanitize_release_group @@ -181,8 +181,6 @@ def _search_show_id(self, series, year=None): logger.info('Searching show ids with %r', params) r = self.session.get(self.server_url + 'search.php', params=params, timeout=10) r.raise_for_status() - if r.status_code == 304: - raise TooManyRequests() soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) # get the suggestion @@ -249,8 +247,13 @@ def query(self, series, season, year=None, country=None): logger.info('Getting the page of show id %d, season %d', show_id, season) r = self.session.get(self.server_url + 'show/%d' % show_id, params={'season': season}, timeout=10) r.raise_for_status() - if r.status_code == 304: - raise TooManyRequests() + + if not r.content: + # Provider returns a status of 304 Not Modified with an empty content + # raise_for_status won't raise exception for that status code + logger.debug('No data returned from provider') + return [] + soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) # loop over subtitle rows @@ -301,6 +304,12 @@ def download_subtitle(self, subtitle): timeout=10) r.raise_for_status() + if not r.content: + # Provider returns a status of 304 Not Modified with an empty content + # raise_for_status won't raise exception for that status code + logger.debug('Unable to download subtitle. No data returned from provider') + return + # detect download limit exceeded if r.headers['Content-Type'] == 'text/html': raise DownloadLimitExceeded diff --git a/subliminal/providers/legendastv.py b/subliminal/providers/legendastv.py index bc5e5c4e3..d35a88b2b 100644 --- a/subliminal/providers/legendastv.py +++ b/subliminal/providers/legendastv.py @@ -18,7 +18,7 @@ from . import ParserBeautifulSoup, Provider from .. import __short_version__ from ..cache import SHOW_EXPIRATION_TIME, region -from ..exceptions import AuthenticationError, ConfigurationError, ProviderError +from ..exceptions import AuthenticationError, ConfigurationError, ProviderError, ServiceUnavailable from ..subtitle import SUBTITLE_EXTENSIONS, Subtitle, fix_line_ending, guess_matches, sanitize from ..video import Episode, Movie @@ -184,7 +184,7 @@ def initialize(self): logger.info('Logging in') data = {'_method': 'POST', 'data[User][username]': self.username, 'data[User][password]': self.password} r = self.session.post(self.server_url + 'login', data, allow_redirects=False, timeout=10) - r.raise_for_status() + raise_for_status(r) soup = ParserBeautifulSoup(r.content, ['html.parser']) if soup.find('div', {'class': 'alert-error'}, string=re.compile(u'Usuário ou senha inválidos')): @@ -198,7 +198,7 @@ def terminate(self): if self.logged_in: logger.info('Logging out') r = self.session.get(self.server_url + 'users/logout', allow_redirects=False, timeout=10) - r.raise_for_status() + raise_for_status(r) logger.debug('Logged out') self.logged_in = False @@ -263,7 +263,7 @@ def search_titles(self, title, season, title_year): logger.info('Searching movie title %r', sanitized_title) r = self.session.get(self.server_url + 'legenda/sugestao/{}'.format(sanitized_title), timeout=10) - r.raise_for_status() + raise_for_status(r) results = json.loads(r.text) # loop over results @@ -332,7 +332,7 @@ def get_archives(self, title_id, language_code): url = self.server_url + 'legenda/busca/-/{language}/-/{page}/{title}'.format( language=language_code, page=page, title=title_id) r = self.session.get(url) - r.raise_for_status() + raise_for_status(r) # parse the results soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) @@ -386,7 +386,7 @@ def download_archive(self, archive): """ logger.info('Downloading archive %s', archive.id) r = self.session.get(self.server_url + 'downloadarquivo/{}'.format(archive.id)) - r.raise_for_status() + raise_for_status(r) # open the archive archive_stream = io.BytesIO(r.content) @@ -491,3 +491,11 @@ def download_subtitle(self, subtitle): # extract subtitle's content subtitle.content = fix_line_ending(subtitle.archive.content.read(subtitle.name)) + + +def raise_for_status(r): + # When site is under maintaince and http status code 200. + if 'Em breve estaremos de volta' in r.text: + raise ServiceUnavailable + else: + r.raise_for_status() diff --git a/subliminal/providers/napiprojekt.py b/subliminal/providers/napiprojekt.py index e1bc0b677..0a1a3aa37 100644 --- a/subliminal/providers/napiprojekt.py +++ b/subliminal/providers/napiprojekt.py @@ -86,16 +86,16 @@ def query(self, language, hash): 'f': hash, 't': get_subhash(hash)} logger.info('Searching subtitle %r', params) - response = self.session.get(self.server_url, params=params, timeout=10) - response.raise_for_status() + r = self.session.get(self.server_url, params=params, timeout=10) + r.raise_for_status() # handle subtitles not found and errors - if response.content[:4] == b'NPc0': + if r.content[:4] == b'NPc0': logger.debug('No subtitles found') return None subtitle = self.subtitle_class(language, hash) - subtitle.content = response.content + subtitle.content = r.content logger.debug('Found subtitle %r', subtitle) return subtitle diff --git a/subliminal/providers/opensubtitles.py b/subliminal/providers/opensubtitles.py index a51c244d6..2cc04d7d0 100644 --- a/subliminal/providers/opensubtitles.py +++ b/subliminal/providers/opensubtitles.py @@ -11,7 +11,8 @@ from . import Provider, TimeoutSafeTransport from .. import __short_version__ -from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError +from ..exceptions import (AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError, + ServiceUnavailable) from ..subtitle import Subtitle, fix_line_ending, guess_matches from ..utils import sanitize from ..video import Episode, Movie @@ -261,11 +262,6 @@ class DisabledUserAgent(OpenSubtitlesError, AuthenticationError): pass -class ServiceUnavailable(OpenSubtitlesError): - """Exception raised when status is '503 Service Unavailable'.""" - pass - - def checked(response): """Check a response status before returning it. diff --git a/subliminal/providers/podnapisi.py b/subliminal/providers/podnapisi.py index 357d900ce..d7d654a73 100644 --- a/subliminal/providers/podnapisi.py +++ b/subliminal/providers/podnapisi.py @@ -114,7 +114,9 @@ def query(self, language, keyword, season=None, episode=None, year=None): pids = set() while True: # query the server - xml = etree.fromstring(self.session.get(self.server_url + 'search/old', params=params, timeout=10).content) + r = self.session.get(self.server_url + 'search/old', params=params, timeout=10) + r.raise_for_status() + xml = etree.fromstring(r.content) # exit if no results if not int(xml.find('pagination/results').text): diff --git a/tests/cassettes/legendastv/test_under_maintenance.yaml b/tests/cassettes/legendastv/test_under_maintenance.yaml new file mode 100644 index 000000000..a0e2a251b --- /dev/null +++ b/tests/cassettes/legendastv/test_under_maintenance.yaml @@ -0,0 +1,41 @@ +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [Subliminal/2.1] + method: GET + uri: http://legendas.tv/legenda/sugestao/man%20of%20steel + response: + body: + string: !!binary | + H4sIAAAAAAAAA3VU207cMBB971cMqSrvqptkUUHQXEAg8YZohWhfEEImmU0Mjh3syS6r0o+p+tAP + 4cfqrLPLRW2kyJ45Y885nrGzrVIXtGwRamrkwbusH0ByVeVBS8fnwcE7gKxGXvYTN22QOBQ1NxYp + DzqahfvBAJEgiQenWKEquYWL71nsXR62hREtQZ8sDwgfKL7lc+69wxYAc27guuL3kPvh8REur9IB + 7D1R29l6dMmuXf6jotCdIjYB9u0o3N+fft7dCbfZ1fifC8jw4u4rr3AucPEiaDTrVEFCq9EYfgw+ + T6TijoY7n65BRVFhkBOeSOytEfPE2Th1YdHqBHNgb2SxFcjtUhUOJdNhukngAGt694jVRK1NGOQv + skld8J5U1BpNutASDmEIjGNrJYPE285cLBZsDB+BRZXWlcSQKy6XJAobFbqJXaZby9JX0uxLZRXS + IMseLy94dcYbfBZ4Ob1KwUYtNy7gTJcYCWXR0DHOtMFRxSdgN4f5czwa5lnsNxiqL4W6g9rgLN+o + mGmXcKDMW+HJFtYezngj5DI/1zdOerIznU7cL4hLUUz2nLW3thgYlDmztJRoa0Rivr18HdxWbNW+ + 8bp/sxtdLgdCpZjDamEeuNPVJnn/afWl0PNKmM/uestyZUMnWMw8FHp6/4moUVQ19aTbhxQabiqh + EtjedSbwjjSsJz34EC5ESXWy66O9sT2dfkg3FyJr1yx7TaFTXamkcJVA44Iy0VTg2igPhlYQkRzu + X0TzWOpKR62qAuDS3dbTZ8gtjdvnHAdf5NOvrdeuE0u80RawcUxVR6ie/jz91hGcNHBjcI6ALsJg + H1MizLUkHkEyfrPLfSfc3dik7l+FAc9iVwNfIF8XV6jVI/QX3eL2o5UEAAA= + headers: + accept-ranges: [bytes] + age: ['0'] + connection: [keep-alive] + content-encoding: [gzip] + content-length: ['671'] + content-type: [text/html] + date: ['Thu, 22 Jun 2017 21:09:05 GMT'] + vary: [Accept-Encoding] + via: [1.1 varnish] + x-backend: [default_director] + x-cache: [MISS] + x-cacheable: ['NO:Not Cacheable'] + x-varnish: ['1338282906'] + status: {code: 200, message: OK} +version: 1 diff --git a/tests/test_legendastv.py b/tests/test_legendastv.py index 47429eaeb..e2cbe51a2 100644 --- a/tests/test_legendastv.py +++ b/tests/test_legendastv.py @@ -4,7 +4,7 @@ from babelfish import Language, language_converters import pytest from vcr import VCR -from subliminal.exceptions import ConfigurationError, AuthenticationError +from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable from subliminal.providers.legendastv import LegendasTVSubtitle, LegendasTVProvider, LegendasTVArchive USERNAME = 'python-subliminal' @@ -318,3 +318,18 @@ def test_download_subtitle(movies): provider.download_subtitle(subtitles[0]) assert subtitles[0].content is not None assert subtitles[0].is_valid() is True + + +@pytest.mark.integration +@vcr.use_cassette +def test_under_maintenance(movies): + """Tests when is under maintenance and http status code 200.""" + video = movies['man_of_steel'] + languages = {Language('eng')} + with LegendasTVProvider() as provider: + try: + provider.list_subtitles(video, languages) + except ServiceUnavailable: + pass + else: + pytest.fail()