Skip to content

Commit

Permalink
Split parsing (#788)
Browse files Browse the repository at this point in the history 
* Split parsing

* Clean up getURL

* Fix #801: 'dict' object has no attribute 'content'

* Fix #803: 'NoneType' object is not iterable

* Fix #728: 'int' object has no attribute 'get'

* Fix #805: substring not found

* Adds back SocketTimeout catch. Its not unknown error and should not be reported as ERROR

and change other logs formats

* fixup

* fixup2

* Allow user to submit error when response is not what we are expecting

TD is a special provider that we are still testing their blocks to scripts

* fixup3

* Add rewritten extratorrent to html providers

* Add torrentday fix

* Fixup extratorrent

* Fixup tests

* one more try

* one more

* one more

* Fix newpct

* Removed SocketTimeout (handled by requests.exceptions.Timeout)

* Fix custom url with ending slash (extratorrent)

* Remove unused import

* Remove unused import

* Remove unused import

* Fix: String statement has no effect

* Fix rarbg, reorder imports

* Small ExtraTorrent improvement

* Invert log to display message first. Not all code are errors

[Rarbg] :: [42fd472] Error 20: No results found

to

[Rarbg] :: [42fd472] No results found. Code: 20

* fixup

* Fix manual search

* Don't add point to rarbg message, as most errors end with !

* Bring sense to getProviderClass, reorder imports

* Updating flake8 whitelist with new paths
  • Loading branch information
@labrys @fernandog
labrys authored and fernandog committed Aug 4, 2016
1 parent 5c22bc0 commit ccc5f4e
Show file tree
Hide file tree
Showing 88 changed files with 6,024 additions and 5,032 deletions.
124 changes: 66 additions & 58 deletions pytest.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,61 +87,68 @@ flake8-ignore =
sickbeard/postProcessor.py D100 D200 D202 D205 D400 D401 E501 F401 I100 I101
sickbeard/processTV.py D100 D101 D102 D103 D202 D205 D400 D401 E127 E128 E265 E302 E501 F401 I100 I101
sickbeard/properFinder.py D100 D101 D102 D200 D202 D400 E501 F401 I100 I101
sickbeard/providers/abnormal.py D100 D102 D204 D400 D403
sickbeard/providers/alpharatio.py D100 D102 D204 D400 D403
sickbeard/providers/animebytes.py D100 D102 D204 D205 D400 D401 D403 E501 F841 I100
sickbeard/providers/anizb.py D100 D102 D400 I100
sickbeard/providers/binsearch.py D100 D101 D102 D202 D204 D400 D401
sickbeard/providers/bitcannon.py D100 D102 D204 D400 D403 I100
sickbeard/providers/bithdtv.py D100 D102 D204 D400 E501
sickbeard/providers/bitsnoop.py D100 D102 D204 D400 D403 E501
sickbeard/providers/bluetigers.py D100 D102 D204 D400
sickbeard/providers/btdigg.py D100 D102 D204 D400
sickbeard/providers/btn.py D100 D102 D204 D400 I100
sickbeard/providers/cpasbien.py D100 D102 D204 D400 E501
sickbeard/providers/danishbits.py D100 D101 D102 D400 D401 D403
sickbeard/providers/elitetorrent.py D100 D102 D204 D400 D403 E501
sickbeard/providers/extratorrent.py D100 D102 D204 D400 D403
sickbeard/providers/freshontv.py D100 D102 D204 D400 E501 I100
sickbeard/providers/gftracker.py D100 D101 D102 D400
sickbeard/providers/hd4free.py D100 D102 D204 D400 F401
sickbeard/providers/hdbits.py D100 D101 D102 D204 D400 E501
sickbeard/providers/hdspace.py D100 D102 D204 D400 D403 E501
sickbeard/providers/hdtorrents.py D100 D102 D204 D400 D401 D403 E501
sickbeard/providers/hounddawgs.py D100 D102 D204 D400 D401 D403
sickbeard/providers/ilovetorrents.py D100 D102 D204 D400 D401 D403
sickbeard/providers/__init__.py D104 E501 I101
sickbeard/providers/iptorrents.py D100 D102 D204 D400 E501 I100
sickbeard/providers/kat.py D100 D102 D204 D400 E501 I100
sickbeard/providers/limetorrents.py D100 D102 D204 D400 D403 I100
sickbeard/providers/morethantv.py D100 D102 D204 D400 D403 I100
sickbeard/providers/newpct.py D100 D102 D200 D202 D204 D205 D400 E501
sickbeard/providers/newznab.py D100 D102 D200 D204 D205 D400 D401 E501 I100 W391
sickbeard/providers/norbits.py D100 D202 D210 D400 I100
sickbeard/providers/nyaatorrents.py D100 D101 D102 D400 D401 D403 E501
sickbeard/providers/omgwtfnzbs.py D100 D101 D102
sickbeard/providers/pretome.py D100 D102 D204 D400
sickbeard/providers/rarbg.py D100 D101 D102 D400 E501 I100
sickbeard/providers/rsstorrent.py D100 D101 D102 E501 I100
sickbeard/providers/scc.py D100 D101 D102 E501 F401 I101
sickbeard/providers/scenetime.py D100 D102 D204 D400 D403 E501
sickbeard/providers/shazbat.py D100 D101 D102 D204 D400
sickbeard/providers/speedcd.py D100 D102 D204 D400 E501
sickbeard/providers/t411.py D100 D102 D204 D400 E501 I100
sickbeard/providers/thepiratebay.py D100 D102 D204 D400 E501 I100
sickbeard/providers/tntvillage.py D100 D102 D204 D400 D403 I100
sickbeard/providers/tokyotoshokan.py D100 D102 D204 D400 D403 E501
sickbeard/providers/torrentbytes.py D100 D102 D204 D400 E501
sickbeard/providers/torrentday.py D100 D102 D204 D400 E501 F401 W291
sickbeard/providers/torrentleech.py D100 D102 D204 D400 D403 E501
sickbeard/providers/torrentproject.py D100 D101 D102 E501 I100
sickbeard/providers/torrentshack.py D100 D102 D204 D400 D403
sickbeard/providers/torrentz.py D100 D102 D204 D400 E501 F401
sickbeard/providers/transmitthenet.py D100 D102 D204 D400 D403 I100
sickbeard/providers/tvchaosuk.py D100 D102 D204 D400 D403 E501
sickbeard/providers/womble.py D100 D101 D102 D204 D400
sickbeard/providers/xthor.py D100 D102 D204 D400
sickbeard/providers/zooqle.py D100 D102 D204 D400
sickbeard/providers/__init__.py D104 F401
sickbeard/providers/nzb/anizb.py D100 D102 D400 I100
sickbeard/providers/nzb/binsearch.py D100 D101 D102 D202 D204 D400 D401
sickbeard/providers/nzb/__init__.py D104
sickbeard/providers/nzb/newznab.py D100 D102 D200 D204 D205 D400 D401 E501 I100 W391
sickbeard/providers/nzb/omgwtfnzbs.py D100 D101 D102
sickbeard/providers/nzb/womble.py D100 D101 D102 D204 D400
sickbeard/providers/torrent/html/abnormal.py D100 D102 D204 D400
sickbeard/providers/torrent/html/alpharatio.py D100 D102 D204 D400
sickbeard/providers/torrent/html/animebytes.py D100 D102 D202 D204 D400 E501 F841 I100
sickbeard/providers/torrent/html/bithdtv.py D100 D102 D204 D400 E501
sickbeard/providers/torrent/html/bluetigers.py D100 D102 D202 D204 D400
sickbeard/providers/torrent/html/cpasbien.py D100 D102 D204 D400 E501
sickbeard/providers/torrent/html/danishbits.py D100 D102 D204 D400
sickbeard/providers/torrent/html/elitetorrent.py D100 D102 D202 D204 D400 E501
sickbeard/providers/torrent/html/extratorrent.py D100 D102
sickbeard/providers/torrent/html/freshontv.py D100 D102 D202 D204 D400 E501
sickbeard/providers/torrent/html/gftracker.py D100 D101 D102 D400
sickbeard/providers/torrent/html/hdspace.py D100 D102 D202 D204 D400
sickbeard/providers/torrent/html/hdtorrents.py D100 D102 D202 D204 D400 E501
sickbeard/providers/torrent/html/hounddawgs.py D100 D102 D202 D204 D400
sickbeard/providers/torrent/html/ilovetorrents.py D100 D102 D202 D204 D400
sickbeard/providers/torrent/html/__init__.py D104
sickbeard/providers/torrent/html/iptorrents.py D100 D102 D202 D204 D400 I100
sickbeard/providers/torrent/html/limetorrents.py D100 D102 D204 D400 I100
sickbeard/providers/torrent/html/morethantv.py D100 D102 D204 D400 I100
sickbeard/providers/torrent/html/newpct.py D100 D102 D200 D202 D204 D205 D400 E501
sickbeard/providers/torrent/html/pretome.py D100 D102 D202 D204 D400
sickbeard/providers/torrent/html/scc.py D100 D101 D102 D202 D400 E501 I101
sickbeard/providers/torrent/html/scenetime.py D100 D102 D202 D204 D400 E501
sickbeard/providers/torrent/html/speedcd.py D100 D102 D204 D400 E501
sickbeard/providers/torrent/html/thepiratebay.py D100 D102 D204 D400 E501 I100
sickbeard/providers/torrent/html/tntvillage.py D100 D102 D202 D204 D400 I100
sickbeard/providers/torrent/html/tokyotoshokan.py D100 D102 D202 D204 D400
sickbeard/providers/torrent/html/torrentbytes.py D100 D102 D202 D204 D400 E501
sickbeard/providers/torrent/html/torrentleech.py D100 D102 D204 D400
sickbeard/providers/torrent/html/torrentshack.py D100 D102 D204 D400
sickbeard/providers/torrent/html/transmitthenet.py D100 D102 D202 D204 D400 I100
sickbeard/providers/torrent/html/tvchaosuk.py D100 D102 D204 D400 E501
sickbeard/providers/torrent/html/xthor.py D100 D102 D204 D400 F841
sickbeard/providers/torrent/html/zooqle.py D100 D102 D204 D400
sickbeard/providers/torrent/__init__.py D104 F401
sickbeard/providers/torrent/json/bitcannon.py D100 D102 D202 D204 D400 I100
sickbeard/providers/torrent/json/btdigg.py D100 D102 D202 D204 D400
sickbeard/providers/torrent/json/btn.py D100 D102 D202 D204 D400 I100
sickbeard/providers/torrent/json/hd4free.py D100 D102 D202 D204 D400 F401
sickbeard/providers/torrent/json/hdbits.py D100 D101 D102 D202 D204 D400 E501 I100
sickbeard/providers/torrent/json/__init__.py D104
sickbeard/providers/torrent/json/norbits.py D100 D202 D210 D400 E501 F821 I100
sickbeard/providers/torrent/json/rarbg.py D100 D102 E501
sickbeard/providers/torrent/json/t411.py D100 D102 D202 D204 D400 E501 I100
sickbeard/providers/torrent/json/torrentday.py D100 D102 D202 D204 D400 E501 F401
sickbeard/providers/torrent/json/torrentproject.py D100 D102 D202 D204 D400 I100
sickbeard/providers/torrent/rss/__init__.py D104
sickbeard/providers/torrent/rss/nyaatorrents.py D100 D101 D102 D202 D400 E501
sickbeard/providers/torrent/rss/rsstorrent.py D100 D101 D102 E501 I100
sickbeard/providers/torrent/rss/shazbat.py D100 D101 D102 D204 D400
sickbeard/providers/torrent/xml/bitsnoop.py D100 D102 D202 D204 D400 E501
sickbeard/providers/torrent/xml/extratorrent.py D100 D102 D202 D204 D400 E501
sickbeard/providers/torrent/xml/__init__.py D104
sickbeard/providers/torrent/xml/kat.py D100 D102 D202 D204 D400 E501 I100
sickbeard/providers/torrent/xml/torrentz.py D100 D102 D202 D204 D400
SickBeard.py D102 D200 D400 D401 E402 E501 I100 I101
sickbeard/rssfeeds.py D100 D103
sickbeard/sab.py D100 D202 D400 D401 I100
Expand All @@ -152,9 +159,9 @@ flake8-ignore =
sickbeard/searchBacklog.py D100 D101 D102 E501 I100 I101
sickbeard/search.py D100 D202 D401 E501 F401 F821 I100 I101
sickbeard/search_queue.py D100 D101 D102 D103 D200 D204 D205 D210 D400 D401 E231 E501 I100 I101
sickbeard/server/api/__init__.py D104
sickbeard/server/api/v1/core.py D100 D101 D102 D200 D201 D202 D204 D205 D208 D210 D400 D401 D403 E501 I100 I101
sickbeard/server/api/v1/__init__.py D104
sickbeard/server/api/__init__.py D104
sickbeard/server/core.py D100 D101 D102 E501 I100 I101
sickbeard/server/__init__.py D104
sickbeard/server/web/config/anime.py D102 D200 D202 D204 D400 I100
Expand All @@ -164,7 +171,7 @@ flake8-ignore =
sickbeard/server/web/config/__init__.py D104 F401 I100
sickbeard/server/web/config/notifications.py D102 D200 D202 D204 D400 E501 I100
sickbeard/server/web/config/post_processing.py D102 D200 D202 D204 D205 D400 I100
sickbeard/server/web/config/providers.py D102 D200 D202 D205 D400 D401 E501 I100
sickbeard/server/web/config/providers.py D102 D200 D202 D205 D400 D401 E501 F821 I100
sickbeard/server/web/config/search.py D102 D200 D202 D204 D400 E501 I100
sickbeard/server/web/config/subtitles.py D102 D200 D204 D400 E501 I100
sickbeard/server/web/core/authentication.py D102 D200 D202 D204 D205 D400 I100
Expand Down Expand Up @@ -204,6 +211,7 @@ flake8-ignore =
sickrage/media/ShowFanArt.py D100 D102 D200 D400
sickrage/media/ShowNetworkLogo.py D100 D102 D200 D400
sickrage/media/ShowPoster.py D100 D102 D200 D400
sickrage/providers/exceptions.py D200 D400
sickrage/providers/GenericProvider.py D100 D101 D102 D202 D205 D400 D401 E501 F841 I100 I101
sickrage/providers/__init__.py D104
sickrage/providers/nzb/__init__.py D104
Expand All @@ -225,7 +233,6 @@ flake8-ignore =
sickrage/system/Shutdown.py D100 D101 D102
sickrage/tagger/episode.py D102 D200 D204 D205 D400 I100 W291 W293
sickrage/tagger/__init__.py D104
tests/*.py D101 D102 D103
tests/api_v1_tests.py D200 D204 D205 D400
tests/common_tests.py D200 D204 D400 E402
tests/config_tests.py D200 D204 D400 E402 E501 I100
Expand All @@ -240,6 +247,7 @@ flake8-ignore =
tests/notifier_tests.py D200 D204 D400 E402 E501 I100
tests/numdict_tests.py D200 D204 D400 D403 E402 E501 F841 I100
tests/pp_tests.py D100 D200 D204 D400 E402 I100
tests/*.py D101 D102 D103
tests/scene_helpers_tests.py D200 D204 D205 D400 E402 E501 I100 I101
tests/search_tests.py D100 D200 D202 D204 D400 E402 I100
tests/sickrage_tests/helper/common_tests.py D200 D400 E402 E501 I100 I101
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
description="Automatic Video Library Manager for TV Shows",
long_description=long_description,
packages=find_packages(),
install_requires=['six', 'requests', 'tornado', 'profilehooks', 'mako', 'subliminal', 'github', 'contextlib2', ],
install_requires=['six', 'requests', 'tornado', 'profilehooks', 'mako', 'subliminal', 'pygithub', 'contextlib2', ],
test_suite="tests",
tests_require=[
'coveralls',
Expand Down
22 changes: 7 additions & 15 deletions sickbeard/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,23 +42,15 @@
from sickbeard import (
searchBacklog, showUpdater, versionChecker, properFinder, auto_postprocessor, subtitles, traktChecker,
)
from sickbeard import db
from sickbeard import helpers
from sickbeard import scheduler
from sickbeard import search_queue
from sickbeard import show_queue
from sickbeard import logger
from sickbeard import naming
from sickbeard import dailysearcher
from sickbeard import db, helpers, scheduler, search_queue, show_queue, logger, naming, dailysearcher
from sickbeard.indexers import indexer_api
from sickbeard.indexers.indexer_exceptions import indexer_shownotfound, indexer_showincomplete, indexer_exception, \
indexer_error, indexer_episodenotfound, indexer_attributenotfound, indexer_seasonnotfound, indexer_userabort
from sickbeard.common import SD
from sickbeard.common import SKIPPED
from sickbeard.common import WANTED
from sickbeard.providers.rsstorrent import TorrentRssProvider
from sickbeard.indexers.indexer_exceptions import (
indexer_shownotfound, indexer_showincomplete, indexer_exception, indexer_error, indexer_episodenotfound,
indexer_attributenotfound, indexer_seasonnotfound, indexer_userabort,
)
from sickbeard.common import SD, SKIPPED, WANTED
from sickbeard.providers import NewznabProvider, TorrentRssProvider
from sickbeard.databases import main_db, cache_db, failed_db
from sickbeard.providers.newznab import NewznabProvider

from sickrage.helper.encoding import ek
from sickrage.helper.exceptions import ex
Expand Down
82 changes: 31 additions & 51 deletions sickbeard/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@
import re
import shutil
import socket
from socket import timeout as SocketTimeout
import ssl
import stat
import tempfile
import time
import traceback
import uuid
import warnings
import xml.etree.ElementTree as ET
import zipfile

Expand Down Expand Up @@ -1413,58 +1413,42 @@ def getURL(url, post_data=None, params=None, headers=None, # pylint:disable=too
"""
Returns data retrieved from the url provider.
"""
try:
response_type = kwargs.pop(u'returns', 'text')
stream = kwargs.pop(u'stream', False)
response_type = kwargs.pop(u'returns', u'response')
stream = kwargs.pop(u'stream', False)
hooks, cookies, verify, proxies = request_defaults(kwargs)
method = u'POST' if post_data else u'GET'

hooks, cookies, verify, proxies = request_defaults(kwargs)
resp = session.request(method, url, data=post_data, params=params, timeout=timeout, allow_redirects=True,
hooks=hooks, stream=stream, headers=headers, cookies=cookies, proxies=proxies,
verify=verify)

if params and isinstance(params, (list, dict)):
for param in params:
if isinstance(params[param], text_type):
params[param] = params[param].encode('utf-8')

if post_data and isinstance(post_data, (list, dict)):
for param in post_data:
if isinstance(post_data[param], text_type):
post_data[param] = post_data[param].encode('utf-8')

resp = session.request(
'POST' if post_data else 'GET', url, data=post_data, params=params,
timeout=timeout, allow_redirects=True, hooks=hooks, stream=stream,
headers=headers, cookies=cookies, proxies=proxies, verify=verify
)

if not resp.ok:
logger.log(u"Requested getURL %s returned status code is %s: %s"
% (url, resp.status_code, http_code_description(resp.status_code)), logger.DEBUG)
return None

except (SocketTimeout, TypeError) as e:
logger.log(u"Connection timed out (sockets) accessing getURL %s Error: %r" % (url, ex(e)), logger.DEBUG)
return None
except (requests.exceptions.HTTPError, requests.exceptions.TooManyRedirects) as e:
logger.log(u"HTTP error in getURL %s Error: %r" % (url, ex(e)), logger.DEBUG)
return None
except requests.exceptions.ConnectionError as e:
logger.log(u"Connection error to getURL %s Error: %r" % (url, ex(e)), logger.DEBUG)
return None
except requests.exceptions.Timeout as e:
logger.log(u"Connection timed out accessing getURL %s Error: %r" % (url, ex(e)), logger.DEBUG)
return None
except requests.exceptions.ContentDecodingError:
logger.log(u"Content-Encoding was gzip, but content was not compressed. getURL: %s" % url, logger.DEBUG)
logger.log(traceback.format_exc(), logger.DEBUG)
return None
if not resp.ok:
logger.log(u'Requested url {url} returned status code {status}: {desc}'.format
(url=url, status=resp.status_code, desc=http_code_description(resp.status_code)), logger.DEBUG)

try:
resp.raise_for_status()
except requests.exceptions.RequestException as e:
logger.log(u'Error requesting url {resp.url}. Error: {msg}'.format(resp=resp, msg=ex(e)), logger.DEBUG)
except Exception as e:
if 'ECONNRESET' in e or (hasattr(e, 'errno') and e.errno == errno.ECONNRESET):
logger.log(u"Connection reseted by peer accessing getURL %s Error: %r" % (url, ex(e)), logger.WARNING)
if u'ECONNRESET' in e or (hasattr(e, u'errno') and e.errno == errno.ECONNRESET):
logger.log(u'Connection reset by peer accessing url {resp.url}. Error: {msg}'.format(resp=resp, msg=ex(e)), logger.WARNING)
else:
logger.log(u"Unknown exception in getURL %s Error: %r" % (url, ex(e)), logger.ERROR)
logger.log(u'Unknown exception in url {resp.url}. Error: {msg}'.format(resp=resp, msg=ex(e)), logger.ERROR)
logger.log(traceback.format_exc(), logger.DEBUG)
return None

return resp if response_type == u'response' or response_type is None else resp.json() if response_type == u'json' else getattr(resp, response_type, resp)
if not response_type or response_type == u'response':
return resp
else:
warnings.warn(u'Returning {0} instead of {1} will be deprecated in the near future!'.format
(response_type, 'response'), PendingDeprecationWarning)
if response_type == u'json':
try:
return resp.json()
except ValueError:
return {}
else:
return getattr(resp, response_type, resp)


def download_file(url, filename, session=None, headers=None, **kwargs): # pylint:disable=too-many-return-statements
Expand Down Expand Up @@ -1501,10 +1485,6 @@ def download_file(url, filename, session=None, headers=None, **kwargs): # pylin
except Exception:
logger.log(u"Problem setting permissions or writing file to: %s" % filename, logger.WARNING)

except (SocketTimeout, TypeError) as e:
remove_file_failed(filename)
logger.log(u"Connection timed out (sockets) while loading download URL %s Error: %r" % (url, ex(e)), logger.WARNING)
return False
except (requests.exceptions.HTTPError, requests.exceptions.TooManyRedirects) as e:
remove_file_failed(filename)
logger.log(u"HTTP error %r while loading download URL %s " % (ex(e), url), logger.WARNING)
Expand Down
Loading

0 comments on commit ccc5f4e

Please sign in to comment.