diff --git a/pytest.ini b/pytest.ini index 82d4f734c4..3d492906ab 100644 --- a/pytest.ini +++ b/pytest.ini @@ -87,61 +87,68 @@ flake8-ignore = sickbeard/postProcessor.py D100 D200 D202 D205 D400 D401 E501 F401 I100 I101 sickbeard/processTV.py D100 D101 D102 D103 D202 D205 D400 D401 E127 E128 E265 E302 E501 F401 I100 I101 sickbeard/properFinder.py D100 D101 D102 D200 D202 D400 E501 F401 I100 I101 - sickbeard/providers/abnormal.py D100 D102 D204 D400 D403 - sickbeard/providers/alpharatio.py D100 D102 D204 D400 D403 - sickbeard/providers/animebytes.py D100 D102 D204 D205 D400 D401 D403 E501 F841 I100 - sickbeard/providers/anizb.py D100 D102 D400 I100 - sickbeard/providers/binsearch.py D100 D101 D102 D202 D204 D400 D401 - sickbeard/providers/bitcannon.py D100 D102 D204 D400 D403 I100 - sickbeard/providers/bithdtv.py D100 D102 D204 D400 E501 - sickbeard/providers/bitsnoop.py D100 D102 D204 D400 D403 E501 - sickbeard/providers/bluetigers.py D100 D102 D204 D400 - sickbeard/providers/btdigg.py D100 D102 D204 D400 - sickbeard/providers/btn.py D100 D102 D204 D400 I100 - sickbeard/providers/cpasbien.py D100 D102 D204 D400 E501 - sickbeard/providers/danishbits.py D100 D101 D102 D400 D401 D403 - sickbeard/providers/elitetorrent.py D100 D102 D204 D400 D403 E501 - sickbeard/providers/extratorrent.py D100 D102 D204 D400 D403 - sickbeard/providers/freshontv.py D100 D102 D204 D400 E501 I100 - sickbeard/providers/gftracker.py D100 D101 D102 D400 - sickbeard/providers/hd4free.py D100 D102 D204 D400 F401 - sickbeard/providers/hdbits.py D100 D101 D102 D204 D400 E501 - sickbeard/providers/hdspace.py D100 D102 D204 D400 D403 E501 - sickbeard/providers/hdtorrents.py D100 D102 D204 D400 D401 D403 E501 - sickbeard/providers/hounddawgs.py D100 D102 D204 D400 D401 D403 - sickbeard/providers/ilovetorrents.py D100 D102 D204 D400 D401 D403 - sickbeard/providers/__init__.py D104 E501 I101 - sickbeard/providers/iptorrents.py D100 D102 D204 D400 E501 I100 - sickbeard/providers/kat.py D100 D102 D204 D400 E501 I100 - sickbeard/providers/limetorrents.py D100 D102 D204 D400 D403 I100 - sickbeard/providers/morethantv.py D100 D102 D204 D400 D403 I100 - sickbeard/providers/newpct.py D100 D102 D200 D202 D204 D205 D400 E501 - sickbeard/providers/newznab.py D100 D102 D200 D204 D205 D400 D401 E501 I100 W391 - sickbeard/providers/norbits.py D100 D202 D210 D400 I100 - sickbeard/providers/nyaatorrents.py D100 D101 D102 D400 D401 D403 E501 - sickbeard/providers/omgwtfnzbs.py D100 D101 D102 - sickbeard/providers/pretome.py D100 D102 D204 D400 - sickbeard/providers/rarbg.py D100 D101 D102 D400 E501 I100 - sickbeard/providers/rsstorrent.py D100 D101 D102 E501 I100 - sickbeard/providers/scc.py D100 D101 D102 E501 F401 I101 - sickbeard/providers/scenetime.py D100 D102 D204 D400 D403 E501 - sickbeard/providers/shazbat.py D100 D101 D102 D204 D400 - sickbeard/providers/speedcd.py D100 D102 D204 D400 E501 - sickbeard/providers/t411.py D100 D102 D204 D400 E501 I100 - sickbeard/providers/thepiratebay.py D100 D102 D204 D400 E501 I100 - sickbeard/providers/tntvillage.py D100 D102 D204 D400 D403 I100 - sickbeard/providers/tokyotoshokan.py D100 D102 D204 D400 D403 E501 - sickbeard/providers/torrentbytes.py D100 D102 D204 D400 E501 - sickbeard/providers/torrentday.py D100 D102 D204 D400 E501 F401 W291 - sickbeard/providers/torrentleech.py D100 D102 D204 D400 D403 E501 - sickbeard/providers/torrentproject.py D100 D101 D102 E501 I100 - sickbeard/providers/torrentshack.py D100 D102 D204 D400 D403 - sickbeard/providers/torrentz.py D100 D102 D204 D400 E501 F401 - sickbeard/providers/transmitthenet.py D100 D102 D204 D400 D403 I100 - sickbeard/providers/tvchaosuk.py D100 D102 D204 D400 D403 E501 - sickbeard/providers/womble.py D100 D101 D102 D204 D400 - sickbeard/providers/xthor.py D100 D102 D204 D400 - sickbeard/providers/zooqle.py D100 D102 D204 D400 + sickbeard/providers/__init__.py D104 F401 + sickbeard/providers/nzb/anizb.py D100 D102 D400 I100 + sickbeard/providers/nzb/binsearch.py D100 D101 D102 D202 D204 D400 D401 + sickbeard/providers/nzb/__init__.py D104 + sickbeard/providers/nzb/newznab.py D100 D102 D200 D204 D205 D400 D401 E501 I100 W391 + sickbeard/providers/nzb/omgwtfnzbs.py D100 D101 D102 + sickbeard/providers/nzb/womble.py D100 D101 D102 D204 D400 + sickbeard/providers/torrent/html/abnormal.py D100 D102 D204 D400 + sickbeard/providers/torrent/html/alpharatio.py D100 D102 D204 D400 + sickbeard/providers/torrent/html/animebytes.py D100 D102 D202 D204 D400 E501 F841 I100 + sickbeard/providers/torrent/html/bithdtv.py D100 D102 D204 D400 E501 + sickbeard/providers/torrent/html/bluetigers.py D100 D102 D202 D204 D400 + sickbeard/providers/torrent/html/cpasbien.py D100 D102 D204 D400 E501 + sickbeard/providers/torrent/html/danishbits.py D100 D102 D204 D400 + sickbeard/providers/torrent/html/elitetorrent.py D100 D102 D202 D204 D400 E501 + sickbeard/providers/torrent/html/extratorrent.py D100 D102 + sickbeard/providers/torrent/html/freshontv.py D100 D102 D202 D204 D400 E501 + sickbeard/providers/torrent/html/gftracker.py D100 D101 D102 D400 + sickbeard/providers/torrent/html/hdspace.py D100 D102 D202 D204 D400 + sickbeard/providers/torrent/html/hdtorrents.py D100 D102 D202 D204 D400 E501 + sickbeard/providers/torrent/html/hounddawgs.py D100 D102 D202 D204 D400 + sickbeard/providers/torrent/html/ilovetorrents.py D100 D102 D202 D204 D400 + sickbeard/providers/torrent/html/__init__.py D104 + sickbeard/providers/torrent/html/iptorrents.py D100 D102 D202 D204 D400 I100 + sickbeard/providers/torrent/html/limetorrents.py D100 D102 D204 D400 I100 + sickbeard/providers/torrent/html/morethantv.py D100 D102 D204 D400 I100 + sickbeard/providers/torrent/html/newpct.py D100 D102 D200 D202 D204 D205 D400 E501 + sickbeard/providers/torrent/html/pretome.py D100 D102 D202 D204 D400 + sickbeard/providers/torrent/html/scc.py D100 D101 D102 D202 D400 E501 I101 + sickbeard/providers/torrent/html/scenetime.py D100 D102 D202 D204 D400 E501 + sickbeard/providers/torrent/html/speedcd.py D100 D102 D204 D400 E501 + sickbeard/providers/torrent/html/thepiratebay.py D100 D102 D204 D400 E501 I100 + sickbeard/providers/torrent/html/tntvillage.py D100 D102 D202 D204 D400 I100 + sickbeard/providers/torrent/html/tokyotoshokan.py D100 D102 D202 D204 D400 + sickbeard/providers/torrent/html/torrentbytes.py D100 D102 D202 D204 D400 E501 + sickbeard/providers/torrent/html/torrentleech.py D100 D102 D204 D400 + sickbeard/providers/torrent/html/torrentshack.py D100 D102 D204 D400 + sickbeard/providers/torrent/html/transmitthenet.py D100 D102 D202 D204 D400 I100 + sickbeard/providers/torrent/html/tvchaosuk.py D100 D102 D204 D400 E501 + sickbeard/providers/torrent/html/xthor.py D100 D102 D204 D400 F841 + sickbeard/providers/torrent/html/zooqle.py D100 D102 D204 D400 + sickbeard/providers/torrent/__init__.py D104 F401 + sickbeard/providers/torrent/json/bitcannon.py D100 D102 D202 D204 D400 I100 + sickbeard/providers/torrent/json/btdigg.py D100 D102 D202 D204 D400 + sickbeard/providers/torrent/json/btn.py D100 D102 D202 D204 D400 I100 + sickbeard/providers/torrent/json/hd4free.py D100 D102 D202 D204 D400 F401 + sickbeard/providers/torrent/json/hdbits.py D100 D101 D102 D202 D204 D400 E501 I100 + sickbeard/providers/torrent/json/__init__.py D104 + sickbeard/providers/torrent/json/norbits.py D100 D202 D210 D400 E501 F821 I100 + sickbeard/providers/torrent/json/rarbg.py D100 D102 E501 + sickbeard/providers/torrent/json/t411.py D100 D102 D202 D204 D400 E501 I100 + sickbeard/providers/torrent/json/torrentday.py D100 D102 D202 D204 D400 E501 F401 + sickbeard/providers/torrent/json/torrentproject.py D100 D102 D202 D204 D400 I100 + sickbeard/providers/torrent/rss/__init__.py D104 + sickbeard/providers/torrent/rss/nyaatorrents.py D100 D101 D102 D202 D400 E501 + sickbeard/providers/torrent/rss/rsstorrent.py D100 D101 D102 E501 I100 + sickbeard/providers/torrent/rss/shazbat.py D100 D101 D102 D204 D400 + sickbeard/providers/torrent/xml/bitsnoop.py D100 D102 D202 D204 D400 E501 + sickbeard/providers/torrent/xml/extratorrent.py D100 D102 D202 D204 D400 E501 + sickbeard/providers/torrent/xml/__init__.py D104 + sickbeard/providers/torrent/xml/kat.py D100 D102 D202 D204 D400 E501 I100 + sickbeard/providers/torrent/xml/torrentz.py D100 D102 D202 D204 D400 SickBeard.py D102 D200 D400 D401 E402 E501 I100 I101 sickbeard/rssfeeds.py D100 D103 sickbeard/sab.py D100 D202 D400 D401 I100 @@ -152,9 +159,9 @@ flake8-ignore = sickbeard/searchBacklog.py D100 D101 D102 E501 I100 I101 sickbeard/search.py D100 D202 D401 E501 F401 F821 I100 I101 sickbeard/search_queue.py D100 D101 D102 D103 D200 D204 D205 D210 D400 D401 E231 E501 I100 I101 + sickbeard/server/api/__init__.py D104 sickbeard/server/api/v1/core.py D100 D101 D102 D200 D201 D202 D204 D205 D208 D210 D400 D401 D403 E501 I100 I101 sickbeard/server/api/v1/__init__.py D104 - sickbeard/server/api/__init__.py D104 sickbeard/server/core.py D100 D101 D102 E501 I100 I101 sickbeard/server/__init__.py D104 sickbeard/server/web/config/anime.py D102 D200 D202 D204 D400 I100 @@ -164,7 +171,7 @@ flake8-ignore = sickbeard/server/web/config/__init__.py D104 F401 I100 sickbeard/server/web/config/notifications.py D102 D200 D202 D204 D400 E501 I100 sickbeard/server/web/config/post_processing.py D102 D200 D202 D204 D205 D400 I100 - sickbeard/server/web/config/providers.py D102 D200 D202 D205 D400 D401 E501 I100 + sickbeard/server/web/config/providers.py D102 D200 D202 D205 D400 D401 E501 F821 I100 sickbeard/server/web/config/search.py D102 D200 D202 D204 D400 E501 I100 sickbeard/server/web/config/subtitles.py D102 D200 D204 D400 E501 I100 sickbeard/server/web/core/authentication.py D102 D200 D202 D204 D205 D400 I100 @@ -204,6 +211,7 @@ flake8-ignore = sickrage/media/ShowFanArt.py D100 D102 D200 D400 sickrage/media/ShowNetworkLogo.py D100 D102 D200 D400 sickrage/media/ShowPoster.py D100 D102 D200 D400 + sickrage/providers/exceptions.py D200 D400 sickrage/providers/GenericProvider.py D100 D101 D102 D202 D205 D400 D401 E501 F841 I100 I101 sickrage/providers/__init__.py D104 sickrage/providers/nzb/__init__.py D104 @@ -225,7 +233,6 @@ flake8-ignore = sickrage/system/Shutdown.py D100 D101 D102 sickrage/tagger/episode.py D102 D200 D204 D205 D400 I100 W291 W293 sickrage/tagger/__init__.py D104 - tests/*.py D101 D102 D103 tests/api_v1_tests.py D200 D204 D205 D400 tests/common_tests.py D200 D204 D400 E402 tests/config_tests.py D200 D204 D400 E402 E501 I100 @@ -240,6 +247,7 @@ flake8-ignore = tests/notifier_tests.py D200 D204 D400 E402 E501 I100 tests/numdict_tests.py D200 D204 D400 D403 E402 E501 F841 I100 tests/pp_tests.py D100 D200 D204 D400 E402 I100 + tests/*.py D101 D102 D103 tests/scene_helpers_tests.py D200 D204 D205 D400 E402 E501 I100 I101 tests/search_tests.py D100 D200 D202 D204 D400 E402 I100 tests/sickrage_tests/helper/common_tests.py D200 D400 E402 E501 I100 I101 diff --git a/setup.py b/setup.py index 922ebb1be6..6c8d5ac656 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ description="Automatic Video Library Manager for TV Shows", long_description=long_description, packages=find_packages(), - install_requires=['six', 'requests', 'tornado', 'profilehooks', 'mako', 'subliminal', 'github', 'contextlib2', ], + install_requires=['six', 'requests', 'tornado', 'profilehooks', 'mako', 'subliminal', 'pygithub', 'contextlib2', ], test_suite="tests", tests_require=[ 'coveralls', diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index ee06373ec4..e034d2c26b 100644 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -42,23 +42,15 @@ from sickbeard import ( searchBacklog, showUpdater, versionChecker, properFinder, auto_postprocessor, subtitles, traktChecker, ) -from sickbeard import db -from sickbeard import helpers -from sickbeard import scheduler -from sickbeard import search_queue -from sickbeard import show_queue -from sickbeard import logger -from sickbeard import naming -from sickbeard import dailysearcher +from sickbeard import db, helpers, scheduler, search_queue, show_queue, logger, naming, dailysearcher from sickbeard.indexers import indexer_api -from sickbeard.indexers.indexer_exceptions import indexer_shownotfound, indexer_showincomplete, indexer_exception, \ - indexer_error, indexer_episodenotfound, indexer_attributenotfound, indexer_seasonnotfound, indexer_userabort -from sickbeard.common import SD -from sickbeard.common import SKIPPED -from sickbeard.common import WANTED -from sickbeard.providers.rsstorrent import TorrentRssProvider +from sickbeard.indexers.indexer_exceptions import ( + indexer_shownotfound, indexer_showincomplete, indexer_exception, indexer_error, indexer_episodenotfound, + indexer_attributenotfound, indexer_seasonnotfound, indexer_userabort, +) +from sickbeard.common import SD, SKIPPED, WANTED +from sickbeard.providers import NewznabProvider, TorrentRssProvider from sickbeard.databases import main_db, cache_db, failed_db -from sickbeard.providers.newznab import NewznabProvider from sickrage.helper.encoding import ek from sickrage.helper.exceptions import ex diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index db0e239c9a..ca09a53057 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -33,13 +33,13 @@ import re import shutil import socket -from socket import timeout as SocketTimeout import ssl import stat import tempfile import time import traceback import uuid +import warnings import xml.etree.ElementTree as ET import zipfile @@ -1413,58 +1413,42 @@ def getURL(url, post_data=None, params=None, headers=None, # pylint:disable=too """ Returns data retrieved from the url provider. """ - try: - response_type = kwargs.pop(u'returns', 'text') - stream = kwargs.pop(u'stream', False) + response_type = kwargs.pop(u'returns', u'response') + stream = kwargs.pop(u'stream', False) + hooks, cookies, verify, proxies = request_defaults(kwargs) + method = u'POST' if post_data else u'GET' - hooks, cookies, verify, proxies = request_defaults(kwargs) + resp = session.request(method, url, data=post_data, params=params, timeout=timeout, allow_redirects=True, + hooks=hooks, stream=stream, headers=headers, cookies=cookies, proxies=proxies, + verify=verify) - if params and isinstance(params, (list, dict)): - for param in params: - if isinstance(params[param], text_type): - params[param] = params[param].encode('utf-8') - - if post_data and isinstance(post_data, (list, dict)): - for param in post_data: - if isinstance(post_data[param], text_type): - post_data[param] = post_data[param].encode('utf-8') - - resp = session.request( - 'POST' if post_data else 'GET', url, data=post_data, params=params, - timeout=timeout, allow_redirects=True, hooks=hooks, stream=stream, - headers=headers, cookies=cookies, proxies=proxies, verify=verify - ) - - if not resp.ok: - logger.log(u"Requested getURL %s returned status code is %s: %s" - % (url, resp.status_code, http_code_description(resp.status_code)), logger.DEBUG) - return None - - except (SocketTimeout, TypeError) as e: - logger.log(u"Connection timed out (sockets) accessing getURL %s Error: %r" % (url, ex(e)), logger.DEBUG) - return None - except (requests.exceptions.HTTPError, requests.exceptions.TooManyRedirects) as e: - logger.log(u"HTTP error in getURL %s Error: %r" % (url, ex(e)), logger.DEBUG) - return None - except requests.exceptions.ConnectionError as e: - logger.log(u"Connection error to getURL %s Error: %r" % (url, ex(e)), logger.DEBUG) - return None - except requests.exceptions.Timeout as e: - logger.log(u"Connection timed out accessing getURL %s Error: %r" % (url, ex(e)), logger.DEBUG) - return None - except requests.exceptions.ContentDecodingError: - logger.log(u"Content-Encoding was gzip, but content was not compressed. getURL: %s" % url, logger.DEBUG) - logger.log(traceback.format_exc(), logger.DEBUG) - return None + if not resp.ok: + logger.log(u'Requested url {url} returned status code {status}: {desc}'.format + (url=url, status=resp.status_code, desc=http_code_description(resp.status_code)), logger.DEBUG) + + try: + resp.raise_for_status() + except requests.exceptions.RequestException as e: + logger.log(u'Error requesting url {resp.url}. Error: {msg}'.format(resp=resp, msg=ex(e)), logger.DEBUG) except Exception as e: - if 'ECONNRESET' in e or (hasattr(e, 'errno') and e.errno == errno.ECONNRESET): - logger.log(u"Connection reseted by peer accessing getURL %s Error: %r" % (url, ex(e)), logger.WARNING) + if u'ECONNRESET' in e or (hasattr(e, u'errno') and e.errno == errno.ECONNRESET): + logger.log(u'Connection reset by peer accessing url {resp.url}. Error: {msg}'.format(resp=resp, msg=ex(e)), logger.WARNING) else: - logger.log(u"Unknown exception in getURL %s Error: %r" % (url, ex(e)), logger.ERROR) + logger.log(u'Unknown exception in url {resp.url}. Error: {msg}'.format(resp=resp, msg=ex(e)), logger.ERROR) logger.log(traceback.format_exc(), logger.DEBUG) - return None - return resp if response_type == u'response' or response_type is None else resp.json() if response_type == u'json' else getattr(resp, response_type, resp) + if not response_type or response_type == u'response': + return resp + else: + warnings.warn(u'Returning {0} instead of {1} will be deprecated in the near future!'.format + (response_type, 'response'), PendingDeprecationWarning) + if response_type == u'json': + try: + return resp.json() + except ValueError: + return {} + else: + return getattr(resp, response_type, resp) def download_file(url, filename, session=None, headers=None, **kwargs): # pylint:disable=too-many-return-statements @@ -1501,10 +1485,6 @@ def download_file(url, filename, session=None, headers=None, **kwargs): # pylin except Exception: logger.log(u"Problem setting permissions or writing file to: %s" % filename, logger.WARNING) - except (SocketTimeout, TypeError) as e: - remove_file_failed(filename) - logger.log(u"Connection timed out (sockets) while loading download URL %s Error: %r" % (url, ex(e)), logger.WARNING) - return False except (requests.exceptions.HTTPError, requests.exceptions.TooManyRedirects) as e: remove_file_failed(filename) logger.log(u"HTTP error %r while loading download URL %s " % (ex(e), url), logger.WARNING) diff --git a/sickbeard/providers/__init__.py b/sickbeard/providers/__init__.py index 18dc1df4a1..5b1c6bd532 100644 --- a/sickbeard/providers/__init__.py +++ b/sickbeard/providers/__init__.py @@ -1,31 +1,38 @@ # coding=utf-8 # Author: Nic Wolfe # +# This file is part of Medusa. # -# This file is part of SickRage. -# -# SickRage is free software: you can redistribute it and/or modify +# Medusa is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# SickRage is distributed in the hope that it will be useful, +# Medusa is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with SickRage. If not, see . +# along with Medusa. If not, see . from os import sys from random import shuffle import sickbeard -from sickbeard.providers import btn, womble, thepiratebay, torrentleech, kat, iptorrents, torrentz, \ - omgwtfnzbs, scc, hdtorrents, torrentday, hdbits, hounddawgs, speedcd, nyaatorrents, bluetigers, xthor, abnormal, torrentbytes, cpasbien,\ - freshontv, morethantv, t411, tokyotoshokan, shazbat, rarbg, alpharatio, tntvillage, binsearch, torrentproject, extratorrent, \ - scenetime, btdigg, transmitthenet, tvchaosuk, bitcannon, pretome, gftracker, hdspace, newpct, elitetorrent, bitsnoop, danishbits, hd4free, limetorrents, \ - norbits, ilovetorrents, anizb, bithdtv, zooqle, animebytes, torrentshack +from sickbeard.providers.nzb import ( + anizb, binsearch, omgwtfnzbs, womble, +) +from sickbeard.providers.torrent import ( + abnormal, alpharatio, animebytes, bitcannon, bithdtv, bitsnoop, bluetigers, btdigg, btn, cpasbien, danishbits, + elitetorrent, extratorrent, freshontv, gftracker, hd4free, hdbits, hdspace, hdtorrents, hounddawgs, ilovetorrents, + iptorrents, kat, limetorrents, morethantv, newpct, norbits, nyaatorrents, pretome, rarbg, scc, scenetime, shazbat, + speedcd, t411, thepiratebay, tntvillage, tokyotoshokan, torrentbytes, torrentday, torrentleech, torrentproject, + torrentshack, torrentz, transmitthenet, tvchaosuk, xthor, zooqle, +) + +from .nzb.newznab import NewznabProvider +from .torrent.rss.rsstorrent import TorrentRssProvider __all__ = [ 'womble', 'btn', 'thepiratebay', 'kat', 'torrentleech', 'scc', 'hdtorrents', @@ -41,30 +48,30 @@ def sortedProviderList(randomize=False): - initialList = sickbeard.providerList + sickbeard.newznabProviderList + sickbeard.torrentRssProviderList - providerDict = dict(zip([x.get_id() for x in initialList], initialList)) + initial_list = sickbeard.providerList + sickbeard.newznabProviderList + sickbeard.torrentRssProviderList + provider_dict = dict(zip([x.get_id() for x in initial_list], initial_list)) - newList = [] + new_list = [] # add all modules in the priority list, in order - for curModule in sickbeard.PROVIDER_ORDER: - if curModule in providerDict: - newList.append(providerDict[curModule]) + for cur_module in sickbeard.PROVIDER_ORDER: + if cur_module in provider_dict: + new_list.append(provider_dict[cur_module]) # add all enabled providers first - for curModule in providerDict: - if providerDict[curModule] not in newList and providerDict[curModule].is_enabled(): - newList.append(providerDict[curModule]) + for cur_module in provider_dict: + if provider_dict[cur_module] not in new_list and provider_dict[cur_module].is_enabled(): + new_list.append(provider_dict[cur_module]) # add any modules that are missing from that list - for curModule in providerDict: - if providerDict[curModule] not in newList: - newList.append(providerDict[curModule]) + for cur_module in provider_dict: + if provider_dict[cur_module] not in new_list: + new_list.append(provider_dict[cur_module]) if randomize: - shuffle(newList) + shuffle(new_list) - return newList + return new_list def makeProviderList(): @@ -73,19 +80,21 @@ def makeProviderList(): def getProviderModule(name): name = name.lower() - prefix = "sickbeard.providers." - if name in __all__ and prefix + name in sys.modules: - return sys.modules[prefix + name] - else: - raise Exception("Can't find " + prefix + name + " in " + "Providers") + prefixes = [ + "sickbeard.providers.nzb.", + "sickbeard.providers.torrent.html.", + "sickbeard.providers.torrent.json.", + "sickbeard.providers.torrent.rss.", + "sickbeard.providers.torrent.xml.", + ] + + for prefix in prefixes: + if name in __all__ and prefix + name in sys.modules: + return sys.modules[prefix + name] + + raise Exception("Can't find " + prefix + name + " in " + "Providers") def getProviderClass(provider_id): - providerMatch = [x for x in - sickbeard.providerList + sickbeard.newznabProviderList + sickbeard.torrentRssProviderList if - x and x.get_id() == provider_id] - - if len(providerMatch) != 1: - return None - else: - return providerMatch[0] + provider_list = sickbeard.providerList + sickbeard.newznabProviderList + sickbeard.torrentRssProviderList + return next((provider for provider in provider_list if provider.get_id() == provider_id), None) diff --git a/sickbeard/providers/animebytes.py b/sickbeard/providers/animebytes.py deleted file mode 100644 index ce0096da4c..0000000000 --- a/sickbeard/providers/animebytes.py +++ /dev/null @@ -1,355 +0,0 @@ -# coding=utf-8 -# Author: p0ps -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import re -import traceback - -from six.moves.urllib_parse import parse_qs - -from requests.compat import urljoin -from requests.utils import dict_from_cookiejar - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size -from sickrage.providers.torrent.TorrentProvider import TorrentProvider -from sickbeard.show_name_helpers import allPossibleShowNames - - -SEASON_PACK = 1 -SINGLE_EP = 2 -MULTI_EP = 3 -MULTI_SEASON = 4 -COMPLETE = 5 -OTHER = 6 - - -class AnimeBytes(TorrentProvider): # pylint: disable=too-many-instance-attributes - """AnimeBytes Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'AnimeBytes') - - # Credentials - self.username = None - self.password = None - - # URLs - self.url = 'https://animebytes.tv/' - self.urls = { - 'login': urljoin(self.url, '/user/login'), - 'search': urljoin(self.url, 'torrents.php'), - 'download': urljoin(self.url, '/torrent/{torrent_id}/download/{passkey}'), - } - - # Proper Strings - self.proper_strings = [] - - # Miscellaneous Options - self.anime_only = True - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, min_time=30) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - AnimeBytes search and parsing - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - _ = age - _ = ep_obj - results = [] - - if self.show and not self.show.is_anime: - return results - - if not self.login(): - return results - - episode = None - season = None - - # Search Params - search_params = { - 'filter_cat[1]': '1', - 'filter_cat[5]': '1', - 'action': 'advanced', - 'search_type': 'title', - 'year': '', - 'year2': '', - 'tags': '', - 'tags_type': '0', - 'sort': 'time_added', - 'way': 'desc', - 'hentai': '2', - 'anime[tv_series]': '1', - 'anime[tv_special]': '1', - 'releasegroup': '', - 'epcount': '', - 'epcount2': '', - 'artbooktitle': '', - } - - for mode in search_strings: - items = [] - logger.log('Search Mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {0}'.format(search_string), - logger.DEBUG) - search_params['searchstr'] = search_string - - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(data, 'html5lib') as html: - torrent_div = html.find('div', class_='thin') - torrent_group = torrent_div.find_all('div', class_='group_cont box anime') - - if not torrent_group: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - for group in torrent_group: - torrent_main = group.find_all('div', class_='group_main') - - for row in torrent_main: - try: - show_name = row.find('span', class_='group_title').find_next('a').get_text() - show_table = row.find('table', class_='torrent_group') - show_info = show_table.find_all('td') - - # A type of release used to determine how to parse the release - # For example a SINGLE_EP should be parsed like: show_name.episode.12.[source].[codec].[release_group] - # A multi ep release shoiuld look like: show_name.episode.1-12.[source].. - release_type = OTHER - - rows_to_skip = 0 - - for index, info in enumerate(show_info): - - if rows_to_skip: - rows_to_skip = rows_to_skip - 1 - continue - - info = info.get_text(strip=True) - - if show_name and info.startswith('[DL]'): - # Set skip next 4 rows, as they are useless - rows_to_skip = 4 - - hrefs = show_info[index].find_all('a') - params = parse_qs(hrefs[0].get('href', '')) - properties_string = hrefs[1].get_text().rstrip(' |').replace('|', '.').replace(' ', '') - properties_string = properties_string.replace('h26410-bit', 'h264.hi10p') # Hack for the h264 10bit stuff - properties = properties_string.split('.') - download_url = self.urls['download'].format(torrent_id=params['id'][0], - passkey=params['torrent_pass'][0]) - if not all([params, properties]): - continue - - tags = '{torrent_source}.{torrent_container}.{torrent_codec}.{torrent_res}.' \ - '{torrent_audio}'.format(torrent_source=properties[0], - torrent_container=properties[1], - torrent_codec=properties[2], - torrent_res=properties[3], - torrent_audio=properties[4]) - - last_field = re.match(r'(.*)\((.*)\)', properties[-1]) - # subs = last_field.group(1) if last_field else '' # We're not doing anything with this for now - release_group = '-{0}'.format(last_field.group(2)) if last_field else '' - - # Construct title based on the release type - - if release_type == SINGLE_EP: - # Create the single episode release_name - # Single.Episode.TV.Show.SXXEXX[Episode.Part].[Episode.Title].TAGS.[LANGUAGE].720p.FORMAT.x264-GROUP - title = '{title}.{season}{episode}.{tags}' \ - '{release_group}'.format(title=show_name, - season='S{0}'.format(season) if season else 'S01', - episode='E{0}'.format(episode), - tags=tags, - release_group=release_group) - if release_type == MULTI_EP: - # Create the multi-episode release_name - # Multiple.Episode.TV.Show.SXXEXX-EXX[Episode.Part].[Episode.Title].TAGS.[LANGUAGE].720p.FORMAT.x264-GROUP - title = '{title}.{season}{multi_episode}.{tags}' \ - '{release_group}'.format(title=show_name, - season='S{0}'.format(season) if season else 'S01', - multi_episode='E01-E{0}'.format(episode), - tags=tags, - release_group=release_group) - if release_type == SEASON_PACK: - # Create the season pack release_name - title = '{title}.{season}.{tags}' \ - '{release_group}'.format(title=show_name, - season='S{0}'.format(season) if season else 'S01', - tags=tags, - release_group=release_group) - - if release_type == MULTI_SEASON: - # Create the multi season pack release_name - # Multiple.Episode.TV.Show.EXX-EXX[Episode.Part].[Episode.Title].TAGS.[LANGUAGE].720p.FORMAT.x264-GROUP - title = '{title}.{episode}.{tags}' \ - '{release_group}'.format(title=show_name, - episode=episode, - tags=tags, - release_group=release_group) - - seeders = show_info[index + 3].get_text() - leechers = show_info[index + 4].get_text() - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the" - ' minimum seeders: {0}. Seeders: {1}'.format - (title, seeders), logger.DEBUG) - continue - - torrent_size = show_info[index + 1].get_text() - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - - # Determine episode, season and type - if info.startswith('Episode'): - # show_name = '{0}.{1}'.format(show_title, info) - episode = re.match('^Episode.([0-9]+)', info).group(1) - release_type = SINGLE_EP - elif info.startswith('Season'): - # Test for MultiSeason pack - if re.match('Season.[0-9]+-[0-9]+.\([0-9-]+\)', info): - # We can read the season AND the episodes, but we can only process multiep. - # So i've chosen to use it like 12-23 or 1-12. - match = re.match('Season.([0-9]+)-([0-9]+).\(([0-9-]+)\)', info) - episode = match.group(3).upper() - season = '{0}-{1}'.format(match.group(1), match.group(2)) - release_type = MULTI_SEASON - else: - season = re.match('Season.([0-9]+)', info).group(1) - # show_name = '{0}.{1}'.format(show_title, info) - release_type = SEASON_PACK - elif re.match('([0-9]+).episodes.*', info): - # This is a season pack, but, let's use it as a multi ep for now - # 13 episodes -> SXXEXX-EXX - episode = re.match('^([0-9]+).episodes.*', info).group(1) - release_type = MULTI_EP - else: - # Row is useless, skip it (eg. only animation studio) - continue - - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - def login(self): - """Login to AnimeBytes, check of the session cookie""" - if (any(dict_from_cookiejar(self.session.cookies).values()) and - dict_from_cookiejar(self.session.cookies).get('session')): - return True - - # Get csrf_token - data = self.get_url(self.urls['login'], returns='text') - with BS4Parser(data, 'html5lib') as html: - csrf_token = html.find('input', {'name': 'csrf_token'}).get('value') - - if not csrf_token: - logger.log("Unable to get csrf_token, can't login", logger.WARNING) - return False - - login_params = { - 'username': self.username, - 'password': self.password, - 'csrf_token': csrf_token, - 'login': 'Log In!', - 'keeplogged_sent': 'true', - } - - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: - logger.log('Unable to connect to provider', logger.WARNING) - return False - - if re.search('You will be banned for 6 hours after your login attempts run out.', response): - logger.log('Invalid username or password. Check your settings', logger.WARNING) - self.session.cookies.clear() - return False - - return True - - def _get_episode_search_strings(self, episode, add_string=''): - """Method override because AnimeBytes doesnt support searching showname + episode number""" - if not episode: - return [] - - search_string = { - 'Episode': [] - } - - for show_name in allPossibleShowNames(episode.show, season=episode.scene_season): - search_string['Episode'].append(show_name.strip()) - - return [search_string] - - def _get_season_search_strings(self, episode): - """Method override because AnimeBytes doesnt support searching showname + season number""" - search_string = { - 'Season': [] - } - - for show_name in allPossibleShowNames(episode.show, season=episode.scene_season): - search_string['Season'].append(show_name.strip()) - - return [search_string] - - -provider = AnimeBytes() diff --git a/sickbeard/providers/bitcannon.py b/sickbeard/providers/bitcannon.py deleted file mode 100644 index 336d3b8aea..0000000000 --- a/sickbeard/providers/bitcannon.py +++ /dev/null @@ -1,159 +0,0 @@ -# coding=utf-8 -# Author: Dustyn Gibson -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import validators -import traceback - -from requests.compat import urljoin - -from sickbeard import logger, tvcache - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class BitCannonProvider(TorrentProvider): - """BitCannon Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'BitCannon') - - # Credentials - self.api_key = None - - # URLs - self.custom_url = None - - # Proper Strings - - # Miscellaneous Options - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, search_params={'RSS': ['tv', 'anime']}) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals - """ - BitCannon search and parsing - - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - url = 'http://localhost:3000/' - if self.custom_url: - if not validators.url(self.custom_url): - logger.log('Invalid custom url set, please check your settings', logger.WARNING) - return results - url = self.custom_url - - # Search Params - search_params = { - 'category': 'anime' if ep_obj and ep_obj.show and ep_obj.show.anime else 'tv', - 'apiKey': self.api_key - } - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - search_params['q'] = search_string - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_url = urljoin(url, 'api/search') - parsed_json = self.get_url(search_url, params=search_params, returns='json') - if not parsed_json: - logger.log('No data returned from provider', logger.DEBUG) - continue - - if not self._check_auth_from_data(parsed_json): - return results - - for result in parsed_json.pop('torrents', {}): - try: - title = result.pop('title', '') - - info_hash = result.pop('infoHash', '') - download_url = 'magnet:?xt=urn:btih:' + info_hash - if not all([title, download_url, info_hash]): - continue - - swarm = result.pop('swarm', None) - if swarm: - seeders = try_int(swarm.pop('seeders', 0)) - leechers = try_int(swarm.pop('leechers', 0)) - else: - seeders = leechers = 0 - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - size = convert_size(result.pop('size', -1)) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - @staticmethod - def _check_auth_from_data(data): - if not all([isinstance(data, dict), - data.pop('status', 200) != 401, - data.pop('message', '') != 'Invalid API key']): - - logger.log('Invalid api key. Check your settings', logger.WARNING) - return False - - return True - - -provider = BitCannonProvider() diff --git a/sickbeard/providers/bitsnoop.py b/sickbeard/providers/bitsnoop.py deleted file mode 100644 index f404d993ec..0000000000 --- a/sickbeard/providers/bitsnoop.py +++ /dev/null @@ -1,152 +0,0 @@ -# coding=utf-8 -# Author: Gonçalo M. (aka duramato/supergonkas) -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import traceback - -from requests.compat import urljoin - -import sickbeard -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class BitSnoopProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """BitSnoop Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'BitSnoop') - - # Credentials - self.public = True - - # URLs - self.url = 'http://bitsnoop.com' - self.urls = { - 'index': self.url, - 'search': urljoin(self.url, '/search/video/'), - 'rss': urljoin(self.url, '/new_video.html?fmt=rss'), - } - - # Proper Strings - self.proper_strings = ['PROPER', 'REPACK'] - - # Miscellaneous Options - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, search_params={'RSS': ['rss']}) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches,too-many-locals - """ - BitSnoop search and parsing - - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_url = (self.urls['rss'], self.urls['search'] + search_string + '/s/d/1/?fmt=rss')[mode != 'RSS'] - data = self.get_url(search_url, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - if not data.startswith(' -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import re -import traceback - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class CpasbienProvider(TorrentProvider): - """Cpasbien Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'Cpasbien') - - # Credentials - self.public = True - - # URLs - self.url = 'http://www.cpasbien.cm' - - # Proper Strings - self.proper_strings = ['PROPER', 'REPACK'] - - # Miscellaneous Options - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals - results = [] - - # Units - units = ['o', 'Ko', 'Mo', 'Go', 'To', 'Po'] - - for mode in search_strings: - items = [] - logger.log('Search Mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - search_url = self.url + '/recherche/' + search_string.replace('.', '-').replace(' ', '-') + '.html,trie-seeds-d' - else: - search_url = self.url + '/view_cat.php?categorie=series&trie=date-d' - - data = self.get_url(search_url, returns='text') - if not data: - continue - - with BS4Parser(data, 'html5lib') as html: - torrent_rows = html(class_=re.compile('ligne[01]')) - for result in torrent_rows: - try: - title = result.find(class_='titre').get_text(strip=True).replace('HDTV', 'HDTV x264-CPasBien') - title = re.sub(r' Saison', ' Season', title, flags=re.IGNORECASE) - tmp = result.find('a')['href'].split('/')[-1].replace('.html', '.torrent').strip() - download_url = (self.url + '/telechargement/%s' % tmp) - if not all([title, download_url]): - continue - - seeders = try_int(result.find(class_='up').get_text(strip=True)) - leechers = try_int(result.find(class_='down').get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = result.find(class_='poid').get_text(strip=True) - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - -provider = CpasbienProvider() diff --git a/sickbeard/providers/extratorrent.py b/sickbeard/providers/extratorrent.py deleted file mode 100644 index 776cbe0a7d..0000000000 --- a/sickbeard/providers/extratorrent.py +++ /dev/null @@ -1,168 +0,0 @@ -# coding=utf-8 -# Author: Gonçalo M. (aka duramato/supergonkas) -# Author: Dustyn Gibson -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import traceback - -from requests.compat import urljoin - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class ExtraTorrentProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """ExtraTorrent Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'ExtraTorrent') - - # Credentials - self.public = True - - # URLs - self.url = 'http://extratorrent.cc' - self.urls = { - 'search': urljoin(self.url, 'search/'), - 'rss': urljoin(self.url, 'view/today/TV.html'), - } - self.custom_url = None - - # Proper Strings - self.proper_strings = ['PROPER', 'REPACK', 'REAL'] - - # Miscellaneous Options - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, min_time=30) # Only poll ExtraTorrent every 30 minutes max - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - ExtraTorrent search and parsing - - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - - # Search Params - search_params = { - 'search': '', - 'new': 1, - 'x': 0, - 'y': 0, - } - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - search_params['search'] = search_string - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - search_url = self.urls['search'] - decrease = 0 - else: - search_url = self.urls['rss'] - # RSS search has one less column - decrease = 1 - - search_url = search_url if not self.custom_url else \ - search_url.replace(self.url + '/', self.custom_url) - - data = self.get_url(search_url, params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', class_='tl') - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one release is found - if len(torrent_rows) < 3 or (len(torrent_rows) == 3 and - torrent_rows[2].get_text() == 'No torrents'): - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - # Skip column headers - for result in torrent_rows[2:]: - - try: - cells = result('td') - - torrent_info = cells[0].find('a') - title = torrent_info.get('title').strip('Download torrent') - download_url = urljoin(self.url, torrent_info.get('href').replace - ('torrent_download', 'download')) - if not all([title, download_url]): - continue - - seeders = try_int(cells[4 - decrease].get_text(), 1) - leechers = try_int(cells[5 - decrease].get_text()) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[3 - decrease].get_text().replace('\xa0', ' ') - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - -provider = ExtraTorrentProvider() diff --git a/sickbeard/providers/hounddawgs.py b/sickbeard/providers/hounddawgs.py deleted file mode 100644 index 81a6717734..0000000000 --- a/sickbeard/providers/hounddawgs.py +++ /dev/null @@ -1,213 +0,0 @@ -# coding=utf-8 -# Author: Idan Gutman -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import re -import traceback - -from requests.compat import urljoin -from requests.utils import dict_from_cookiejar - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class HoundDawgsProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """HoundDawgs Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'HoundDawgs') - - # Credentials - self.username = None - self.password = None - - # URLs - self.url = 'https://hounddawgs.org' - self.urls = { - 'base_url': self.url, - 'search': urljoin(self.url, 'torrents.php'), - 'login': urljoin(self.url, 'login.php'), - } - - # Proper Strings - - # Miscellaneous Options - self.freeleech = None - self.ranked = None - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - HoundDawgs search and parsing - - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - if not self.login(): - return results - - # Search Params - search_params = { - 'filter_cat[85]': 1, - 'filter_cat[58]': 1, - 'filter_cat[57]': 1, - 'filter_cat[74]': 1, - 'filter_cat[92]': 1, - 'filter_cat[93]': 1, - 'order_by': 's3', - 'order_way': 'desc', - 'type': '', - 'userid': '', - 'searchstr': '', - 'searchimdb': '', - 'searchtags': '' - } - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_params['searchstr'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - str_table_start = "Login :: HoundDawgs', response) \ - or re.search('Dine cookies er ikke aktiveret.', response): - logger.log('Invalid username or password. Check your settings', logger.WARNING) - return False - - return True - - -provider = HoundDawgsProvider() diff --git a/sickbeard/providers/kat.py b/sickbeard/providers/kat.py deleted file mode 100644 index 13cbcd0a32..0000000000 --- a/sickbeard/providers/kat.py +++ /dev/null @@ -1,165 +0,0 @@ -# coding=utf-8 -# Author: Dustyn Gibson -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import traceback -import validators - -from requests.compat import urljoin -from sickbeard.bs4_parser import BS4Parser - -import sickbeard -from sickbeard import logger, tvcache - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class KatProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """KAT Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'KickAssTorrents') - - # Credentials - self.public = True - - # URLs - self.url = 'https://kat.cr' - self.urls = { - 'search': urljoin(self.url, '%s/'), - } - self.custom_url = None - - # Proper Strings - - # Miscellaneous Options - self.confirmed = True - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, search_params={'RSS': ['tv', 'anime']}) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals, too-many-statements - results = [] - - anime = (self.show and self.show.anime) or (ep_obj and ep_obj.show and ep_obj.show.anime) or False - search_params = { - 'q': '', - 'field': 'seeders', - 'sorder': 'desc', - 'rss': 1, - 'category': ('tv', 'anime')[anime] - } - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - search_params['q'] = search_string if mode != 'RSS' else '' - search_params['field'] = 'seeders' if mode != 'RSS' else 'time_add' - - if mode != 'RSS': - logger.log('Search string: {0}'.format(search_string), - logger.DEBUG) - - search_url = self.urls['search'] % ('usearch' if mode != 'RSS' else search_string) - if self.custom_url: - if not validators.url(self.custom_url): - logger.log('Invalid custom url: {0}'.format(self.custom_url), logger.WARNING) - return results - search_url = urljoin(self.custom_url, search_url.split(self.url)[1]) - - data = self.get_url(search_url, params=search_params, returns='text') - if not data: - logger.log('URL did not return data, maybe try a custom url, or a different one', logger.DEBUG) - continue - - if not data.startswith('. - -from __future__ import unicode_literals - -import re -import traceback - -from requests.compat import urljoin, quote -from requests.utils import dict_from_cookiejar - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class SCCProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'SceneAccess') - - # Credentials - self.username = None - self.password = None - - # URLs - self.url = 'https://sceneaccess.eu' - self.urls = { - 'base_url': self.url, - 'login': urljoin(self.url, 'login'), - 'detail': urljoin(self.url, 'details?id=%s'), - 'search': urljoin(self.url, 'all?search=%s&method=1&%s'), - 'download': urljoin(self.url, '%s') - } - - # Proper Strings - - # Miscellaneous Options - self.categories = { - 'Season': 'c26=26&c44=44&c45=45', # Archive, non-scene HD, non-scene SD; need to include non-scene because WEB-DL packs get added to those categories - 'Episode': 'c17=17&c27=27&c33=33&c34=34&c44=44&c45=45', # TV HD, TV SD, non-scene HD, non-scene SD, foreign XviD, foreign x264 - 'RSS': 'c17=17&c26=26&c27=27&c33=33&c34=34&c44=44&c45=45' # Season + Episode - } - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self) # only poll SCC every 20 minutes max - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals,too-many-branches, too-many-statements - results = [] - if not self.login(): - return results - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {0}'.format(search_string), logger.DEBUG) - - search_url = self.urls['search'] % (quote(search_string), self.categories[mode]) - - data = self.get_url(search_url, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', attrs={'id': 'torrents-table'}) - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one Release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - for result in torrent_table('tr')[1:]: - try: - link = result.find('td', attrs={'class': 'ttr_name'}).find('a') - url = result.find('td', attrs={'class': 'td_dl'}).find('a') - - title = link.string - if re.search(r'\.\.\.', title): - data = self.get_url(urljoin(self.url, link['href']), returns='text') - if data: - with BS4Parser(data) as details_html: - title = re.search("(?<=').+(?SceneAccess \| Login', response): - logger.log('Invalid username or password. Check your settings', logger.WARNING) - return False - - return True - - @staticmethod - def _is_section(section, text): - title = r'.+? \| %s' % section - return re.search(title, text, re.IGNORECASE) - - -provider = SCCProvider() diff --git a/sickbeard/providers/thepiratebay.py b/sickbeard/providers/thepiratebay.py deleted file mode 100644 index 9c4556eeab..0000000000 --- a/sickbeard/providers/thepiratebay.py +++ /dev/null @@ -1,188 +0,0 @@ -# coding=utf-8 -# Author: Dustyn Gibson -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import re -import traceback -import validators - -from requests.compat import urljoin - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class ThePirateBayProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """ThePirateBay Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'ThePirateBay') - - # Credentials - self.public = True - - # URLs - self.url = 'https://thepiratebay.org' - self.urls = { - 'rss': urljoin(self.url, 'tv/latest'), - 'search': urljoin(self.url, 's/'), # Needs trailing / - } - self.custom_url = None - - # Proper Strings - - # Miscellaneous Options - self.confirmed = True - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, min_time=20) # only poll ThePirateBay every 20 minutes max - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements - results = [] - """ - 205 = SD, 208 = HD, 200 = All Videos - https://pirateproxy.pl/s/?q=Game of Thrones&type=search&orderby=7&page=0&category=200 - """ - search_params = { - 'q': '', - 'type': 'search', - 'orderby': 7, - 'page': 0, - 'category': 200 - } - - # Units - units = ['B', 'KIB', 'MIB', 'GIB', 'TIB', 'PIB'] - - def process_column_header(th): - result = '' - if th.a: - result = th.a.get_text(strip=True) - if not result: - result = th.get_text(strip=True) - return result - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - search_url = self.urls['search'] if mode != 'RSS' else self.urls['rss'] - if self.custom_url: - if not validators.url(self.custom_url): - logger.log('Invalid custom url: {0}'.format(self.custom_url), logger.WARNING) - return results - search_url = urljoin(self.custom_url, search_url.split(self.url)[1]) - - if mode != 'RSS': - search_params['q'] = search_string - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - data = self.get_url(search_url, params=search_params, returns='text') - else: - data = self.get_url(search_url, returns='text') - - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', id='searchResult') - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - labels = [process_column_header(label) for label in torrent_rows[0]('th')] - - # Skip column headers - for result in torrent_rows[1:]: - try: - cells = result('td') - - title = result.find(class_='detName') - title = title.get_text(strip=True) if title else None - download_url = result.find(title='Download this torrent using magnet') - download_url = download_url['href'] + self._custom_trackers if download_url else None - if download_url and 'magnet:?' not in download_url: - logger.log('Invalid ThePirateBay proxy please try another one', logger.DEBUG) - continue - if not all([title, download_url]): - continue - - seeders = try_int(cells[labels.index('SE')].get_text(strip=True)) - leechers = try_int(cells[labels.index('LE')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - # Accept Torrent only from Good People for every Episode Search - if self.confirmed and not result.find(alt=re.compile(r'VIP|Trusted')): - if mode != 'RSS': - logger.log("Found result {0} but that doesn't seem like a trusted" - " result so I'm ignoring it".format(title), logger.DEBUG) - continue - - # Convert size after all possible skip scenarios - torrent_size = cells[labels.index('Name')].find(class_='detDesc').get_text(strip=True).split(', ')[1] - torrent_size = re.sub(r'Size ([\d.]+).+([KMGT]iB)', r'\1 \2', torrent_size) - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - -provider = ThePirateBayProvider() diff --git a/sickbeard/providers/tokyotoshokan.py b/sickbeard/providers/tokyotoshokan.py deleted file mode 100644 index 09809ef52c..0000000000 --- a/sickbeard/providers/tokyotoshokan.py +++ /dev/null @@ -1,156 +0,0 @@ -# coding=utf-8 -# Author: Mr_Orange -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import re -import traceback - -from requests.compat import urljoin - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class TokyoToshokanProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """TokyoToshokan Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'TokyoToshokan') - - # Credentials - self.public = True - - # URLs - self.url = 'http://tokyotosho.info/' - self.urls = { - 'search': urljoin(self.url, 'search.php'), - 'rss': urljoin(self.url, 'rss.php'), - } - - # Proper Strings - - # Miscellaneous Options - self.supports_absolute_numbering = True - self.anime_only = True - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, min_time=15) # only poll TokyoToshokan every 15 minutes max - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - TokyoToshokan search and parsing - - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - if self.show and not self.show.is_anime: - return results - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_params = { - 'terms': search_string, - 'type': 1, # get anime types - } - - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(data, 'html5lib') as soup: - torrent_table = soup.find('table', class_='listing') - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - a = 1 if len(torrent_rows[0]('td')) < 2 else 0 - - # Skip column headers - for top, bot in zip(torrent_rows[a::2], torrent_rows[a + 1::2]): - try: - desc_top = top.find('td', class_='desc-top') - title = desc_top.get_text(strip=True) if desc_top else None - download_url = desc_top.find('a')['href'] if desc_top else None - if not all([title, download_url]): - continue - - stats = bot.find('td', class_='stats').get_text(strip=True) - sl = re.match(r'S:(?P\d+)L:(?P\d+)C:(?:\d+)ID:(?:\d+)', stats.replace(' ', '')) - seeders = try_int(sl.group('seeders')) if sl else 0 - leechers = try_int(sl.group('leechers')) if sl else 0 - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - desc_bottom = bot.find('td', class_='desc-bot').get_text(strip=True) - size = convert_size(desc_bottom.split('|')[1].strip('Size: ')) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - -provider = TokyoToshokanProvider() diff --git a/sickbeard/providers/torrent/__init__.py b/sickbeard/providers/torrent/__init__.py new file mode 100644 index 0000000000..fdb9545aab --- /dev/null +++ b/sickbeard/providers/torrent/__init__.py @@ -0,0 +1,62 @@ +# coding=utf-8 + +from .html import ( + abnormal, + alpharatio, + animebytes, + bithdtv, + bluetigers, + cpasbien, + danishbits, + elitetorrent, + extratorrent, + freshontv, + gftracker, + hdspace, + hdtorrents, + hounddawgs, + ilovetorrents, + iptorrents, + limetorrents, + morethantv, + newpct, + pretome, + scc, + scenetime, + speedcd, + thepiratebay, + tntvillage, + tokyotoshokan, + torrentbytes, + torrentleech, + torrentshack, + transmitthenet, + tvchaosuk, + xthor, + zooqle, +) + +from .json import ( + bitcannon, + btdigg, + btn, + hd4free, + hdbits, + norbits, + rarbg, + t411, + torrentday, + torrentproject, +) + +from .rss import ( + nyaatorrents, + rsstorrent, + shazbat, +) + +from .xml import ( + bitsnoop, + kat, + torrentz, +) diff --git a/sickbeard/providers/torrent/html/__init__.py b/sickbeard/providers/torrent/html/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sickbeard/providers/abnormal.py b/sickbeard/providers/torrent/html/abnormal.py similarity index 52% rename from sickbeard/providers/abnormal.py rename to sickbeard/providers/torrent/html/abnormal.py index 19de730f04..fcd98fe635 100644 --- a/sickbeard/providers/abnormal.py +++ b/sickbeard/providers/torrent/html/abnormal.py @@ -63,9 +63,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - ABNormal search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -90,11 +90,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'way': 'DESC', # Both ASC and DESC are available for sort direction } - # Units - units = ['O', 'KO', 'MO', 'GO', 'TO', 'PO'] - for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -105,75 +101,91 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_params['order'] = 'Seeders' search_params['search'] = re.sub(r'[()]', '', search_string) - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find(class_='torrent_table') - torrent_rows = torrent_table('tr') if torrent_table else [] + results += self.parse(response.text, mode) - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + # Units + units = ['O', 'KO', 'MO', 'GO', 'TO', 'PO'] + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find(class_='torrent_table') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Catégorie, Release, Date, DL, Size, C, S, L + labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')] + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): + continue + + try: + title = cells[labels.index('Release')].get_text(strip=True) + download = cells[labels.index('DL')].find('a', class_='tooltip')['href'] + download_url = urljoin(self.url, download) + if not all([title, download_url]): continue - # Catégorie, Release, Date, DL, Size, C, S, L - labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')] - - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - if len(cells) < len(labels): - continue - - try: - title = cells[labels.index('Release')].get_text(strip=True) - download = cells[labels.index('DL')].find('a', class_='tooltip')['href'] - download_url = urljoin(self.url, download) - if not all([title, download_url]): - continue - - seeders = try_int(cells[labels.index('S')].get_text(strip=True)) - leechers = try_int(cells[labels.index('L')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - size_index = labels.index('Size') if 'Size' in labels else labels.index('Taille') - torrent_size = cells[size_index].get_text() - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(cells[labels.index('S')].get_text(strip=True)) + leechers = try_int(cells[labels.index('L')].get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + size_index = labels.index('Size') if 'Size' in labels else labels.index('Taille') + torrent_size = cells[size_index].get_text() + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -182,12 +194,12 @@ def login(self): 'password': self.password, } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if not re.search('torrents.php', response): + if not re.search('torrents.php', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/alpharatio.py b/sickbeard/providers/torrent/html/alpharatio.py similarity index 52% rename from sickbeard/providers/alpharatio.py rename to sickbeard/providers/torrent/html/alpharatio.py index 4ea71a35ec..0aeedcdaaf 100644 --- a/sickbeard/providers/alpharatio.py +++ b/sickbeard/providers/torrent/html/alpharatio.py @@ -52,20 +52,20 @@ def __init__(self): # Proper Strings self.proper_strings = ['PROPER', 'REPACK'] + # Miscellaneous Options + # Torrent Stats self.minseed = None self.minleech = None - # Miscellaneous Options - # Cache self.cache = tvcache.TVCache(self) def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - AlphaRatio search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -81,9 +81,37 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'filter_cat[2]': 1, 'filter_cat[3]': 1, 'filter_cat[4]': 1, - 'filter_cat[5]': 1 + 'filter_cat[5]': 1, } + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['searchstr'] = search_string + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ # Units units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] @@ -95,86 +123,69 @@ def process_column_header(td): result = td.get_text(strip=True) return result - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + items = [] - for search_string in search_strings[mode]: + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', id='torrent_table') + torrent_rows = torrent_table('tr') if torrent_table else [] - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items - search_params['searchstr'] = search_string - search_url = self.urls['search'] - data = self.get_url(search_url, params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue + # '', '', 'Name /Year', 'Files', 'Time', 'Size', 'Snatches', 'Seeders', 'Leechers' + labels = [process_column_header(label) for label in torrent_rows[0]('td')] - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', id='torrent_table') - torrent_rows = torrent_table('tr') if torrent_table else [] + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): + continue - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + try: + title = cells[labels.index('Name /Year')].find('a', dir='ltr').get_text(strip=True) + download = cells[labels.index('Name /Year')].find('a', title='Download')['href'] + download_url = urljoin(self.url, download) + if not all([title, download_url]): continue - # '', '', 'Name /Year', 'Files', 'Time', 'Size', 'Snatches', 'Seeders', 'Leechers' - labels = [process_column_header(label) for label in torrent_rows[0]('td')] - - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - if len(cells) < len(labels): - continue - - try: - title = cells[labels.index('Name /Year')].find('a', dir='ltr').get_text(strip=True) - download = cells[labels.index('Name /Year')].find('a', title='Download')['href'] - download_url = urljoin(self.url, download) - if not all([title, download_url]): - continue - - seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) - leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[labels.index('Size')].get_text(strip=True) - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) + leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[labels.index('Size')].get_text(strip=True) + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -185,13 +196,13 @@ def login(self): 'remember_me': 'on', } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if any([re.search('Invalid Username/password', response), - re.search('Login :: AlphaRatio.cc', response)]): + if any([re.search('Invalid Username/password', response.text), + re.search('Login :: AlphaRatio.cc', response.text)]): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/torrent/html/animebytes.py b/sickbeard/providers/torrent/html/animebytes.py new file mode 100644 index 0000000000..9c49e47f7d --- /dev/null +++ b/sickbeard/providers/torrent/html/animebytes.py @@ -0,0 +1,369 @@ +# coding=utf-8 +# Author: p0ps +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from six.moves.urllib_parse import parse_qs + +from requests.compat import urljoin +from requests.utils import dict_from_cookiejar + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size +from sickrage.providers.torrent.TorrentProvider import TorrentProvider +from sickbeard.show_name_helpers import allPossibleShowNames + + +SEASON_PACK = 1 +SINGLE_EP = 2 +MULTI_EP = 3 +MULTI_SEASON = 4 +COMPLETE = 5 +OTHER = 6 + + +class AnimeBytes(TorrentProvider): # pylint: disable=too-many-instance-attributes + """AnimeBytes Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'AnimeBytes') + + # Credentials + self.username = None + self.password = None + + # URLs + self.url = 'https://animebytes.tv/' + self.urls = { + 'login': urljoin(self.url, '/user/login'), + 'search': urljoin(self.url, 'torrents.php'), + 'download': urljoin(self.url, '/torrent/{torrent_id}/download/{passkey}'), + } + + # Proper Strings + self.proper_strings = [] + + # Miscellaneous Options + self.anime_only = True + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, min_time=30) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + _ = age + _ = ep_obj + results = [] + + if self.show and not self.show.is_anime: + return results + + if not self.login(): + return results + + # Search Params + search_params = { + 'filter_cat[1]': '1', + 'filter_cat[5]': '1', + 'action': 'advanced', + 'search_type': 'title', + 'year': '', + 'year2': '', + 'tags': '', + 'tags_type': '0', + 'sort': 'time_added', + 'way': 'desc', + 'hentai': '2', + 'anime[tv_series]': '1', + 'anime[tv_special]': '1', + 'releasegroup': '', + 'epcount': '', + 'epcount2': '', + 'artbooktitle': '', + } + + for mode in search_strings: + logger.log('Search Mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + search_params['searchstr'] = search_string + + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + episode = None + season = None + + with BS4Parser(data, 'html5lib') as html: + torrent_div = html.find('div', class_='thin') + torrent_group = torrent_div('div', class_='group_cont box anime') + + # Continue only if at least one release is found + if not torrent_group: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for group in torrent_group: + torrent_rows = group('div', class_='group_main') + + for row in torrent_rows: + try: + show_name = row.find('span', class_='group_title').find_next('a').get_text() + show_table = row.find('table', class_='torrent_group') + show_info = show_table('td') + + # A type of release used to determine how to parse the release + # For example a SINGLE_EP should be parsed like: show_name.episode.12.[source].[codec].[release_group] + # A multi ep release should look like: show_name.episode.1-12.[source].. + release_type = OTHER + + rows_to_skip = 0 + + for index, info in enumerate(show_info): + + if rows_to_skip: + rows_to_skip -= 1 + continue + + info = info.get_text(strip=True) + + if show_name and info.startswith('[DL]'): + # Set skip next 4 rows, as they are useless + rows_to_skip = 4 + + hrefs = show_info[index]('a') + params = parse_qs(hrefs[0].get('href', '')) + properties_string = hrefs[1].get_text().rstrip(' |').replace('|', '.').replace(' ', '') + properties_string = properties_string.replace('h26410-bit', 'h264.hi10p') # Hack for the h264 10bit stuff + properties = properties_string.split('.') + download_url = self.urls['download'].format(torrent_id=params['id'][0], + passkey=params['torrent_pass'][0]) + if not all([params, properties]): + continue + + tags = '{torrent_source}.{torrent_container}.{torrent_codec}.{torrent_res}.' \ + '{torrent_audio}'.format(torrent_source=properties[0], + torrent_container=properties[1], + torrent_codec=properties[2], + torrent_res=properties[3], + torrent_audio=properties[4]) + + last_field = re.match(r'(.*)\((.*)\)', properties[-1]) + # subs = last_field.group(1) if last_field else '' # We're not doing anything with this for now + release_group = '-{0}'.format(last_field.group(2)) if last_field else '' + + # Construct title based on the release type + + if release_type == SINGLE_EP: + # Create the single episode release_name + # Single.Episode.TV.Show.SXXEXX[Episode.Part].[Episode.Title].TAGS.[LANGUAGE].720p.FORMAT.x264-GROUP + title = '{title}.{season}{episode}.{tags}' \ + '{release_group}'.format(title=show_name, + season='S{0}'.format(season) if season else 'S01', + episode='E{0}'.format(episode), + tags=tags, + release_group=release_group) + if release_type == MULTI_EP: + # Create the multi-episode release_name + # Multiple.Episode.TV.Show.SXXEXX-EXX[Episode.Part].[Episode.Title].TAGS.[LANGUAGE].720p.FORMAT.x264-GROUP + title = '{title}.{season}{multi_episode}.{tags}' \ + '{release_group}'.format(title=show_name, + season='S{0}'.format(season) if season else 'S01', + multi_episode='E01-E{0}'.format(episode), + tags=tags, + release_group=release_group) + if release_type == SEASON_PACK: + # Create the season pack release_name + title = '{title}.{season}.{tags}' \ + '{release_group}'.format(title=show_name, + season='S{0}'.format(season) if season else 'S01', + tags=tags, + release_group=release_group) + + if release_type == MULTI_SEASON: + # Create the multi season pack release_name + # Multiple.Episode.TV.Show.EXX-EXX[Episode.Part].[Episode.Title].TAGS.[LANGUAGE].720p.FORMAT.x264-GROUP + title = '{title}.{episode}.{tags}' \ + '{release_group}'.format(title=show_name, + episode=episode, + tags=tags, + release_group=release_group) + + seeders = show_info[index + 3].get_text() + leechers = show_info[index + 4].get_text() + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the" + ' minimum seeders: {0}. Seeders: {1}'.format + (title, seeders), logger.DEBUG) + continue + + torrent_size = show_info[index + 1].get_text() + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + + # Determine episode, season and type + if info.startswith('Episode'): + # show_name = '{0}.{1}'.format(show_title, info) + episode = re.match('^Episode.([0-9]+)', info).group(1) + release_type = SINGLE_EP + elif info.startswith('Season'): + # Test for MultiSeason pack + if re.match('Season.[0-9]+-[0-9]+.\([0-9-]+\)', info): + # We can read the season AND the episodes, but we can only process multiep. + # So i've chosen to use it like 12-23 or 1-12. + match = re.match('Season.([0-9]+)-([0-9]+).\(([0-9-]+)\)', info) + episode = match.group(3).upper() + season = '{0}-{1}'.format(match.group(1), match.group(2)) + release_type = MULTI_SEASON + else: + season = re.match('Season.([0-9]+)', info).group(1) + # show_name = '{0}.{1}'.format(show_title, info) + release_type = SEASON_PACK + elif re.match('([0-9]+).episodes.*', info): + # This is a season pack, but, let's use it as a multi ep for now + # 13 episodes -> SXXEXX-EXX + episode = re.match('^([0-9]+).episodes.*', info).group(1) + release_type = MULTI_EP + else: + # Row is useless, skip it (eg. only animation studio) + continue + + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + def login(self): + """Login method used for logging in before doing search and torrent downloads.""" + if (any(dict_from_cookiejar(self.session.cookies).values()) and + dict_from_cookiejar(self.session.cookies).get('session')): + return True + + # Get csrf_token + response = self.get_url(self.urls['login'], returns='resposne') + with BS4Parser(response.text, 'html5lib') as html: + csrf_token = html.find('input', {'name': 'csrf_token'}).get('value') + + if not csrf_token: + logger.log("Unable to get csrf_token, can't login", logger.WARNING) + return False + + login_params = { + 'username': self.username, + 'password': self.password, + 'csrf_token': csrf_token, + 'login': 'Log In!', + 'keeplogged_sent': 'true', + } + + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: + logger.log('Unable to connect to provider', logger.WARNING) + return False + + if re.search('You will be banned for 6 hours after your login attempts run out.', response.text): + logger.log('Invalid username or password. Check your settings', logger.WARNING) + self.session.cookies.clear() + return False + + return True + + def _get_episode_search_strings(self, episode, add_string=''): + """Method override because AnimeBytes doesnt support searching showname + episode number""" + if not episode: + return [] + + search_string = { + 'Episode': [] + } + + for show_name in allPossibleShowNames(episode.show, season=episode.scene_season): + search_string['Episode'].append(show_name.strip()) + + return [search_string] + + def _get_season_search_strings(self, episode): + """Method override because AnimeBytes doesnt support searching showname + season number""" + search_string = { + 'Season': [] + } + + for show_name in allPossibleShowNames(episode.show, season=episode.scene_season): + search_string['Season'].append(show_name.strip()) + + return [search_string] + + +provider = AnimeBytes() diff --git a/sickbeard/providers/bithdtv.py b/sickbeard/providers/torrent/html/bithdtv.py similarity index 51% rename from sickbeard/providers/bithdtv.py rename to sickbeard/providers/torrent/html/bithdtv.py index 8348413c76..33e9cea6c1 100644 --- a/sickbeard/providers/bithdtv.py +++ b/sickbeard/providers/torrent/html/bithdtv.py @@ -55,14 +55,18 @@ def __init__(self): # Proper Strings + # Miscellaneous Options + + # Torrent Stats + # Cache self.cache = tvcache.TVCache(self, min_time=10) # Only poll BitHDTV every 10 minutes max def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - BIT HDTV search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -76,92 +80,96 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'cat': 10, } - # Units - units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] - for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: if mode != 'RSS': - search_params['search'] = search_string logger.log('Search string: {search}'.format (search=search_string), logger.DEBUG) + search_params['search'] = search_string if mode == 'Season': search_params['cat'] = 12 - response = self.get_url(self.urls['search'], params=search_params, returns='response') - if not response or not response.text: + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(response.text, 'html.parser') as html: # Use html.parser, since html5parser has issues with this site. - all_tables = html('table', width='750') # Get the last table with a width of 750px. - if all_tables: - result_table = all_tables[-1] - else: - continue + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. - torrent_rows = result_table('tr') if result_table else [] + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS - # Continue only if at least one Release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + :return: A list of items found + """ + # Units + units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + + items = [] + + with BS4Parser(data, 'html.parser') as html: # Use html.parser, since html5parser has issues with this site. + tables = html('table', width='750') # Get the last table with a width of 750px. + torrent_table = tables[-1] if tables else [] + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if self.freeleech and not row.get('bgcolor'): + continue + + try: + title = cells[2].find('a')['title'] + download_url = urljoin(self.url, cells[0].find('a')['href']) + if not all([title, download_url]): continue - # Skip column headers - for result in torrent_rows[1:]: - freeleech = result.get('bgcolor') - if self.freeleech and not freeleech: - continue - - try: - cells = result('td') - - title = cells[2].find('a')['title'] - download_url = urljoin(self.url, cells[0].find('a')['href']) - if not all([title, download_url]): - continue - - seeders = try_int(cells[8].get_text(strip=True)) - leechers = try_int(cells[9].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = '{size} {unit}'.format(size=cells[6].contents[0], unit=cells[6].contents[1].get_text()) - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(cells[8].get_text(strip=True)) + leechers = try_int(cells[9].get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = '{size} {unit}'.format(size=cells[6].contents[0], unit=cells[6].contents[1].get_text()) + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): """Login method used for logging in before doing search and torrent downloads.""" @@ -173,13 +181,13 @@ def login(self): 'password': self.password, } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) self.session.cookies.clear() return False - if '

Login failed!

' in response: + if '

Login failed!

' in response.text: logger.log('Invalid username or password. Check your settings', logger.WARNING) self.session.cookies.clear() return False diff --git a/sickbeard/providers/bluetigers.py b/sickbeard/providers/torrent/html/bluetigers.py similarity index 53% rename from sickbeard/providers/bluetigers.py rename to sickbeard/providers/torrent/html/bluetigers.py index 5c12dcb83d..9d10fc7015 100644 --- a/sickbeard/providers/bluetigers.py +++ b/sickbeard/providers/torrent/html/bluetigers.py @@ -64,9 +64,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals """ - BLUETIGERS search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -88,8 +88,8 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] - logger.log('Search Mode: {0}'.format(mode), logger.DEBUG) + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + for search_string in search_strings[mode]: if mode != 'RSS': @@ -97,65 +97,80 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man (search=search_string), logger.DEBUG) search_params['search'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(data, 'html5lib') as html: - result_linkz = html('a', href=re.compile('torrents-details')) + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_rows = html('a', href=re.compile('torrents-details')) - # Continue only if at least one release is found - if not result_linkz: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + # Continue only if at least one release is found + if not torrent_rows: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for row in torrent_rows: + try: + title = row.text + download_url = self.urls['base_url'] + row['href'] + download_url = download_url.replace('torrents-details', 'download') + if not all([title, download_url]): continue - for link in result_linkz: - try: - title = link.text - download_url = self.urls['base_url'] + link['href'] - download_url = download_url.replace('torrents-details', 'download') - if not all([title, download_url]): - continue - - # FIXME - seeders = 1 - leechers = 0 - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - # FIXME - size = -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + # FIXME + seeders = 1 + leechers = 0 - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + # FIXME + size = -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -165,19 +180,20 @@ def login(self): 'take_login': '1' } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: - check_login = self.get_url(self.urls['base_url'], returns='text') - if re.search('account-logout.php', check_login): - return True - else: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: + check_login = self.get_url(self.urls['base_url'], returns='response') + if not re.search('account-logout.php', check_login.text): logger.log('Unable to connect to provider', logger.WARNING) return False + else: + return True - if re.search('account-login.php', response): + if re.search('account-login.php', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False return True + provider = BlueTigersProvider() diff --git a/sickbeard/providers/torrent/html/cpasbien.py b/sickbeard/providers/torrent/html/cpasbien.py new file mode 100644 index 0000000000..c0e1b418f6 --- /dev/null +++ b/sickbeard/providers/torrent/html/cpasbien.py @@ -0,0 +1,148 @@ +# coding=utf-8 +# Author: Guillaume Serre +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class CpasbienProvider(TorrentProvider): + """Cpasbien Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'Cpasbien') + + # Credentials + self.public = True + + # URLs + self.url = 'http://www.cpasbien.cm' + + # Proper Strings + self.proper_strings = ['PROPER', 'REPACK'] + + # Miscellaneous Options + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + search_url = self.url + '/recherche/' + search_string.replace('.', '-').replace(' ', '-') + '.html,trie-seeds-d' + else: + search_url = self.url + '/view_cat.php?categorie=series&trie=date-d' + + response = self.get_url(search_url, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + # Units + units = ['o', 'Ko', 'Mo', 'Go', 'To', 'Po'] + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_rows = html(class_=re.compile('ligne[01]')) + for row in torrent_rows: + try: + title = row.find(class_='titre').get_text(strip=True).replace('HDTV', 'HDTV x264-CPasBien') + title = re.sub(r' Saison', ' Season', title, flags=re.IGNORECASE) + tmp = row.find('a')['href'].split('/')[-1].replace('.html', '.torrent').strip() + download_url = (self.url + '/telechargement/%s' % tmp) + if not all([title, download_url]): + continue + + seeders = try_int(row.find(class_='up').get_text(strip=True)) + leechers = try_int(row.find(class_='down').get_text(strip=True)) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = row.find(class_='poid').get_text(strip=True) + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + +provider = CpasbienProvider() diff --git a/sickbeard/providers/danishbits.py b/sickbeard/providers/torrent/html/danishbits.py similarity index 50% rename from sickbeard/providers/danishbits.py rename to sickbeard/providers/torrent/html/danishbits.py index de2f4a62ba..5505674b61 100644 --- a/sickbeard/providers/danishbits.py +++ b/sickbeard/providers/torrent/html/danishbits.py @@ -31,7 +31,7 @@ class DanishbitsProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - + """Danishbits Torrent provider""" def __init__(self): # Provider Init @@ -41,11 +41,6 @@ def __init__(self): self.username = None self.password = None - # Torrent Stats - self.minseed = 0 - self.minleech = 0 - self.freeleech = True - # URLs self.url = 'https://danishbits.org' self.urls = { @@ -56,17 +51,20 @@ def __init__(self): # Proper Strings # Miscellaneous Options + self.freeleech = True # Torrent Stats + self.minseed = 0 + self.minleech = 0 # Cache self.cache = tvcache.TVCache(self, min_time=10) # Only poll Danishbits every 10 minutes max def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - DanishBits search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -82,6 +80,34 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'search': '', } + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['search'] = search_string + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ # Units units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] @@ -93,89 +119,73 @@ def process_column_header(td): result = td.get_text(strip=True) return result - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + items = [] - for search_string in search_strings[mode]: + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', id='torrent_table') + torrent_rows = torrent_table('tr') if torrent_table else [] - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items - search_params['search'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) + # Literal: Navn, Størrelse, Kommentarer, Tilføjet, Snatches, Seeders, Leechers + # Translation: Name, Size, Comments, Added, Snatches, Seeders, Leechers + labels = [process_column_header(label) for label in torrent_rows[0]('td')] + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', id='torrent_table') - torrent_rows = torrent_table('tr') if torrent_table else [] + try: + title = row.find(class_='croptorrenttext').get_text(strip=True) + download_url = self.url + row.find(title='Direkte download link')['href'] + if not all([title, download_url]): + continue + + seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) + leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) continue - # Literal: Navn, Størrelse, Kommentarer, Tilføjet, Snatches, Seeders, Leechers - # Translation: Name, Size, Comments, Added, Snatches, Seeders, Leechers - labels = [process_column_header(label) for label in torrent_rows[0]('td')] - - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - if len(cells) < len(labels): - continue - - try: - title = result.find(class_='croptorrenttext').get_text(strip=True) - download_url = self.url + result.find(title='Direkte download link')['href'] - if not all([title, download_url]): - continue - - seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) - leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - freeleech = result.find(class_='freeleech') - if self.freeleech and not freeleech: - continue - - torrent_size = cells[labels.index('Størrelse')].contents[0] - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + freeleech = row.find(class_='freeleech') + if self.freeleech and not freeleech: + continue - return results + torrent_size = cells[labels.index('Størrelse')].contents[0] + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -187,13 +197,13 @@ def login(self): 'login': 'Login', } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) self.session.cookies.clear() return False - if 'Login :: Danishbits.org' in response: + if 'Login :: Danishbits.org' in response.text: logger.log('Invalid username or password. Check your settings', logger.WARNING) self.session.cookies.clear() return False diff --git a/sickbeard/providers/elitetorrent.py b/sickbeard/providers/torrent/html/elitetorrent.py similarity index 58% rename from sickbeard/providers/elitetorrent.py rename to sickbeard/providers/torrent/html/elitetorrent.py index 5d2a6bbef4..d72bd77144 100644 --- a/sickbeard/providers/elitetorrent.py +++ b/sickbeard/providers/torrent/html/elitetorrent.py @@ -60,9 +60,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - EliteTorrent search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -82,7 +82,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) # Only search if user conditions are true @@ -98,63 +97,76 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_string = re.sub(r'S0*(\d*)E(\d*)', r'\1x\2', search_string) search_params['buscar'] = search_string.strip() if mode != 'RSS' else '' - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', class_='fichas-listado') - torrent_rows = torrent_table('tr') if torrent_table else [] + results += self.parse(response.text, mode) - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', class_='fichas-listado') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Skip column headers + for row in torrent_rows[1:]: + try: + title = self._process_title(row.find('a', class_='nombre')['title']) + download_url = self.urls['base_url'] + row.find('a')['href'] + if not all([title, download_url]): continue - # Skip column headers - for row in torrent_rows[1:]: - try: - title = self._process_title(row.find('a', class_='nombre')['title']) - download_url = self.urls['base_url'] + row.find('a')['href'] - if not all([title, download_url]): - continue - - seeders = try_int(row.find('td', class_='semillas').get_text(strip=True)) - leechers = try_int(row.find('td', class_='clientes').get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - size = -1 # Provider does not provide size - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(row.find('td', class_='semillas').get_text(strip=True)) + leechers = try_int(row.find('td', class_='clientes').get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + size = -1 # Provider does not provide size + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items @staticmethod def _process_title(title): diff --git a/sickbeard/providers/torrent/html/extratorrent.py b/sickbeard/providers/torrent/html/extratorrent.py new file mode 100644 index 0000000000..9fcf9b02ac --- /dev/null +++ b/sickbeard/providers/torrent/html/extratorrent.py @@ -0,0 +1,181 @@ +# coding=utf-8 +# Author: Gonçalo M. (aka duramato/supergonkas) +# Author: Dustyn Gibson +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import traceback + +from requests.compat import urljoin + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class ExtraTorrentProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """ExtraTorrent Torrent provider.""" + + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'ExtraTorrent') + + # Credentials + self.public = True + + # URLs + self.url = 'http://extratorrent.cc' + self.urls = { + 'search': urljoin(self.url, 'search/'), + 'rss': urljoin(self.url, 'view/today/TV.html'), + } + self.custom_url = None + + # Proper Strings + self.proper_strings = ['PROPER', 'REPACK', 'REAL'] + + # Miscellaneous Options + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, min_time=30) # Only poll ExtraTorrent every 30 minutes max + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results. + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + + # Search Params + search_params = { + 'search': '', + 'new': 1, + 'x': 0, + 'y': 0, + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + search_params['search'] = search_string + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + search_url = self.urls['search'] + else: + search_params = None + search_url = self.urls['rss'] + + search_url = search_url if not self.custom_url else \ + search_url.replace(self.url, self.custom_url.rstrip('/')) + + response = self.get_url(search_url, params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + :return: A list of items found + """ + # RSS search has one less column + decrease = 1 if mode == 'RSS' else 0 + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', class_='tl') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 3 or (len(torrent_rows) == 3 and + torrent_rows[2].get_text() == 'No torrents'): + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Skip column headers + for result in torrent_rows[2:]: + + try: + cells = result('td') + + torrent_info = cells[0].find('a') + title = torrent_info.get('title').strip('Download torrent') + download_url = urljoin(self.url, torrent_info.get('href').replace + ('torrent_download', 'download')) + if not all([title, download_url]): + continue + + seeders = try_int(cells[4 - decrease].get_text(), 1) + leechers = try_int(cells[5 - decrease].get_text()) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[3 - decrease].get_text().replace('\xa0', ' ') + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + continue + + return items + + +provider = ExtraTorrentProvider() diff --git a/sickbeard/providers/freshontv.py b/sickbeard/providers/torrent/html/freshontv.py similarity index 58% rename from sickbeard/providers/freshontv.py rename to sickbeard/providers/torrent/html/freshontv.py index 3bdf658db0..4aebf1dca9 100644 --- a/sickbeard/providers/freshontv.py +++ b/sickbeard/providers/torrent/html/freshontv.py @@ -70,7 +70,15 @@ def __init__(self): # Cache self.cache = tvcache.TVCache(self) - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ results = [] if not self.login(): return results @@ -78,7 +86,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man freeleech = '3' if self.freeleech else '0' for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -88,19 +95,18 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man (search=search_string), logger.DEBUG) search_url = self.urls['search'] % (freeleech, search_string) - - init_html = self.get_url(search_url, returns='text') + response = self.get_url(search_url, returns='response') max_page_number = 0 - if not init_html: + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(init_html, 'html5lib') as init_soup: + with BS4Parser(response.text, 'html5lib') as html: try: # Check to see if there is more than 1 page of results - pager = init_soup.find('div', {'class': 'pager'}) + pager = html.find('div', {'class': 'pager'}) if pager: page_links = pager('a', href=True) else: @@ -125,86 +131,97 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man (traceback.format_exc()), logger.ERROR) continue - data_response_list = [init_html] + responses = [response] # Freshon starts counting pages from zero, even though it displays numbers from 1 if max_page_number > 1: - for i in range(1, max_page_number): - + for x in range(1, max_page_number): time.sleep(1) - page_search_url = search_url + '&page=' + text_type(i) - # '.log('Search string: ' + page_search_url, logger.DEBUG) - page_html = self.get_url(page_search_url, returns='text') + page_search_url = '{url}&page={x}'.format(url=search_url, x=x) + page = self.get_url(page_search_url, returns='response') - if not page_html: + if not page: continue - data_response_list.append(page_html) + responses.append(page) - for data_response in data_response_list: + for response in responses: + results += self.parse(response.text, mode) - with BS4Parser(data_response, 'html5lib') as html: - torrent_rows = html('tr', class_=re.compile('torrent_[0-9]*')) + return results - # Continue only if a Release is found - if not torrent_rows: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue + def parse(self, data, mode): + """ + Parse search results for items. - for individual_torrent in torrent_rows: - - try: - # skip if torrent has been nuked due to poor quality - if individual_torrent.find('img', alt='Nuked') is not None: - continue - - title = individual_torrent.find('a', class_='torrent_name_link')['title'] - details_url = individual_torrent.find('a', class_='torrent_name_link')['href'] - torrent_id = int((re.match('.*?([0-9]+)$', details_url).group(1)).strip()) - download_url = self.urls['download'] % (text_type(torrent_id)) - if not all([title, download_url]): - continue - - seeders = try_int(individual_torrent.find('td', class_='table_seeders').find('span').get_text(strip=True), 1) - leechers = try_int(individual_torrent.find('td', class_='table_leechers').find('a').get_text(strip=True), 0) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = individual_torrent.find('td', class_='table_size').get_text(strip=True) - torrent_size = re.split('(\d+.?\d+)', text_type(torrent_size), 1) - torrent_size = '{0} {1}'.format(torrent_size[1], torrent_size[2]) - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS - return results + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_rows = html('tr', class_=re.compile('torrent_[0-9]*')) + + # Continue only if at least one release is found + if not torrent_rows: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for row in torrent_rows: + + try: + # skip if torrent has been nuked due to poor quality + if row.find('img', alt='Nuked') is not None: + continue + + title = row.find('a', class_='torrent_name_link')['title'] + details_url = row.find('a', class_='torrent_name_link')['href'] + torrent_id = int((re.match('.*?([0-9]+)$', details_url).group(1)).strip()) + download_url = self.urls['download'] % (text_type(torrent_id)) + if not all([title, download_url]): + continue + + seeders = try_int(row.find('td', class_='table_seeders').find('span').get_text(strip=True), 1) + leechers = try_int(row.find('td', class_='table_leechers').find('a').get_text(strip=True), 0) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = row.find('td', class_='table_size').get_text(strip=True) + torrent_size = re.split('(\d+.?\d+)', text_type(torrent_size), 1) + torrent_size = '{0} {1}'.format(torrent_size[1], torrent_size[2]) + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -216,12 +233,12 @@ def login(self): if self._uid and self._hash: add_dict_to_cookiejar(self.session.cookies, self.cookies) else: - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('/logout.php', response): + if re.search('/logout.php', response.text): try: if dict_from_cookiejar(self.session.cookies)['uid'] and \ dict_from_cookiejar(self.session.cookies)['pass']: @@ -236,12 +253,12 @@ def login(self): return False else: - if re.search('Username does not exist in the userbase or the account is not confirmed yet.', response) or \ + if re.search('Username does not exist in the userbase or the account is not confirmed yet.', response.text) or \ re.search('Username or password is incorrect. If you have an account here please use the' - ' recovery system or try again.', response): + ' recovery system or try again.', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) - if re.search('DDoS protection by CloudFlare', response): + if re.search('DDoS protection by CloudFlare', response.text): logger.log('Unable to login to provider due to CloudFlare DDoS javascript check', logger.WARNING) return False diff --git a/sickbeard/providers/gftracker.py b/sickbeard/providers/torrent/html/gftracker.py similarity index 55% rename from sickbeard/providers/gftracker.py rename to sickbeard/providers/torrent/html/gftracker.py index ce4216eccd..126211ad59 100644 --- a/sickbeard/providers/gftracker.py +++ b/sickbeard/providers/torrent/html/gftracker.py @@ -65,9 +65,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - GFT search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -89,6 +89,34 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'search': '', } + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['search'] = search_string + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ # Units units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] @@ -100,85 +128,71 @@ def process_column_header(td): result = td.get_text(strip=True) return result - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + items = [] - for search_string in search_strings[mode]: + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('div', id='torrentBrowse') + torrent_rows = torrent_table('tr') if torrent_table else [] - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items - search_params['search'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) + labels = [process_column_header(label) for label in torrent_rows[0]('td')] + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('div', id='torrentBrowse') - torrent_rows = torrent_table('tr') if torrent_table else [] + try: - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + title_anchor = cells[labels.index('Name')].find('a').find_next('a') or \ + cells[labels.index('Name')].find('a') + title = title_anchor.get('title') if title_anchor else None + download_url = urljoin(self.url, cells[labels.index('DL')].find('a')['href']) + if not all([title, download_url]): continue - labels = [process_column_header(label) for label in torrent_rows[0]('td')] - - # Skip column headers - for result in torrent_rows[1:]: - - try: - cells = result('td') - - title_anchor = cells[labels.index('Name')].find('a').find_next('a') or \ - cells[labels.index('Name')].find('a') - title = title_anchor.get('title') if title_anchor else None - download_url = urljoin(self.url, cells[labels.index('DL')].find('a')['href']) - if not all([title, download_url]): - continue - - peers = cells[labels.index('S/L')].get_text(strip=True).split('/', 1) - seeders = try_int(peers[0]) - leechers = try_int(peers[1]) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[labels.index('Size/Snatched')].get_text(strip=True).split('/', 1)[0] - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + peers = cells[labels.index('S/L')].get_text(strip=True).split('/', 1) + seeders = try_int(peers[0]) + leechers = try_int(peers[1]) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[labels.index('Size/Snatched')].get_text(strip=True).split('/', 1)[0] + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -188,13 +202,13 @@ def login(self): } # Initialize session with a GET to have cookies - self.get_url(self.url, returns='text') - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + self.get_url(self.url, returns='response') + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('Username or password incorrect', response): + if re.search('Username or password incorrect', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/hdspace.py b/sickbeard/providers/torrent/html/hdspace.py similarity index 58% rename from sickbeard/providers/hdspace.py rename to sickbeard/providers/torrent/html/hdspace.py index 7cdbc96b4d..3d1eb302a9 100644 --- a/sickbeard/providers/hdspace.py +++ b/sickbeard/providers/torrent/html/hdspace.py @@ -64,9 +64,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - HDSpace search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -85,7 +85,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -96,7 +95,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_params['search'] = search_string response = self.get_url(self.urls['search'], params=search_params, returns='response') - if not response or not response.text or 'please try later' in response.text: + if not response.text or 'please try later' in response.text: logger.log('No data returned from provider', logger.DEBUG) continue @@ -106,70 +105,84 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man try: index = response.text.index('
. + +from __future__ import unicode_literals + +import re +import traceback + +from requests.compat import urljoin +from requests.utils import dict_from_cookiejar + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class HoundDawgsProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """HoundDawgs Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'HoundDawgs') + + # Credentials + self.username = None + self.password = None + + # URLs + self.url = 'https://hounddawgs.org' + self.urls = { + 'base_url': self.url, + 'search': urljoin(self.url, 'torrents.php'), + 'login': urljoin(self.url, 'login.php'), + } + + # Proper Strings + + # Miscellaneous Options + self.freeleech = None + self.ranked = None + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + if not self.login(): + return results + + # Search Params + search_params = { + 'filter_cat[85]': 1, + 'filter_cat[58]': 1, + 'filter_cat[57]': 1, + 'filter_cat[74]': 1, + 'filter_cat[92]': 1, + 'filter_cat[93]': 1, + 'order_by': 's3', + 'order_way': 'desc', + 'type': '', + 'userid': '', + 'searchstr': '', + 'searchimdb': '', + 'searchtags': '' + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['searchstr'] = search_string + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + str_table_start = "
Login :: HoundDawgs', response.text), + re.search('Dine cookies er ikke aktiveret.', response.text)], ): + logger.log('Invalid username or password. Check your settings', logger.WARNING) + return False + + return True + + +provider = HoundDawgsProvider() diff --git a/sickbeard/providers/ilovetorrents.py b/sickbeard/providers/torrent/html/ilovetorrents.py similarity index 52% rename from sickbeard/providers/ilovetorrents.py rename to sickbeard/providers/torrent/html/ilovetorrents.py index 59f14d5a52..d715ebbbf6 100644 --- a/sickbeard/providers/ilovetorrents.py +++ b/sickbeard/providers/torrent/html/ilovetorrents.py @@ -1,8 +1,6 @@ # coding=utf-8 # Author: Gonçalo M. (aka duramato/supergonkas) # -# URL: https://sickrage.github.io -# # This file is part of Medusa. # # Medusa is free software: you can redistribute it and/or modify @@ -66,9 +64,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - ILoveTorrents search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -83,84 +81,98 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: + if mode != 'RSS': logger.log('Search string: {search}'.format (search=search_string), logger.DEBUG) search_params['search'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(data, 'html.parser') as html: - torrent_table = html.find('table', class_='koptekst') - torrent_rows = torrent_table('tr') if torrent_table else [] + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + with BS4Parser(data, 'html.parser') as html: + torrent_table = html.find('table', class_='koptekst') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < 11: + continue + + try: + link = cells[1].find('a') + title = link.getText() + download_url = self.urls['download'].format(link=cells[2].find('a')['href']) + if not all([title, download_url]): continue - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - if len(cells) < 11: - continue - - try: - link = cells[1].find('a') - title = link.getText() - download_url = self.urls['download'].format(link=cells[2].find('a')['href']) - if not all([title, download_url]): - continue - - seeders = int(cells[10].getText().replace(',', '')) - leechers = int(cells[11].getText().replace(',', '')) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - # Use same failsafe as Bitsoup - if seeders >= 32768 or leechers >= 32768: - continue - - torrent_size = cells[8].getText() - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = int(cells[10].getText().replace(',', '')) + leechers = int(cells[11].getText().replace(',', '')) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + # Use same failsafe as Bitsoup + if seeders >= 32768 or leechers >= 32768: + continue + + torrent_size = cells[8].getText() + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -171,12 +183,12 @@ def login(self): 'submit': 'Welcome to ILT' } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('Username or password incorrect', response): + if re.search('Username or password incorrect', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/iptorrents.py b/sickbeard/providers/torrent/html/iptorrents.py similarity index 55% rename from sickbeard/providers/iptorrents.py rename to sickbeard/providers/torrent/html/iptorrents.py index e578078cf9..4108ce688a 100644 --- a/sickbeard/providers/iptorrents.py +++ b/sickbeard/providers/torrent/html/iptorrents.py @@ -64,7 +64,15 @@ def __init__(self): # Cache self.cache = tvcache.TVCache(self, min_time=10) # Only poll IPTorrents every 10 minutes max - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ results = [] if not self.login(): return results @@ -72,7 +80,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man freeleech = '&free=on' if self.freeleech else '' for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -85,67 +92,82 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_url = self.urls['search'] % (self.categories, freeleech, search_string) search_url += ';o=seeders' if mode != 'RSS' else '' - data = self.get_url(search_url, returns='text') - if not data: + response = self.get_url(search_url, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - data = re.sub(r'(?im)', '', data, 0) - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', attrs={'class': 'torrents'}) - torrents = torrent_table('tr') if torrent_table else [] + data = re.sub(r'(?im)', '', response.text, 0) - # Continue only if at least one release is found - if len(torrents) < 2 or html.find(text='No Torrents Found!'): - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + results += self.parse(data, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', attrs={'class': 'torrents'}) + torrents = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrents) < 2 or html.find(text='No Torrents Found!'): + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Skip column headers + for row in torrents[1:]: + try: + title = row('td')[1].find('a').text + download_url = self.urls['base_url'] + row('td')[3].find('a')['href'] + if not all([title, download_url]): continue - # Skip column headers - for result in torrents[1:]: - try: - title = result('td')[1].find('a').text - download_url = self.urls['base_url'] + result('td')[3].find('a')['href'] - if not all([title, download_url]): - continue - - seeders = int(result.find('td', attrs={'class': 'ac t_seeders'}).text) - leechers = int(result.find('td', attrs={'class': 'ac t_leechers'}).text) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = result('td')[5].text - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = int(row.find('td', attrs={'class': 'ac t_seeders'}).text) + leechers = int(row.find('td', attrs={'class': 'ac t_leechers'}).text) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = row('td')[5].text + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -155,19 +177,20 @@ def login(self): 'login': 'submit', } - self.get_url(self.urls['login'], returns='text') - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + # Initialize session with a GET to have cookies + self.get_url(self.urls['login'], returns='response') + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False # Invalid username and password combination - if re.search('Invalid username and password combination', response): + if re.search('Invalid username and password combination', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False # You tried too often, please try again after 2 hours! - if re.search('You tried too often', response): + if re.search('You tried too often', response.text): logger.log('You tried too often, please try again after 2 hours!' ' Disable IPTorrents for at least 2 hours', logger.WARNING) return False diff --git a/sickbeard/providers/limetorrents.py b/sickbeard/providers/torrent/html/limetorrents.py similarity index 88% rename from sickbeard/providers/limetorrents.py rename to sickbeard/providers/torrent/html/limetorrents.py index 97f0b83ace..63268b6350 100644 --- a/sickbeard/providers/limetorrents.py +++ b/sickbeard/providers/torrent/html/limetorrents.py @@ -40,7 +40,7 @@ class LimeTorrentsProvider(TorrentProvider): # pylint: disable=too-many-instanc """LimeTorrents Torrent provider""" def __init__(self): - # Provider Inits + # Provider Init TorrentProvider.__init__(self, 'LimeTorrents') # Credentials @@ -68,11 +68,11 @@ def __init__(self): # Cache self.cache = tvcache.TVCache(self, min_time=10) - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches,too-many-locals + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - ABNormal search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -83,20 +83,20 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: - if mode == 'RSS': - search_url = self.urls['rss'].format(page=1) - else: - logger.log("Search string: {0}".format(search_string), logger.DEBUG) + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) search_url = self.urls['search'].format(query=search_string) + else: + search_url = self.urls['rss'].format(page=1) - data = self.get_url(search_url, returns='text') - if not data: + response = self.get_url(search_url, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - items = self.parse(data, mode) - if items: - results += items + results += self.parse(response.text, mode) return results @@ -116,21 +116,21 @@ def parse(self, data, mode): if mode != 'RSS' and torrent_table and len(torrent_table) < 2: logger.log(u'Data returned from provider does not contain any torrents', logger.DEBUG) - return + return items torrent_table = torrent_table[0 if mode == 'RSS' else 1] torrent_rows = torrent_table('tr') # Skip the first row, since it isn't a valid result - for result in torrent_rows[1:]: - cells = result('td') + for row in torrent_rows[1:]: + cells = row('td') try: - verified = result('img', title='Verified torrent') + verified = row('img', title='Verified torrent') if self.confirmed and not verified: continue - url = result.find('a', rel='nofollow') - title_info = result('a') + url = row.find('a', rel='nofollow') + title_info = row('a') info = title_info[1]['href'] if not all([url, title_info, info]): continue @@ -178,7 +178,6 @@ def parse(self, data, mode): except (AttributeError, TypeError, KeyError, ValueError, IndexError): logger.log('Failed parsing provider. Traceback: {0!r}'.format (traceback.format_exc()), logger.ERROR) - continue return items diff --git a/sickbeard/providers/morethantv.py b/sickbeard/providers/torrent/html/morethantv.py similarity index 55% rename from sickbeard/providers/morethantv.py rename to sickbeard/providers/torrent/html/morethantv.py index 21c75ccabb..49c16ba2e5 100644 --- a/sickbeard/providers/morethantv.py +++ b/sickbeard/providers/torrent/html/morethantv.py @@ -66,9 +66,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - MoreThanTV search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -87,6 +87,35 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'searchstr': '', } + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['searchstr'] = search_string + + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ # Units units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] @@ -98,88 +127,71 @@ def process_column_header(td): result = td.get_text(strip=True) return result - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + items = [] - for search_string in search_strings[mode]: + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', class_='torrent_table') + torrent_rows = torrent_table('tr') if torrent_table else [] - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items - search_params['searchstr'] = search_string + labels = [process_column_header(label) for label in torrent_rows[0]('td')] - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', class_='torrent_table') - torrent_rows = torrent_table('tr') if torrent_table else [] + try: + # skip if torrent has been nuked due to poor quality + if row.find('img', alt='Nuked'): + continue - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + title = row.find('a', title='View torrent').get_text(strip=True) + download_url = urljoin(self.url, row.find('span', title='Download').parent['href']) + if not all([title, download_url]): continue - labels = [process_column_header(label) for label in torrent_rows[0]('td')] - - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - if len(cells) < len(labels): - continue - - try: - # skip if torrent has been nuked due to poor quality - if result.find('img', alt='Nuked'): - continue - - title = result.find('a', title='View torrent').get_text(strip=True) - download_url = urljoin(self.url, result.find('span', title='Download').parent['href']) - if not all([title, download_url]): - continue - - seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) - leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[labels.index('Size')].get_text(strip=True) - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) + leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[labels.index('Size')].get_text(strip=True) + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -190,12 +202,12 @@ def login(self): 'login': 'Log in', } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('Your username or password was incorrect.', response): + if re.search('Your username or password was incorrect.', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/newpct.py b/sickbeard/providers/torrent/html/newpct.py similarity index 70% rename from sickbeard/providers/newpct.py rename to sickbeard/providers/torrent/html/newpct.py index fd8a099ef0..b99cf9c353 100644 --- a/sickbeard/providers/newpct.py +++ b/sickbeard/providers/torrent/html/newpct.py @@ -59,14 +59,13 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - Newpct search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) """ - results = [] # Only search if user conditions are true @@ -83,7 +82,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) # Only search if user conditions are true @@ -100,64 +98,75 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man (search=search_string), logger.DEBUG) search_params['q'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', id='categoryTable') - torrent_rows = torrent_table('tr') if torrent_table else [] + results += self.parse(response.text, mode) - # Continue only if at least one release is found - if len(torrent_rows) < 3: # Headers + 1 Torrent + Pagination - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue + return results + + def parse(self, data, mode): + """ + Parse search results for items. - # 'Fecha', 'Título', 'Tamaño', '' - # Date, Title, Size - labels = [label.get_text(strip=True) for label in torrent_rows[0]('th')] - - # Skip column headers - for row in torrent_rows[1:-1]: - cells = row('td') - if len(cells) < len(labels): - continue - - try: - torrent_row = row.find('a') - title = self._process_title(torrent_row.get('title', '')) - download_url = torrent_row.get('href', '') - if not all([title, download_url]): - continue - - seeders = 1 # Provider does not provide seeders - leechers = 0 # Provider does not provide leechers - torrent_size = cells[labels.index('Tamaño')].get_text(strip=True) - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS - return results + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', id='categoryTable') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 3: # Headers + 1 Torrent + Pagination + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # 'Fecha', 'Título', 'Tamaño', '' + # Date, Title, Size + labels = [label.get_text(strip=True) for label in torrent_rows[0]('th')] + + # Skip column headers + for row in torrent_rows[1:-1]: + cells = row('td') + + try: + torrent_row = row.find('a') + title = self._process_title(torrent_row.get('title', '')) + download_url = torrent_row.get('href', '') + if not all([title, download_url]): + continue + + seeders = 1 # Provider does not provide seeders + leechers = 0 # Provider does not provide leechers + torrent_size = cells[labels.index('Tamaño')].get_text(strip=True) + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items @staticmethod def _process_title(title): @@ -215,8 +224,8 @@ def download_result(self, result): for url in urls: # Search results don't return torrent files directly, it returns show sheets so we must parse showSheet to access torrent. - data = self.get_url(url, returns='text') - url_torrent = re.search(r'http://tumejorserie.com/descargar/.+\.torrent', data, re.DOTALL).group() + response = self.get_url(url, returns='response') + url_torrent = re.search(r'http://tumejorserie.com/descargar/.+\.torrent', response.text, re.DOTALL).group() if url_torrent.startswith('http'): self.headers.update({'Referer': '/'.join(url_torrent.split('/')[:3]) + '/'}) diff --git a/sickbeard/providers/pretome.py b/sickbeard/providers/torrent/html/pretome.py similarity index 51% rename from sickbeard/providers/pretome.py rename to sickbeard/providers/torrent/html/pretome.py index dbcda2881d..fffcde109c 100644 --- a/sickbeard/providers/pretome.py +++ b/sickbeard/providers/torrent/html/pretome.py @@ -69,9 +69,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - Pretome search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -81,7 +81,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man return results for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -91,82 +90,96 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man (search=search_string), logger.DEBUG) search_url = self.urls['search'] % (quote(search_string), self.categories) - data = self.get_url(search_url, returns='text') - if not data: + response = self.get_url(search_url, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(data, 'html5lib') as html: - # Continue only if one Release is found - empty = html.find('h2', text='No .torrents fit this filter criteria') - if empty: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] - torrent_table = html.find('table', attrs={'style': 'border: none; width: 100%;'}) - if not torrent_table: - logger.log('Could not find table of torrents', logger.ERROR) + with BS4Parser(data, 'html5lib') as html: + # Continue only if one Release is found + empty = html.find('h2', text='No .torrents fit this filter criteria') + if empty: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + torrent_table = html.find('table', attrs={'style': 'border: none; width: 100%;'}) + if not torrent_table: + logger.log('Could not find table of torrents', logger.ERROR) + return items + + torrent_rows = torrent_table('tr', attrs={'class': 'browse'}) + + for row in torrent_rows: + cells = row('td') + try: + size = None + link = cells[1].find('a', attrs={'style': 'font-size: 1.25em; font-weight: bold;'}) + + torrent_id = link['href'].replace('details.php?id=', '') + + if link.get('title', ''): + title = link['title'] + else: + title = link.contents[0] + + download_url = self.urls['download'] % (torrent_id, link.contents[0]) + if not all([title, download_url]): continue - torrent_rows = torrent_table('tr', attrs={'class': 'browse'}) - - for result in torrent_rows: - cells = result('td') - try: - size = None - link = cells[1].find('a', attrs={'style': 'font-size: 1.25em; font-weight: bold;'}) - - torrent_id = link['href'].replace('details.php?id=', '') - - if link.get('title', ''): - title = link['title'] - else: - title = link.contents[0] - - download_url = self.urls['download'] % (torrent_id, link.contents[0]) - if not all([title, download_url]): - continue - - seeders = int(cells[9].contents[0]) - leechers = int(cells[10].contents[0]) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - # Need size for failed downloads handling - if size is None: - torrent_size = cells[7].text - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = int(cells[9].contents[0]) + leechers = int(cells[10].contents[0]) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + # Need size for failed downloads handling + if size is None: + torrent_size = cells[7].text + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -176,12 +189,12 @@ def login(self): 'login_pin': self.pin, } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('Username or password incorrect', response): + if re.search('Username or password incorrect', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/torrent/html/scc.py b/sickbeard/providers/torrent/html/scc.py new file mode 100644 index 0000000000..4b610e8cf2 --- /dev/null +++ b/sickbeard/providers/torrent/html/scc.py @@ -0,0 +1,203 @@ +# coding=utf-8 +# Author: Idan Gutman +# Modified by jkaberg, https://github.com/jkaberg for SceneAccess +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from requests.compat import urljoin, quote +from requests.utils import dict_from_cookiejar + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class SCCProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'SceneAccess') + + # Credentials + self.username = None + self.password = None + + # URLs + self.url = 'https://sceneaccess.eu' + self.urls = { + 'base_url': self.url, + 'login': urljoin(self.url, 'login'), + 'detail': urljoin(self.url, 'details?id=%s'), + 'search': urljoin(self.url, 'all?search=%s&method=1&%s'), + 'download': urljoin(self.url, '%s') + } + + # Proper Strings + + # Miscellaneous Options + self.categories = { + 'Season': 'c26=26&c44=44&c45=45', # Archive, non-scene HD, non-scene SD; need to include non-scene because WEB-DL packs get added to those categories + 'Episode': 'c17=17&c27=27&c33=33&c34=34&c44=44&c45=45', # TV HD, TV SD, non-scene HD, non-scene SD, foreign XviD, foreign x264 + 'RSS': 'c17=17&c26=26&c27=27&c33=33&c34=34&c44=44&c45=45' # Season + Episode + } + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self) # only poll SCC every 20 minutes max + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + if not self.login(): + return results + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_url = self.urls['search'] % (quote(search_string), self.categories[mode]) + response = self.get_url(search_url, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', attrs={'id': 'torrents-table'}) + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for row in torrent_rows[1:]: + try: + link = row.find('td', attrs={'class': 'ttr_name'}).find('a') + url = row.find('td', attrs={'class': 'td_dl'}).find('a') + + title = link.string + if re.search(r'\.\.\.', title): + response = self.get_url(urljoin(self.url, link['href']), returns='response') + if response.text: + with BS4Parser(response.text) as details_html: + title = re.search("(?<=').+(?SceneAccess \| Login', response.text), ]): + logger.log('Invalid username or password. Check your settings', logger.WARNING) + return False + + return True + + @staticmethod + def _is_section(section, text): + title = r'.+? \| %s' % section + return re.search(title, text, re.IGNORECASE) + + +provider = SCCProvider() diff --git a/sickbeard/providers/scenetime.py b/sickbeard/providers/torrent/html/scenetime.py similarity index 50% rename from sickbeard/providers/scenetime.py rename to sickbeard/providers/torrent/html/scenetime.py index d35ed61d6a..7678046d73 100644 --- a/sickbeard/providers/scenetime.py +++ b/sickbeard/providers/torrent/html/scenetime.py @@ -63,9 +63,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - SceneTime search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -88,7 +88,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -99,78 +98,92 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_params['search'] = search_string response = self.get_url(self.urls['search'], params=search_params, returns='response') - if not response or not response.text: + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(response.text, 'html5lib') as html: - torrent_table = html.find('div', id='torrenttable') - torrent_rows = [] - if torrent_table: - torrent_rows = torrent_table.select('tr') + results += self.parse(response.text, mode) - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('div', id='torrenttable') + torrent_rows = [] + if torrent_table: + torrent_rows = torrent_table.select('tr') + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Scenetime apparently uses different number of cells in #torrenttable based + # on who you are. This works around that by extracting labels from the first + # and using their index to find the correct download/seeders/leechers td. + labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')] + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): + continue + + try: + link = cells[labels.index('Name')].find('a') + torrent_id = link['href'].replace('details.php?id=', '').split('&')[0] + title = link.get_text(strip=True) + download_url = self.urls['download'].format(torrent_id, '{0}.torrent'.format(title.replace(' ', '.'))) + if not all([title, download_url]): continue - # Scenetime apparently uses different number of cells in #torrenttable based - # on who you are. This works around that by extracting labels from the first - # and using their index to find the correct download/seeders/leechers td. - labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')] - - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - if len(cells) < len(labels): - continue - - try: - link = cells[labels.index('Name')].find('a') - torrent_id = link['href'].replace('details.php?id=', '').split('&')[0] - title = link.get_text(strip=True) - download_url = self.urls['download'].format(torrent_id, '{0}.torrent'.format(title.replace(' ', '.'))) - if not all([title, download_url]): - continue - - seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) - leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[labels.index('Size')].get_text() - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) + leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[labels.index('Size')].get_text() + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -179,12 +192,12 @@ def login(self): 'password': self.password, } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('Username or password incorrect', response): + if re.search('Username or password incorrect', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/speedcd.py b/sickbeard/providers/torrent/html/speedcd.py similarity index 52% rename from sickbeard/providers/speedcd.py rename to sickbeard/providers/torrent/html/speedcd.py index 3d85e9e878..9ae04fd8ee 100644 --- a/sickbeard/providers/speedcd.py +++ b/sickbeard/providers/torrent/html/speedcd.py @@ -63,6 +63,14 @@ def __init__(self): self.cache = tvcache.TVCache(self) def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ results = [] if not self.login(): return results @@ -78,8 +86,37 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'c52': 1, # TV/B-Ray 'c55': 1, # TV/Kids 'search': '', + 'freeleech': 'on' if self.freeleech else None } + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['search'] = search_string + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ # Units units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] @@ -93,88 +130,69 @@ def process_column_header(td): result = td.get_text(strip=True) return result - if self.freeleech: - search_params['freeleech'] = 'on' + items = [] - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('div', class_='boxContent') + torrent_table = torrent_table.find('table') if torrent_table else None + torrent_rows = torrent_table('tr') if torrent_table else [] - for search_string in search_strings[mode]: + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) + labels = [process_column_header(label) for label in torrent_rows[0]('th')] - search_params['search'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('div', class_='boxContent') - torrent_table = torrent_table.find('table') if torrent_table else None - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + try: + title = cells[labels.index('Title')].find('a', class_='torrent').get_text() + download_url = urljoin(self.url, cells[labels.index('Download')].find(title='Download').parent['href']) + if not all([title, download_url]): continue - labels = [process_column_header(label) for label in torrent_rows[0]('th')] - - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - if len(cells) < len(labels): - continue - - try: - title = cells[labels.index('Title')].find('a', class_='torrent').get_text() - download_url = urljoin(self.url, cells[labels.index('Download')].find(title='Download').parent['href']) - if not all([title, download_url]): - continue - - seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) - leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[labels.index('Size')].get_text() - torrent_size = torrent_size[:-2] + ' ' + torrent_size[-2:] - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) + leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[labels.index('Size')].get_text() + torrent_size = torrent_size[:-2] + ' ' + torrent_size[-2:] + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -183,12 +201,12 @@ def login(self): 'password': self.password, } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('Incorrect username or Password. Please try again.', response): + if re.search('Incorrect username or Password. Please try again.', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/torrent/html/thepiratebay.py b/sickbeard/providers/torrent/html/thepiratebay.py new file mode 100644 index 0000000000..885cd8fff6 --- /dev/null +++ b/sickbeard/providers/torrent/html/thepiratebay.py @@ -0,0 +1,208 @@ +# coding=utf-8 +# Author: Dustyn Gibson +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback +import validators + +from requests.compat import urljoin + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class ThePirateBayProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """ThePirateBay Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'ThePirateBay') + + # Credentials + self.public = True + + # URLs + self.url = 'https://thepiratebay.org' + self.urls = { + 'rss': urljoin(self.url, 'tv/latest'), + 'search': urljoin(self.url, 's/'), # Needs trailing / + } + self.custom_url = None + + # Proper Strings + + # Miscellaneous Options + self.confirmed = True + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, min_time=20) # only poll ThePirateBay every 20 minutes max + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + + # 205 = SD, 208 = HD, 200 = All Videos + # https://pirateproxy.pl/s/?q=Game of Thrones&type=search&orderby=7&page=0&category=200 + + search_params = { + 'q': '', + 'type': 'search', + 'orderby': 7, + 'page': 0, + 'category': 200 + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + search_url = self.urls['search'] if mode != 'RSS' else self.urls['rss'] + if self.custom_url: + if not validators.url(self.custom_url): + logger.log('Invalid custom url: {0}'.format(self.custom_url), logger.WARNING) + return results + search_url = urljoin(self.custom_url, search_url.split(self.url)[1]) + + if mode != 'RSS': + search_params['q'] = search_string + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + else: + search_params = {} + + response = self.get_url(search_url, params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + # Units + units = ['B', 'KIB', 'MIB', 'GIB', 'TIB', 'PIB'] + + def process_column_header(th): + result = '' + if th.a: + result = th.a.get_text(strip=True) + if not result: + result = th.get_text(strip=True) + return result + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', id='searchResult') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + labels = [process_column_header(label) for label in torrent_rows[0]('th')] + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): + continue + + try: + title = row.find(class_='detName') + title = title.get_text(strip=True) if title else None + download_url = row.find(title='Download this torrent using magnet') + download_url = download_url['href'] + self._custom_trackers if download_url else None + if download_url and 'magnet:?' not in download_url: + logger.log('Invalid ThePirateBay proxy please try another one', logger.DEBUG) + continue + if not all([title, download_url]): + continue + + seeders = try_int(cells[labels.index('SE')].get_text(strip=True)) + leechers = try_int(cells[labels.index('LE')].get_text(strip=True)) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + # Accept Torrent only from Good People for every Episode Search + if self.confirmed and not row.find(alt=re.compile(r'VIP|Trusted')): + if mode != 'RSS': + logger.log("Found result {0} but that doesn't seem like a trusted" + " result so I'm ignoring it".format(title), logger.DEBUG) + continue + + # Convert size after all possible skip scenarios + torrent_size = cells[labels.index('Name')].find(class_='detDesc').get_text(strip=True).split(', ')[1] + torrent_size = re.sub(r'Size ([\d.]+).+([KMGT]iB)', r'\1 \2', torrent_size) + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + +provider = ThePirateBayProvider() diff --git a/sickbeard/providers/tntvillage.py b/sickbeard/providers/torrent/html/tntvillage.py similarity index 61% rename from sickbeard/providers/tntvillage.py rename to sickbeard/providers/torrent/html/tntvillage.py index 973bd0ac37..7630cb776f 100644 --- a/sickbeard/providers/tntvillage.py +++ b/sickbeard/providers/torrent/html/tntvillage.py @@ -71,9 +71,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - TNTVillage search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -82,6 +82,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man if not self.login(): return results + # Search Params search_params = { 'act': 'allreleases', 'filter': 'eng ' if self.engrelease else '', @@ -89,7 +90,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -101,82 +101,97 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_params['cat'] = None response = self.get_url(self.url, params=search_params, returns='response') - if not response or not response.text: + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(response.text, 'html5lib') as html: - torrent_table = html.find('table', class_='copyright') - torrent_rows = torrent_table('tr') if torrent_table else [] + results += self.parse(response.text, mode) - # Continue only if at least one release is found - if len(torrent_rows) < 3: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', class_='copyright') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 3: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Skip column headers + for row in torrent_table('tr')[1:]: + cells = row('td') + + try: + if not cells: continue - # Skip column headers - for result in torrent_table('tr')[1:]: - try: - cells = result('td') - if not cells: - continue - - last_cell_anchor = cells[-1].find('a') - if not last_cell_anchor: - continue - params = parse_qs(last_cell_anchor.get('href', '')) - download_url = self.urls['download'].format(params['pid'][0]) if \ - params.get('pid') else None - title = self._process_title(cells[0], cells[1], mode) - if not all([title, download_url]): - continue - - info_cell = cells[3].find_all('td') - leechers = info_cell[0].find('span').get_text(strip=True) - leechers = try_int(leechers) - seeders = info_cell[1].find('span').get_text() - seeders = try_int(seeders, 1) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - if self._has_only_subs(title) and not self.subtitle: - logger.log('Torrent is only subtitled, skipping: {0}'.format - (title), logger.DEBUG) - continue - - torrent_size = info_cell[3].find('span').get_text() + ' GB' - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + last_cell_anchor = cells[-1].find('a') + if not last_cell_anchor: + continue + params = parse_qs(last_cell_anchor.get('href', '')) + download_url = self.urls['download'].format(params['pid'][0]) if \ + params.get('pid') else None + title = self._process_title(cells[0], cells[1], mode) + if not all([title, download_url]): + continue - return results + info_cell = cells[3]('td') + leechers = info_cell[0].find('span').get_text(strip=True) + leechers = try_int(leechers) + seeders = info_cell[1].find('span').get_text() + seeders = try_int(seeders, 1) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + if self._has_only_subs(title) and not self.subtitle: + logger.log('Torrent is only subtitled, skipping: {0}'.format + (title), logger.DEBUG) + continue + + torrent_size = info_cell[3].find('span').get_text() + ' GB' + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if len(self.session.cookies) > 1: cookies_dict = dict_from_cookiejar(self.session.cookies) if cookies_dict['pass_hash'] != '0' and cookies_dict['member_id'] != '0': @@ -189,13 +204,13 @@ def login(self): 'submit': 'Connettiti al Forum', } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('Sono stati riscontrati i seguenti errori', response) or \ - re.search('Connettiti', response): + if any([re.search('Sono stati riscontrati i seguenti errori', response.text), + re.search('Connettiti', response.text), ]): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/torrent/html/tokyotoshokan.py b/sickbeard/providers/torrent/html/tokyotoshokan.py new file mode 100644 index 0000000000..cd92d5a4b2 --- /dev/null +++ b/sickbeard/providers/torrent/html/tokyotoshokan.py @@ -0,0 +1,168 @@ +# coding=utf-8 +# Author: Mr_Orange +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from requests.compat import urljoin + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class TokyoToshokanProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """TokyoToshokan Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'TokyoToshokan') + + # Credentials + self.public = True + + # URLs + self.url = 'http://tokyotosho.info/' + self.urls = { + 'search': urljoin(self.url, 'search.php'), + 'rss': urljoin(self.url, 'rss.php'), + } + + # Proper Strings + + # Miscellaneous Options + self.supports_absolute_numbering = True + self.anime_only = True + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, min_time=15) # only poll TokyoToshokan every 15 minutes max + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + if self.show and not self.show.is_anime: + return results + + # Search Params + search_params = { + 'type': 1, # get anime types + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['terms'] = search_string + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as soup: + torrent_table = soup.find('table', class_='listing') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + a = 1 if len(torrent_rows[0]('td')) < 2 else 0 + + # Skip column headers + for top, bot in zip(torrent_rows[a::2], torrent_rows[a + 1::2]): + try: + desc_top = top.find('td', class_='desc-top') + title = desc_top.get_text(strip=True) if desc_top else None + download_url = desc_top.find('a')['href'] if desc_top else None + if not all([title, download_url]): + continue + + stats = bot.find('td', class_='stats').get_text(strip=True) + sl = re.match(r'S:(?P\d+)L:(?P\d+)C:(?:\d+)ID:(?:\d+)', stats.replace(' ', '')) + seeders = try_int(sl.group('seeders')) if sl else 0 + leechers = try_int(sl.group('leechers')) if sl else 0 + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + desc_bottom = bot.find('td', class_='desc-bot').get_text(strip=True) + size = convert_size(desc_bottom.split('|')[1].strip('Size: ')) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + +provider = TokyoToshokanProvider() diff --git a/sickbeard/providers/torrent/html/torrentbytes.py b/sickbeard/providers/torrent/html/torrentbytes.py new file mode 100644 index 0000000000..9ed58d7c22 --- /dev/null +++ b/sickbeard/providers/torrent/html/torrentbytes.py @@ -0,0 +1,207 @@ +# coding=utf-8 +# Author: Idan Gutman +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from requests.compat import urljoin +from requests.utils import dict_from_cookiejar + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class TorrentBytesProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """TorrentBytes Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'TorrentBytes') + + # Credentials + self.username = None + self.password = None + + # URLs + self.url = 'https://www.torrentbytes.net' + self.urls = { + 'login': urljoin(self.url, 'takelogin.php'), + 'search': urljoin(self.url, 'browse.php') + } + + # Proper Strings + self.proper_strings = ['PROPER', 'REPACK'] + + # Miscellaneous Options + self.freeleech = False + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + if not self.login(): + return results + + # Search Params + search_params = { + 'c41': 1, + 'c33': 1, + 'c38': 1, + 'c32': 1, + 'c37': 1 + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['search'] = search_string + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', border='1') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # "Type", "Name", Files", "Comm.", "Added", "TTL", "Size", "Snatched", "Seeders", "Leechers" + labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')] + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + + if len(cells) < len(labels): + continue + + try: + download_url = urljoin(self.url, cells[labels.index('Name')].find('a', href=re.compile(r'download.php\?id='))['href']) + title_element = cells[labels.index('Name')].find('a', href=re.compile(r'details.php\?id=')) + title = title_element.get('title', '') or title_element.get_text(strip=True) + if not all([title, download_url]): + continue + + if self.freeleech: + # Free leech torrents are marked with green [F L] in the title (i.e. [F L]) + freeleech = cells[labels.index('Name')].find('font', color='green') + if not freeleech or freeleech.get_text(strip=True) != '[F\xa0L]': + continue + + seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) + leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[labels.index('Size')].get_text(strip=True) + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + def login(self): + """Login method used for logging in before doing search and torrent downloads.""" + if any(dict_from_cookiejar(self.session.cookies).values()): + return True + + login_params = { + 'username': self.username, + 'password': self.password, + 'login': 'Log in!', + } + + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: + logger.log('Unable to connect to provider', logger.WARNING) + return False + + if re.search('Username or password incorrect', response.text): + logger.log('Invalid username or password. Check your settings', logger.WARNING) + return False + + return True + + +provider = TorrentBytesProvider() diff --git a/sickbeard/providers/torrentleech.py b/sickbeard/providers/torrent/html/torrentleech.py similarity index 55% rename from sickbeard/providers/torrentleech.py rename to sickbeard/providers/torrent/html/torrentleech.py index 7341eda9dd..46cd5292c4 100644 --- a/sickbeard/providers/torrentleech.py +++ b/sickbeard/providers/torrent/html/torrentleech.py @@ -63,9 +63,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - TorrentLeech search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -77,19 +77,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man # TV, Episodes, BoxSets, Episodes HD, Animation, Anime, Cartoons # 2,26,27,32,7,34,35 - # Units - units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] - - def process_column_header(td): - result = '' - if td.a: - result = td.a.get('title') - if not result: - result = td.get_text(strip=True) - return result - for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -109,69 +97,92 @@ def process_column_header(td): 'categories': ','.join(categories), 'query': search_string } - - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', id='torrenttable') - torrent_rows = torrent_table('tr') if torrent_table else [] + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + # Units + units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + + def process_column_header(td): + result = '' + if td.a: + result = td.a.get('title') + if not result: + result = td.get_text(strip=True) + return result + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', id='torrenttable') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + labels = [process_column_header(label) for label in torrent_rows[0]('th')] + + # Skip column headers + for row in torrent_rows[1:]: + try: + title = row.find('td', class_='name').find('a').get_text(strip=True) + download_url = urljoin(self.url, row.find('td', class_='quickdownload').find('a')['href']) + if not all([title, download_url]): continue - labels = [process_column_header(label) for label in torrent_rows[0]('th')] - - # Skip column headers - for result in torrent_rows[1:]: - try: - title = result.find('td', class_='name').find('a').get_text(strip=True) - download_url = urljoin(self.url, result.find('td', class_='quickdownload').find('a')['href']) - if not all([title, download_url]): - continue - - seeders = try_int(result.find('td', class_='seeders').get_text(strip=True)) - leechers = try_int(result.find('td', class_='leechers').get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = result('td')[labels.index('Size')].get_text() - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(row.find('td', class_='seeders').get_text(strip=True)) + leechers = try_int(row.find('td', class_='leechers').get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = row('td')[labels.index('Size')].get_text() + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -182,12 +193,13 @@ def login(self): 'remember_me': 'on', } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if re.search('Invalid Username/password', response) or re.search('Login :: TorrentLeech.org', response): + if any([re.search('Invalid Username/password', response.text), + re.search('Login :: TorrentLeech.org', response.text), ]): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/torrentshack.py b/sickbeard/providers/torrent/html/torrentshack.py similarity index 59% rename from sickbeard/providers/torrentshack.py rename to sickbeard/providers/torrent/html/torrentshack.py index bc43e84e4c..b2c17ca09a 100644 --- a/sickbeard/providers/torrentshack.py +++ b/sickbeard/providers/torrent/html/torrentshack.py @@ -42,10 +42,6 @@ def __init__(self): self.username = None self.password = None - # Torrent Stats - self.minseed = 0 - self.minleech = 0 - # URLs self.url = 'https://www.torrentshack.me' self.urls = { @@ -56,14 +52,20 @@ def __init__(self): # Proper Strings self.proper_strings = ['PROPER', 'REPACK', 'REAL'] + # Miscellaneous Options + + # Torrent Stats + self.minseed = 0 + self.minleech = 0 + # Cache self.cache = tvcache.TVCache(self, min_time=20) # Only poll TorrentShack every 20 minutes max def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - TorrentShack search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -92,79 +94,92 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'filter_cat[980]': 1, } - # Units - units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] - for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: if mode != 'RSS': - search_params['searchstr'] = search_string logger.log('Search string: {search}'.format (search=search_string), logger.DEBUG) + search_params['searchstr'] = search_string response = self.get_url(self.urls['search'], params=search_params, returns='response') - if not response or not response.text: + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(response.text, 'html5lib') as html: - torrent_rows = html.find_all('tr', class_='torrent') - if not torrent_rows: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + # Units + units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_rows = html.find_all('tr', class_='torrent') + + # Continue only if at least one release is found + if not torrent_rows: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for result in torrent_rows: + cells = result('td') + + try: + title = cells[1].find('span', class_='torrent_name_link').get_text() + download_url = cells[1].find('span', class_='torrent_handle_links') + download_url = download_url.find('a').find_next('a').get('href') + download_url = urljoin(self.url, download_url) + if not all([title, download_url]): continue - for result in torrent_rows: - try: - cells = result('td') - - title = cells[1].find('span', class_='torrent_name_link').get_text() - download_url = cells[1].find('span', class_='torrent_handle_links') - download_url = download_url.find('a').find_next('a').get('href') - download_url = urljoin(self.url, download_url) - if not all([title, download_url]): - continue - - seeders = try_int(cells[6].get_text()) - leechers = try_int(cells[7].get_text()) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[4].get_text() - torrent_size = re.search('\d+.\d+.\w+', torrent_size).group(0) - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(cells[6].get_text()) + leechers = try_int(cells[7].get_text()) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[4].get_text() + torrent_size = re.search('\d+.\d+.\w+', torrent_size).group(0) + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): """Login method used for logging in before doing search and torrent downloads.""" @@ -174,7 +189,6 @@ def login(self): login_params = { 'username': self.username, 'password': self.password, - 'keeplogged': '1', 'login': 'Login', } diff --git a/sickbeard/providers/torrent/html/transmitthenet.py b/sickbeard/providers/torrent/html/transmitthenet.py new file mode 100644 index 0000000000..c8c9f7fdf7 --- /dev/null +++ b/sickbeard/providers/torrent/html/transmitthenet.py @@ -0,0 +1,222 @@ +# coding=utf-8 +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from requests.utils import dict_from_cookiejar +from requests.compat import urljoin + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import try_int +from sickrage.helper.exceptions import AuthException +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class TransmitTheNetProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """TransmitTheNet Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'TransmitTheNet') + + # Credentials + self.username = None + self.password = None + + # URLs + self.url = 'https://transmithe.net/' + self.urls = { + 'login': urljoin(self.url, '/login.php'), + 'search': urljoin(self.url, '/torrents.php'), + } + + # Proper Strings + + # Miscellaneous Options + self.freeleech = None + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + if not self.login(): + return results + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params = { + 'searchtext': search_string, + 'filter_freeleech': (0, 1)[self.freeleech is True], + 'order_by': ('seeders', 'time')[mode == 'RSS'], + 'order_way': 'desc', + } + + if not search_string: + del search_params['searchtext'] + + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', {'id': 'torrent_table'}) + + # Continue only if at least one release is found + if not torrent_table: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + torrent_rows = torrent_table('tr', {'class': 'torrent'}) + + # Continue only if one Release is found + if not torrent_rows: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for row in torrent_rows: + try: + freeleech = row.find('img', alt='Freeleech') is not None + if self.freeleech and not freeleech: + continue + + download_item = row.find('a', {'title': [ + 'Download Torrent', # Download link + 'Previously Grabbed Torrent File', # Already Downloaded + 'Currently Seeding Torrent', # Seeding + 'Currently Leeching Torrent', # Leeching + ]}) + + if not download_item: + continue + + download_url = urljoin(self.url, download_item['href']) + + temp_anchor = row.find('a', {'data-src': True}) + title = temp_anchor['data-src'].rsplit('.', 1)[0] + if not all([title, download_url]): + continue + + cells = row('td') + seeders = try_int(cells[8].text.strip()) + leechers = try_int(cells[9].text.strip()) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + size = temp_anchor['data-filesize'] or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + def login(self): + """Login method used for logging in before doing search and torrent downloads.""" + if any(dict_from_cookiejar(self.session.cookies).values()): + return True + + login_params = { + 'username': self.username, + 'password': self.password, + 'keeplogged': 'on', + 'login': 'Login' + } + + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: + logger.log('Unable to connect to provider', logger.WARNING) + return False + + if any([re.search('Username Incorrect', response.text), + re.search('Password Incorrect', response.text), ]): + logger.log('Invalid username or password. Check your settings', logger.WARNING) + return False + + return True + + def _check_auth(self): + + if not self.username or not self.password: + raise AuthException('Your authentication credentials for {0} are missing,' + ' check your config.'.format(self.name)) + + return True + + +provider = TransmitTheNetProvider() diff --git a/sickbeard/providers/torrent/html/tvchaosuk.py b/sickbeard/providers/torrent/html/tvchaosuk.py new file mode 100644 index 0000000000..0186ee723b --- /dev/null +++ b/sickbeard/providers/torrent/html/tvchaosuk.py @@ -0,0 +1,228 @@ +# coding=utf-8 +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from requests.compat import urljoin + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.helper.exceptions import AuthException +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class TVChaosUKProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """TVChaosUK Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'TvChaosUK') + + # Credentials + self.username = None + self.password = None + + # URLs + self.url = 'https://www.tvchaosuk.com' + self.urls = { + 'login': urljoin(self.url, 'takelogin.php'), + 'index': urljoin(self.url, 'index.php'), + 'search': urljoin(self.url, 'browse.php'), + } + + # Proper Strings + + # Miscellaneous Options + self.freeleech = None + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + if not self.login(): + return results + + # Search Params + search_params = { + 'do': 'search', + 'search_type': 't_name', + 'category': 0, + 'include_dead_torrents': 'no', + 'submit': 'search', + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode == 'Season': + search_string = re.sub(r'(.*)S0?', r'\1Series ', search_string) + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['keywords'] = search_string + response = self.get_url(self.urls['search'], post_data=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode, keywords=search_string) + + return results + + def parse(self, data, mode, **kwargs): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + # Units + units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + + items = [] + + keywords = kwargs.pop('keywords', None) + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find(id='sortabletable') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + labels = [label.img['title'] if label.img else label.get_text(strip=True) for label in torrent_rows[0]('td')] + + # Skip column headers + for row in torrent_rows[1:]: + try: + if self.freeleech and not row.find('img', alt=re.compile('Free Torrent')): + continue + + title = row.find(class_='tooltip-content') + title = title.div.get_text(strip=True) if title else None + download_url = row.find(title='Click to Download this Torrent!') + download_url = download_url.parent['href'] if download_url else None + if not all([title, download_url]): + continue + + seeders = try_int(row.find(title='Seeders').get_text(strip=True)) + leechers = try_int(row.find(title='Leechers').get_text(strip=True)) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + # Chop off tracker/channel prefix or we cant parse the result! + if mode != 'RSS' and keywords: + show_name_first_word = re.search(r'^[^ .]+', keywords).group() + if not title.startswith(show_name_first_word): + title = re.sub(r'.*(' + show_name_first_word + '.*)', r'\1', title) + + # Change title from Series to Season, or we can't parse + if mode == 'Season': + title = re.sub(r'(.*)(?i)Series', r'\1Season', title) + + # Strip year from the end or we can't parse it! + title = re.sub(r'(.*)[\. ]?\(\d{4}\)', r'\1', title) + title = re.sub(r'\s+', r' ', title) + + torrent_size = row('td')[labels.index('Size')].get_text(strip=True) + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title + '.hdtv.x264', + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + def login(self): + """Login method used for logging in before doing search and torrent downloads.""" + if len(self.session.cookies) >= 4: + return True + + login_params = { + 'username': self.username, + 'password': self.password, + 'logout': 'no', + 'submit': 'LOGIN', + 'returnto': '/browse.php', + } + + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: + logger.log('Unable to connect to provider', logger.WARNING) + return False + + if re.search('Error: Username or password incorrect!', response.text): + logger.log('Invalid username or password. Check your settings', logger.WARNING) + return False + + return True + + def _check_auth(self): + if self.username and self.password: + return True + + raise AuthException('Your authentication credentials for {0} are missing,' + ' check your config.'.format(self.name)) + + +provider = TVChaosUKProvider() diff --git a/sickbeard/providers/xthor.py b/sickbeard/providers/torrent/html/xthor.py similarity index 55% rename from sickbeard/providers/xthor.py rename to sickbeard/providers/torrent/html/xthor.py index 76b7de7f9f..9e3f08af3d 100644 --- a/sickbeard/providers/xthor.py +++ b/sickbeard/providers/torrent/html/xthor.py @@ -64,9 +64,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - Xthor search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -90,17 +90,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'c34': 1, # Sport 34 } - # Units - units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] - - def process_column_header(td): - result = '' - if td.a: - result = td.a.get('title', td.a.get_text(strip=True)) - if not result: - result = td.get_text(strip=True) - return result - for mode in search_strings: items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) @@ -115,75 +104,99 @@ def process_column_header(td): (search=search_string), logger.DEBUG) search_params['search'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: logger.log('No data returned from provider', logger.DEBUG) continue - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', class_='table2 table-bordered2') - torrent_rows = [] - if torrent_table: - torrent_rows = torrent_table('tr') + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + # Units + units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + + def process_column_header(td): + result = '' + if td.a: + result = td.a.get('title', td.a.get_text(strip=True)) + if not result: + result = td.get_text(strip=True) + return result + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', class_='table2 table-bordered2') + torrent_rows = [] + if torrent_table: + torrent_rows = torrent_table('tr') + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Catégorie, Nom du Torrent, (Download), (Bookmark), Com., Taille, Compl�t�, Seeders, Leechers + labels = [process_column_header(label) for label in torrent_rows[0]('td')] + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + if len(cells) < len(labels): + continue + + try: + title = cells[labels.index('Nom du Torrent')].get_text(strip=True) + download_url = self.url + '/' + row.find('a', href=re.compile('download.php'))['href'] + if not all([title, download_url]): continue - # Catégorie, Nom du Torrent, (Download), (Bookmark), Com., Taille, Compl�t�, Seeders, Leechers - labels = [process_column_header(label) for label in torrent_rows[0]('td')] - - # Skip column headers - for row in torrent_rows[1:]: - cells = row('td') - if len(cells) < len(labels): - continue - - try: - title = cells[labels.index('Nom du Torrent')].get_text(strip=True) - download_url = self.url + '/' + row.find('a', href=re.compile('download.php'))['href'] - if not all([title, download_url]): - continue - - seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) - leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[labels.index('Taille')].get_text() - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) + leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[labels.index('Taille')].get_text() + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): + """Login method used for logging in before doing search and torrent downloads.""" if any(dict_from_cookiejar(self.session.cookies).values()): return True @@ -193,12 +206,12 @@ def login(self): 'submitme': 'X' } - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: + response = self.get_url(self.urls['login'], post_data=login_params, returns='response') + if not response.text: logger.log('Unable to connect to provider', logger.WARNING) return False - if not re.search('donate.php', response): + if not re.search('donate.php', response.text): logger.log('Invalid username or password. Check your settings', logger.WARNING) return False diff --git a/sickbeard/providers/torrent/html/zooqle.py b/sickbeard/providers/torrent/html/zooqle.py new file mode 100644 index 0000000000..2eaaf75e07 --- /dev/null +++ b/sickbeard/providers/torrent/html/zooqle.py @@ -0,0 +1,174 @@ +# coding=utf-8 +# Author: medariox +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import traceback + +from requests.compat import urljoin + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class ZooqleProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """Zooqle Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'Zooqle') + + # Credentials + self.public = True + + # URLs + self.url = 'https://zooqle.com/' + self.urls = { + 'search': urljoin(self.url, '/search'), + } + + # Proper Strings + self.proper_strings = ['PROPER', 'REPACK', 'REAL'] + + # Miscellaneous Options + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, min_time=15) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + + # Search Params + search_params = { + 'q': '* category:TV', + 's': 'dt', + 'v': 't', + 'sd': 'd', + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + search_params = {'q': '{0} category:TV'.format(search_string)} + + response = self.get_url(self.urls['search'], params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + + results += self.parse(response.text, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + # Units + units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + + items = [] + + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('div', class_='panel-body') + torrent_rows = torrent_table('tr') if torrent_table else [] + + # Continue only if at least one release is found + if len(torrent_rows) < 2: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Skip column headers + for row in torrent_rows[1:]: + cells = row('td') + + try: + title = cells[1].find('a').get_text() + magnet = cells[2].find('a')['href'] + download_url = '{magnet}{trackers}'.format(magnet=magnet, + trackers=self._custom_trackers) + if not all([title, download_url]): + continue + + seeders = 1 + leechers = 0 + if len(cells) > 6: + peers = cells[6].find('div') + if peers and peers.get('title'): + peers = peers['title'].replace(',', '').split(' | ', 1) + seeders = try_int(peers[0].strip('Seeders: ')) + leechers = try_int(peers[1].strip('Leechers: ')) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = cells[4].get_text(strip=True) + size = convert_size(torrent_size, units=units) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + +provider = ZooqleProvider() diff --git a/sickbeard/providers/torrent/json/__init__.py b/sickbeard/providers/torrent/json/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sickbeard/providers/torrent/json/bitcannon.py b/sickbeard/providers/torrent/json/bitcannon.py new file mode 100644 index 0000000000..d0844ca0d9 --- /dev/null +++ b/sickbeard/providers/torrent/json/bitcannon.py @@ -0,0 +1,184 @@ +# coding=utf-8 +# Author: Dustyn Gibson +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import validators +import traceback + +from requests.compat import urljoin +from requests.exceptions import RequestException + +from sickbeard import logger, tvcache + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class BitCannonProvider(TorrentProvider): + """BitCannon Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'BitCannon') + + # Credentials + self.api_key = None + + # URLs + self.custom_url = None + + # Proper Strings + + # Miscellaneous Options + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + cache_params = {'RSS': ['tv', 'anime']} + self.cache = tvcache.TVCache(self, search_params=cache_params) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + url = 'http://localhost:3000/' + if self.custom_url: + if not validators.url(self.custom_url): + logger.log('Invalid custom url set, please check your settings', logger.WARNING) + return results + url = self.custom_url + + # Search Params + search_params = { + 'category': 'anime' if ep_obj and ep_obj.show and ep_obj.show.anime else 'tv', + 'apiKey': self.api_key, + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + search_params['q'] = search_string + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_url = urljoin(url, 'api/search') + try: + response = self.get_url(search_url, params=search_params, returns='response') + response.raise_for_status() + except RequestException as msg: + logger.log(u'Error while connecting to provider: {error}'.format(error=msg), logger.ERROR) + continue + + if not response.content: + logger.log('No data returned from provider', logger.DEBUG) + continue + + try: + jdata = response.json() + except ValueError: # also catches JSONDecodeError if simplejson is installed + logger.log('No data returned from provider', logger.DEBUG) + continue + + if not self._check_auth_from_data(jdata): + continue + + results += self.parse(jdata, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + torrent_rows = data.pop('torrents', {}) + + # Skip column headers + for row in torrent_rows: + try: + title = row.pop('title', '') + info_hash = row.pop('infoHash', '') + download_url = 'magnet:?xt=urn:btih:' + info_hash + if not all([title, download_url, info_hash]): + continue + + swarm = row.pop('swarm', {}) + seeders = try_int(swarm.pop('seeders', 0)) + leechers = try_int(swarm.pop('leechers', 0)) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + size = convert_size(row.pop('size', -1)) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + @staticmethod + def _check_auth_from_data(data): + if not all([isinstance(data, dict), + data.pop('status', 200) != 401, + data.pop('message', '') != 'Invalid API key']): + + logger.log('Invalid api key. Check your settings', logger.WARNING) + return False + + return True + + +provider = BitCannonProvider() diff --git a/sickbeard/providers/btdigg.py b/sickbeard/providers/torrent/json/btdigg.py similarity index 50% rename from sickbeard/providers/btdigg.py rename to sickbeard/providers/torrent/json/btdigg.py index d6ca2cb61a..eff2654f2b 100644 --- a/sickbeard/providers/btdigg.py +++ b/sickbeard/providers/torrent/json/btdigg.py @@ -34,6 +34,7 @@ def __init__(self): # Provider Init TorrentProvider.__init__(self, 'BTDigg') + # Credentials self.public = True # URLs @@ -52,13 +53,21 @@ def __init__(self): self.minseed = None self.minleech = None + # Cache # Use this hacky way for RSS search since most results will use this codecs cache_params = {'RSS': ['x264', 'x264.HDTV', '720.HDTV.x264']} - # Only poll BTDigg every 30 minutes max, since BTDigg takes some time to crawl self.cache = tvcache.TVCache(self, min_time=30, search_params=cache_params) - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ results = [] # Search Params @@ -68,16 +77,15 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: search_params['q'] = search_string - if mode != 'RSS': - search_params['order'] = 0 logger.log('Search string: {search}'.format (search=search_string), logger.DEBUG) + search_params['order'] = 0 + if self.custom_url: # if not validators.url(self.custom_url): # logger.log('Invalid custom url set, please check your settings', logger.WARNING) @@ -86,65 +94,79 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man else: search_url = self.urls['api'] - jdata = self.get_url(search_url, params=search_params, returns='json') - if not jdata: + response = self.get_url(search_url, params=search_params, returns='response') + if not response.content: logger.log('No data returned from provider', logger.DEBUG) continue - for torrent in jdata: - try: - title = torrent.pop('name', '') - download_url = torrent.pop('magnet') + self._custom_trackers if torrent['magnet'] else None - if not all([title, download_url]): - continue - - if float(torrent.pop('ff')): - logger.log("Ignoring result for {0} since it's been" - ' reported as fake (level = {1})'.format - (title, torrent['ff']), logger.DEBUG) - continue - - if not int(torrent.pop('files')): - logger.log('Ignoring result for {0} because it has no files'.format - (title), logger.DEBUG) - continue - - leechers = torrent.pop('leechers', 0) - seeders = torrent.pop('seeders', 1) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = torrent.pop('size') - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + results += self.parse(response.json(), mode) return results + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + torrent_rows = data + + for row in torrent_rows: + try: + title = row.pop('name', '') + download_url = row.pop('magnet') + self._custom_trackers if row['magnet'] else None + if not all([title, download_url]): + continue + + if float(row.pop('ff')): + logger.log("Ignoring result for {0} since it's been" + ' reported as fake (level = {1})'.format + (title, row['ff']), logger.DEBUG) + continue + + if not int(row.pop('files')): + logger.log('Ignoring result for {0} because it has no files'.format + (title), logger.DEBUG) + continue + + leechers = row.pop('leechers', 0) + seeders = row.pop('seeders', 1) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = row.pop('size') + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items provider = BTDiggProvider() diff --git a/sickbeard/providers/btn.py b/sickbeard/providers/torrent/json/btn.py similarity index 83% rename from sickbeard/providers/btn.py rename to sickbeard/providers/torrent/json/btn.py index 0ccaab2a3b..f1f512508b 100644 --- a/sickbeard/providers/btn.py +++ b/sickbeard/providers/torrent/json/btn.py @@ -23,7 +23,7 @@ import time import sickbeard -from six import iteritems +from six import itervalues from sickbeard import logger, scene_exceptions, tvcache from sickbeard.common import cpu_presets @@ -65,9 +65,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint:disable=too-many-locals """ - BTN search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -77,10 +77,11 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint:disable=too-many return results # Search Params - search_params = {'age': '<=10800'} # Results from the past 3 hours + search_params = { + 'age': '<=10800', # Results from the past 3 hours + } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) if mode != 'RSS': @@ -94,49 +95,62 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint:disable=too-many response = self._api_call(self.api_key, search_params) if not response: logger.log('No data returned from provider', logger.DEBUG) - return results + continue if not self._check_auth_from_data(response): return results - found_torrents = response.get('torrents', {}) + results += self.parse(response.get('torrents', {}), mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. - for _, torrent_info in iteritems(found_torrents): - (title, download_url) = self._process_title_and_url(torrent_info) + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS - if not all([title, download_url]): - continue + :return: A list of items found + """ - seeders = torrent_info.get('Seeders', 1) - leechers = torrent_info.get('Leechers', 0) + items = [] - # Filter unseeded torrent + torrent_rows = itervalues(data) - if seeders < min(self.minseed, 1): - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue + for row in torrent_rows: + title, download_url = self._process_title_and_url(row) - size = torrent_info.get('Size') or -1 + if not all([title, download_url]): + continue - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) + seeders = row.get('Seeders', 1) + leechers = row.get('Leechers', 0) - items.append(item) + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue - results += items + size = row.get('Size') or -1 - return results + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + + return items def _check_auth(self): diff --git a/sickbeard/providers/hd4free.py b/sickbeard/providers/torrent/json/hd4free.py similarity index 63% rename from sickbeard/providers/hd4free.py rename to sickbeard/providers/torrent/json/hd4free.py index f367a59a1b..96f66cd0dd 100644 --- a/sickbeard/providers/hd4free.py +++ b/sickbeard/providers/torrent/json/hd4free.py @@ -59,9 +59,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - HD4Free search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -79,7 +79,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -91,8 +90,9 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man else: search_params['search'] = None + response = self.get_url(self.urls['search'], params=search_params, returns='response') try: - jdata = self.get_url(self.urls['search'], params=search_params, returns='json') + jdata = response.json() except ValueError: logger.log('No data returned from provider', logger.DEBUG) continue @@ -106,7 +106,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man if error: logger.log('{0}'.format(error), logger.DEBUG) return results - try: if jdata['0']['total_results'] == 0: logger.log('Provider has no results for this search', logger.DEBUG) @@ -114,50 +113,65 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man except KeyError: continue - for i in jdata: - try: - title = jdata[i]['release_name'] - download_url = jdata[i]['download_url'] - if not all([title, download_url]): - continue - - seeders = jdata[i]['seeders'] - leechers = jdata[i]['leechers'] - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = str(jdata[i]['size']) + ' MB' - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + results += self.parse(jdata, mode) return results + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + torrent_rows = data + + for row in torrent_rows: + try: + title = torrent_rows[row]['release_name'] + download_url = torrent_rows[row]['download_url'] + if not all([title, download_url]): + continue + + seeders = torrent_rows[row]['seeders'] + leechers = torrent_rows[row]['leechers'] + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = str(torrent_rows[row]['size']) + ' MB' + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + def _check_auth(self): if self.username and self.api_key: return True diff --git a/sickbeard/providers/hdbits.py b/sickbeard/providers/torrent/json/hdbits.py similarity index 79% rename from sickbeard/providers/hdbits.py rename to sickbeard/providers/torrent/json/hdbits.py index 1107992e93..9038427177 100644 --- a/sickbeard/providers/hdbits.py +++ b/sickbeard/providers/torrent/json/hdbits.py @@ -18,7 +18,7 @@ from __future__ import unicode_literals import datetime -import json +import sickbeard.providers.torrent.json from requests.compat import urlencode, urljoin @@ -56,8 +56,15 @@ def __init__(self): # Cache self.cache = HDBitsCache(self, min_time=15) # only poll HDBits every 15 minutes max - def search(self, search_strings, age=0, ep_obj=None): + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals + """ + Search a provider and parse the results + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ # FIXME results = [] @@ -65,21 +72,38 @@ def search(self, search_strings, age=0, ep_obj=None): self._check_auth() - parsed_json = self.get_url(self.urls['search'], post_data=search_strings, returns='json') - if not parsed_json: - return [] + response = self.get_url(self.urls['search'], post_data=search_strings, returns='response') + if not response.content: + logger.log('No data returned from provider', logger.DEBUG) + return results - if self._check_auth_from_data(parsed_json): - if parsed_json and 'data' in parsed_json: - items = parsed_json['data'] - else: - logger.log("Resulting JSON from provider isn't correct, not parsing it", logger.ERROR) - items = [] + if not self._check_auth_from_data(response): + return results + else: + results += self.parse(response.json(), None) + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + torrent_rows = data.get('data') + if not torrent_rows: + logger.log("Resulting JSON from provider isn't correct, not parsing it", logger.ERROR) + + for row in torrent_rows: + items.append(row) - for item in items: - results.append(item) # FIXME SORTING - return results + return items def _check_auth(self): @@ -182,7 +206,7 @@ def _make_post_data_json(self, show=None, episode=None, season=None, search_term if search_term: post_data['search'] = search_term - return json.dumps(post_data) + return sickbeard.providers.torrent.json.dumps(post_data) class HDBitsCache(tvcache.TVCache): @@ -191,7 +215,7 @@ def _getRSSData(self): results = [] try: - parsed_json = self.provider.getURL(self.provider.urls['rss'], post_data=self.provider._make_post_data_json(), returns='json') + parsed_json = self.provider.get_url(self.provider.urls['rss'], post_data=self.provider._make_post_data_json(), returns='json') if self.provider._check_auth_from_data(parsed_json): results = parsed_json['data'] diff --git a/sickbeard/providers/norbits.py b/sickbeard/providers/torrent/json/norbits.py similarity index 51% rename from sickbeard/providers/norbits.py rename to sickbeard/providers/torrent/json/norbits.py index 6a0a0f28b3..f30199496b 100644 --- a/sickbeard/providers/norbits.py +++ b/sickbeard/providers/torrent/json/norbits.py @@ -18,7 +18,7 @@ from __future__ import unicode_literals import traceback -import json +import sickbeard.providers.torrent.json from requests.compat import urljoin from requests.compat import urlencode @@ -61,13 +61,18 @@ def __init__(self): # Cache self.cache = tvcache.TVCache(self, min_time=20) # only poll Norbits every 15 minutes max - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals - """ Do the actual searching and JSON parsing""" + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals + """ + Search a provider and parse the results + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ results = [] for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -84,67 +89,82 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } self._check_auth() - parsed_json = self.get_url(self.urls['search'], - post_data=json.dumps(post_data), - returns='json') + response = self.get_url(self.urls['search'], post_data=sickbeard.providers.torrent.json.dumps(post_data), returns='response') + if not response.content: + logger.log('No data returned from provider', logger.DEBUG) + continue - if not parsed_json: + if self._check_auth_from_data(response.json()): return results - if self._check_auth_from_data(parsed_json): - json_items = parsed_json.get('data', '') - if not json_items: - logger.log('Resulting JSON from provider is not correct, ' - 'not parsing it', logger.ERROR) - - # Skip column headers - for item in json_items.get('torrents', []): - try: - title = item.pop('name', '') - download_url = '{0}{1}'.format( - self.urls['download'], - urlencode({'id': item.pop('id', ''), 'passkey': self.passkey})) - - if not all([title, download_url]): - continue - - seeders = try_int(item.pop('seeders', 0)) - leechers = try_int(item.pop('leechers', 0)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - info_hash = item.pop('info_hash', '') - size = convert_size(item.pop('size', -1), -1) - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': info_hash, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + json_items = parsed_json + if not json_items: + logger.log('Resulting JSON from provider is not correct, ' + 'not parsing it', logger.ERROR) + + results += self.parse(response.json(), mode) return results + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + data.get('data', '') + torrent_rows = data.get('torrents', []) + + # Skip column headers + for row in torrent_rows: + try: + title = row.pop('name', '') + download_url = '{0}{1}'.format( + self.urls['download'], + urlencode({'id': row.pop('id', ''), 'passkey': self.passkey})) + + if not all([title, download_url]): + continue + + seeders = try_int(row.pop('seeders', 0)) + leechers = try_int(row.pop('leechers', 0)) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + info_hash = row.pop('info_hash', '') + size = convert_size(row.pop('size', -1), -1) + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': info_hash, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + def _check_auth(self): if not self.username or not self.passkey: diff --git a/sickbeard/providers/rarbg.py b/sickbeard/providers/torrent/json/rarbg.py similarity index 55% rename from sickbeard/providers/rarbg.py rename to sickbeard/providers/torrent/json/rarbg.py index 187bb9d98e..131bccdf8a 100644 --- a/sickbeard/providers/rarbg.py +++ b/sickbeard/providers/torrent/json/rarbg.py @@ -18,9 +18,9 @@ from __future__ import unicode_literals -import traceback import datetime import time +import traceback from sickbeard import logger, tvcache from sickbeard.indexers.indexer_config import INDEXER_TVDB @@ -30,6 +30,7 @@ class RarbgProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """RARBG Torrent provider.""" def __init__(self): @@ -63,9 +64,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals, too-many-statements """ - RARBG search and parsing + Search a provider and parse the results. - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -84,6 +85,8 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man 'format': 'json_extended', 'ranked': try_int(self.ranked), 'token': self.token, + 'sort': 'last', + 'mode': 'list', } if ep_obj is not None: @@ -94,29 +97,21 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man ep_indexer = None for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) if mode == 'RSS': - search_params['sort'] = 'last' - search_params['mode'] = 'list' - search_params.pop('search_string', None) - search_params.pop('search_tvdb', None) + search_params['search_string'] = None + search_params['search_tvdb'] = None else: search_params['sort'] = self.sorting if self.sorting else 'seeders' search_params['mode'] = 'search' - - if ep_indexer == INDEXER_TVDB and ep_indexerid: - search_params['search_tvdb'] = ep_indexerid - else: - search_params.pop('search_tvdb', None) + search_params['search_tvdb'] = ep_indexerid if ep_indexer == INDEXER_TVDB and ep_indexerid else None for search_string in search_strings[mode]: - if mode != 'RSS': - search_params['search_string'] = search_string - logger.log('Search string: {0}'.format(search_string), - logger.DEBUG) + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + search_params['search_string'] = search_string # Check if token is still valid before search if not self.login(): @@ -126,73 +121,96 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man # Changing to 5 because of server clock desync time.sleep(5) - data = self.get_url(self.urls['api'], params=search_params, returns='json') - if not isinstance(data, dict): + search_url = self.urls['api'] + response = self.get_url(search_url, params=search_params, returns='response') + if not response.content: + logger.log('No data returned from provider', logger.DEBUG) + continue + + try: + jdata = response.json() + except ValueError: logger.log('No data returned from provider', logger.DEBUG) continue - error = data.get('error') - error_code = data.get('error_code') - # Don't log when {'error':'No results found','error_code':20} - # List of errors: https://github.com/rarbg/torrentapi/issues/1#issuecomment-114763312 + error = jdata.get('error') + error_code = jdata.get('error_code') if error: - if error_code == 5: - # 5 = Too many requests per second - logger.log('{0}. Error code: {1}'.format(error, error_code), logger.INFO) - elif error_code not in (8, 10, 12, 14, 20): + # List of errors: https://github.com/rarbg/torrentapi/issues/1#issuecomment-114763312 + if error_code not in (8, 10, 12, 14, 20): # 8, 10, 12, 14 = Cant find * in database. Are you sure this * exists? # 20 = No results found - logger.log('{0} Error code: {1}'.format(error, error_code), logger.WARNING) + log_level = logger.WARNING + elif error_code == 5: + # 5 = Too many requests per second + log_level = logger.INFO + else: + log_level = logger.DEBUG + logger.log('{msg} Code: {code}'.format(msg=error, code=error_code), log_level) continue - torrent_results = data.get('torrent_results') - if not torrent_results: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + results += self.parse(jdata, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + items = [] + + torrent_rows = data.get('torrent_results', {}) + + if not torrent_rows: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + # Skip column headers + for row in torrent_rows: + try: + title = row.pop('title') + download_url = row.pop('download') + if not all([title, download_url]): continue - for item in torrent_results: - try: - title = item.pop('title') - download_url = item.pop('download') - if not all([title, download_url]): - continue - - seeders = item.pop('seeders') - leechers = item.pop('leechers') - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = item.pop('size', -1) - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + seeders = row.pop('seeders') + leechers = row.pop('leechers') - return results + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = row.pop('size', -1) + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): if self.token and self.token_expires and datetime.datetime.now() < self.token_expires: diff --git a/sickbeard/providers/t411.py b/sickbeard/providers/torrent/json/t411.py similarity index 57% rename from sickbeard/providers/t411.py rename to sickbeard/providers/torrent/json/t411.py index 712aecb5bf..3aa91c12dc 100644 --- a/sickbeard/providers/t411.py +++ b/sickbeard/providers/torrent/json/t411.py @@ -71,9 +71,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - T411 search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -83,85 +83,99 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man return results for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: - if mode != 'RSS': logger.log('Search string: {search}'.format (search=search_string), logger.DEBUG) search_urls = ([self.urls['search'] % (search_string, u) for u in self.subcategories], [self.urls['rss']])[mode == 'RSS'] for search_url in search_urls: - data = self.get_url(search_url, returns='json') - if not data: + response = self.get_url(search_url, returns='response') + + if not response.content: logger.log('No data returned from provider', logger.DEBUG) continue - if 'torrents' not in data and mode != 'RSS': - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + try: + jdata = response.json() + except ValueError: # also catches JSONDecodeError if simplejson is installed + logger.log('No data returned from provider', logger.DEBUG) continue - torrents = data['torrents'] if mode != 'RSS' else data + results += self.parse(jdata, mode) - if not torrents: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue + return results - for torrent in torrents: - if mode == 'RSS' and 'category' in torrent and try_int(torrent['category'], 0) not in self.subcategories: - continue - - try: - title = torrent['name'] - torrent_id = torrent['id'] - download_url = (self.urls['download'] % torrent_id) - if not all([title, download_url]): - continue - - seeders = try_int(torrent['seeders']) - leechers = try_int(torrent['leechers']) - verified = bool(torrent['isVerified']) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - if self.confirmed and not verified and mode != 'RSS': - logger.log("Found result {0} but that doesn't seem like a verified" - " result so I'm ignoring it".format(title), logger.DEBUG) - continue - - torrent_size = torrent['size'] - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + def parse(self, data, mode): + """ + Parse search results for items. - return results + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + torrent_rows = data.get('torrents') if mode != 'RSS' else data + + if not torrent_rows: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for row in torrent_rows: + if mode == 'RSS' and 'category' in row and try_int(row['category'], 0) not in self.subcategories: + continue + + try: + title = row['name'] + torrent_id = row['id'] + download_url = (self.urls['download'] % torrent_id) + if not all([title, download_url]): + continue + + seeders = try_int(row['seeders']) + leechers = try_int(row['leechers']) + verified = bool(row['isVerified']) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + if self.confirmed and not verified and mode != 'RSS': + logger.log("Found result {0} but that doesn't seem like a verified" + " result so I'm ignoring it".format(title), logger.DEBUG) + continue + + torrent_size = row['size'] + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items def login(self): if self.token is not None: diff --git a/sickbeard/providers/torrentday.py b/sickbeard/providers/torrent/json/torrentday.py similarity index 59% rename from sickbeard/providers/torrentday.py rename to sickbeard/providers/torrent/json/torrentday.py index 7e43d17676..a7307be7a8 100644 --- a/sickbeard/providers/torrentday.py +++ b/sickbeard/providers/torrent/json/torrentday.py @@ -67,17 +67,23 @@ def __init__(self): # Cache self.cache = tvcache.TVCache(self, min_time=10) # Only poll IPTorrents every 10 minutes max - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ results = [] if not self.login(): return results for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: - if mode != 'RSS': logger.log('Search string: {search}'.format (search=search_string), logger.DEBUG) @@ -97,69 +103,85 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u'Error while connecting to provider: {error}'.format(error=msg), logger.ERROR) continue + if not response.content: + logger.log('No data returned from provider', logger.DEBUG) + continue + try: jdata = response.json() except ValueError: # also catches JSONDecodeError if simplejson is installed - logger.log('Data returned from provider is not json', logger.ERROR) - self.session.cookies.clear() + logger.log('No data returned from provider', logger.DEBUG) continue - try: - cn = jdata.get('Fs', [dict()])[0].get('Cn', {}) - torrents = cn.get('torrents', []) if cn else [] - except (AttributeError, TypeError, KeyError, ValueError, IndexError) as e: - # If TorrentDay changes their website issue will be opened so we can fix fast - # and not wait user notice it's not downloading torrents from there - logger.log('TorrentDay response: {0}. Error: {1!r}'.format(jdata, e), logger.ERROR) + results += self.parse(jdata, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + try: + initial_data = data.get('Fs', [dict()])[0].get('Cn', {}) + torrent_rows = initial_data.get('torrents', []) if initial_data else None + except (AttributeError, TypeError, KeyError, ValueError, IndexError) as e: + # If TorrentDay changes their website issue will be opened so we can fix fast + # and not wait user notice it's not downloading torrents from there + logger.log('TorrentDay response: {0}. Error: {1!r}'.format(data, e), logger.ERROR) + torrent_rows = None + + if not torrent_rows: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for row in torrent_rows: + try: + title = re.sub(r'\[.*=.*\].*\[/.*\]', '', row['name']) if row['name'] else None + download_url = urljoin(self.urls['download'], '{}/{}'.format(row['id'], row['fname'])) if row['id'] and row['fname'] else None + if not all([title, download_url]): continue - if not torrents: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + seeders = int(row['seed']) if row['seed'] else 1 + leechers = int(row['leech']) if row['leech'] else 0 + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) continue - for torrent in torrents: - try: - title = re.sub(r'\[.*=.*\].*\[/.*\]', '', torrent['name']) if torrent['name'] else None - download_url = urljoin(self.urls['download'], '{}/{}'.format(torrent['id'], torrent['fname'])) if torrent['id'] and torrent['fname'] else None - if not all([title, download_url]): - continue - - seeders = int(torrent['seed']) if torrent['seed'] else 1 - leechers = int(torrent['leech']) if torrent['leech'] else 0 - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = torrent['size'] - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + torrent_size = row['size'] + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) - return results + return items def login(self): if dict_from_cookiejar(self.session.cookies).get('uid') and dict_from_cookiejar(self.session.cookies).get('pass'): @@ -179,7 +201,7 @@ def login(self): } response = self.get_url(self.urls['login'], post_data=login_params, returns='response') - if response.status_code != 200: + if not (response.content and response.status_code == 200): logger.log('Unable to connect to provider', logger.WARNING) return False @@ -187,7 +209,7 @@ def login(self): logger.log('Too many login access attempts', logger.WARNING) return False - if (dict_from_cookiejar(self.session.cookies).get('uid') and + if (dict_from_cookiejar(self.session.cookies).get('uid') and dict_from_cookiejar(self.session.cookies).get('uid') in response.text): return True else: diff --git a/sickbeard/providers/torrent/json/torrentproject.py b/sickbeard/providers/torrent/json/torrentproject.py new file mode 100644 index 0000000000..406ac51e84 --- /dev/null +++ b/sickbeard/providers/torrent/json/torrentproject.py @@ -0,0 +1,171 @@ +# coding=utf-8 +# Author: Gonçalo M. (aka duramato/supergonkas) +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import validators +import traceback + +from sickbeard import logger, tvcache +from sickbeard.common import USER_AGENT + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class TorrentProjectProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """TorrentProject Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'TorrentProject') + + # Credentials + self.public = True + + # URLs + self.url = 'https://torrentproject.se/' + self.custom_url = None + + # Proper Strings + + # Miscellaneous Options + self.headers.update({'User-Agent': USER_AGENT}) + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, search_params={'RSS': ['0day']}) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + + search_params = { + 'out': 'json', + 'filter': 2101, + 'showmagnets': 'on', + 'num': 150 + } + + for mode in search_strings: # Mode = RSS, Season, Episode + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_params['s'] = search_string + + if self.custom_url: + if not validators.url(self.custom_url): + logger.log('Invalid custom url set, please check your settings', logger.WARNING) + return results + search_url = self.custom_url + else: + search_url = self.url + + response = self.get_url(search_url, params=search_params, returns='response') + + try: + jdata = response.json() + except ValueError: # also catches JSONDecodeError if simplejson is installed + logger.log('No data returned from provider', logger.DEBUG) + continue + + if not jdata: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + continue + + results += self.parse(jdata, mode) + + return results + + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + torrent_rows = data + + if not int(torrent_rows.pop('total_found', 0)) > 0: + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + return items + + for row in torrent_rows: + try: + title = torrent_rows[row].get('title') + download_url = torrent_rows[row].get('magnet') + if not all([title, download_url]): + continue + + download_url += self._custom_trackers + seeders = try_int(torrent_rows[row].get('seeds'), 1) + leechers = try_int(torrent_rows[row].get('leechs'), 0) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = torrent_rows[row].get('torrent_size') + size = convert_size(torrent_size) or -1 + torrent_hash = torrent_rows[row].get('torrent_hash') + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': torrent_hash, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + +provider = TorrentProjectProvider() diff --git a/sickbeard/providers/torrent/rss/__init__.py b/sickbeard/providers/torrent/rss/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sickbeard/providers/nyaatorrents.py b/sickbeard/providers/torrent/rss/nyaatorrents.py similarity index 50% rename from sickbeard/providers/nyaatorrents.py rename to sickbeard/providers/torrent/rss/nyaatorrents.py index e4c2c650e3..8156fe1944 100644 --- a/sickbeard/providers/nyaatorrents.py +++ b/sickbeard/providers/torrent/rss/nyaatorrents.py @@ -40,6 +40,8 @@ def __init__(self): # URLs self.url = 'http://www.nyaa.se' + # Proper Strings + # Miscellaneous Options self.supports_absolute_numbering = True self.anime_only = True @@ -55,9 +57,9 @@ def __init__(self): def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches """ - NyaaTorrents search and parsing + Search a provider and parse the results - :param search_string: A dict with mode (key) and the search value (value) + :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) @@ -75,7 +77,6 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } for mode in search_strings: - items = [] logger.log('Search mode: {0}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: @@ -87,63 +88,76 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_params['term'] = search_string data = self.cache.getRSSFeed(self.url, params=search_params)['entries'] if not data: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + logger.log('No data returned from provider', logger.DEBUG) continue - for curItem in data: - try: - title = curItem['title'] - download_url = curItem['link'] - if not all([title, download_url]): - continue - - item_info = self.regex.search(curItem['summary']) - if not item_info: - logger.log('There was a problem parsing an item summary, skipping: {0}'.format - (title), logger.DEBUG) - continue - - seeders, leechers, torrent_size, verified = item_info.groups() - seeders = try_int(seeders) - leechers = try_int(leechers) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - if self.confirmed and not verified and mode != 'RSS': - logger.log("Found result {0} but that doesn't seem like a verified" - " result so I'm ignoring it".format(title), logger.DEBUG) - continue - - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items + results += self.parse(data, mode) return results + def parse(self, data, mode): + """ + Parse search results for items. + + :param data: The raw response from a search + :param mode: The current mode used to search, e.g. RSS + + :return: A list of items found + """ + + items = [] + + for result in data: + try: + title = result['title'] + download_url = result['link'] + if not all([title, download_url]): + continue + + item_info = self.regex.search(result['summary']) + if not item_info: + logger.log('There was a problem parsing an item summary, skipping: {0}'.format + (title), logger.DEBUG) + continue + + seeders, leechers, torrent_size, verified = item_info.groups() + seeders = try_int(seeders) + leechers = try_int(leechers) + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + if self.confirmed and not verified and mode != 'RSS': + logger.log("Found result {0} but that doesn't seem like a verified" + " result so I'm ignoring it".format(title), logger.DEBUG) + continue + + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + provider = NyaaProvider() diff --git a/sickbeard/providers/rsstorrent.py b/sickbeard/providers/torrent/rss/rsstorrent.py similarity index 97% rename from sickbeard/providers/rsstorrent.py rename to sickbeard/providers/torrent/rss/rsstorrent.py index accc1dfdaa..37942cca99 100644 --- a/sickbeard/providers/rsstorrent.py +++ b/sickbeard/providers/torrent/rss/rsstorrent.py @@ -38,13 +38,18 @@ def __init__(self, name, url, cookies='', # pylint: disable=too-many-arguments titleTAG='title', search_mode='eponly', search_fallback=False, enable_daily=False, enable_backlog=False, enable_manualsearch=False): + # Provider Init TorrentProvider.__init__(self, name) - self.cache = TorrentRssCache(self, min_time=15) + # Credentials + + # URLs self.url = url.rstrip('/') - self.supports_backlog = False + # Proper Strings + # Miscellaneous Options + self.supports_backlog = False self.search_mode = search_mode self.search_fallback = search_fallback self.enable_daily = enable_daily @@ -54,6 +59,11 @@ def __init__(self, name, url, cookies='', # pylint: disable=too-many-arguments self.cookies = cookies self.titleTAG = titleTAG + # Torrent Stats + + # Cache + self.cache = TorrentRssCache(self, min_time=15) + def _get_title_and_url(self, item): title = item.get(self.titleTAG, '').replace(' ', '.') diff --git a/sickbeard/providers/shazbat.py b/sickbeard/providers/torrent/rss/shazbat.py similarity index 100% rename from sickbeard/providers/shazbat.py rename to sickbeard/providers/torrent/rss/shazbat.py diff --git a/sickbeard/providers/torrent/xml/__init__.py b/sickbeard/providers/torrent/xml/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sickbeard/providers/torrent/xml/bitsnoop.py b/sickbeard/providers/torrent/xml/bitsnoop.py new file mode 100644 index 0000000000..5d79096fcf --- /dev/null +++ b/sickbeard/providers/torrent/xml/bitsnoop.py @@ -0,0 +1,163 @@ +# coding=utf-8 +# Author: Gonçalo M. (aka duramato/supergonkas) +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import traceback + +from requests.compat import urljoin + +import sickbeard +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class BitSnoopProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """BitSnoop Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'BitSnoop') + + # Credentials + self.public = True + + # URLs + self.url = 'http://bitsnoop.com' + self.urls = { + 'base': self.url, + 'rss': urljoin(self.url, '/new_video.html?fmt=rss'), + 'search': urljoin(self.url, '/search/video/'), + } + + # Proper Strings + self.proper_strings = ['PROPER', 'REPACK'] + + # Miscellaneous Options + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, search_params={'RSS': ['rss']}) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches,too-many-locals + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_url = (self.urls['rss'], self.urls['search'] + search_string + '/s/d/1/?fmt=rss')[mode != 'RSS'] + response = self.get_url(search_url, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + elif not response.text.startswith(' +# Author: Dustyn Gibson +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from requests.compat import urljoin + +import sickbeard +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser +from sickbeard.common import USER_AGENT + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class ExtraTorrentProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """ExtraTorrent Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'ExtraTorrent') + + # Credentials + self.public = True + + # URLs + self.url = 'http://extratorrent.cc' + self.urls = { + 'base': self.url, + 'rss': urljoin(self.url, 'rss.xml'), + } + self.custom_url = None + + # Proper Strings + + # Miscellaneous Options + self.headers.update({'User-Agent': USER_AGENT}) + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, min_time=30) # Only poll ExtraTorrent every 30 minutes max + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + + # Search Params + search_params = { + 'cid': 8, + 'type': 'rss', + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + search_params['type'] = 'search' + + search_params['search'] = search_string + search_url = self.urls['rss'] if not self.custom_url else self.urls['rss'].replace(self.urls['base'], self.custom_url) + response = self.get_url(search_url, params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + elif not response.text.startswith('$', '', row.find('title').get_text(strip=True)) + + if sickbeard.TORRENT_METHOD == 'blackhole': + enclosure = row.find('enclosure') # Backlog doesnt have enclosure + download_url = enclosure['url'] if enclosure else row.find('link').next.strip() + download_url = re.sub(r'(.*)/torrent/(.*).html', r'\1/download/\2.torrent', download_url) + else: + info_hash = row.find('info_hash') + if not info_hash: + continue + info_hash = info_hash.get_text(strip=True) + download_url = 'magnet:?xt=urn:btih:' + info_hash + '&dn=' + title + self._custom_trackers + + if not all([title, download_url]): + continue + + seeders = row.find('seeders') + seeders = try_int(seeders.get_text(strip=True)) if seeders else 1 + leechers = row.find('leechers') + leechers = try_int(leechers.get_text(strip=True)) if leechers else 0 + + # Filter unseeded torrent + if seeders < min(self.minseed, 1): + if mode != 'RSS': + logger.log("Discarding torrent because it doesn't meet the " + "minimum seeders: {0}. Seeders: {1}".format + (title, seeders), logger.DEBUG) + continue + + torrent_size = row.find('size') + torrent_size = torrent_size.get_text() if torrent_size else None + size = convert_size(torrent_size) or -1 + + item = { + 'title': title, + 'link': download_url, + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'pubdate': None, + 'hash': None, + } + if mode != 'RSS': + logger.log('Found result: {0} with {1} seeders and {2} leechers'.format + (title, seeders, leechers), logger.DEBUG) + + items.append(item) + except (AttributeError, TypeError, KeyError, ValueError, IndexError): + logger.log('Failed parsing provider. Traceback: {0!r}'.format + (traceback.format_exc()), logger.ERROR) + + return items + + +provider = ExtraTorrentProvider() diff --git a/sickbeard/providers/torrent/xml/kat.py b/sickbeard/providers/torrent/xml/kat.py new file mode 100644 index 0000000000..5d91f115ef --- /dev/null +++ b/sickbeard/providers/torrent/xml/kat.py @@ -0,0 +1,187 @@ +# coding=utf-8 +# Author: Dustyn Gibson +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import traceback +import validators + +from requests.compat import urljoin +from sickbeard.bs4_parser import BS4Parser + +import sickbeard +from sickbeard import logger, tvcache + +from sickrage.helper.common import convert_size, try_int +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class KatProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """KAT Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, 'KickAssTorrents') + + # Credentials + self.public = True + + # URLs + self.url = 'https://kat.cr' + self.urls = { + 'base': self.url, + 'search': urljoin(self.url, '%s/'), + } + self.custom_url = None + + # Proper Strings + + # Miscellaneous Options + self.confirmed = True + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, search_params={'RSS': ['tv', 'anime']}) + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Used to check if show is anime + :returns: A list of search results (structure) + """ + results = [] + + anime = (self.show and self.show.anime) or (ep_obj and ep_obj.show and ep_obj.show.anime) or False + + # Search Params + search_params = { + 'q': '', + 'field': 'time_add', + 'sorder': 'desc', + 'rss': 1, + 'category': 'anime' if anime else 'tv' + } + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + search_params['field'] = 'seeders' + search_params['q'] = search_string + + search_url = self.urls['search'] % ('usearch' if mode != 'RSS' else search_string) + if self.custom_url: + if not validators.url(self.custom_url): + logger.log('Invalid custom url: {0}'.format(self.custom_url), logger.WARNING) + return results + search_url = urljoin(self.custom_url, search_url.split(self.url)[1]) + + response = self.get_url(search_url, params=search_params, returns='response') + if not response.text: + logger.log('No data returned from provider, maybe try a custom url, or a different one', logger.DEBUG) + continue + elif not response.text.startswith(' +# +# This file is part of Medusa. +# +# Medusa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Medusa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Medusa. If not, see . + +from __future__ import unicode_literals + +import re +import traceback + +from requests.compat import urljoin + +from sickbeard import logger, tvcache +from sickbeard.bs4_parser import BS4Parser +from sickbeard.common import USER_AGENT + +from sickrage.helper.common import convert_size +from sickrage.providers.torrent.TorrentProvider import TorrentProvider + + +class TorrentzProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes + """Torrentz Torrent provider""" + def __init__(self): + + # Provider Init + TorrentProvider.__init__(self, "Torrentz") + + # Credentials + self.public = True + + # URLs + self.url = 'https://torrentz.eu/' + self.urls = { + 'base': self.url, + 'verified': urljoin(self.url, 'feed_verified'), + 'feed': urljoin(self.url, 'feed'), + } + + # Proper Strings + + # Miscellaneous Options + self.confirmed = True + self.headers.update({'User-Agent': USER_AGENT}) + + # Torrent Stats + self.minseed = None + self.minleech = None + + # Cache + self.cache = tvcache.TVCache(self, min_time=15) # only poll Torrentz every 15 minutes max + + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches + """ + Search a provider and parse the results + + :param search_strings: A dict with mode (key) and the search value (value) + :param age: Not used + :param ep_obj: Not used + :returns: A list of search results (structure) + """ + results = [] + + for mode in search_strings: + logger.log('Search mode: {0}'.format(mode), logger.DEBUG) + + for search_string in search_strings[mode]: + if mode != 'RSS': + logger.log('Search string: {search}'.format + (search=search_string), logger.DEBUG) + + search_url = self.urls['verified'] if self.confirmed else self.urls['feed'] + response = self.get_url(search_url, params={'q': search_string}, returns='response') + if not response.text: + logger.log('No data returned from provider', logger.DEBUG) + continue + elif not response.text.startswith('. - -from __future__ import unicode_literals - -import re -import traceback - -from requests.compat import urljoin -from requests.utils import dict_from_cookiejar - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class TorrentBytesProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """TorrentBytes Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'TorrentBytes') - - # Credentials - self.username = None - self.password = None - - # URLs - self.url = 'https://www.torrentbytes.net' - self.urls = { - 'login': urljoin(self.url, 'takelogin.php'), - 'search': urljoin(self.url, 'browse.php') - } - - # Proper Strings - self.proper_strings = ['PROPER', 'REPACK'] - - # Miscellaneous Options - self.freeleech = False - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements - results = [] - if not self.login(): - return results - - # Search Params - search_params = { - 'c41': 1, - 'c33': 1, - 'c38': 1, - 'c32': 1, - 'c37': 1 - } - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_params['search'] = search_string - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', border='1') - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - # "Type", "Name", Files", "Comm.", "Added", "TTL", "Size", "Snatched", "Seeders", "Leechers" - labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')] - - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - if len(cells) < len(labels): - continue - - try: - download_url = urljoin(self.url, cells[labels.index('Name')].find('a', href=re.compile(r'download.php\?id='))['href']) - title_element = cells[labels.index('Name')].find('a', href=re.compile(r'details.php\?id=')) - title = title_element.get('title', '') or title_element.get_text(strip=True) - if not all([title, download_url]): - continue - - if self.freeleech: - # Free leech torrents are marked with green [F L] in the title (i.e. [F L]) - freeleech = cells[labels.index('Name')].find('font', color='green') - if not freeleech or freeleech.get_text(strip=True) != '[F\xa0L]': - continue - - seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) - leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[labels.index('Size')].get_text(strip=True) - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - def login(self): - if any(dict_from_cookiejar(self.session.cookies).values()): - return True - - login_params = { - 'username': self.username, - 'password': self.password, - 'login': 'Log in!', - } - - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: - logger.log('Unable to connect to provider', logger.WARNING) - return False - - if re.search('Username or password incorrect', response): - logger.log('Invalid username or password. Check your settings', logger.WARNING) - return False - - return True - - -provider = TorrentBytesProvider() diff --git a/sickbeard/providers/torrentproject.py b/sickbeard/providers/torrentproject.py deleted file mode 100644 index 8426e64c02..0000000000 --- a/sickbeard/providers/torrentproject.py +++ /dev/null @@ -1,139 +0,0 @@ -# coding=utf-8 -# Author: Gonçalo M. (aka duramato/supergonkas) -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import validators -import traceback - -from sickbeard import logger, tvcache -from sickbeard.common import USER_AGENT - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class TorrentProjectProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'TorrentProject') - - # Credentials - self.public = True - - # URLs - self.url = 'https://torrentproject.se/' - self.custom_url = None - - # Proper Strings - - # Miscellaneous Options - self.headers.update({'User-Agent': USER_AGENT}) - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, search_params={'RSS': ['0day']}) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements - results = [] - - search_params = { - 'out': 'json', - 'filter': 2101, - 'showmagnets': 'on', - 'num': 150 - } - - for mode in search_strings: # Mode = RSS, Season, Episode - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_params['s'] = search_string - - if self.custom_url: - if not validators.url(self.custom_url): - logger.log('Invalid custom url set, please check your settings', logger.WARNING) - return results - search_url = self.custom_url - else: - search_url = self.url - - torrents = self.get_url(search_url, params=search_params, returns='json') - if not (torrents and int(torrents.pop('total_found', 0)) > 0): - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - for result in torrents: - try: - title = torrents[result].get('title') - download_url = torrents[result].get('magnet') - if not all([title, download_url]): - continue - - download_url += self._custom_trackers - seeders = try_int(torrents[result].get('seeds'), 1) - leechers = try_int(torrents[result].get('leechs'), 0) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = torrents[result].get('torrent_size') - size = convert_size(torrent_size) or -1 - torrent_hash = torrents[result].get('torrent_hash') - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': torrent_hash, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - -provider = TorrentProjectProvider() diff --git a/sickbeard/providers/torrentz.py b/sickbeard/providers/torrentz.py deleted file mode 100644 index c296e64a20..0000000000 --- a/sickbeard/providers/torrentz.py +++ /dev/null @@ -1,139 +0,0 @@ -# coding=utf-8 -# Author: Dustyn Gibson -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import re -import traceback - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser -from sickbeard.common import USER_AGENT - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class TorrentzProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """Torrentz Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, "Torrentz") - - # Credentials - self.public = True - self.confirmed = True - - # URLs - self.url = 'https://torrentz.eu/' - self.urls = { - 'verified': 'https://torrentz.eu/feed_verified', - 'feed': 'https://torrentz.eu/feed', - 'base': self.url, - } - - # Proper Strings - self.headers.update({'User-Agent': USER_AGENT}) - - # Miscellaneous Options - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, min_time=15) # only poll Torrentz every 15 minutes max - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals - results = [] - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - search_url = self.urls['verified'] if self.confirmed else self.urls['feed'] - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - data = self.get_url(search_url, params={'q': search_string}, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - if not data.startswith(". - -from __future__ import unicode_literals - -import re -import traceback - -from requests.utils import dict_from_cookiejar -from requests.compat import urljoin - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import try_int -from sickrage.helper.exceptions import AuthException -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class TransmitTheNetProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """TransmitTheNet Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'TransmitTheNet') - - # Credentials - self.username = None - self.password = None - - # URLs - self.url = 'https://transmithe.net/' - self.urls = { - 'login': urljoin(self.url, '/login.php'), - 'search': urljoin(self.url, '/torrents.php'), - } - - # Proper Strings - - # Miscellaneous Options - self.freeleech = None - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - TransmitTheNet search and parsing - - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - if not self.login(): - return results - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_params = { - 'searchtext': search_string, - 'filter_freeleech': (0, 1)[self.freeleech is True], - 'order_by': ('seeders', 'time')[mode == 'RSS'], - 'order_way': 'desc' - } - - if not search_string: - del search_params['searchtext'] - - data = self.get_url(self.urls['search'], params=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', {'id': 'torrent_table'}) - if not torrent_table: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - torrent_rows = torrent_table('tr', {'class': 'torrent'}) - - # Continue only if one Release is found - if not torrent_rows: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - for torrent_row in torrent_rows: - try: - freeleech = torrent_row.find('img', alt='Freeleech') is not None - if self.freeleech and not freeleech: - continue - - download_item = torrent_row.find('a', {'title': [ - 'Download Torrent', # Download link - 'Previously Grabbed Torrent File', # Already Downloaded - 'Currently Seeding Torrent', # Seeding - 'Currently Leeching Torrent', # Leeching - ]}) - - if not download_item: - continue - - download_url = urljoin(self.url, download_item['href']) - - temp_anchor = torrent_row.find('a', {'data-src': True}) - title = temp_anchor['data-src'].rsplit('.', 1)[0] - if not all([title, download_url]): - continue - - cells = torrent_row('td') - seeders = try_int(cells[8].text.strip()) - leechers = try_int(cells[9].text.strip()) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - size = temp_anchor['data-filesize'] or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - def login(self): - if any(dict_from_cookiejar(self.session.cookies).values()): - return True - - login_params = { - 'username': self.username, - 'password': self.password, - 'keeplogged': 'on', - 'login': 'Login' - } - - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: - logger.log('Unable to connect to provider', logger.WARNING) - return False - - if re.search('Username Incorrect', response) or re.search('Password Incorrect', response): - logger.log('Invalid username or password. Check your settings', logger.WARNING) - return False - - return True - - def _check_auth(self): - - if not self.username or not self.password: - raise AuthException('Your authentication credentials for {0} are missing,' - ' check your config.'.format(self.name)) - - return True - - -provider = TransmitTheNetProvider() diff --git a/sickbeard/providers/tvchaosuk.py b/sickbeard/providers/tvchaosuk.py deleted file mode 100644 index 79f7f4570f..0000000000 --- a/sickbeard/providers/tvchaosuk.py +++ /dev/null @@ -1,212 +0,0 @@ -# coding=utf-8 -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import re -import traceback - -from requests.compat import urljoin - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.helper.exceptions import AuthException -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class TVChaosUKProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """TVChaosUK Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'TvChaosUK') - - # Credentials - self.username = None - self.password = None - - # URLs - self.url = 'https://www.tvchaosuk.com' - self.urls = { - 'login': urljoin(self.url, 'takelogin.php'), - 'index': urljoin(self.url, 'index.php'), - 'search': urljoin(self.url, 'browse.php'), - } - - # Proper Strings - - # Miscellaneous Options - self.freeleech = None - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - TVChaosUK search and parsing - - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - if not self.login(): - return results - - # Search Params - search_params = { - 'do': 'search', - 'search_type': 't_name', - 'category': 0, - 'include_dead_torrents': 'no', - 'submit': 'search' - } - - # Units - units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode == 'Season': - search_string = re.sub(r'(.*)S0?', r'\1Series ', search_string) - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_params['keywords'] = search_string - data = self.get_url(self.urls['search'], post_data=search_params, returns='text') - if not data: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find(id='sortabletable') - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - labels = [label.img['title'] if label.img else label.get_text(strip=True) for label in torrent_rows[0]('td')] - for torrent in torrent_rows[1:]: - try: - if self.freeleech and not torrent.find('img', alt=re.compile('Free Torrent')): - continue - - title = torrent.find(class_='tooltip-content') - title = title.div.get_text(strip=True) if title else None - download_url = torrent.find(title='Click to Download this Torrent!') - download_url = download_url.parent['href'] if download_url else None - if not all([title, download_url]): - continue - - seeders = try_int(torrent.find(title='Seeders').get_text(strip=True)) - leechers = try_int(torrent.find(title='Leechers').get_text(strip=True)) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - # Chop off tracker/channel prefix or we cant parse the result! - if mode != 'RSS' and search_params['keywords']: - show_name_first_word = re.search(r'^[^ .]+', search_params['keywords']).group() - if not title.startswith(show_name_first_word): - title = re.sub(r'.*(' + show_name_first_word + '.*)', r'\1', title) - - # Change title from Series to Season, or we can't parse - if mode == 'Season': - title = re.sub(r'(.*)(?i)Series', r'\1Season', title) - - # Strip year from the end or we can't parse it! - title = re.sub(r'(.*)[\. ]?\(\d{4}\)', r'\1', title) - title = re.sub(r'\s+', r' ', title) - - torrent_size = torrent('td')[labels.index('Size')].get_text(strip=True) - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title + '.hdtv.x264', - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - def login(self): - if len(self.session.cookies) >= 4: - return True - - login_params = { - 'username': self.username, - 'password': self.password, - 'logout': 'no', - 'submit': 'LOGIN', - 'returnto': '/browse.php', - } - - response = self.get_url(self.urls['login'], post_data=login_params, returns='text') - if not response: - logger.log('Unable to connect to provider', logger.WARNING) - return False - - if re.search('Error: Username or password incorrect!', response): - logger.log('Invalid username or password. Check your settings', logger.WARNING) - return False - - return True - - def _check_auth(self): - if self.username and self.password: - return True - - raise AuthException('Your authentication credentials for {0} are missing,' - ' check your config.'.format(self.name)) - - -provider = TVChaosUKProvider() diff --git a/sickbeard/providers/zooqle.py b/sickbeard/providers/zooqle.py deleted file mode 100644 index 758270c2ce..0000000000 --- a/sickbeard/providers/zooqle.py +++ /dev/null @@ -1,163 +0,0 @@ -# coding=utf-8 -# Author: medariox -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import traceback - -from requests.compat import urljoin - -from sickbeard import logger, tvcache -from sickbeard.bs4_parser import BS4Parser - -from sickrage.helper.common import convert_size, try_int -from sickrage.providers.torrent.TorrentProvider import TorrentProvider - - -class ZooqleProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """Zooqle Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'Zooqle') - - # Credentials - self.public = True - - # URLs - self.url = 'https://zooqle.com/' - self.urls = { - 'search': urljoin(self.url, '/search'), - } - - # Proper Strings - self.proper_strings = ['PROPER', 'REPACK', 'REAL'] - - # Miscellaneous Options - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tvcache.TVCache(self, min_time=15) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - Zooqle search and parsing - - :param search_string: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - - # Search Params - search_params = { - 'q': '* category:TV', - 's': 'dt', - 'v': 't', - 'sd': 'd', - } - - # Units - units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] - - for mode in search_strings: - items = [] - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - search_params = {'q': '{0} category:TV'.format(search_string)} - - response = self.get_url(self.urls['search'], params=search_params, returns='response') - if not response or not response.text: - logger.log('No data returned from provider', logger.DEBUG) - continue - - with BS4Parser(response.text, 'html5lib') as html: - torrent_table = html.find('div', class_='panel-body') - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one Release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - continue - - # Skip column headers - for result in torrent_rows[1:]: - cells = result('td') - - try: - title = cells[1].find('a').get_text() - magnet = cells[2].find('a')['href'] - download_url = '{magnet}{trackers}'.format(magnet=magnet, - trackers=self._custom_trackers) - if not all([title, download_url]): - continue - - seeders = 1 - leechers = 0 - if len(cells) > 6: - peers = cells[6].find('div') - if peers and peers.get('title'): - peers = peers['title'].replace(',', '').split(' | ', 1) - seeders = try_int(peers[0].strip('Seeders: ')) - leechers = try_int(peers[1].strip('Leechers: ')) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = cells[4].get_text(strip=True) - size = convert_size(torrent_size, units=units) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - continue - - results += items - - return results - - -provider = ZooqleProvider() diff --git a/sickbeard/server/web/config/providers.py b/sickbeard/server/web/config/providers.py index a07715bbb1..71ddb740b2 100644 --- a/sickbeard/server/web/config/providers.py +++ b/sickbeard/server/web/config/providers.py @@ -13,7 +13,7 @@ from sickbeard import ( config, logger, ui, ) -from sickbeard.providers import newznab, rsstorrent +from sickbeard.providers import NewznabProvider from sickrage.helper.common import try_int from sickrage.helper.encoding import ek from sickrage.providers.GenericProvider import GenericProvider @@ -51,7 +51,7 @@ def canAddNewznabProvider(name): provider_dict = dict(zip([x.get_id() for x in sickbeard.newznabProviderList], sickbeard.newznabProviderList)) - temp_provider = newznab.NewznabProvider(name, '') + temp_provider = NewznabProvider(name, '') if temp_provider.get_id() in provider_dict: return json.dumps({'error': 'Provider Name already exists as {name}'.format(name=provider_dict[temp_provider.get_id()].name)}) @@ -84,7 +84,7 @@ def saveNewznabProvider(name, url, key=''): return '|'.join([provider_dict[name].get_id(), provider_dict[name].config_string()]) else: - new_provider = newznab.NewznabProvider(name, url, key=key) + new_provider = NewznabProvider(name, url, key=key) sickbeard.newznabProviderList.append(new_provider) return '|'.join([new_provider.get_id(), new_provider.config_string()]) @@ -111,7 +111,7 @@ def getNewznabCategories(name, url, key): # providerDict = dict(zip([x.get_id() for x in sickbeard.newznabProviderList], sickbeard.newznabProviderList)) # Get newznabprovider obj with provided name - temp_provider = newznab.NewznabProvider(name, url, key) + temp_provider = NewznabProvider(name, url, key) success, tv_categories, error = temp_provider.get_newznab_categories() @@ -147,7 +147,7 @@ def canAddTorrentRssProvider(name, url, cookies, titleTAG): provider_dict = dict( zip([x.get_id() for x in sickbeard.torrentRssProviderList], sickbeard.torrentRssProviderList)) - temp_provider = rsstorrent.TorrentRssProvider(name, url, cookies, titleTAG) + temp_provider = TorrentRSSProvider(name, url, cookies, titleTAG) if temp_provider.get_id() in provider_dict: return json.dumps({'error': 'Exists as {name}'.format(name=provider_dict[temp_provider.get_id()].name)}) @@ -178,7 +178,7 @@ def saveTorrentRssProvider(name, url, cookies, titleTAG): return '|'.join([provider_dict[name].get_id(), provider_dict[name].config_string()]) else: - new_provider = rsstorrent.TorrentRssProvider(name, url, cookies, titleTAG) + new_provider = TorrentRSSProvider(name, url, cookies, titleTAG) sickbeard.torrentRssProviderList.append(new_provider) return '|'.join([new_provider.get_id(), new_provider.config_string()]) @@ -225,7 +225,7 @@ def saveProviders(self, newznab_string='', torrentrss_string='', provider_order= cur_name, cur_url, cur_key, cur_cat = curNewznabProviderStr.split('|') cur_url = config.clean_url(cur_url) - new_provider = newznab.NewznabProvider(cur_name, cur_url, key=cur_key, catIDs=cur_cat) + new_provider = NewznabProvider(cur_name, cur_url, key=cur_key, catIDs=cur_cat) cur_id = new_provider.get_id() @@ -292,7 +292,7 @@ def saveProviders(self, newznab_string='', torrentrss_string='', provider_order= cur_name, cur_url, cur_cookies, cur_title_tag = curTorrentRssProviderStr.split('|') cur_url = config.clean_url(cur_url) - new_provider = rsstorrent.TorrentRssProvider(cur_name, cur_url, cur_cookies, cur_title_tag) + new_provider = TorrentRSSProvider(cur_name, cur_url, cur_cookies, cur_title_tag) cur_id = new_provider.get_id() diff --git a/sickrage/helper/exceptions.py b/sickrage/helper/exceptions.py index 85038a9a05..059cedeb26 100644 --- a/sickrage/helper/exceptions.py +++ b/sickrage/helper/exceptions.py @@ -65,7 +65,7 @@ class SickRageException(Exception): class AuthException(SickRageException): """ - Your authentication information are incorrect + Authentication information is incorrect """ @@ -77,7 +77,7 @@ class CantRefreshShowException(SickRageException): class CantRemoveShowException(SickRageException): """ - The show can't removed right now + The show can't be removed right now """ diff --git a/sickrage/providers/exceptions.py b/sickrage/providers/exceptions.py new file mode 100644 index 0000000000..ea7f059c8c --- /dev/null +++ b/sickrage/providers/exceptions.py @@ -0,0 +1,19 @@ +# coding=utf-8 + +""" +Exceptions raised by Medusa providers +""" + +from __future__ import unicode_literals + + +class ProviderError(Exception): + """A generic provider exception occurred.""" + + +class AuthenticationError(ProviderError): + """Provider authentication failed.""" + + +class ParsingError(ProviderError): + """Provider parsing failed.""" diff --git a/tests/feedparser_tests.py b/tests/feedparser_tests.py index 5cca764782..72f8dd9e0b 100644 --- a/tests/feedparser_tests.py +++ b/tests/feedparser_tests.py @@ -13,7 +13,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))) sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from sickbeard.providers.womble import provider as womble +from sickbeard.providers.nzb.womble import provider as womble class FeedParserTests(unittest.TestCase): diff --git a/tests/test_lib.py b/tests/test_lib.py index 916c3dddfb..b45c4b262b 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -52,7 +52,7 @@ from configobj import ConfigObj from sickbeard import db, providers from sickbeard.databases import cache_db, failed_db, main_db -from sickbeard.providers.newznab import NewznabProvider +from sickbeard.providers.nzb.newznab import NewznabProvider from sickbeard.tv import TVEpisode import shutil_custom # pylint: disable=import-error import sickbeard diff --git a/tests/torrent_tests.py b/tests/torrent_tests.py index 2e060578bf..309e486478 100644 --- a/tests/torrent_tests.py +++ b/tests/torrent_tests.py @@ -34,7 +34,7 @@ from bs4 import BeautifulSoup from sickbeard.helpers import getURL, make_session -from sickbeard.providers.bitcannon import BitCannonProvider +from sickbeard.providers.torrent.json.bitcannon import BitCannonProvider from sickbeard.tv import TVEpisode, TVShow import tests.test_lib as test from six.moves.urllib_parse import urljoin