From 8695768ddc77b28c451bcd7b6f2f0fbbc299bd46 Mon Sep 17 00:00:00 2001 From: miigotu Date: Wed, 17 Feb 2016 09:45:10 -0800 Subject: [PATCH 01/34] Fix searches with ABNormal Replace double quotes with single quotes as per our decided convention ABNormal, sort by Time for rss, or Seeders for everything else. --- sickbeard/providers/abnormal.py | 75 +++++++++++++++++---------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/sickbeard/providers/abnormal.py b/sickbeard/providers/abnormal.py index 9fbd146ad8..d31bb8dcbe 100644 --- a/sickbeard/providers/abnormal.py +++ b/sickbeard/providers/abnormal.py @@ -36,7 +36,7 @@ class ABNormalProvider(TorrentProvider): # pylint: disable=too-many-instance-at def __init__(self): # Provider Init - TorrentProvider.__init__(self, "ABNormal") + TorrentProvider.__init__(self, 'ABNormal') # Credentials self.username = None @@ -48,14 +48,14 @@ def __init__(self): self.minleech = None # URLs - self.url = "https://abnormal.ws" + self.url = 'https://abnormal.ws' self.urls = { - "login": urljoin(self.url, "login.php"), - "search": urljoin(self.url, "torrents.php"), + 'login': urljoin(self.url, 'login.php'), + 'search': urljoin(self.url, 'torrents.php'), } # Proper Strings - self.proper_strings = ["PROPER"] + self.proper_strings = ['PROPER'] # Cache self.cache = tvcache.TVCache(self, min_time=30) @@ -65,17 +65,17 @@ def login(self): return True login_params = { - "username": self.username, - "password": self.password, + 'username': self.username, + 'password': self.password, } - response = self.get_url(self.urls["login"], post_data=login_params, timeout=30, returns="text") + response = self.get_url(self.urls['login'], post_data=login_params, timeout=30, returns='text') if not response: - logger.log("Unable to connect to provider", logger.WARNING) + logger.log('Unable to connect to provider', logger.WARNING) return False - if not re.search("torrents.php", response): - logger.log("Invalid username or password. Check your settings", logger.WARNING) + if not re.search('torrents.php', response): + logger.log('Invalid username or password. Check your settings', logger.WARNING) return False return True @@ -87,71 +87,72 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man # Search Params search_params = { - "cat[]": ["TV|SD|VOSTFR", "TV|HD|VOSTFR", "TV|SD|VF", "TV|HD|VF", "TV|PACK|FR", "TV|PACK|VOSTFR", "TV|EMISSIONS", "ANIME"], - # Sorting: by time. Available parameters: ReleaseName, Seeders, Leechers, Snatched, Size - "order": "Time", - # Both ASC and DESC are available - "way": "DESC" + 'cat[]': ['TV|SD|VOSTFR', 'TV|HD|VOSTFR', 'TV|SD|VF', 'TV|HD|VF', 'TV|PACK|FR', 'TV|PACK|VOSTFR', 'TV|EMISSIONS', 'ANIME'], + # Both ASC and DESC are available for sort direction + 'way': 'DESC' } # Units - units = ["O", "KO", "MO", "GO", "TO", "PO"] + units = ['O', 'KO', 'MO', 'GO', 'TO', 'PO'] for mode in search_strings: items = [] - logger.log("Search Mode: {}".format(mode), logger.DEBUG) + logger.log('Search Mode: {}'.format(mode), logger.DEBUG) for search_string in search_strings[mode]: - if mode != "RSS": - logger.log("Search string: {}".format(search_string.decode("utf-8")), + if mode != 'RSS': + logger.log('Search string: {}'.format(search_string.decode('utf-8')), logger.DEBUG) - search_params["search"] = search_string - data = self.get_url(self.urls["search"], params=search_params, returns="text") + # Sorting: Available parameters: ReleaseName, Seeders, Leechers, Snatched, Size + search_params['order'] = ('Seeders', 'Time')[mode == 'RSS'] + search_params['search'] = re.sub(r'[()]', '', search_string) + data = self.get_url(self.urls['search'], params=search_params, returns='text') if not data: continue - with BS4Parser(data, "html5lib") as html: - torrent_table = html.find("table", class_=re.compile("torrent_table cats")) - torrent_rows = torrent_table.find_all("tr") if torrent_table else [] + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find(class_='torrent_table') + torrent_rows = torrent_table.find_all('tr') if torrent_table else [] # Continue only if at least one Release is found if len(torrent_rows) < 2: - logger.log("Data returned from provider does not contain any torrents", logger.DEBUG) + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) continue # Catégorie, Release, Date, DL, Size, C, S, L - labels = [label.get_text(strip=True) for label in torrent_rows[0].find_all("td")] + labels = [label.get_text(strip=True) for label in torrent_rows[0].find_all('td')] # Skip column headers for result in torrent_rows[1:]: - cells = result.find_all("td") + cells = result.find_all('td') if len(cells) < len(labels): continue try: - title = cells[labels.index("Release")].get_text(strip=True) - download_url = urljoin(self.url, cells[labels.index("DL")].find("a", class_="tooltip")["href"]) + title = cells[labels.index('Release')].get_text(strip=True) + download_url = urljoin(self.url, cells[labels.index('DL')].find('a', class_='tooltip')['href']) if not all([title, download_url]): continue - seeders = try_int(cells[labels.index("S")].get_text(strip=True)) - leechers = try_int(cells[labels.index("L")].get_text(strip=True)) + seeders = try_int(cells[labels.index('S')].get_text(strip=True)) + leechers = try_int(cells[labels.index('L')].get_text(strip=True)) # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: - if mode != "RSS": - logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {} (S:{} L:{})".format + if mode != 'RSS': + logger.log('Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {} (S:{} L:{})'.format (title, seeders, leechers), logger.DEBUG) continue - torrent_size = cells[labels.index("Size")].get_text() + size_index = labels.index('Size') if 'Size' in labels else labels.index('Taille') + torrent_size = cells[size_index].get_text() size = convert_size(torrent_size, units=units) or -1 item = title, download_url, size, seeders, leechers - if mode != "RSS": - logger.log("Found result: {} with {} seeders and {} leechers".format + if mode != 'RSS': + logger.log('Found result: {} with {} seeders and {} leechers'.format (title, seeders, leechers), logger.DEBUG) items.append(item) From a445380e00b0bcc36d500c4542b2145cb424cf71 Mon Sep 17 00:00:00 2001 From: miigotu Date: Wed, 17 Feb 2016 19:12:20 -0800 Subject: [PATCH 02/34] NewPCT: * Unicode literals * Fix string quotes * Fix get_url override * Conformity and cleanup --- sickbeard/providers/newpct.py | 138 ++++++++++++++-------------------- 1 file changed, 56 insertions(+), 82 deletions(-) diff --git a/sickbeard/providers/newpct.py b/sickbeard/providers/newpct.py index 3b261dbe85..7bffdd4ccd 100644 --- a/sickbeard/providers/newpct.py +++ b/sickbeard/providers/newpct.py @@ -18,9 +18,9 @@ # You should have received a copy of the GNU General Public License # along with SickRage. If not, see . +from __future__ import unicode_literals +from requests.compat import urljoin import re -from six.moves import urllib -import traceback from sickbeard import helpers from sickbeard import logger, tvcache @@ -34,34 +34,32 @@ class newpctProvider(TorrentProvider): def __init__(self): - TorrentProvider.__init__(self, "Newpct") + TorrentProvider.__init__(self, 'Newpct') self.onlyspasearch = None - self.cache = tvcache.TVCache(self, min_time=10) - # Unsupported - # self.minseed = None - # self.minleech = None + self.url = 'http://www.newpct.com' + self.urls = {'search': urljoin(self.url, 'index.php')} - self.urls = { - 'base_url': 'http://www.newpct.com', - 'search': 'http://www.newpct.com/index.php' - } - - self.url = self.urls['base_url'] + self.cache = tvcache.TVCache(self, min_time=20) + def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals """ Search query: http://www.newpct.com/index.php?l=doSearch&q=fringe&category_=All&idioma_=1&bus_de_=All q => Show name - category_ = Category "Shows" (767) + category_ = Category 'Shows' (767) idioma_ = Language Spanish (1) bus_de_ = Date from (All, hoy) """ + results = [] - self.search_params = { + # Only search if user conditions are true + lang_info = '' if not ep_obj or not ep_obj.show else ep_obj.show.lang + + search_params = { 'l': 'doSearch', 'q': '', 'category_': 'All', @@ -69,103 +67,79 @@ def __init__(self): 'bus_de_': 'All' } - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals - - results = [] - - # Only search if user conditions are true - lang_info = '' if not ep_obj or not ep_obj.show else ep_obj.show.lang - for mode in search_strings: items = [] - logger.log(u"Search Mode: {}".format(mode), logger.DEBUG) + logger.log('Search Mode: {}'.format(mode), logger.DEBUG) # Only search if user conditions are true if self.onlyspasearch and lang_info != 'es' and mode != 'RSS': - logger.log(u"Show info is not spanish, skipping provider search", logger.DEBUG) + logger.log('Show info is not spanish, skipping provider search', logger.DEBUG) continue + search_params['bus_de_'] = 'All' if mode != 'RSS' else 'hoy' + for search_string in search_strings[mode]: if mode != 'RSS': - logger.log(u"Search string: {}".format(search_string.decode("utf-8")), + logger.log('Search string: {}'.format(search_string.decode('utf-8')), logger.DEBUG) - self.search_params['q'] = search_string.strip() if mode != 'RSS' else '' - self.search_params['bus_de_'] = 'All' if mode != 'RSS' else 'hoy' + search_params['q'] = search_string - search_url = self.urls['search'] + '?' + urllib.parse.urlencode(self.search_params) - logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - - data = self.get_url(search_url, timeout=30) + data = self.get_url(self.urls['search'], params=search_params, timeout=30, returns='text') if not data: continue - try: - with BS4Parser(data, 'html5lib') as html: - torrent_tbody = html.find('tbody') - - if torrent_tbody is None: - logger.log(u"Data returned from provider does not contain any torrents", logger.DEBUG) - continue - - torrent_table = torrent_tbody.findAll('tr') - if not torrent_table: - logger.log(u"Torrent table does not have any rows", logger.DEBUG) - continue - - for row in torrent_table[:-1]: - try: - torrent_row = row.findAll('a')[0] - - download_url = torrent_row.get('href', '') - - title = self._processTitle(torrent_row.get('title', '')) - - # Provider does not provide seeders/leechers - seeders = 1 - leechers = 0 - torrent_size = row.findAll('td')[2].text - - size = convert_size(torrent_size) or -1 - except (AttributeError, TypeError): - continue - + with BS4Parser(data, 'html5lib') as html: + torrent_table = html.find('table', id='categoryTable') + torrent_rows = torrent_table.find_all('tr') if torrent_table else [] + + # Continue only if at least one Release is found + if len(torrent_rows) < 3: # Headers + 1 Torrent + Pagination + logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) + continue + + # 'Fecha', 'Título', 'Tamaño', '' + # Date, Title, Size + labels = [label.get_text(strip=True) for label in torrent_rows[0].find_all('th')] + for row in torrent_rows[1:-1]: + try: + cells = row.find_all('td') + + torrent_row = row.find('a') + title = self._processTitle(torrent_row.get('title', '')) + download_url = torrent_row.get('href', '') if not all([title, download_url]): continue - # Filter unseeded torrent (Unsupported) - # if seeders < self.minseed or leechers < self.minleech: - # if mode != 'RSS': - # logger.log(u"Discarding torrent because it doesn't meet the minimum seeders or leechers: {} (S:{} L:{})".format(title, seeders, leechers), logger.DEBUG) - # continue + # Provider does not provide seeders/leechers + seeders = 1 + leechers = 0 + torrent_size = cells[labels.index('Tamaño')].get_text(strip=True) + size = convert_size(torrent_size) or -1 item = title, download_url, size, seeders, leechers if mode != 'RSS': - logger.log(u"Found result: %s " % title, logger.DEBUG) + logger.log('Found result: {}'.format(title), logger.DEBUG) items.append(item) - - except Exception: - logger.log(u"Failed parsing provider. Traceback: %s" % traceback.format_exc(), logger.WARNING) - - # For each search mode sort all the items by seeders if available (Unsupported) - # items.sort(key=lambda tup: tup[3], reverse=True) + except (AttributeError, TypeError): + continue results += items return results - def get_url(self, url, post_data=None, params=None, timeout=30, json=False, need_bytes=False): # pylint: disable=too-many-arguments + def get_url(self, url, post_data=None, params=None, timeout=30, json=False, need_bytes=False, **kwargs): # pylint: disable=too-many-arguments """ need_bytes=True when trying access to torrent info (For calling torrent client). Previously we must parse the URL to get torrent file """ if need_bytes: - data = helpers.getURL(url, headers=self.headers, timeout=timeout, session=self.session, returns='json') + data = super(newpctProvider, self).get_url(url, post_data=post_data, params=params, timeout=timeout, json=json, kwargs=kwargs) url = re.search(r'http://tumejorserie.com/descargar/.+\.torrent', data, re.DOTALL).group() - return helpers.getURL(url, post_data=post_data, params=params, headers=self.headers, timeout=timeout, - session=self.session, json=json, need_bytes=need_bytes) + return super(newpctProvider, self).get_url(url, post_data=post_data, params=params, timeout=timeout, + json=json, need_bytes=need_bytes, kwargs=kwargs) def download_result(self, result): """ @@ -186,24 +160,24 @@ def download_result(self, result): if url_torrent.startswith('http'): self.headers.update({'Referer': '/'.join(url_torrent.split('/')[:3]) + '/'}) - logger.log(u"Downloading a result from " + self.name + " at " + url) + logger.log('Downloading a result from {}'.format(url)) if helpers.download_file(url_torrent, filename, session=self.session, headers=self.headers): if self._verify_download(filename): - logger.log(u"Saved result to " + filename, logger.INFO) + logger.log('Saved result to {}'.format(filename), logger.INFO) return True else: - logger.log(u"Could not download %s" % url, logger.WARNING) + logger.log('Could not download {}'.format(url), logger.WARNING) helpers.remove_file_failed(filename) if len(urls): - logger.log(u"Failed to download any results", logger.WARNING) + logger.log('Failed to download any results', logger.WARNING) return False @staticmethod def _processTitle(title): - # Remove "Mas informacion sobre " literal from title + # Remove 'Mas informacion sobre ' literal from title title = title[22:] # Quality - Use re module to avoid case sensitive problems with replace From e143bb94e4365474baa7304e3a33f26d45d2acd4 Mon Sep 17 00:00:00 2001 From: Indigo744 Date: Thu, 18 Feb 2016 21:51:20 +0100 Subject: [PATCH 03/34] Add OCS network logo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added OCS (Orange Ciné Series) logo, a french network. --- gui/slick/images/network/ocs.png | Bin 0 -> 3501 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 gui/slick/images/network/ocs.png diff --git a/gui/slick/images/network/ocs.png b/gui/slick/images/network/ocs.png new file mode 100644 index 0000000000000000000000000000000000000000..afae58e058972b2fb5580b370d19dfa5476cb218 GIT binary patch literal 3501 zcmV;e4N~%nP)004R>004l5008;`004mK004C`008P>0026e000+ooVrmw00006 zVoOIv00000008+zyMF)x010qNS#tmYE+PN`E+PS{;@y$}01XjIL_t(&-tAa)!feCQBR{))|4vFd+L0>$FNHp!aTmcAAhmnIXbZE4pCo~N)jLq zAddkC0ihsZ11td64OjNe+abqKo#V^De@%OMZhfs`uU_~lHorTWN>Ao0HUJHP;Q@pI z%pf=funNeOPUSrE?honSU%zH_?0%tL1BCW{Hj3Z+%>w0AQLZomcr>t3+7BJ+0+tPF z0&aG%LY_Q#s(1LEn=Yn&XyyEgyyUg%YK6Zm`2f~+fJFP)V8F3}MuzM41ntSMJsmgp zskd7>s;(-@`SZi?D4V~yN6`UzHLwuv`4q%15V9eg0B7Kk0`BmveuVGICdhX5JjDgf4uD4uk| z)i&IlezsDX_VL}Co@fgIaBI#c%Y{!b)G8`6DcJ)i3M)(mxHh=a1lNavKnQ3w0oREk zNkEbT?ZSnQ{Py#2I@P5qu{u~K`>&#RU!141B*Yt0B?*9K0s#{o2!-(*Oh7te%!ti7 z&I6^k&L@}sVWzArH2}aYJ=Rlf{^w-H2e76`4QN?#HJ{_8bv@gfpN$0o6#$&I|Cl5W6bz#TJ)Dcp*0(~CAh>xWHdD^90x3aM^z|@Nxc4SS zT3|iK@(CK0L%WB_)eHX?+MN8 zJxJe|Nhc*M2a8{stsdPu99~b@rkVz7GC|uueSqt$KUdw_^C`{?w>&HqMDjhNkmJCW z0`7y;f31vpWWJu5ZUcZ@RixP)KYEfEPoJaMHlV6uV4*^2Hr%btA95>Byrp#RTO&^$ z>QDRD-6?#C7&gK-7=oLWUd>Pbb%OWWpRWa00RWns>b?BlOS9;MKR+dO8)z;O42Xb) zdvI$$fNga8o8FL!eTXW?f7GR+`(F&XxOje4^N9#N7Za04Mk6yBE%N zQsur4BdDw>hrB8{Cg-`yfjfG_GVg!Cy$Wsp|| zoiFKS7p%Aqz)~FEd6THC_i}Z_7j zkO~IDGT|n5sMB=CqjJ%nks@e=6Ft$ib}YDw@gaD=U}*Vs5DytZ5KtT@seem?<|bdfa?@%s$!sC;QyO-TUdj=cgdE=eM3t zeVd$`Z#CHUUWJ$_DXT$sE6xlIM^a1$wj{V)&tfIL%Mn~m(z+Z)M)wjF9~uBHx=bn{ zD$jRw{O8g{Re2`b-l8^utlYz%9)rs+2Cb%a4+6Mo10?$8Z{c{411qG$h}0HlK-C}= z0s6i?ll|r6c-QK29A~)5=w2wh_dn3Ia=`{k?#Ft1zxoy|app)a47Jq(>;jaUabE9~ z8@Lz=tz*wdj!W`0w_axfps_kp)K_#N>BhCb*dj@m-0?hqgoxa0_sP+}{JR?fC}^V9 zbxjJ`Fn|ODBHjZ<+KM=LqMy2P`Gd;*CziO=?%EODxb)r@V=HwRRMIqXtR@VQraPA+ z!ilL3a9gjD0AMv4=>8LE*gr`#*-h{@|!^yZ1W<({8Pp0&wkjo!z=J-^B=6M|T-i zQ`w2y@i2^p2FFM%mqKl5t$l<{ZDr>lYk(F*jQGVq&y>GCE9Y$J=S-PD!5w+)Vmmwc zn5%ki*$@Jn+N20G*lR#BC3w?E52#93-~HIV<@#2r8iD+$`$&ILE`a^*9ZUxGG(4u__I}%9k ze3lc^%Bb}C$Z(ztkfN{g0@xC*)! zbpQw=UICxjbB)S*H7V^R6}HiDh)~@n{gGCE4z_PwoM*pe?TBFU)qI8TGzWQIU+8HXZ@M z9eu}g49H(67C!q95u&%N6E>4Q<&7}_)~HvGsRFPCz?M)&$pUxoLmw#L?6@AjX08mW z>Cgg|al#pxze8s9C_^w>^B4kTD4_N4Pi$QK_la#u2G=i|Kx^N9m^=!&g#`ho4epdx zPD4ho0sxVdw}In47TI>x`icS~20r=I3!&pX2mQbRtKOL+Dl3vHP60Fx&@_;y0Ro_I zxy6(-Xg?cXNdnP*Lx3cp*o`@D@jRWcDCexcC&}hksX&em+m{p~00{l#*_q8xFlnTBauUJ733{Mo38C)BJGvS%p-bwRb001}e zonpWK(8ubw)#F>u0zh~cNbgoI2jBKN6z3#C(NR-9RP0$fUa+e~i!uR@1=n&RXMFZE z${W3zrVKr-oH?Ew9RybfaYu0sz#TaIU!W z1kGG?qbI$)zj@g@wsR1_GIo<1s7_{In}H~3f^C3x0!aor0QnTKs>fh6xGn@|#4YQ^ zv=8s{cvFMck{Qp^qL*G52{G>#5j^SfIx?$wArjNe(O92C=Z;>Z*g+MZ7-vZrvcU#F zJ^#O2Dwuh2?F~HrzSVktO&lk*o!L3ub^_4|iIso}CqBLjFMc%E7(8Zk+ii2j8NEsO8+2FKSIVnVA)0#;Q?zuWL>KfL&J?Z_gUHQropPU!_eW zWL*e$PVO-=dG2H_v2)!I-jA9J3-ZM0e|d%uZWv3|RmnmCq>FID0RYewhdbSb8;y${L} Date: Sat, 20 Feb 2016 15:47:01 -0800 Subject: [PATCH 04/34] Plex: Always return True when getting auth token if username/password aren't configured This was causing failures to update Plex server when no username and password were specified since we were failing to get the auth token --- sickbeard/__init__.py | 5 +---- sickbeard/notifiers/plex.py | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index 58dea8110f..f871f27083 100644 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -351,7 +351,6 @@ PLEX_CLIENT_HOST = None PLEX_SERVER_USERNAME = None PLEX_SERVER_PASSWORD = None -PLEX_SERVER_NO_AUTH = False USE_PLEX_CLIENT = False PLEX_CLIENT_USERNAME = None @@ -601,7 +600,7 @@ def initialize(consoleLogging=True): # pylint: disable=too-many-locals, too-man USE_KODI, KODI_ALWAYS_ON, KODI_NOTIFY_ONSNATCH, KODI_NOTIFY_ONDOWNLOAD, KODI_NOTIFY_ONSUBTITLEDOWNLOAD, KODI_UPDATE_FULL, KODI_UPDATE_ONLYFIRST, \ KODI_UPDATE_LIBRARY, KODI_HOST, KODI_USERNAME, KODI_PASSWORD, BACKLOG_FREQUENCY, \ USE_TRAKT, TRAKT_USERNAME, TRAKT_ACCESS_TOKEN, TRAKT_REFRESH_TOKEN, TRAKT_REMOVE_WATCHLIST, TRAKT_SYNC_WATCHLIST, TRAKT_REMOVE_SHOW_FROM_SICKRAGE, TRAKT_METHOD_ADD, TRAKT_START_PAUSED, traktCheckerScheduler, TRAKT_USE_RECOMMENDED, TRAKT_SYNC, TRAKT_SYNC_REMOVE, TRAKT_DEFAULT_INDEXER, TRAKT_REMOVE_SERIESLIST, TRAKT_TIMEOUT, TRAKT_BLACKLIST_NAME, \ - USE_PLEX_SERVER, PLEX_SERVER_NO_AUTH, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \ + USE_PLEX_SERVER, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \ PLEX_SERVER_HOST, PLEX_SERVER_TOKEN, PLEX_CLIENT_HOST, PLEX_SERVER_USERNAME, PLEX_SERVER_PASSWORD, PLEX_SERVER_HTTPS, MIN_BACKLOG_FREQUENCY, SKIP_REMOVED_FILES, ALLOWED_EXTENSIONS, \ USE_EMBY, EMBY_HOST, EMBY_APIKEY, \ showUpdateScheduler, __INITIALIZED__, INDEXER_DEFAULT_LANGUAGE, EP_DEFAULT_DELETED_STATUS, LAUNCH_BROWSER, TRASH_REMOVE_SHOW, TRASH_ROTATE_LOGS, SORT_ARTICLE, \ @@ -1024,7 +1023,6 @@ def path_leaf(path): PLEX_CLIENT_USERNAME = check_setting_str(CFG, 'Plex', 'plex_client_username', '', censor_log=True) PLEX_CLIENT_PASSWORD = check_setting_str(CFG, 'Plex', 'plex_client_password', '', censor_log=True) PLEX_SERVER_HTTPS = bool(check_setting_int(CFG, 'Plex', 'plex_server_https', 0)) - PLEX_SERVER_NO_AUTH = bool(check_setting_int(CFG, 'Plex', 'plex_server_no_auth', 0)) USE_EMBY = bool(check_setting_int(CFG, 'Emby', 'use_emby', 0)) EMBY_HOST = check_setting_str(CFG, 'Emby', 'emby_host', '') @@ -1936,7 +1934,6 @@ def save_config(): # pylint: disable=too-many-statements, too-many-branches new_config['Plex']['plex_client_host'] = PLEX_CLIENT_HOST new_config['Plex']['plex_server_username'] = PLEX_SERVER_USERNAME new_config['Plex']['plex_server_password'] = helpers.encrypt(PLEX_SERVER_PASSWORD, ENCRYPTION_VERSION) - new_config['Plex']['plex_server_no_auth'] = int(PLEX_SERVER_NO_AUTH) new_config['Plex']['use_plex_client'] = int(USE_PLEX_CLIENT) new_config['Plex']['plex_client_username'] = PLEX_CLIENT_USERNAME diff --git a/sickbeard/notifiers/plex.py b/sickbeard/notifiers/plex.py index ea71d5a6ac..0c26a41606 100644 --- a/sickbeard/notifiers/plex.py +++ b/sickbeard/notifiers/plex.py @@ -130,6 +130,7 @@ def update_library(self, ep_obj=None, host=None, # pylint: disable=too-many-arg return False if not self.get_token(username, password, plex_server_token): + logger.log(u'PLEX: Error getting auth token for Plex Media Server, check your settings', logger.WARNING) return False file_location = '' if not ep_obj else ep_obj.location @@ -220,7 +221,7 @@ def get_token(self, username=None, password=None, plex_server_token=None): return True if not (username and password): - return sickbeard.PLEX_SERVER_NO_AUTH + return True logger.log(u'PLEX: fetching plex.tv credentials for user: ' + username, logger.DEBUG) From dde73eaa9d4e83128519583bc52d2d77c08c4545 Mon Sep 17 00:00:00 2001 From: miigotu Date: Mon, 22 Feb 2016 11:30:49 -0800 Subject: [PATCH 05/34] Use a decorator to prevent processing a dir more than once at a time, regardless of caller Fixes https://github.com/SickRage/sickrage-issues/issues/1028 --- sickbeard/processTV.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/sickbeard/processTV.py b/sickbeard/processTV.py index 142bcd7efc..6a0327123f 100644 --- a/sickbeard/processTV.py +++ b/sickbeard/processTV.py @@ -20,6 +20,7 @@ import os import stat +from functools import wraps import sickbeard from sickbeard import postProcessor @@ -142,7 +143,25 @@ def logHelper(logMessage, logLevel=logger.INFO): return logMessage + u"\n" +def OneRunPP(): + isRunning = [False] + + def decorate(func): + @wraps(func) + def func_wrapper(*args, **kargs): + if isRunning[0]: + return logHelper(u'Post processor is already running', logger.ERROR) + + isRunning[0] = True + ret = func(*args, **kargs) + isRunning[0] = False + return ret + return func_wrapper + return decorate + + # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals +@OneRunPP() def processDir(dirName, nzbName=None, process_method=None, force=False, is_priority=None, delete_on=False, failed=False, proc_type="auto"): """ Scans through the files in dirName and processes whatever media files it finds @@ -633,7 +652,7 @@ def subtitles_enabled(video): :param video: video filename to be parsed """ - + try: parse_result = NameParser().parse(video, cache_result=True) except (InvalidNameException, InvalidShowException): From 72b9c0be41b8b3edbf5aa61be02a1b25576b62dc Mon Sep 17 00:00:00 2001 From: miigotu Date: Tue, 23 Feb 2016 11:03:23 -0800 Subject: [PATCH 06/34] Fixes https://github.com/SickRage/sickrage-issues/issues/1046 --- sickbeard/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 1b3f54ed80..3acfbe352a 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -353,7 +353,7 @@ def copyFile(srcFile, destFile): try: ek(shutil.copyfile, srcFile, destFile) except (SpecialFileError, Error) as error: - logger.log(error, logger.WARNING) + logger.log(u'{}'.format(error), logger.WARNING) except Exception as error: logger.log(u'{}'.format(error), logger.ERROR) else: From 28a2af78ed3bdb3d94ade88dea06b6c23e5e672b Mon Sep 17 00:00:00 2001 From: Thor Jacobsen Date: Tue, 23 Feb 2016 22:06:12 +0100 Subject: [PATCH 07/34] fix: Danishbits provider now works again Aparently, BS4 `class_` lookup does not work when element has more than one class? Anywho, replaced the lookup with `id` --- sickbeard/providers/danishbits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sickbeard/providers/danishbits.py b/sickbeard/providers/danishbits.py index d9d3c0bb4e..8b07e72d18 100644 --- a/sickbeard/providers/danishbits.py +++ b/sickbeard/providers/danishbits.py @@ -126,7 +126,7 @@ def process_column_header(td): continue with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', class_='torrent_table') + torrent_table = html.find('table', id='torrent_table') torrent_rows = torrent_table.find_all('tr') if torrent_table else [] # Continue only if at least one Release is found From 5dd1042d0839c7b87cc0fd705ca4e49e5cdc1195 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 24 Feb 2016 08:38:33 -0300 Subject: [PATCH 08/34] Add more threads to LOGFILTER and increase number of lines as we don't do pagination --- sickbeard/webserve.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 472e3faafa..8c724a45e9 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -5225,7 +5225,7 @@ def clearerrors(self, level=logger.ERROR): return self.redirect("/errorlogs/viewlog/") - def viewlog(self, minLevel=logger.INFO, logFilter="", logSearch=None, maxLines=500): + def viewlog(self, minLevel=logger.INFO, logFilter="", logSearch=None, maxLines=1000): def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): @@ -5285,6 +5285,10 @@ def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): 'SEARCHQUEUE-MANUAL': u'Search Queue (Manual)', 'SEARCHQUEUE-RETRY': u'Search Queue (Retry/Failed)', 'SEARCHQUEUE-RSS': u'Search Queue (RSS)', + 'SHOWQUEUE-FORCE-UPDATE': u'Search Queue (Forced Update)', + 'SHOWQUEUE-UPDATE': u'Search Queue (Update)', + 'SHOWQUEUE-REFRESH': u'Search Queue (Refresh)', + 'SHOWQUEUE-FORCE-REFRESH': u'Search Queue (Forced Refresh)', 'FINDPROPERS': u'Find Propers', 'POSTPROCESSER': u'Postprocesser', 'FINDSUBTITLES': u'Find Subtitles', From b658719ced9935b84e7fe69431230dc656f75600 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 24 Feb 2016 09:36:45 -0300 Subject: [PATCH 09/34] Always write debug/db lines --- sickbeard/webserve.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 472e3faafa..17c801dc16 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -5241,8 +5241,6 @@ def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): if match: level = match.group(7) logName = match.group(8) - if not sickbeard.DEBUG and (level == 'DEBUG' or level == 'DB'): - continue if level not in logger.LOGGING_LEVELS: lastLine = False continue From dd3184cd500ccd88cfd424c0a0a0d8bc850f8039 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 24 Feb 2016 10:03:12 -0300 Subject: [PATCH 10/34] Prevent logging "No results found" message --- sickbeard/providers/rarbg.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sickbeard/providers/rarbg.py b/sickbeard/providers/rarbg.py index 07edef7b24..f8434eeff6 100644 --- a/sickbeard/providers/rarbg.py +++ b/sickbeard/providers/rarbg.py @@ -127,8 +127,12 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man continue error = data.get("error") + error_code = data.get("error_code") + # Don't log when {"error":"No results found","error_code":20} + # List of errors: https://github.com/rarbg/torrentapi/issues/1#issuecomment-114763312 if error: - logger.log(error) + if try_int(error_code) != 20: + logger.log(error) continue torrent_results = data.get("torrent_results") From 7644bd874b60776d29c64f1f9ede9d9b607ee82c Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 24 Feb 2016 15:39:43 -0300 Subject: [PATCH 11/34] Update URLs --- sickbeard/__init__.py | 2 +- sickbeard/indexers/indexer_config.py | 2 +- sickbeard/webserve.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index 7b3b352d44..2de5ab24f0 100644 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -138,7 +138,7 @@ GIT_PATH = None DEVELOPER = False -NEWS_URL = 'https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/news.md' +NEWS_URL = 'https://api.pymedusa.com/news.md' NEWS_LAST_READ = None NEWS_LATEST = None NEWS_UNREAD = 0 diff --git a/sickbeard/indexers/indexer_config.py b/sickbeard/indexers/indexer_config.py index 3e28bb20f3..10b1e90309 100644 --- a/sickbeard/indexers/indexer_config.py +++ b/sickbeard/indexers/indexer_config.py @@ -30,7 +30,7 @@ 'trakt_id': 'tvdb_id', 'xem_origin': 'tvdb', 'icon': 'thetvdb16.png', - 'scene_loc': 'https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/scene_exceptions/scene_exceptions.json', + 'scene_loc': 'https://api.pymedusa.com/scene_exceptions.json', 'show_url': 'http://thetvdb.com/?tab=series&id=', 'base_url': 'http://thetvdb.com/api/%(apikey)s/series/' } diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index b3c9bd94e0..6a16cdffb5 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -2234,7 +2234,7 @@ def __init__(self, *args, **kwargs): def index(self): try: - changes = helpers.getURL('https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/CHANGES.md', session=requests.Session(), returns='text') + changes = helpers.getURL('https://api.pymedusa.com/changelog.md', session=requests.Session(), returns='text') except Exception: logger.log(u'Could not load changes from repo, giving a link!', logger.DEBUG) changes = 'Could not load changes from the repo. [Click here for CHANGES.md](https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/CHANGES.md)' From 62fef40eb65d5c4e6314a8378c514a18de643dad Mon Sep 17 00:00:00 2001 From: Labrys Date: Wed, 24 Feb 2016 17:00:29 -0500 Subject: [PATCH 12/34] Fix echoing logs * Add echo kwarg to get_url so logging can be selectively disabled. Disable echoing on newznab search. --- sickbeard/providers/newznab.py | 4 ++-- sickrage/providers/GenericProvider.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index 648b305b33..ed03a93c51 100644 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -283,7 +283,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: {url}".format(url=search_url), logger.DEBUG) time.sleep(cpu_presets[sickbeard.CPU_PRESET]) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False) if not data: break @@ -301,7 +301,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: %s" % search_url, logger.DEBUG) time.sleep(cpu_presets[sickbeard.CPU_PRESET]) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False) if not data: break diff --git a/sickrage/providers/GenericProvider.py b/sickrage/providers/GenericProvider.py index fb3e244577..c165cea5fd 100644 --- a/sickrage/providers/GenericProvider.py +++ b/sickrage/providers/GenericProvider.py @@ -344,7 +344,6 @@ def get_result(self, episodes): @staticmethod def get_url_hook(response, **kwargs): - _ = kwargs logger.log(u'{} URL: {} [Status: {}]'.format (response.request.method, response.request.url, response.status_code), logger.DEBUG) @@ -352,7 +351,8 @@ def get_url_hook(response, **kwargs): logger.log(u'With post data: {}'.format(response.request.body), logger.DEBUG) def get_url(self, url, post_data=None, params=None, timeout=30, json=False, need_bytes=False, **kwargs): # pylint: disable=too-many-arguments, - kwargs['hooks'] = {'response': self.get_url_hook} + if kwargs.pop('echo', True): + kwargs['hooks'] = {'response': self.get_url_hook} return getURL(url, post_data, params, self.headers, timeout, self.session, json, need_bytes, **kwargs) def image_name(self): From f432b0a4d45d4a8cf38c0ff0e4fb0c21d0ac3d08 Mon Sep 17 00:00:00 2001 From: miigotu Date: Wed, 24 Feb 2016 23:30:23 -0800 Subject: [PATCH 13/34] Gui had wrong replacement chars for scene numbers in renamer --- gui/slick/views/config_postProcessing.mako | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gui/slick/views/config_postProcessing.mako b/gui/slick/views/config_postProcessing.mako index 28cf077369..050b586cb5 100644 --- a/gui/slick/views/config_postProcessing.mako +++ b/gui/slick/views/config_postProcessing.mako @@ -321,7 +321,7 @@   - %0XMS + %0XS 02 @@ -341,7 +341,7 @@   - %0XME + %0XE 03 @@ -919,12 +919,12 @@ XEM Season Number: - %XMS + %XS 2   - %0XMS + %0XS 02 @@ -944,7 +944,7 @@   - %0XME + %0XE 03 From 68720a5f6efee84f1b581b1eba650cd73e50cd53 Mon Sep 17 00:00:00 2001 From: miigotu Date: Thu, 25 Feb 2016 00:48:37 -0800 Subject: [PATCH 14/34] Disable show update on start until it can be improved --- SickBeard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SickBeard.py b/SickBeard.py index b0f4a2a1f4..c997af094d 100755 --- a/SickBeard.py +++ b/SickBeard.py @@ -368,7 +368,7 @@ def start(self): # pylint: disable=too-many-branches,too-many-statements failed_history.trimHistory() # Check for metadata indexer updates for shows (sets the next aired ep!) - sickbeard.showUpdateScheduler.forceRun() + # sickbeard.showUpdateScheduler.forceRun() # Launch browser if sickbeard.LAUNCH_BROWSER and not (self.no_launch or self.run_as_daemon): From 93c88caf49fd3748beb0c84a8d10256ddeb6f55a Mon Sep 17 00:00:00 2001 From: miigotu Date: Thu, 25 Feb 2016 00:51:22 -0800 Subject: [PATCH 15/34] Missed a few replace-map typos --- gui/slick/views/config_postProcessing.mako | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gui/slick/views/config_postProcessing.mako b/gui/slick/views/config_postProcessing.mako index 050b586cb5..46bf5b0a3b 100644 --- a/gui/slick/views/config_postProcessing.mako +++ b/gui/slick/views/config_postProcessing.mako @@ -316,7 +316,7 @@ XEM Season Number: - %XMS + %XS 2 @@ -336,7 +336,7 @@ XEM Episode Number: - %XME + %XE 3 @@ -939,7 +939,7 @@ XEM Episode Number: - %XME + %XE 3 From 0dadb35b5261b82b97942b7c91b3cae351266619 Mon Sep 17 00:00:00 2001 From: neoatomic Date: Thu, 25 Feb 2016 14:38:22 +0100 Subject: [PATCH 16/34] Added provider icons for french-adn.com & torrentshack.me --- gui/slick/images/providers/french-adn.png | Bin 0 -> 588 bytes gui/slick/images/providers/french-adn_com.png | Bin 0 -> 588 bytes gui/slick/images/providers/torrentshack.png | Bin 0 -> 630 bytes gui/slick/images/providers/torrentshack_me.png | Bin 0 -> 630 bytes 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 gui/slick/images/providers/french-adn.png create mode 100644 gui/slick/images/providers/french-adn_com.png create mode 100644 gui/slick/images/providers/torrentshack.png create mode 100644 gui/slick/images/providers/torrentshack_me.png diff --git a/gui/slick/images/providers/french-adn.png b/gui/slick/images/providers/french-adn.png new file mode 100644 index 0000000000000000000000000000000000000000..521edbf50a6516c85e7f0f8fb51896927f116311 GIT binary patch literal 588 zcmV-S0<-;zP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D02p*dSaefwW^{L9 za%BK;VQFr3E^cLXAT%y8E;(#7eog=Y0li5?K~y+TbyCYt!cY*szu-a=14W_@wuGjl zlD?r&pe+apw1EYP2_$NOKXC8P4{+ncm2qXFet;k5!4g|dxS39-bLY&QnYpB1uPcgD zsZ^@fsw7F!hGD2b2TpjPBafzOx~>y^@Z#S!-TTq#VL03(;hm+aqPVGm#JxuUbG7=~ zDmTxTAfKKP$8ls?M!Al2@$o+>Dc4U@fj`OgF-L3Uvz-+Ws#jzw^D#}zU_Rgr?Y&J{NG|RHZVv%7Oj^mJ;U;=0wdf*3fg=weFv$iNU zb-iQTq3@lWdPSDkQ^I^AN^QHly@%y%;CMU_E*na8cLHAefk@2>^7@ z?Z*`sx9{`$TP(9jUoaU4%-56zK>(&@c4ENTJKoz#rBdm1I+Mvz6qU>6(E7S*>;a1d zwC@cPQ}SgxgRzv$W#szp6GVt@+YzO{n*dCFEx%tXm2id0AOZTLQ5#xwwgM6b^A6rEfGE;EeYMo4BqE0BT`z2|yD7BnAL%5d#Cq aFpO^$tOnq~Am_{g0000Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D02p*dSaefwW^{L9 za%BK;VQFr3E^cLXAT%y8E;(#7eog=Y0li5?K~y+TbyCYt!cY*szu-a=14W_@wuGjl zlD?r&pe+apw1EYP2_$NOKXC8P4{+ncm2qXFet;k5!4g|dxS39-bLY&QnYpB1uPcgD zsZ^@fsw7F!hGD2b2TpjPBafzOx~>y^@Z#S!-TTq#VL03(;hm+aqPVGm#JxuUbG7=~ zDmTxTAfKKP$8ls?M!Al2@$o+>Dc4U@fj`OgF-L3Uvz-+Ws#jzw^D#}zU_Rgr?Y&J{NG|RHZVv%7Oj^mJ;U;=0wdf*3fg=weFv$iNU zb-iQTq3@lWdPSDkQ^I^AN^QHly@%y%;CMU_E*na8cLHAefk@2>^7@ z?Z*`sx9{`$TP(9jUoaU4%-56zK>(&@c4ENTJKoz#rBdm1I+Mvz6qU>6(E7S*>;a1d zwC@cPQ}SgxgRzv$W#szp6GVt@+YzO{n*dCFEx%tXm2id0AOZTLQ5#xwwgM6b^A6rEfGE;EeYMo4BqE0BT`z2|yD7BnAL%5d#Cq aFpO^$tOnq~Am_{g0000JP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGjU;qFQU;%n~MU4Oe00(qQO+^RZ1sVV> z04kM)+5i9m7<5HgbW?9;ba!ELWdLwtX>N2bZe?^JG%heMIczh2P5=M_)Ja4^R5(v9 zl4)|{Fc5_`$s7i}%v4!2|H9%WCxe`ZU}MRE&61hF6T+G1YbjNYY0K~Rd);a^&C6s~ zBxaT7MVzlIJHFJ+!OwB9j0H%2jEi!W@ zf4WHWDw51zuD=EA5_Ux7Yoor_KR!1SA;*ZK zf!*;YHtfk;5;NpzKGuMqiThf8Zl0m}Ddi%xwNiNR2fFuCyTj0QVh`N??J3EY%rMfG zV!$$cH;N5AG9^EsvskwyZgKK&?eAC5;iz!UQvMrs!XEstyTBxWY@YDLKs{$EJp=+H z7eqe<0(^gi0}7ALPi7+oQ?eQh{Mq9laJUCb8}9VCJ$`hDkFCAl-xQW8n&wNnX<4LM zfff;EFPzG5vZO3RR{UZqvYK4(Tfc|`Kd)7XJP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGjU;qFQU;%n~MU4Oe00(qQO+^RZ1sVV> z04kM)+5i9m7<5HgbW?9;ba!ELWdLwtX>N2bZe?^JG%heMIczh2P5=M_)Ja4^R5(v9 zl4)|{Fc5_`$s7i}%v4!2|H9%WCxe`ZU}MRE&61hF6T+G1YbjNYY0K~Rd);a^&C6s~ zBxaT7MVzlIJHFJ+!OwB9j0H%2jEi!W@ zf4WHWDw51zuD=EA5_Ux7Yoor_KR!1SA;*ZK zf!*;YHtfk;5;NpzKGuMqiThf8Zl0m}Ddi%xwNiNR2fFuCyTj0QVh`N??J3EY%rMfG zV!$$cH;N5AG9^EsvskwyZgKK&?eAC5;iz!UQvMrs!XEstyTBxWY@YDLKs{$EJp=+H z7eqe<0(^gi0}7ALPi7+oQ?eQh{Mq9laJUCb8}9VCJ$`hDkFCAl-xQW8n&wNnX<4LM zfff;EFPzG5vZO3RR{UZqvYK4(Tfc|`Kd)7X Date: Thu, 25 Feb 2016 11:04:13 -0500 Subject: [PATCH 17/34] Pull from upstream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Merge from upstream * Use a decorator to prevent processing a dir more than once at a time, regardless of caller * Fixes https://github.com/SickRage/sickrage-issues/issues/1028 *Plex: Always return True when getting auth token if username/password aren't configured * This was causing failures to update Plex server when no username and password were specified since we were failing to get the auth token * Fix searches with ABNormal * Replace double quotes with single quotes as per our decided convention * ABNormal, sort by Time for rss, or Seeders for everything else. * Add OCS network logo * Added OCS (Orange Ciné Series) logo, a french network. * Fixes https://github.com/SickRage/sickrage-issues/issues/1046 * fix: Danishbits provider now works again * Aparently, BS4 `class_` lookup does not work when element has more than one class? * Anywho, replaced the lookup with `id` * Gui had wrong replacement chars for scene numbers in renamer * Missed a few replace-map typos --- gui/slick/images/network/ocs.png | Bin 0 -> 3501 bytes gui/slick/views/config_postProcessing.mako | 16 ++++++++-------- sickbeard/__init__.py | 5 +---- sickbeard/helpers.py | 2 +- sickbeard/notifiers/plex.py | 3 ++- sickbeard/processTV.py | 19 +++++++++++++++++++ sickbeard/providers/danishbits.py | 2 +- 7 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 gui/slick/images/network/ocs.png diff --git a/gui/slick/images/network/ocs.png b/gui/slick/images/network/ocs.png new file mode 100644 index 0000000000000000000000000000000000000000..afae58e058972b2fb5580b370d19dfa5476cb218 GIT binary patch literal 3501 zcmV;e4N~%nP)004R>004l5008;`004mK004C`008P>0026e000+ooVrmw00006 zVoOIv00000008+zyMF)x010qNS#tmYE+PN`E+PS{;@y$}01XjIL_t(&-tAa)!feCQBR{))|4vFd+L0>$FNHp!aTmcAAhmnIXbZE4pCo~N)jLq zAddkC0ihsZ11td64OjNe+abqKo#V^De@%OMZhfs`uU_~lHorTWN>Ao0HUJHP;Q@pI z%pf=funNeOPUSrE?honSU%zH_?0%tL1BCW{Hj3Z+%>w0AQLZomcr>t3+7BJ+0+tPF z0&aG%LY_Q#s(1LEn=Yn&XyyEgyyUg%YK6Zm`2f~+fJFP)V8F3}MuzM41ntSMJsmgp zskd7>s;(-@`SZi?D4V~yN6`UzHLwuv`4q%15V9eg0B7Kk0`BmveuVGICdhX5JjDgf4uD4uk| z)i&IlezsDX_VL}Co@fgIaBI#c%Y{!b)G8`6DcJ)i3M)(mxHh=a1lNavKnQ3w0oREk zNkEbT?ZSnQ{Py#2I@P5qu{u~K`>&#RU!141B*Yt0B?*9K0s#{o2!-(*Oh7te%!ti7 z&I6^k&L@}sVWzArH2}aYJ=Rlf{^w-H2e76`4QN?#HJ{_8bv@gfpN$0o6#$&I|Cl5W6bz#TJ)Dcp*0(~CAh>xWHdD^90x3aM^z|@Nxc4SS zT3|iK@(CK0L%WB_)eHX?+MN8 zJxJe|Nhc*M2a8{stsdPu99~b@rkVz7GC|uueSqt$KUdw_^C`{?w>&HqMDjhNkmJCW z0`7y;f31vpWWJu5ZUcZ@RixP)KYEfEPoJaMHlV6uV4*^2Hr%btA95>Byrp#RTO&^$ z>QDRD-6?#C7&gK-7=oLWUd>Pbb%OWWpRWa00RWns>b?BlOS9;MKR+dO8)z;O42Xb) zdvI$$fNga8o8FL!eTXW?f7GR+`(F&XxOje4^N9#N7Za04Mk6yBE%N zQsur4BdDw>hrB8{Cg-`yfjfG_GVg!Cy$Wsp|| zoiFKS7p%Aqz)~FEd6THC_i}Z_7j zkO~IDGT|n5sMB=CqjJ%nks@e=6Ft$ib}YDw@gaD=U}*Vs5DytZ5KtT@seem?<|bdfa?@%s$!sC;QyO-TUdj=cgdE=eM3t zeVd$`Z#CHUUWJ$_DXT$sE6xlIM^a1$wj{V)&tfIL%Mn~m(z+Z)M)wjF9~uBHx=bn{ zD$jRw{O8g{Re2`b-l8^utlYz%9)rs+2Cb%a4+6Mo10?$8Z{c{411qG$h}0HlK-C}= z0s6i?ll|r6c-QK29A~)5=w2wh_dn3Ia=`{k?#Ft1zxoy|app)a47Jq(>;jaUabE9~ z8@Lz=tz*wdj!W`0w_axfps_kp)K_#N>BhCb*dj@m-0?hqgoxa0_sP+}{JR?fC}^V9 zbxjJ`Fn|ODBHjZ<+KM=LqMy2P`Gd;*CziO=?%EODxb)r@V=HwRRMIqXtR@VQraPA+ z!ilL3a9gjD0AMv4=>8LE*gr`#*-h{@|!^yZ1W<({8Pp0&wkjo!z=J-^B=6M|T-i zQ`w2y@i2^p2FFM%mqKl5t$l<{ZDr>lYk(F*jQGVq&y>GCE9Y$J=S-PD!5w+)Vmmwc zn5%ki*$@Jn+N20G*lR#BC3w?E52#93-~HIV<@#2r8iD+$`$&ILE`a^*9ZUxGG(4u__I}%9k ze3lc^%Bb}C$Z(ztkfN{g0@xC*)! zbpQw=UICxjbB)S*H7V^R6}HiDh)~@n{gGCE4z_PwoM*pe?TBFU)qI8TGzWQIU+8HXZ@M z9eu}g49H(67C!q95u&%N6E>4Q<&7}_)~HvGsRFPCz?M)&$pUxoLmw#L?6@AjX08mW z>Cgg|al#pxze8s9C_^w>^B4kTD4_N4Pi$QK_la#u2G=i|Kx^N9m^=!&g#`ho4epdx zPD4ho0sxVdw}In47TI>x`icS~20r=I3!&pX2mQbRtKOL+Dl3vHP60Fx&@_;y0Ro_I zxy6(-Xg?cXNdnP*Lx3cp*o`@D@jRWcDCexcC&}hksX&em+m{p~00{l#*_q8xFlnTBauUJ733{Mo38C)BJGvS%p-bwRb001}e zonpWK(8ubw)#F>u0zh~cNbgoI2jBKN6z3#C(NR-9RP0$fUa+e~i!uR@1=n&RXMFZE z${W3zrVKr-oH?Ew9RybfaYu0sz#TaIU!W z1kGG?qbI$)zj@g@wsR1_GIo<1s7_{In}H~3f^C3x0!aor0QnTKs>fh6xGn@|#4YQ^ zv=8s{cvFMck{Qp^qL*G52{G>#5j^SfIx?$wArjNe(O92C=Z;>Z*g+MZ7-vZrvcU#F zJ^#O2Dwuh2?F~HrzSVktO&lk*o!L3ub^_4|iIso}CqBLjFMc%E7(8Zk+ii2j8NEsO8+2FKSIVnVA)0#;Q?zuWL>KfL&J?Z_gUHQropPU!_eW zWL*e$PVO-=dG2H_v2)!I-jA9J3-ZM0e|d%uZWv3|RmnmCq>FID0RYewhdbSb8;y${L} XEM Season Number: - %XMS + %XS 2   - %0XMS + %0XS 02 @@ -336,12 +336,12 @@ XEM Episode Number: - %XME + %XE 3   - %0XME + %0XE 03 @@ -919,12 +919,12 @@ XEM Season Number: - %XMS + %XS 2   - %0XMS + %0XS 02 @@ -939,12 +939,12 @@ XEM Episode Number: - %XME + %XE 3   - %0XME + %0XE 03 diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index 2de5ab24f0..775c1c4450 100644 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -351,7 +351,6 @@ PLEX_CLIENT_HOST = None PLEX_SERVER_USERNAME = None PLEX_SERVER_PASSWORD = None -PLEX_SERVER_NO_AUTH = False USE_PLEX_CLIENT = False PLEX_CLIENT_USERNAME = None @@ -601,7 +600,7 @@ def initialize(consoleLogging=True): # pylint: disable=too-many-locals, too-man USE_KODI, KODI_ALWAYS_ON, KODI_NOTIFY_ONSNATCH, KODI_NOTIFY_ONDOWNLOAD, KODI_NOTIFY_ONSUBTITLEDOWNLOAD, KODI_UPDATE_FULL, KODI_UPDATE_ONLYFIRST, \ KODI_UPDATE_LIBRARY, KODI_HOST, KODI_USERNAME, KODI_PASSWORD, BACKLOG_FREQUENCY, \ USE_TRAKT, TRAKT_USERNAME, TRAKT_ACCESS_TOKEN, TRAKT_REFRESH_TOKEN, TRAKT_REMOVE_WATCHLIST, TRAKT_SYNC_WATCHLIST, TRAKT_REMOVE_SHOW_FROM_SICKRAGE, TRAKT_METHOD_ADD, TRAKT_START_PAUSED, traktCheckerScheduler, TRAKT_USE_RECOMMENDED, TRAKT_SYNC, TRAKT_SYNC_REMOVE, TRAKT_DEFAULT_INDEXER, TRAKT_REMOVE_SERIESLIST, TRAKT_TIMEOUT, TRAKT_BLACKLIST_NAME, \ - USE_PLEX_SERVER, PLEX_SERVER_NO_AUTH, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \ + USE_PLEX_SERVER, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \ PLEX_SERVER_HOST, PLEX_SERVER_TOKEN, PLEX_CLIENT_HOST, PLEX_SERVER_USERNAME, PLEX_SERVER_PASSWORD, PLEX_SERVER_HTTPS, MIN_BACKLOG_FREQUENCY, SKIP_REMOVED_FILES, ALLOWED_EXTENSIONS, \ USE_EMBY, EMBY_HOST, EMBY_APIKEY, \ showUpdateScheduler, __INITIALIZED__, INDEXER_DEFAULT_LANGUAGE, EP_DEFAULT_DELETED_STATUS, LAUNCH_BROWSER, TRASH_REMOVE_SHOW, TRASH_ROTATE_LOGS, SORT_ARTICLE, \ @@ -1024,7 +1023,6 @@ def path_leaf(path): PLEX_CLIENT_USERNAME = check_setting_str(CFG, 'Plex', 'plex_client_username', '', censor_log=True) PLEX_CLIENT_PASSWORD = check_setting_str(CFG, 'Plex', 'plex_client_password', '', censor_log=True) PLEX_SERVER_HTTPS = bool(check_setting_int(CFG, 'Plex', 'plex_server_https', 0)) - PLEX_SERVER_NO_AUTH = bool(check_setting_int(CFG, 'Plex', 'plex_server_no_auth', 0)) USE_EMBY = bool(check_setting_int(CFG, 'Emby', 'use_emby', 0)) EMBY_HOST = check_setting_str(CFG, 'Emby', 'emby_host', '') @@ -1936,7 +1934,6 @@ def save_config(): # pylint: disable=too-many-statements, too-many-branches new_config['Plex']['plex_client_host'] = PLEX_CLIENT_HOST new_config['Plex']['plex_server_username'] = PLEX_SERVER_USERNAME new_config['Plex']['plex_server_password'] = helpers.encrypt(PLEX_SERVER_PASSWORD, ENCRYPTION_VERSION) - new_config['Plex']['plex_server_no_auth'] = int(PLEX_SERVER_NO_AUTH) new_config['Plex']['use_plex_client'] = int(USE_PLEX_CLIENT) new_config['Plex']['plex_client_username'] = PLEX_CLIENT_USERNAME diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 8c3beee9e8..910298180d 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -353,7 +353,7 @@ def copyFile(srcFile, destFile): try: ek(shutil.copyfile, srcFile, destFile) except (SpecialFileError, Error) as error: - logger.log(error, logger.WARNING) + logger.log(u'{}'.format(error), logger.WARNING) except Exception as error: logger.log(u'{}'.format(error), logger.ERROR) else: diff --git a/sickbeard/notifiers/plex.py b/sickbeard/notifiers/plex.py index 9a6d0c655c..9855e9217b 100644 --- a/sickbeard/notifiers/plex.py +++ b/sickbeard/notifiers/plex.py @@ -130,6 +130,7 @@ def update_library(self, ep_obj=None, host=None, # pylint: disable=too-many-arg return False if not self.get_token(username, password, plex_server_token): + logger.log(u'PLEX: Error getting auth token for Plex Media Server, check your settings', logger.WARNING) return False file_location = '' if not ep_obj else ep_obj.location @@ -220,7 +221,7 @@ def get_token(self, username=None, password=None, plex_server_token=None): return True if not (username and password): - return sickbeard.PLEX_SERVER_NO_AUTH + return True logger.log(u'PLEX: fetching plex.tv credentials for user: ' + username, logger.DEBUG) diff --git a/sickbeard/processTV.py b/sickbeard/processTV.py index 43fa0c12a7..6dba5c8df4 100644 --- a/sickbeard/processTV.py +++ b/sickbeard/processTV.py @@ -19,6 +19,7 @@ import os import stat +from functools import wraps import sickbeard from sickbeard import postProcessor @@ -141,7 +142,25 @@ def logHelper(logMessage, logLevel=logger.INFO): return logMessage + u"\n" +def OneRunPP(): + isRunning = [False] + + def decorate(func): + @wraps(func) + def func_wrapper(*args, **kargs): + if isRunning[0]: + return logHelper(u'Post processor is already running', logger.ERROR) + + isRunning[0] = True + ret = func(*args, **kargs) + isRunning[0] = False + return ret + return func_wrapper + return decorate + + # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals +@OneRunPP() def processDir(dirName, nzbName=None, process_method=None, force=False, is_priority=None, delete_on=False, failed=False, proc_type="auto"): """ Scans through the files in dirName and processes whatever media files it finds diff --git a/sickbeard/providers/danishbits.py b/sickbeard/providers/danishbits.py index 44696e05f5..ad5fec2d35 100644 --- a/sickbeard/providers/danishbits.py +++ b/sickbeard/providers/danishbits.py @@ -126,7 +126,7 @@ def process_column_header(td): continue with BS4Parser(data, 'html5lib') as html: - torrent_table = html.find('table', class_='torrent_table') + torrent_table = html.find('table', id='torrent_table') torrent_rows = torrent_table.find_all('tr') if torrent_table else [] # Continue only if at least one Release is found From 87f3bb02704263c044d2d2c703ea12fc46ef5581 Mon Sep 17 00:00:00 2001 From: medariox Date: Thu, 25 Feb 2016 17:21:04 +0100 Subject: [PATCH 18/34] Update subliminal to latest version (c3c0c45) # Refactor some code # Improves legendastv # Fixes podnapisi url --- lib/subliminal/__init__.py | 9 +- lib/subliminal/cli.py | 9 +- lib/subliminal/converters/thesubdb.py | 3 +- lib/subliminal/core.py | 275 ++++++++++------- lib/subliminal/extensions.py | 104 +++++++ lib/subliminal/providers/addic7ed.py | 3 +- lib/subliminal/providers/legendastv.py | 18 +- lib/subliminal/providers/opensubtitles.py | 19 +- lib/subliminal/providers/podnapisi.py | 3 +- lib/subliminal/providers/subscenter.py | 3 +- lib/subliminal/providers/tvsubtitles.py | 3 +- lib/subliminal/refiners/metadata.py | 100 +++++++ lib/subliminal/refiners/omdb.py | 30 +- lib/subliminal/refiners/tvdb.py | 25 +- lib/subliminal/score.py | 2 +- lib/subliminal/subtitle.py | 25 +- lib/subliminal/utils.py | 122 ++++++++ lib/subliminal/video.py | 340 +--------------------- 18 files changed, 616 insertions(+), 477 deletions(-) create mode 100644 lib/subliminal/extensions.py create mode 100644 lib/subliminal/refiners/metadata.py create mode 100644 lib/subliminal/utils.py diff --git a/lib/subliminal/__init__.py b/lib/subliminal/__init__.py index 5acf64ab23..e79e013f43 100644 --- a/lib/subliminal/__init__.py +++ b/lib/subliminal/__init__.py @@ -9,12 +9,13 @@ import logging from .core import (AsyncProviderPool, ProviderPool, check_video, download_best_subtitles, download_subtitles, - list_subtitles, provider_manager, refiner_manager, save_subtitles) + list_subtitles, save_subtitles, scan_video, scan_videos) from .cache import region from .exceptions import Error, ProviderError +from .extensions import provider_manager, refiner_manager from .providers import Provider -from .score import compute_score -from .subtitle import Subtitle -from .video import SUBTITLE_EXTENSIONS, VIDEO_EXTENSIONS, Episode, Movie, Video, scan_video, scan_videos +from .score import compute_score, get_scores +from .subtitle import SUBTITLE_EXTENSIONS, Subtitle +from .video import VIDEO_EXTENSIONS, Episode, Movie, Video logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/lib/subliminal/cli.py b/lib/subliminal/cli.py index 1dc96f648d..572df2f3ed 100644 --- a/lib/subliminal/cli.py +++ b/lib/subliminal/cli.py @@ -18,9 +18,8 @@ from dogpile.core import ReadWriteMutex from six.moves import configparser -from subliminal import (AsyncProviderPool, Episode, Movie, Video, __version__, check_video, provider_manager, region, - save_subtitles, scan_video, scan_videos) -from subliminal.score import compute_score, get_scores +from subliminal import (AsyncProviderPool, Episode, Movie, Video, __version__, check_video, compute_score, get_scores, + provider_manager, region, save_subtitles, scan_video, scan_videos) logger = logging.getLogger(__name__) @@ -46,7 +45,7 @@ def release_write_lock(self): class Config(object): - """A :class:`~configparser.SafeConfigParser` wrapper to store configuration. + """A :class:`~configparser.ConfigParser` wrapper to store configuration. Interaction with the configuration is done with the properties. @@ -229,6 +228,7 @@ def subliminal(ctx, addic7ed, legendastv, opensubtitles, subscenter, cache_dir, # configure logging if debug: handler = logging.StreamHandler() + # TODO: change format to something nicer (use colorlogs + funcName) handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) logging.getLogger('subliminal').addHandler(handler) logging.getLogger('subliminal').setLevel(logging.DEBUG) @@ -378,6 +378,7 @@ def download(obj, provider, language, age, directory, encoding, single, force, h hearing_impaired=hearing_impaired, only_one=single) downloaded_subtitles[v] = subtitles + # TODO: warn about discarded providers # save subtitles total_subtitles = 0 for v, subtitles in downloaded_subtitles.items(): diff --git a/lib/subliminal/converters/thesubdb.py b/lib/subliminal/converters/thesubdb.py index c9b879c5c7..58051afb8f 100644 --- a/lib/subliminal/converters/thesubdb.py +++ b/lib/subliminal/converters/thesubdb.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from babelfish import LanguageReverseConverter -from subliminal.exceptions import ConfigurationError + +from ..exceptions import ConfigurationError class TheSubDBConverter(LanguageReverseConverter): diff --git a/lib/subliminal/core.py b/lib/subliminal/core.py index a9bfbe7678..6e90830294 100644 --- a/lib/subliminal/core.py +++ b/lib/subliminal/core.py @@ -1,117 +1,27 @@ # -*- coding: utf-8 -*- from collections import defaultdict from concurrent.futures import ThreadPoolExecutor +from datetime import datetime import io import itertools import logging import operator import os.path -from pkg_resources import EntryPoint import socket from babelfish import Language +from guessit import guessit import requests -from stevedore import ExtensionManager +from .extensions import provider_manager, refiner_manager from .score import compute_score as default_compute_score -from .subtitle import get_subtitle_path +from .subtitle import SUBTITLE_EXTENSIONS, get_subtitle_path +from .utils import hash_napiprojekt, hash_opensubtitles, hash_thesubdb +from .video import VIDEO_EXTENSIONS, Episode, Movie, Video logger = logging.getLogger(__name__) -class RegistrableExtensionManager(ExtensionManager): - """:class:~stevedore.extensions.ExtensionManager` with support for registration. - - It allows loading of internal extensions without setup and registering/unregistering additional extensions. - - Loading is done in this order: - - * Entry point extensions - * Internal extensions - * Registered extensions - - """ - def __init__(self, namespace, internal_extensions): - #: Registered extensions with entry point syntax - self.registered_extensions = [] - - #: Internal extensions with entry point syntax - self.internal_extensions = internal_extensions - - super(RegistrableExtensionManager, self).__init__(namespace) - - def _find_entry_points(self, namespace): - # default extensions - eps = super(RegistrableExtensionManager, self)._find_entry_points(namespace) - - # internal extensions - for iep in self.internal_extensions: - ep = EntryPoint.parse(iep) - if ep.name not in [e.name for e in eps]: - eps.append(ep) - - # registered extensions - for rep in self.registered_extensions: - ep = EntryPoint.parse(rep) - if ep.name not in [e.name for e in eps]: - eps.append(ep) - - return eps - - def register(self, entry_point): - """Register an extension - - :param str entry_point: extension to register (entry point syntax) - :raise: ValueError if already registered - - """ - if entry_point in self.registered_extensions: - raise ValueError('Extension already registered') - - ep = EntryPoint.parse(entry_point) - if ep.name in self.names(): - raise ValueError('An extension with the same name already exist') - - ext = self._load_one_plugin(ep, False, (), {}, False) - self.extensions.append(ext) - if self._extensions_by_name is not None: - self._extensions_by_name[ext.name] = ext - self.registered_extensions.insert(0, entry_point) - - def unregister(self, entry_point): - """Unregister a provider - - :param str entry_point: provider to unregister (entry point syntax) - - """ - if entry_point not in self.registered_extensions: - raise ValueError('Extension not registered') - - ep = EntryPoint.parse(entry_point) - self.registered_extensions.remove(entry_point) - if self._extensions_by_name is not None: - del self._extensions_by_name[ep.name] - for i, ext in enumerate(self.extensions): - if ext.name == ep.name: - del self.extensions[i] - break - - -provider_manager = RegistrableExtensionManager('subliminal.providers', [ - 'addic7ed = subliminal.providers.addic7ed:Addic7edProvider', - 'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider', - 'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider', - 'subscenter = subliminal.providers.subscenter:SubsCenterProvider', - 'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider', - 'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider' -]) - -refiner_manager = RegistrableExtensionManager('subliminal.refiners', [ - 'tvdb = subliminal.refiners.tvdb:refine', - 'omdb = subliminal.refiners.omdb:refine' -]) - - class ProviderPool(object): """A pool of providers with the same API as a single :class:`~subliminal.providers.Provider`. @@ -284,7 +194,7 @@ def download_best_subtitles(self, subtitles, video, languages, min_score=0, hear :param int min_score: minimum score for a subtitle to be downloaded. :param bool hearing_impaired: hearing impaired preference. :param bool only_one: download only one subtitle, not one per language. - :param function compute_score: function that takes `subtitle` and `video` as positional arguments, + :param compute_score: function that takes `subtitle` and `video` as positional arguments, `hearing_impaired` as keyword argument and returns the score. :return: downloaded subtitles. :rtype: list of :class:`~subliminal.subtitle.Subtitle` @@ -405,6 +315,175 @@ def check_video(video, languages=None, age=None, undefined=False): return True +def search_external_subtitles(path, directory=None): + """Search for external subtitles from a video `path` and their associated language. + + Unless `directory` is provided, search will be made in the same directory as the video file. + + :param str path: path to the video. + :param str directory: directory to search for subtitles. + :return: found subtitles with their languages. + :rtype: dict + + """ + # split path + dirpath, filename = os.path.split(path) + dirpath = dirpath or '.' + fileroot, fileext = os.path.splitext(filename) + + # search for subtitles + subtitles = {} + for p in os.listdir(directory or dirpath): + # keep only valid subtitle filenames + if not p.startswith(fileroot) or not p.endswith(SUBTITLE_EXTENSIONS): + continue + + # extract the potential language code + language_code = p[len(fileroot):-len(os.path.splitext(p)[1])].replace(fileext, '').replace('_', '-')[1:] + + # default language is undefined + language = Language('und') + + # attempt to parse the language code + if language_code: + try: + language = Language.fromietf(language_code) + except ValueError: + logger.error('Cannot parse language code %r', language_code) + + subtitles[p] = language + + logger.debug('Found subtitles %r', subtitles) + + return subtitles + + +def scan_video(path, subtitles=True, subtitles_dir=None, movie_refiners=('metadata', 'omdb'), + episode_refiners=('metadata', 'tvdb', 'omdb'), **kwargs): + """Scan a video and its subtitle languages from a video `path`. + + Use the :mod:`~subliminal.refiners` to find additional information for the video. + + :param str path: existing path to the video. + :param bool subtitles: scan for subtitles with the same name. + :param str subtitles_dir: directory to search for subtitles. + :param tuple movie_refiners: refiners to use for movies. + :param tuple episode_refiners: refiners to use for episodes. + :param \*\*kwargs: parameters for refiners. + :return: the scanned video. + :rtype: :class:`~subliminal.video.Video` + + """ + # check for non-existing path + if not os.path.exists(path): + raise ValueError('Path does not exist') + + # check video extension + if not path.endswith(VIDEO_EXTENSIONS): + raise ValueError('%s is not a valid video extension' % os.path.splitext(path)[1]) + + dirpath, filename = os.path.split(path) + logger.info('Scanning video %r in %r', filename, dirpath) + + # guess + video = Video.fromguess(path, guessit(path)) + + # refine + refiners = () + if isinstance(video, Episode): + refiners = movie_refiners or () + elif isinstance(video, Movie): + refiners = episode_refiners or () + for refiner in refiners: + logger.info('Refining video with %s', refiner) + try: + refiner_manager[refiner].plugin(video, **kwargs) + except: + logger.exception('Failed to refine video') + + # size and hashes + video.size = os.path.getsize(path) + if video.size > 10485760: + logger.debug('Size is %d', video.size) + video.hashes['opensubtitles'] = hash_opensubtitles(path) + video.hashes['thesubdb'] = hash_thesubdb(path) + video.hashes['napiprojekt'] = hash_napiprojekt(path) + logger.debug('Computed hashes %r', video.hashes) + else: + logger.warning('Size is lower than 10MB: hashes not computed') + + # external subtitles + if subtitles: + video.subtitle_languages |= set(search_external_subtitles(path, directory=subtitles_dir).values()) + + return video + + +def scan_videos(path, age=None, **kwargs): + """Scan `path` for videos and their subtitles. + + :param str path: existing directory path to scan. + :param datetime.timedelta age: maximum age of the video. + :param \*\*kwargs: parameters for the :func:`scan_video` function. + :return: the scanned videos. + :rtype: list of :class:`~subliminal.video.Video` + + """ + # check for non-existing path + if not os.path.exists(path): + raise ValueError('Path does not exist') + + # check for non-directory path + if not os.path.isdir(path): + raise ValueError('Path is not a directory') + + # walk the path + videos = [] + for dirpath, dirnames, filenames in os.walk(path): + logger.debug('Walking directory %s', dirpath) + + # remove badly encoded and hidden dirnames + for dirname in list(dirnames): + if dirname.startswith('.'): + logger.debug('Skipping hidden dirname %r in %r', dirname, dirpath) + dirnames.remove(dirname) + + # scan for videos + for filename in filenames: + # filter on videos + if not filename.endswith(VIDEO_EXTENSIONS): + continue + + # skip hidden files + if filename.startswith('.'): + logger.debug('Skipping hidden filename %r in %r', filename, dirpath) + continue + + # reconstruct the file path + filepath = os.path.join(dirpath, filename) + + # skip links + if os.path.islink(filepath): + logger.debug('Skipping link %r in %r', filename, dirpath) + continue + + # skip old files + if age and datetime.utcnow() - datetime.utcfromtimestamp(os.path.getmtime(filepath)) > age: + logger.debug('Skipping old file %r in %r', filename, dirpath) + continue + + # scan video + try: + video = scan_video(filepath, **kwargs) + except ValueError: # pragma: no cover + logger.exception('Error scanning video') + continue + + videos.append(video) + + return videos + + def list_subtitles(videos, languages, pool_class=ProviderPool, **kwargs): """List subtitles. @@ -479,7 +558,7 @@ def download_best_subtitles(videos, languages, min_score=0, hearing_impaired=Fal :param int min_score: minimum score for a subtitle to be downloaded. :param bool hearing_impaired: hearing impaired preference. :param bool only_one: download only one subtitle, not one per language. - :param function compute_score: function that takes `subtitle` and `video` as positional arguments, + :param compute_score: function that takes `subtitle` and `video` as positional arguments, `hearing_impaired` as keyword argument and returns the score. :param pool_class: class to use as provider pool. :type: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar diff --git a/lib/subliminal/extensions.py b/lib/subliminal/extensions.py new file mode 100644 index 0000000000..a15f8fd1cb --- /dev/null +++ b/lib/subliminal/extensions.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +from pkg_resources import EntryPoint + +from stevedore import ExtensionManager + + +class RegistrableExtensionManager(ExtensionManager): + """:class:~stevedore.extensions.ExtensionManager` with support for registration. + + It allows loading of internal extensions without setup and registering/unregistering additional extensions. + + Loading is done in this order: + + * Entry point extensions + * Internal extensions + * Registered extensions + + :param str namespace: namespace argument for :class:~stevedore.extensions.ExtensionManager`. + :param list internal_extensions: internal extensions to use with entry point syntax. + :param \*\*kwargs: additional parameters for the :class:~stevedore.extensions.ExtensionManager` constructor. + + """ + def __init__(self, namespace, internal_extensions, **kwargs): + #: Registered extensions with entry point syntax + self.registered_extensions = [] + + #: Internal extensions with entry point syntax + self.internal_extensions = internal_extensions + + super(RegistrableExtensionManager, self).__init__(namespace, **kwargs) + + def _find_entry_points(self, namespace): + # copy of default extensions + eps = list(super(RegistrableExtensionManager, self)._find_entry_points(namespace)) + + # internal extensions + for iep in self.internal_extensions: + ep = EntryPoint.parse(iep) + if ep.name not in [e.name for e in eps]: + eps.append(ep) + + # registered extensions + for rep in self.registered_extensions: + ep = EntryPoint.parse(rep) + if ep.name not in [e.name for e in eps]: + eps.append(ep) + + return eps + + def register(self, entry_point): + """Register an extension + + :param str entry_point: extension to register (entry point syntax) + :raise: ValueError if already registered + + """ + if entry_point in self.registered_extensions: + raise ValueError('Extension already registered') + + ep = EntryPoint.parse(entry_point) + if ep.name in self.names(): + raise ValueError('An extension with the same name already exist') + + ext = self._load_one_plugin(ep, False, (), {}, False) + self.extensions.append(ext) + if self._extensions_by_name is not None: + self._extensions_by_name[ext.name] = ext + self.registered_extensions.insert(0, entry_point) + + def unregister(self, entry_point): + """Unregister a provider + + :param str entry_point: provider to unregister (entry point syntax) + + """ + if entry_point not in self.registered_extensions: + raise ValueError('Extension not registered') + + ep = EntryPoint.parse(entry_point) + self.registered_extensions.remove(entry_point) + if self._extensions_by_name is not None: + del self._extensions_by_name[ep.name] + for i, ext in enumerate(self.extensions): + if ext.name == ep.name: + del self.extensions[i] + break + + +#: Provider manager +provider_manager = RegistrableExtensionManager('subliminal.providers', [ + 'addic7ed = subliminal.providers.addic7ed:Addic7edProvider', + 'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider', + 'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider', + 'subscenter = subliminal.providers.subscenter:SubsCenterProvider', + 'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider', + 'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider' +]) + +#: Refiner manager +refiner_manager = RegistrableExtensionManager('subliminal.refiners', [ + 'metadata = subliminal.refiners.metadata:refine', + 'tvdb = subliminal.refiners.tvdb:refine', + 'omdb = subliminal.refiners.omdb:refine' +]) diff --git a/lib/subliminal/providers/addic7ed.py b/lib/subliminal/providers/addic7ed.py index a358c1a2b3..22f9e9f28c 100644 --- a/lib/subliminal/providers/addic7ed.py +++ b/lib/subliminal/providers/addic7ed.py @@ -11,7 +11,8 @@ from ..cache import SHOW_EXPIRATION_TIME, region from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, TooManyRequests from ..subtitle import Subtitle, fix_line_ending, guess_matches -from ..video import Episode, sanitize +from ..utils import sanitize +from ..video import Episode logger = logging.getLogger(__name__) diff --git a/lib/subliminal/providers/legendastv.py b/lib/subliminal/providers/legendastv.py index 1609d0eb4d..6fe9bc894b 100644 --- a/lib/subliminal/providers/legendastv.py +++ b/lib/subliminal/providers/legendastv.py @@ -15,8 +15,8 @@ from . import ParserBeautifulSoup, Provider from ..cache import region, EPISODE_EXPIRATION_TIME, SHOW_EXPIRATION_TIME from ..exceptions import AuthenticationError, ConfigurationError -from ..subtitle import Subtitle, fix_line_ending, guess_matches, sanitize -from ..video import Episode, Movie, SUBTITLE_EXTENSIONS +from ..subtitle import SUBTITLE_EXTENSIONS, Subtitle, fix_line_ending, guess_matches, sanitize +from ..video import Episode, Movie TIMEOUT = 10 @@ -170,10 +170,12 @@ def search_candidates(self, title, season, episode, year): :rtype: list of dict """ - keyword = sanitize(title) - logger.info('Searching candidates using the keyword %s', keyword) - r = self.session.get('%s/legenda/sugestao/%s' % (self.server_url, keyword), timeout=TIMEOUT) - r.raise_for_status() + results = dict() + for keyword in {sanitize(title), title.lower().replace(':', '')}: + logger.info('Searching candidates using the keyword %s', keyword) + r = self.session.get('%s/legenda/sugestao/%s' % (self.server_url, keyword), timeout=TIMEOUT) + r.raise_for_status() + results.update({item['_id']: item for item in json.loads(r.text)}) # get the shows/movies out of the suggestions. # json sample: @@ -224,8 +226,6 @@ def search_candidates(self, title, season, episode, year): # imdb_id: Sometimes it appears as a number and sometimes as a string prefixed with tt # temporada: Sometimes is ``null`` and season information should be extracted from dsc_nome_br - results = json.loads(r.text) - # type, title, series, season, year: should follow guessit properties names mapping = dict( id='id_filme', @@ -252,7 +252,7 @@ def search_candidates(self, title, season, episode, year): imdb_re = re.compile('t{0,2}(\d+)') candidates = [] - for result in results: + for result in results.values(): entry = result['_source'] item = {k: entry.get(v) for k, v in mapping.items()} item['type'] = type_map.get(item.get('type'), 'movie') diff --git a/lib/subliminal/providers/opensubtitles.py b/lib/subliminal/providers/opensubtitles.py index 746f576da2..f5beee87c4 100644 --- a/lib/subliminal/providers/opensubtitles.py +++ b/lib/subliminal/providers/opensubtitles.py @@ -13,7 +13,8 @@ from .. import __short_version__ from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError from ..subtitle import Subtitle, fix_line_ending, guess_matches -from ..video import Episode, Movie, sanitize +from ..utils import sanitize +from ..video import Episode, Movie logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ class OpenSubtitlesSubtitle(Subtitle): series_re = re.compile('^"(?P.*)" (?P.*)$') def __init__(self, language, hearing_impaired, page_link, subtitle_id, matched_by, movie_kind, hash, movie_name, - movie_release_name, movie_year, movie_imdb_id, series_season, series_episode, encoding): + movie_release_name, movie_year, movie_imdb_id, series_season, series_episode, filename, encoding): super(OpenSubtitlesSubtitle, self).__init__(language, hearing_impaired, page_link, encoding) self.subtitle_id = subtitle_id self.matched_by = matched_by @@ -35,6 +36,7 @@ def __init__(self, language, hearing_impaired, page_link, subtitle_id, matched_b self.movie_imdb_id = movie_imdb_id self.series_season = series_season self.series_episode = series_episode + self.filename = filename @property def id(self): @@ -53,6 +55,9 @@ def get_matches(self, video): # episode if isinstance(video, Episode) and self.movie_kind == 'episode': + # tag match, assume series, year, season and episode matches + if self.matched_by == 'tag': + matches |= {'series', 'year', 'season', 'episode'} # series if video.series and sanitize(self.series_name) == sanitize(video.series): matches.add('series') @@ -70,6 +75,7 @@ def get_matches(self, video): matches.add('title') # guess matches |= guess_matches(video, guessit(self.movie_release_name, {'type': 'episode'})) + matches |= guess_matches(video, guessit(self.filename, {'type': 'episode'})) # hash if 'opensubtitles' in video.hashes and self.hash == video.hashes['opensubtitles']: if 'series' in matches and 'season' in matches and 'episode' in matches: @@ -78,6 +84,9 @@ def get_matches(self, video): logger.debug('Match on hash discarded') # movie elif isinstance(video, Movie) and self.movie_kind == 'movie': + # tag match, assume title and year matches + if self.matched_by == 'tag': + matches |= {'title', 'year'} # title if video.title and sanitize(self.movie_name) == sanitize(video.title): matches.add('title') @@ -86,6 +95,7 @@ def get_matches(self, video): matches.add('year') # guess matches |= guess_matches(video, guessit(self.movie_release_name, {'type': 'movie'})) + matches |= guess_matches(video, guessit(self.filename, {'type': 'movie'})) # hash if 'opensubtitles' in video.hashes and self.hash == video.hashes['opensubtitles']: if 'title' in matches: @@ -179,12 +189,13 @@ def query(self, languages, hash=None, size=None, imdb_id=None, query=None, seaso movie_imdb_id = 'tt' + subtitle_item['IDMovieImdb'] series_season = int(subtitle_item['SeriesSeason']) if subtitle_item['SeriesSeason'] else None series_episode = int(subtitle_item['SeriesEpisode']) if subtitle_item['SeriesEpisode'] else None + filename = subtitle_item['SubFileName'] encoding = subtitle_item.get('SubEncoding') or None subtitle = OpenSubtitlesSubtitle(language, hearing_impaired, page_link, subtitle_id, matched_by, movie_kind, hash, movie_name, movie_release_name, movie_year, movie_imdb_id, - series_season, series_episode, encoding) - logger.debug('Found subtitle %r', subtitle) + series_season, series_episode, filename, encoding) + logger.debug('Found subtitle %r by %s', subtitle, matched_by) subtitles.append(subtitle) return subtitles diff --git a/lib/subliminal/providers/podnapisi.py b/lib/subliminal/providers/podnapisi.py index 6b78ae1783..803fc92e67 100644 --- a/lib/subliminal/providers/podnapisi.py +++ b/lib/subliminal/providers/podnapisi.py @@ -19,7 +19,8 @@ from .. import __short_version__ from ..exceptions import ProviderError from ..subtitle import Subtitle, fix_line_ending, guess_matches -from ..video import Episode, Movie, sanitize +from ..utils import sanitize +from ..video import Episode, Movie logger = logging.getLogger(__name__) diff --git a/lib/subliminal/providers/subscenter.py b/lib/subliminal/providers/subscenter.py index ee1f877e41..c88c466454 100644 --- a/lib/subliminal/providers/subscenter.py +++ b/lib/subliminal/providers/subscenter.py @@ -14,7 +14,8 @@ from ..cache import SHOW_EXPIRATION_TIME, region from ..exceptions import AuthenticationError, ConfigurationError, ProviderError from ..subtitle import Subtitle, fix_line_ending, guess_matches -from ..video import Episode, Movie, sanitize +from ..utils import sanitize +from ..video import Episode, Movie logger = logging.getLogger(__name__) diff --git a/lib/subliminal/providers/tvsubtitles.py b/lib/subliminal/providers/tvsubtitles.py index d3c8e85eeb..48df399915 100644 --- a/lib/subliminal/providers/tvsubtitles.py +++ b/lib/subliminal/providers/tvsubtitles.py @@ -13,7 +13,8 @@ from ..cache import EPISODE_EXPIRATION_TIME, SHOW_EXPIRATION_TIME, region from ..exceptions import ProviderError from ..subtitle import Subtitle, fix_line_ending, guess_matches -from ..video import Episode, sanitize +from ..utils import sanitize +from ..video import Episode logger = logging.getLogger(__name__) diff --git a/lib/subliminal/refiners/metadata.py b/lib/subliminal/refiners/metadata.py new file mode 100644 index 0000000000..898d69523e --- /dev/null +++ b/lib/subliminal/refiners/metadata.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +import logging +import os + +from babelfish import Error as BabelfishError, Language +from enzyme import MKV + +logger = logging.getLogger(__name__) + + +def refine(video, embedded_subtitles=True, **kwargs): + """Refine a video by searching its metadata. + + Several :class:`~subliminal.video.Video` attributes can be found: + + * :attr:`~subliminal.video.Video.resolution` + * :attr:`~subliminal.video.Video.video_codec` + * :attr:`~subliminal.video.Video.audio_codec` + * :attr:`~subliminal.video.Video.subtitle_languages` + + :param video: the video to refine. + :param bool embedded_subtitles: search for embedded subtitles. + + """ + # skip non existing videos + if not video.exists: + return + + # check extensions + extension = os.path.splitext(video.name)[1] + if extension == '.mkv': + with open(video.name, 'rb') as f: + mkv = MKV(f) + + # main video track + if mkv.video_tracks: + video_track = mkv.video_tracks[0] + + # resolution + if video_track.height in (480, 720, 1080): + if video_track.interlaced: + video.resolution = '%di' % video_track.height + else: + video.resolution = '%dp' % video_track.height + logger.debug('Found resolution %s', video.resolution) + + # video codec + if video_track.codec_id == 'V_MPEG4/ISO/AVC': + video.video_codec = 'h264' + logger.debug('Found video_codec %s', video.video_codec) + elif video_track.codec_id == 'V_MPEG4/ISO/SP': + video.video_codec = 'DivX' + logger.debug('Found video_codec %s', video.video_codec) + elif video_track.codec_id == 'V_MPEG4/ISO/ASP': + video.video_codec = 'XviD' + logger.debug('Found video_codec %s', video.video_codec) + else: + logger.warning('MKV has no video track') + + # main audio track + if mkv.audio_tracks: + audio_track = mkv.audio_tracks[0] + # audio codec + if audio_track.codec_id == 'A_AC3': + video.audio_codec = 'AC3' + logger.debug('Found audio_codec %s', video.audio_codec) + elif audio_track.codec_id == 'A_DTS': + video.audio_codec = 'DTS' + logger.debug('Found audio_codec %s', video.audio_codec) + elif audio_track.codec_id == 'A_AAC': + video.audio_codec = 'AAC' + logger.debug('Found audio_codec %s', video.audio_codec) + else: + logger.warning('MKV has no audio track') + + # subtitle tracks + if mkv.subtitle_tracks: + if embedded_subtitles: + embedded_subtitle_languages = set() + for st in mkv.subtitle_tracks: + if st.language: + try: + embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) + except BabelfishError: + logger.error('Embedded subtitle track language %r is not a valid language', st.language) + embedded_subtitle_languages.add(Language('und')) + elif st.name: + try: + embedded_subtitle_languages.add(Language.fromname(st.name)) + except BabelfishError: + logger.debug('Embedded subtitle track name %r is not a valid language', st.name) + embedded_subtitle_languages.add(Language('und')) + else: + embedded_subtitle_languages.add(Language('und')) + logger.debug('Found embedded subtitle %r', embedded_subtitle_languages) + video.subtitle_languages |= embedded_subtitle_languages + else: + logger.debug('MKV has no subtitle track') + else: + logger.debug('Unsupported video extension %s', extension) diff --git a/lib/subliminal/refiners/omdb.py b/lib/subliminal/refiners/omdb.py index c3ade2ea09..6726995381 100644 --- a/lib/subliminal/refiners/omdb.py +++ b/lib/subliminal/refiners/omdb.py @@ -6,7 +6,8 @@ from .. import __short_version__ from ..cache import REFINER_EXPIRATION_TIME, region -from ..video import Episode, Movie, sanitize +from ..video import Episode, Movie +from ..utils import sanitize logger = logging.getLogger(__name__) @@ -88,8 +89,29 @@ def search(title, type, year): return all_results -def refine(video): +def refine(video, **kwargs): + """Refine a video by searching `OMDb API `_. + + Several :class:`~subliminal.video.Episode` attributes can be found: + + * :attr:`~subliminal.video.Episode.series` + * :attr:`~subliminal.video.Episode.year` + * :attr:`~subliminal.video.Episode.series_imdb_id` + + Similarly, for a :class:`~subliminal.video.Movie`: + + * :attr:`~subliminal.video.Movie.title` + * :attr:`~subliminal.video.Movie.year` + * :attr:`~subliminal.video.Video.imdb_id` + + :param video: the video to refine. + + """ if isinstance(video, Episode): + # exit if the information is complete + if video.series_imdb_id: + return + # search the series results = search(video.series, 'series', video.year) if not results: @@ -126,6 +148,10 @@ def refine(video): video.series_imdb_id = result['imdbID'] elif isinstance(video, Movie): + # exit if the information is complete + if video.imdb_id: + return + # search the movie results = search(video.title, 'movie', video.year) if not results: diff --git a/lib/subliminal/refiners/tvdb.py b/lib/subliminal/refiners/tvdb.py index 130d5a359d..b659d2bb24 100644 --- a/lib/subliminal/refiners/tvdb.py +++ b/lib/subliminal/refiners/tvdb.py @@ -230,11 +230,34 @@ def get_series_episode(series_id, season, episode): return tvdb_client.get_episode(result['data'][0]['id']) -def refine(video): +def refine(video, **kwargs): + """Refine a video by searching `TheTVDB `_. + + .. note:: + + This refiner only work for instances of :class:`~subliminal.video.Episode`. + + Several attributes can be found: + + * :attr:`~subliminal.video.Episode.series` + * :attr:`~subliminal.video.Episode.year` + * :attr:`~subliminal.video.Episode.series_imdb_id` + * :attr:`~subliminal.video.Episode.series_tvdb_id` + * :attr:`~subliminal.video.Episode.title` + * :attr:`~subliminal.video.Video.imdb_id` + * :attr:`~subliminal.video.Episode.tvdb_id` + + :param video: the video to refine. + + """ # only deal with Episode videos if not isinstance(video, Episode): return + # exit if the information is complete + if video.series_tvdb_id and video.tvdb_id: + return + # search the series logger.info('Searching series %r', video.series) results = search_series(video.series.lower()) diff --git a/lib/subliminal/score.py b/lib/subliminal/score.py index edd7bcd11c..0449c7fa2a 100755 --- a/lib/subliminal/score.py +++ b/lib/subliminal/score.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ This module provides the default implementation of the `compute_score` parameter in -:meth:`~subliminal.api.ProviderPool.download_best_subtitles` and :func:`~subliminal.api.download_best_subtitles`. +:meth:`~subliminal.core.ProviderPool.download_best_subtitles` and :func:`~subliminal.core.download_best_subtitles`. .. note:: diff --git a/lib/subliminal/subtitle.py b/lib/subliminal/subtitle.py index 6f59643baa..2e08576e79 100644 --- a/lib/subliminal/subtitle.py +++ b/lib/subliminal/subtitle.py @@ -1,15 +1,19 @@ # -*- coding: utf-8 -*- +import codecs import logging import os import chardet -from codecs import lookup import pysrt -from .video import Episode, Movie, sanitize, sanitize_release_group +from .video import Episode, Movie +from .utils import sanitize, sanitize_release_group logger = logging.getLogger(__name__) +#: Subtitle extensions +SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl') + class Subtitle(object): """Base class for subtitle. @@ -19,7 +23,7 @@ class Subtitle(object): :param bool hearing_impaired: whether or not the subtitle is hearing impaired. :param page_link: URL of the web page from which the subtitle can be downloaded. :type page_link: str - :param encoding: Text encoding of the subtitle + :param encoding: Text encoding of the subtitle. :type encoding: str """ @@ -40,24 +44,23 @@ def __init__(self, language, hearing_impaired=False, page_link=None, encoding=No self.content = None #: Encoding to decode with when accessing :attr:`text` + self.encoding = None + + # validate the encoding if encoding: try: - # set encoding to canonical codec name - self.encoding = lookup(encoding).name + self.encoding = codecs.lookup(encoding).name except (TypeError, LookupError): - logger.debug('Unsupported encoding "%s", setting to None', encoding) - self.encoding = None - else: - self.encoding = None + logger.debug('Unsupported encoding %s', encoding) @property def id(self): - """Unique identifier of the subtitle.""" + """Unique identifier of the subtitle""" raise NotImplementedError @property def text(self): - """Content as string. + """Content as string If :attr:`encoding` is None, the encoding is guessed with :meth:`guess_encoding` diff --git a/lib/subliminal/utils.py b/lib/subliminal/utils.py new file mode 100644 index 0000000000..0a4fbab0aa --- /dev/null +++ b/lib/subliminal/utils.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +from datetime import datetime +import hashlib +import os +import re +import struct + + +def hash_opensubtitles(video_path): + """Compute a hash using OpenSubtitles' algorithm. + + :param str video_path: path of the video. + :return: the hash. + :rtype: str + + """ + bytesize = struct.calcsize(b'' % (self.__class__.__name__, self.title) return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) - - -def search_external_subtitles(path, directory=None): - """Search for external subtitles from a video `path` and their associated language. - - Unless `directory` is provided, search will be made in the same directory as the video file. - - :param str path: path to the video. - :param str directory: directory to search for subtitles. - :return: found subtitles with their languages. - :rtype: dict - - """ - dirpath, filename = os.path.split(path) - dirpath = dirpath or '.' - fileroot, fileext = os.path.splitext(filename) - subtitles = {} - for p in os.listdir(directory or dirpath): - # keep only valid subtitle filenames - if not p.startswith(fileroot) or not p.endswith(SUBTITLE_EXTENSIONS): - continue - - # extract the potential language code - language_code = p[len(fileroot):-len(os.path.splitext(p)[1])].replace(fileext, '').replace('_', '-')[1:] - - # default language is undefined - language = Language('und') - - # attempt to parse - if language_code: - try: - language = Language.fromietf(language_code) - except ValueError: - logger.error('Cannot parse language code %r', language_code) - - subtitles[p] = language - - logger.debug('Found subtitles %r', subtitles) - - return subtitles - - -def scan_video(path, subtitles=True, embedded_subtitles=True, subtitles_dir=None): - """Scan a video and its subtitle languages from a video `path`. - - :param str path: existing path to the video. - :param bool subtitles: scan for subtitles with the same name. - :param bool embedded_subtitles: scan for embedded subtitles. - :param str subtitles_dir: directory to search for subtitles. - :return: the scanned video. - :rtype: :class:`Video` - - """ - # check for non-existing path - if not os.path.exists(path): - raise ValueError('Path does not exist') - - # check video extension - if not path.endswith(VIDEO_EXTENSIONS): - raise ValueError('%s is not a valid video extension' % os.path.splitext(path)[1]) - - dirpath, filename = os.path.split(path) - logger.info('Scanning video %r in %r', filename, dirpath) - - # guess - video = Video.fromguess(path, guessit(path)) - - # size and hashes - video.size = os.path.getsize(path) - if video.size > 10485760: - logger.debug('Size is %d', video.size) - video.hashes['opensubtitles'] = hash_opensubtitles(path) - video.hashes['thesubdb'] = hash_thesubdb(path) - video.hashes['napiprojekt'] = hash_napiprojekt(path) - logger.debug('Computed hashes %r', video.hashes) - else: - logger.warning('Size is lower than 10MB: hashes not computed') - - # external subtitles - if subtitles: - video.subtitle_languages |= set(search_external_subtitles(path, directory=subtitles_dir).values()) - - # video metadata with enzyme - try: - if filename.endswith('.mkv'): - with open(path, 'rb') as f: - mkv = MKV(f) - - # main video track - if mkv.video_tracks: - video_track = mkv.video_tracks[0] - - # resolution - if video_track.height in (480, 720, 1080): - if video_track.interlaced: - video.resolution = '%di' % video_track.height - else: - video.resolution = '%dp' % video_track.height - logger.debug('Found resolution %s with enzyme', video.resolution) - - # video codec - if video_track.codec_id == 'V_MPEG4/ISO/AVC': - video.video_codec = 'h264' - logger.debug('Found video_codec %s with enzyme', video.video_codec) - elif video_track.codec_id == 'V_MPEG4/ISO/SP': - video.video_codec = 'DivX' - logger.debug('Found video_codec %s with enzyme', video.video_codec) - elif video_track.codec_id == 'V_MPEG4/ISO/ASP': - video.video_codec = 'XviD' - logger.debug('Found video_codec %s with enzyme', video.video_codec) - else: - logger.warning('MKV has no video track') - - # main audio track - if mkv.audio_tracks: - audio_track = mkv.audio_tracks[0] - # audio codec - if audio_track.codec_id == 'A_AC3': - video.audio_codec = 'AC3' - logger.debug('Found audio_codec %s with enzyme', video.audio_codec) - elif audio_track.codec_id == 'A_DTS': - video.audio_codec = 'DTS' - logger.debug('Found audio_codec %s with enzyme', video.audio_codec) - elif audio_track.codec_id == 'A_AAC': - video.audio_codec = 'AAC' - logger.debug('Found audio_codec %s with enzyme', video.audio_codec) - else: - logger.warning('MKV has no audio track') - - # subtitle tracks - if mkv.subtitle_tracks: - if embedded_subtitles: - embedded_subtitle_languages = set() - for st in mkv.subtitle_tracks: - if st.language: - try: - embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) - except BabelfishError: - logger.error('Embedded subtitle track language %r is not a valid language', st.language) - embedded_subtitle_languages.add(Language('und')) - elif st.name: - try: - embedded_subtitle_languages.add(Language.fromname(st.name)) - except BabelfishError: - logger.debug('Embedded subtitle track name %r is not a valid language', st.name) - embedded_subtitle_languages.add(Language('und')) - else: - embedded_subtitle_languages.add(Language('und')) - logger.debug('Found embedded subtitle %r with enzyme', embedded_subtitle_languages) - video.subtitle_languages |= embedded_subtitle_languages - else: - logger.debug('MKV has no subtitle track') - - except: - logger.exception('Parsing video metadata with enzyme failed') - - return video - - -def scan_videos(path, age=None, **kwargs): - """Scan `path` for videos and their subtitles. - - :param str path: existing directory path to scan. - :param datetime.timedelta age: maximum age of the video. - :param \*\*kwargs: parameters for the :func:`scan_video` function. - :return: the scanned videos. - :rtype: list of :class:`Video` - - """ - # check for non-existing path - if not os.path.exists(path): - raise ValueError('Path does not exist') - - # check for non-directory path - if not os.path.isdir(path): - raise ValueError('Path is not a directory') - - # walk the path - videos = [] - for dirpath, dirnames, filenames in os.walk(path): - logger.debug('Walking directory %s', dirpath) - - # remove badly encoded and hidden dirnames - for dirname in list(dirnames): - if dirname.startswith('.'): - logger.debug('Skipping hidden dirname %r in %r', dirname, dirpath) - dirnames.remove(dirname) - - # scan for videos - for filename in filenames: - # filter on videos - if not filename.endswith(VIDEO_EXTENSIONS): - continue - - # skip hidden files - if filename.startswith('.'): - logger.debug('Skipping hidden filename %r in %r', filename, dirpath) - continue - - # reconstruct the file path - filepath = os.path.join(dirpath, filename) - - # skip links - if os.path.islink(filepath): - logger.debug('Skipping link %r in %r', filename, dirpath) - continue - - # skip old files - if age and datetime.utcnow() - datetime.utcfromtimestamp(os.path.getmtime(filepath)) > age: - logger.debug('Skipping old file %r in %r', filename, dirpath) - continue - - # scan video - try: - video = scan_video(filepath, **kwargs) - except ValueError: # pragma: no cover - logger.exception('Error scanning video') - continue - - videos.append(video) - - return videos - - -def hash_opensubtitles(video_path): - """Compute a hash using OpenSubtitles' algorithm. - - :param str video_path: path of the video. - :return: the hash. - :rtype: str - - """ - bytesize = struct.calcsize(b' Date: Thu, 25 Feb 2016 17:51:19 +0100 Subject: [PATCH 19/34] Fix subtitles indent --- sickbeard/subtitles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sickbeard/subtitles.py b/sickbeard/subtitles.py index ae87984fb3..2a853174ad 100644 --- a/sickbeard/subtitles.py +++ b/sickbeard/subtitles.py @@ -384,7 +384,8 @@ def subtitles_download_in_pp(): # pylint: disable=too-many-locals, too-many-bra if subtitle_language not in sickbeard.SUBTITLES_LANGUAGES: try: os.remove(os.path.join(root, filename)) - logger.log(u"Deleted '{}' because we don't want subtitle language '{}'. We only want '{}' language(s)".format(filename, subtitle_language, ', '.join(sickbeard.SUBTITLES_LANGUAGES)), logger.DEBUG) + logger.log(u"Deleted '{}' because we don't want subtitle language '{}'. We only want '{}' language(s)".format + (filename, subtitle_language, ','.join(sickbeard.SUBTITLES_LANGUAGES)), logger.DEBUG) except Exception as error: logger.log(u"Couldn't delete subtitle: {}. Error: {}".format(filename, ex(error)), logger.DEBUG) From 1a9c7102445a461f98c9c036d56ea00887e7625d Mon Sep 17 00:00:00 2001 From: Labrys Date: Thu, 25 Feb 2016 13:32:34 -0500 Subject: [PATCH 20/34] Fix double logging --- sickbeard/providers/bitsnoop.py | 2 +- sickbeard/providers/cpasbien.py | 2 +- sickbeard/providers/danishbits.py | 2 +- sickbeard/providers/elitetorrent.py | 2 +- sickbeard/providers/extratorrent.py | 2 +- sickbeard/providers/freshontv.py | 2 +- sickbeard/providers/gftracker.py | 2 +- sickbeard/providers/hd4free.py | 2 +- sickbeard/providers/hdbits.py | 2 +- sickbeard/providers/hdspace.py | 2 +- sickbeard/providers/hdtorrents.py | 2 +- sickbeard/providers/iptorrents.py | 2 +- sickbeard/providers/limetorrents.py | 2 +- sickbeard/providers/morethantv.py | 2 +- sickbeard/providers/newznab.py | 4 ++-- sickbeard/providers/phxbit.py | 2 +- sickbeard/providers/pretome.py | 2 +- sickbeard/providers/scc.py | 2 +- sickbeard/providers/scenetime.py | 2 +- sickbeard/providers/speedcd.py | 2 +- sickbeard/providers/strike.py | 2 +- sickbeard/providers/t411.py | 2 +- sickbeard/providers/tntvillage.py | 2 +- sickbeard/providers/tokyotoshokan.py | 2 +- sickbeard/providers/torrentproject.py | 2 +- sickbeard/providers/torrentz.py | 2 +- sickbeard/providers/transmitthenet.py | 2 +- sickbeard/providers/xthor.py | 2 +- 28 files changed, 29 insertions(+), 29 deletions(-) diff --git a/sickbeard/providers/bitsnoop.py b/sickbeard/providers/bitsnoop.py index 3c5913481a..c59754197f 100644 --- a/sickbeard/providers/bitsnoop.py +++ b/sickbeard/providers/bitsnoop.py @@ -67,7 +67,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/cpasbien.py b/sickbeard/providers/cpasbien.py index c69ac63d16..dda3ae9930 100644 --- a/sickbeard/providers/cpasbien.py +++ b/sickbeard/providers/cpasbien.py @@ -57,7 +57,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_url = self.url + '/view_cat.php?categorie=series&trie=date-d' logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: continue diff --git a/sickbeard/providers/danishbits.py b/sickbeard/providers/danishbits.py index ad5fec2d35..79ecd13d3a 100644 --- a/sickbeard/providers/danishbits.py +++ b/sickbeard/providers/danishbits.py @@ -120,7 +120,7 @@ def process_column_header(td): search_url = "%s?%s" % (self.urls['search'], urlencode(search_params)) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/elitetorrent.py b/sickbeard/providers/elitetorrent.py index 357433b14b..8640eb8073 100644 --- a/sickbeard/providers/elitetorrent.py +++ b/sickbeard/providers/elitetorrent.py @@ -91,7 +91,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_url = self.urls['search'] + '?' + urllib.parse.urlencode(self.search_params) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url, timeout=30) + data = self.get_url(search_url, echo=False, returns='text') if not data: continue diff --git a/sickbeard/providers/extratorrent.py b/sickbeard/providers/extratorrent.py index d4eafcc419..e121da8374 100644 --- a/sickbeard/providers/extratorrent.py +++ b/sickbeard/providers/extratorrent.py @@ -68,7 +68,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url, params=self.search_params) + data = self.get_url(search_url, params=self.search_params, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/freshontv.py b/sickbeard/providers/freshontv.py index 2ada794e0e..72432d3110 100644 --- a/sickbeard/providers/freshontv.py +++ b/sickbeard/providers/freshontv.py @@ -121,7 +121,7 @@ def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many search_url = self.urls['search'] % (freeleech, search_string) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - init_html = self.get_url(search_url) + init_html = self.get_url(search_url, echo=False, returns='text') max_page_number = 0 if not init_html: diff --git a/sickbeard/providers/gftracker.py b/sickbeard/providers/gftracker.py index aa359e424e..66524885b1 100644 --- a/sickbeard/providers/gftracker.py +++ b/sickbeard/providers/gftracker.py @@ -133,7 +133,7 @@ def process_column_header(td): search_url = "%s?%s" % (self.urls['search'], urlencode(search_params)) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/hd4free.py b/sickbeard/providers/hd4free.py index d46d4f17dc..f7f7fdda11 100644 --- a/sickbeard/providers/hd4free.py +++ b/sickbeard/providers/hd4free.py @@ -80,7 +80,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_url = self.urls['search'] + "?" + urlencode(search_params) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - jdata = self.get_url(search_url, json=True) + jdata = self.get_url(search_url, json=True, echo=False) if not jdata: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/hdbits.py b/sickbeard/providers/hdbits.py index ddf2da2e66..11d32b3895 100644 --- a/sickbeard/providers/hdbits.py +++ b/sickbeard/providers/hdbits.py @@ -88,7 +88,7 @@ def search(self, search_params, age=0, ep_obj=None): self._check_auth() - parsedJSON = self.get_url(self.urls['search'], post_data=search_params, json=True) + parsedJSON = self.get_url(self.urls['search'], post_data=search_params, json=True, echo=False) if not parsedJSON: return [] diff --git a/sickbeard/providers/hdspace.py b/sickbeard/providers/hdspace.py index d1bf17098d..90bc69ae07 100644 --- a/sickbeard/providers/hdspace.py +++ b/sickbeard/providers/hdspace.py @@ -105,7 +105,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search string: {}".format(search_string.decode("utf-8")), logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data or 'please try later' in data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/hdtorrents.py b/sickbeard/providers/hdtorrents.py index 8aa8d2d200..f56e301d47 100644 --- a/sickbeard/providers/hdtorrents.py +++ b/sickbeard/providers/hdtorrents.py @@ -103,7 +103,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data or 'please try later' in data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/iptorrents.py b/sickbeard/providers/iptorrents.py index cd0bd19854..ca7b9ff38c 100644 --- a/sickbeard/providers/iptorrents.py +++ b/sickbeard/providers/iptorrents.py @@ -106,7 +106,7 @@ def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many search_url += ';o=seeders' if mode != 'RSS' else '' logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: continue diff --git a/sickbeard/providers/limetorrents.py b/sickbeard/providers/limetorrents.py index 9e484a1e12..1cd5721c76 100644 --- a/sickbeard/providers/limetorrents.py +++ b/sickbeard/providers/limetorrents.py @@ -69,7 +69,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/morethantv.py b/sickbeard/providers/morethantv.py index 914e9dd821..f7d1b3b3ba 100644 --- a/sickbeard/providers/morethantv.py +++ b/sickbeard/providers/morethantv.py @@ -133,7 +133,7 @@ def process_column_header(td): search_url = "%s?%s" % (self.urls['search'], urlencode(search_params)) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index ed03a93c51..5c8cc5fa31 100644 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -283,7 +283,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: {url}".format(url=search_url), logger.DEBUG) time.sleep(cpu_presets[sickbeard.CPU_PRESET]) - data = self.get_url(search_url, echo=False) + data = self.get_url(search_url, echo=False, returns='text') if not data: break @@ -301,7 +301,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: %s" % search_url, logger.DEBUG) time.sleep(cpu_presets[sickbeard.CPU_PRESET]) - data = self.get_url(search_url, echo=False) + data = self.get_url(search_url, echo=False, returns='text') if not data: break diff --git a/sickbeard/providers/phxbit.py b/sickbeard/providers/phxbit.py index e6bcdbdad9..b0e6b402f0 100644 --- a/sickbeard/providers/phxbit.py +++ b/sickbeard/providers/phxbit.py @@ -119,7 +119,7 @@ def process_column_header(td): search_url = self.urls['search'] + urlencode(search_params) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: continue diff --git a/sickbeard/providers/pretome.py b/sickbeard/providers/pretome.py index a0fbcfc3b2..f7c2bd22b1 100644 --- a/sickbeard/providers/pretome.py +++ b/sickbeard/providers/pretome.py @@ -100,7 +100,7 @@ def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many search_url = self.urls['search'] % (quote(search_string), self.categories) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: continue diff --git a/sickbeard/providers/scc.py b/sickbeard/providers/scc.py index 8a1eccadd3..ca395a0d73 100644 --- a/sickbeard/providers/scc.py +++ b/sickbeard/providers/scc.py @@ -107,7 +107,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man try: logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') time.sleep(cpu_presets[sickbeard.CPU_PRESET]) except Exception as e: logger.log(u"Unable to fetch data. Error: %s" % repr(e), logger.WARNING) diff --git a/sickbeard/providers/scenetime.py b/sickbeard/providers/scenetime.py index 20e11a32e9..3c11373df2 100644 --- a/sickbeard/providers/scenetime.py +++ b/sickbeard/providers/scenetime.py @@ -88,7 +88,7 @@ def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many search_url = self.urls['search'] % (quote(search_string), self.categories) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: continue diff --git a/sickbeard/providers/speedcd.py b/sickbeard/providers/speedcd.py index dd6b680112..db9620a1e1 100644 --- a/sickbeard/providers/speedcd.py +++ b/sickbeard/providers/speedcd.py @@ -129,7 +129,7 @@ def process_column_header(td): search_url = "%s?%s" % (self.urls['search'], urlencode(search_params)) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: continue diff --git a/sickbeard/providers/strike.py b/sickbeard/providers/strike.py index 5ef1bcca20..a9c60d6f57 100644 --- a/sickbeard/providers/strike.py +++ b/sickbeard/providers/strike.py @@ -49,7 +49,7 @@ def search(self, search_strings, age=0, ep_obj=None): search_url = self.url + "api/v2/torrents/search/?category=TV&phrase=" + search_string logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - jdata = self.get_url(search_url, json=True) + jdata = self.get_url(search_url, json=True, echo=False) if not jdata: logger.log(u"No data returned from provider", logger.DEBUG) return [] diff --git a/sickbeard/providers/t411.py b/sickbeard/providers/t411.py index 2611f05309..a5b94b45bd 100644 --- a/sickbeard/providers/t411.py +++ b/sickbeard/providers/t411.py @@ -100,7 +100,7 @@ def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many search_urlS = ([self.urls['search'] % (search_string, u) for u in self.subcategories], [self.urls['rss']])[mode == 'RSS'] for search_url in search_urlS: logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url, json=True) + data = self.get_url(search_url, json=True, echo=False) if not data: continue diff --git a/sickbeard/providers/tntvillage.py b/sickbeard/providers/tntvillage.py index 9f52749b55..27f401dadd 100644 --- a/sickbeard/providers/tntvillage.py +++ b/sickbeard/providers/tntvillage.py @@ -312,7 +312,7 @@ def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many logger.DEBUG) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/tokyotoshokan.py b/sickbeard/providers/tokyotoshokan.py index 9168e89acd..47913cf0e3 100644 --- a/sickbeard/providers/tokyotoshokan.py +++ b/sickbeard/providers/tokyotoshokan.py @@ -71,7 +71,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man } logger.log(u"Search URL: %s" % self.urls['search'] + '?' + urlencode(search_params), logger.DEBUG) - data = self.get_url(self.urls['search'], params=search_params) + data = self.get_url(self.urls['search'], params=search_params, echo=False, returns='text') if not data: continue diff --git a/sickbeard/providers/torrentproject.py b/sickbeard/providers/torrentproject.py index d9e08954ad..1f32691bf2 100644 --- a/sickbeard/providers/torrentproject.py +++ b/sickbeard/providers/torrentproject.py @@ -71,7 +71,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_url = posixpath.join(self.custom_url, search_url.split(self.url)[1].lstrip('/')) # Must use posixpath logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - torrents = self.get_url(search_url, json=True) + torrents = self.get_url(search_url, json=True, echo=False) if not (torrents and "total_found" in torrents and int(torrents["total_found"]) > 0): logger.log(u"Data returned from provider does not contain any torrents", logger.DEBUG) continue diff --git a/sickbeard/providers/torrentz.py b/sickbeard/providers/torrentz.py index 7263a32497..cf8122e109 100644 --- a/sickbeard/providers/torrentz.py +++ b/sickbeard/providers/torrentz.py @@ -81,7 +81,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/transmitthenet.py b/sickbeard/providers/transmitthenet.py index 9c8946546e..9e90ea2d89 100644 --- a/sickbeard/providers/transmitthenet.py +++ b/sickbeard/providers/transmitthenet.py @@ -115,7 +115,7 @@ def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-man search_url = self.urls['search'] + "?" + urlencode(search_params) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(self.urls['search'], params=search_params) + data = self.get_url(self.urls['search'], params=search_params, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue diff --git a/sickbeard/providers/xthor.py b/sickbeard/providers/xthor.py index 28249a9732..43d01fc766 100644 --- a/sickbeard/providers/xthor.py +++ b/sickbeard/providers/xthor.py @@ -134,7 +134,7 @@ def process_column_header(td): search_url = self.urls['search'] + urlencode(search_params) logger.log(u"Search URL: %s" % search_url, logger.DEBUG) - data = self.get_url(search_url) + data = self.get_url(search_url, echo=False, returns='text') if not data: logger.log(u"No data returned from provider", logger.DEBUG) continue From 2d7e56186af8543ad6ff3b39e2c2851b01db1fbe Mon Sep 17 00:00:00 2001 From: miigotu Date: Thu, 25 Feb 2016 18:45:38 -0800 Subject: [PATCH 21/34] 50 Threads? Really? This fix halves VIRT alloc, to halve it again add export MALLOC_ARENA_MAX=2 (or 1) to your runscript on linux) --- sickbeard/webserve.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 16c552611c..8086c830e6 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -88,8 +88,12 @@ from tornado.web import RequestHandler, HTTPError, authenticated from tornado.gen import coroutine from tornado.ioloop import IOLoop -from tornado.concurrent import run_on_executor + from concurrent.futures import ThreadPoolExecutor +from tornado.process import cpu_count + +from tornado.concurrent import run_on_executor + from mako.runtime import UNDEFINED mako_lookup = None @@ -255,7 +259,7 @@ def __init__(self, *args, **kwargs): super(WebHandler, self).__init__(*args, **kwargs) self.io_loop = IOLoop.current() - executor = ThreadPoolExecutor(50) + executor = ThreadPoolExecutor(cpu_count()) @authenticated @coroutine From cff117de52f8b51859688820600cb19ef2920364 Mon Sep 17 00:00:00 2001 From: labrys Date: Wed, 24 Feb 2016 05:48:29 -0500 Subject: [PATCH 22/34] Move logging commit hash Re-enable skipped tests --- sickbeard/logger.py | 14 ++++++++++---- tests/scene_helpers_tests.py | 4 ---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sickbeard/logger.py b/sickbeard/logger.py index 1d6132db7e..6672af4d4c 100644 --- a/sickbeard/logger.py +++ b/sickbeard/logger.py @@ -196,10 +196,16 @@ def log(self, msg, level=INFO, *args, **kwargs): :param kwargs: to pass to logger """ cur_thread = threading.currentThread().getName() - if sickbeard.CUR_COMMIT_HASH and len(sickbeard.CUR_COMMIT_HASH) > 6 and level in [ERROR, WARNING]: - msg += ' [%s]' % sickbeard.CUR_COMMIT_HASH[:7] - - message = '%s :: %s' % (cur_thread, msg) + cur_hash = '[{}] '.format(sickbeard.CUR_COMMIT_HASH[:7]) if all([ + sickbeard.CUR_COMMIT_HASH, + len(sickbeard.CUR_COMMIT_HASH) > 6, + ]) else '' + + message = '{thread} :: {hash}{message}'.format( + thread = cur_thread, + hash = cur_hash, + message = msg + ) # Change the SSL error to a warning with a link to information about how to fix it. # Check for u'error [SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:590)' diff --git a/tests/scene_helpers_tests.py b/tests/scene_helpers_tests.py index d2d65442f5..7ae4361010 100644 --- a/tests/scene_helpers_tests.py +++ b/tests/scene_helpers_tests.py @@ -92,14 +92,12 @@ def test_scene_ex_empty(self): """ self.assertEqual(scene_exceptions.get_scene_exceptions(0), []) - @unittest.skip('Waiting new github page to get exceptions') def test_scene_ex_babylon_5(self): """ Test scene exceptions for Babylon 5 """ self.assertEqual(sorted(scene_exceptions.get_scene_exceptions(70726)), ['Babylon 5', 'Babylon5']) - @unittest.skip('Waiting new github page to get exceptions') def test_scene_ex_by_name(self): """ Test scene exceptions by name @@ -109,14 +107,12 @@ def test_scene_ex_by_name(self): self.assertEqual(scene_exceptions.get_scene_exception_by_name('babylon 5'), (70726, -1)) self.assertEqual(scene_exceptions.get_scene_exception_by_name('Carlos 2010'), (164451, -1)) - @unittest.skip('Waiting new github page to get exceptions') def test_scene_ex_by_name_empty(self): """ Test scene exceptions by name are empty """ self.assertEqual(scene_exceptions.get_scene_exception_by_name('nothing useful'), (None, None)) - @unittest.skip('Waiting new github page to get exceptions') def test_scene_ex_reset_name_cache(self): """ Test scene exceptions reset name cache From 6512bdde296ab56bcd9003d3971179001f46f255 Mon Sep 17 00:00:00 2001 From: X O Date: Fri, 26 Feb 2016 22:53:03 +1030 Subject: [PATCH 23/34] Add mako indenting --- .editorconfig | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 7b38c166a0..1d19a922d2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,11 +13,16 @@ trim_trailing_whitespace = true [*.{js,py}] charset = utf-8 -# 4 space indentation +# Python styling [*.py] indent_style = space indent_size = 4 +# Mako styling +[*.mako] +indent_style = space +indent_size = 4 + # Indentation override for all JS under lib directory [*.js] indent_style = space From 428165f3e98389e38f6577b3e119083871156b5f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Fri, 26 Feb 2016 07:33:44 -0500 Subject: [PATCH 24/34] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 93f72893c2..3c2ad6e8a6 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -SickRage [![Build Status](https://travis-ci.org/PyMedusa/SickRage.svg?branch=develop)](https://travis-ci.org/PyMedusa/SickRage) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/PyMedusa/SickRage-issues.svg)](http://isitmaintained.com/project/PyMedusa/SickRage-issues "Average time to resolve an issue") [![Percentage of issues still open](http://isitmaintained.com/badge/open/PyMedusa/SickRage-issues.svg)](http://isitmaintained.com/project/PyMedusa/SickRage-issues "Percentage of issues still open") [![Coverage Status](https://coveralls.io/repos/PyMedusa/SickRage/badge.svg?branch=develop&service=github)](https://coveralls.io/github/PyMedusa/SickRage?branch=develop) +SickRage [![Build Status](https://travis-ci.org/pymedusa/SickRage.svg?branch=develop)](https://travis-ci.org/pymedusa/SickRage) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/pymedusa/SickRage-issues.svg)](http://isitmaintained.com/project/pymedusa/SickRage-issues "Average time to resolve an issue") [![Percentage of issues still open](http://isitmaintained.com/badge/open/pymedusa/SickRage-issues.svg)](http://isitmaintained.com/project/pymedusa/SickRage-issues "Percentage of issues still open") [![Coverage Status](https://coveralls.io/repos/pymedusa/SickRage/badge.svg?branch=develop&service=github)](https://coveralls.io/github/pymedusa/SickRage?branch=develop) ===== Automatic Video Library Manager for TV Shows. It watches for new episodes of your favorite shows, and when they are posted it does its magic. From 70b84fb01a87c672bc56bfcb58eb470503b7e792 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 23 Feb 2016 09:15:17 -0300 Subject: [PATCH 25/34] Improve SR startup: only run CheckVersion at startup --- sickbeard/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index 775c1c4450..79c32c4337 100644 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -1465,11 +1465,12 @@ def path_leaf(path): run_delay=update_interval) # processors + update_interval = datetime.timedelta(minutes=AUTOPOSTPROCESSER_FREQUENCY) autoPostProcesserScheduler = scheduler.Scheduler(auto_postprocessor.PostProcessor(), - cycleTime=datetime.timedelta( - minutes=AUTOPOSTPROCESSER_FREQUENCY), + cycleTime=update_interval, threadName="POSTPROCESSER", - silent=not PROCESS_AUTOMATICALLY) + silent=not PROCESS_AUTOMATICALLY, + run_delay=update_interval) traktCheckerScheduler = scheduler.Scheduler(traktChecker.TraktChecker(), cycleTime=datetime.timedelta(hours=1), From 424205854d22a4e2146f9b637fc70c569369a66f Mon Sep 17 00:00:00 2001 From: Labrys Date: Fri, 26 Feb 2016 18:05:36 -0500 Subject: [PATCH 26/34] Update git org --- sickbeard/__init__.py | 2 +- sickbeard/logger.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index 79c32c4337..b803b09fd5 100644 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -131,7 +131,7 @@ GIT_REMOTE = '' GIT_REMOTE_URL = '' CUR_COMMIT_BRANCH = '' -GIT_ORG = 'SickRage' +GIT_ORG = 'pymedusa' GIT_REPO = 'SickRage' GIT_USERNAME = None GIT_PASSWORD = None diff --git a/sickbeard/logger.py b/sickbeard/logger.py index 6672af4d4c..8df2e40d52 100644 --- a/sickbeard/logger.py +++ b/sickbeard/logger.py @@ -266,8 +266,8 @@ def submit_errors(self): # pylint: disable=too-many-branches,too-many-locals self.submitter_running = True - gh_org = sickbeard.GIT_ORG or 'SickRage' - gh_repo = 'sickrage-issues' + gh_org = sickbeard.GIT_ORG + gh_repo = sickbeard.GIT_REPO git = Github(login_or_token=sickbeard.GIT_USERNAME, password=sickbeard.GIT_PASSWORD, user_agent='SickRage') @@ -337,7 +337,7 @@ def submit_errors(self): # pylint: disable=too-many-branches,too-many-locals cur_error.message, '```', '---', - '_STAFF NOTIFIED_: @SickRage/owners @SickRage/moderators', + '_STAFF NOTIFIED_: @SickRage/support @SickRage/moderators', ] message = '\n'.join(msg) From 23a3feda55d71fcff2fc72d6aeca2d8cdffc8b0f Mon Sep 17 00:00:00 2001 From: Labrys Date: Fri, 26 Feb 2016 14:53:15 -0500 Subject: [PATCH 27/34] Add preferred words --- gui/slick/views/config_search.mako | 27 ++++++++++++++++++++++++++- sickbeard/__init__.py | 26 ++++++++++++++++++-------- sickbeard/search.py | 14 +++++++++++++- sickbeard/webserve.py | 24 ++++++++++-------------- 4 files changed, 67 insertions(+), 24 deletions(-) diff --git a/gui/slick/views/config_search.mako b/gui/slick/views/config_search.mako index 32b9aac81e..8d98414a50 100644 --- a/gui/slick/views/config_search.mako +++ b/gui/slick/views/config_search.mako @@ -125,6 +125,31 @@ +
+ +
+ +
+ +
+ +
- + \ No newline at end of file diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index b803b09fd5..33404844b3 100644 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -138,7 +138,7 @@ GIT_PATH = None DEVELOPER = False -NEWS_URL = 'https://api.pymedusa.com/news.md' +NEWS_URL = 'https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/news.md' NEWS_LAST_READ = None NEWS_LATEST = None NEWS_UNREAD = 0 @@ -351,6 +351,7 @@ PLEX_CLIENT_HOST = None PLEX_SERVER_USERNAME = None PLEX_SERVER_PASSWORD = None +PLEX_SERVER_NO_AUTH = False USE_PLEX_CLIENT = False PLEX_CLIENT_USERNAME = None @@ -553,6 +554,10 @@ IGNORE_WORDS = "german,french,core2hd,dutch,swedish,reenc,MrLss" +PREFERED_WORDS = "" + +UNDESIRED_WORDS = "" + TRACKERS_LIST = "udp://coppersurfer.tk:6969/announce,udp://open.demonii.com:1337," TRACKERS_LIST += "udp://exodus.desync.com:6969,udp://9.rarbg.me:2710/announce," TRACKERS_LIST += "udp://glotorrents.pw:6969/announce,udp://tracker.openbittorrent.com:80/announce," @@ -600,7 +605,7 @@ def initialize(consoleLogging=True): # pylint: disable=too-many-locals, too-man USE_KODI, KODI_ALWAYS_ON, KODI_NOTIFY_ONSNATCH, KODI_NOTIFY_ONDOWNLOAD, KODI_NOTIFY_ONSUBTITLEDOWNLOAD, KODI_UPDATE_FULL, KODI_UPDATE_ONLYFIRST, \ KODI_UPDATE_LIBRARY, KODI_HOST, KODI_USERNAME, KODI_PASSWORD, BACKLOG_FREQUENCY, \ USE_TRAKT, TRAKT_USERNAME, TRAKT_ACCESS_TOKEN, TRAKT_REFRESH_TOKEN, TRAKT_REMOVE_WATCHLIST, TRAKT_SYNC_WATCHLIST, TRAKT_REMOVE_SHOW_FROM_SICKRAGE, TRAKT_METHOD_ADD, TRAKT_START_PAUSED, traktCheckerScheduler, TRAKT_USE_RECOMMENDED, TRAKT_SYNC, TRAKT_SYNC_REMOVE, TRAKT_DEFAULT_INDEXER, TRAKT_REMOVE_SERIESLIST, TRAKT_TIMEOUT, TRAKT_BLACKLIST_NAME, \ - USE_PLEX_SERVER, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \ + USE_PLEX_SERVER, PLEX_SERVER_NO_AUTH, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \ PLEX_SERVER_HOST, PLEX_SERVER_TOKEN, PLEX_CLIENT_HOST, PLEX_SERVER_USERNAME, PLEX_SERVER_PASSWORD, PLEX_SERVER_HTTPS, MIN_BACKLOG_FREQUENCY, SKIP_REMOVED_FILES, ALLOWED_EXTENSIONS, \ USE_EMBY, EMBY_HOST, EMBY_APIKEY, \ showUpdateScheduler, __INITIALIZED__, INDEXER_DEFAULT_LANGUAGE, EP_DEFAULT_DELETED_STATUS, LAUNCH_BROWSER, TRASH_REMOVE_SHOW, TRASH_ROTATE_LOGS, SORT_ARTICLE, \ @@ -629,7 +634,7 @@ def initialize(consoleLogging=True): # pylint: disable=too-many-locals, too-man NEWZBIN, NEWZBIN_USERNAME, NEWZBIN_PASSWORD, GIT_PATH, MOVE_ASSOCIATED_FILES, SYNC_FILES, POSTPONE_IF_SYNC_FILES, POSTPONE_IF_NO_SUBS, dailySearchScheduler, NFO_RENAME, \ GUI_NAME, HOME_LAYOUT, HISTORY_LAYOUT, DISPLAY_SHOW_SPECIALS, COMING_EPS_LAYOUT, COMING_EPS_SORT, COMING_EPS_DISPLAY_PAUSED, COMING_EPS_MISSED_RANGE, FUZZY_DATING, TRIM_ZERO, DATE_PRESET, TIME_PRESET, TIME_PRESET_W_SECONDS, THEME_NAME, \ POSTER_SORTBY, POSTER_SORTDIR, HISTORY_LIMIT, CREATE_MISSING_SHOW_DIRS, ADD_SHOWS_WO_DIR, \ - METADATA_WDTV, METADATA_TIVO, METADATA_MEDE8ER, IGNORE_WORDS, TRACKERS_LIST, IGNORED_SUBS_LIST, REQUIRE_WORDS, CALENDAR_UNPROTECTED, CALENDAR_ICONS, NO_RESTART, \ + METADATA_WDTV, METADATA_TIVO, METADATA_MEDE8ER, IGNORE_WORDS, PREFERED_WORDS, UNDESIRED_WORDS, TRACKERS_LIST, IGNORED_SUBS_LIST, REQUIRE_WORDS, CALENDAR_UNPROTECTED, CALENDAR_ICONS, NO_RESTART, \ USE_SUBTITLES, SUBTITLES_LANGUAGES, SUBTITLES_DIR, SUBTITLES_SERVICES_LIST, SUBTITLES_SERVICES_ENABLED, SUBTITLES_HISTORY, SUBTITLES_FINDER_FREQUENCY, SUBTITLES_MULTI, SUBTITLES_DOWNLOAD_IN_PP, SUBTITLES_KEEP_ONLY_WANTED, EMBEDDED_SUBTITLES_ALL, SUBTITLES_EXTRA_SCRIPTS, SUBTITLES_PERFECT_MATCH, subtitlesFinderScheduler, \ SUBTITLES_HEARING_IMPAIRED, ADDIC7ED_USER, ADDIC7ED_PASS, LEGENDASTV_USER, LEGENDASTV_PASS, OPENSUBTITLES_USER, OPENSUBTITLES_PASS, \ USE_FAILED_DOWNLOADS, DELETE_FAILED, ANON_REDIRECT, LOCALHOST_IP, DEBUG, DBDEBUG, DEFAULT_PAGE, PROXY_SETTING, PROXY_INDEXERS, \ @@ -1023,6 +1028,7 @@ def path_leaf(path): PLEX_CLIENT_USERNAME = check_setting_str(CFG, 'Plex', 'plex_client_username', '', censor_log=True) PLEX_CLIENT_PASSWORD = check_setting_str(CFG, 'Plex', 'plex_client_password', '', censor_log=True) PLEX_SERVER_HTTPS = bool(check_setting_int(CFG, 'Plex', 'plex_server_https', 0)) + PLEX_SERVER_NO_AUTH = bool(check_setting_int(CFG, 'Plex', 'plex_server_no_auth', 0)) USE_EMBY = bool(check_setting_int(CFG, 'Emby', 'use_emby', 0)) EMBY_HOST = check_setting_str(CFG, 'Emby', 'emby_host', '') @@ -1203,6 +1209,8 @@ def path_leaf(path): GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '') IGNORE_WORDS = check_setting_str(CFG, 'General', 'ignore_words', IGNORE_WORDS) + PREFERED_WORDS = check_setting_str(CFG, 'General', 'prefered_words', PREFERED_WORDS) + UNDESIRED_WORDS = check_setting_str(CFG, 'General', 'undesired_words', UNDESIRED_WORDS) TRACKERS_LIST = check_setting_str(CFG, 'General', 'trackers_list', TRACKERS_LIST) REQUIRE_WORDS = check_setting_str(CFG, 'General', 'require_words', REQUIRE_WORDS) IGNORED_SUBS_LIST = check_setting_str(CFG, 'General', 'ignored_subs_list', IGNORED_SUBS_LIST) @@ -1465,12 +1473,11 @@ def path_leaf(path): run_delay=update_interval) # processors - update_interval = datetime.timedelta(minutes=AUTOPOSTPROCESSER_FREQUENCY) autoPostProcesserScheduler = scheduler.Scheduler(auto_postprocessor.PostProcessor(), - cycleTime=update_interval, + cycleTime=datetime.timedelta( + minutes=AUTOPOSTPROCESSER_FREQUENCY), threadName="POSTPROCESSER", - silent=not PROCESS_AUTOMATICALLY, - run_delay=update_interval) + silent=not PROCESS_AUTOMATICALLY) traktCheckerScheduler = scheduler.Scheduler(traktChecker.TraktChecker(), cycleTime=datetime.timedelta(hours=1), @@ -1749,6 +1756,8 @@ def save_config(): # pylint: disable=too-many-statements, too-many-branches new_config['General']['extra_scripts'] = '|'.join(EXTRA_SCRIPTS) new_config['General']['git_path'] = GIT_PATH new_config['General']['ignore_words'] = IGNORE_WORDS + new_config['General']['prefered_words'] = PREFERED_WORDS + new_config['General']['undesired_words'] = UNDESIRED_WORDS new_config['General']['trackers_list'] = TRACKERS_LIST new_config['General']['require_words'] = REQUIRE_WORDS new_config['General']['ignored_subs_list'] = IGNORED_SUBS_LIST @@ -1935,6 +1944,7 @@ def save_config(): # pylint: disable=too-many-statements, too-many-branches new_config['Plex']['plex_client_host'] = PLEX_CLIENT_HOST new_config['Plex']['plex_server_username'] = PLEX_SERVER_USERNAME new_config['Plex']['plex_server_password'] = helpers.encrypt(PLEX_SERVER_PASSWORD, ENCRYPTION_VERSION) + new_config['Plex']['plex_server_no_auth'] = int(PLEX_SERVER_NO_AUTH) new_config['Plex']['use_plex_client'] = int(USE_PLEX_CLIENT) new_config['Plex']['plex_client_username'] = PLEX_CLIENT_USERNAME @@ -2177,4 +2187,4 @@ def launchBrowser(protocol='http', startPort=None, web_root='/'): try: webbrowser.open(browserURL, 1, 1) except Exception: - logger.log(u"Unable to launch a browser", logger.ERROR) + logger.log(u"Unable to launch a browser", logger.ERROR) \ No newline at end of file diff --git a/sickbeard/search.py b/sickbeard/search.py index c8f30923f2..f9fbdfb2a5 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -246,6 +246,12 @@ def pickBestResult(results, show): # pylint: disable=too-many-branches cur_result.provider.name): logger.log(cur_result.name + u" has previously failed, rejecting it") continue + prefered_words = '' + if sickbeard.PREFERED_WORDS: + prefered_words = sickbeard.PREFERED_WORDS.lower().split(',') + undesired_words = '' + if sickbeard.UNDESIRED_WORDS: + undesired_words = sickbeard.UNDESIRED_WORDS.lower().split(',') if not bestResult: bestResult = cur_result @@ -254,6 +260,9 @@ def pickBestResult(results, show): # pylint: disable=too-many-branches elif cur_result.quality in anyQualities and bestResult.quality not in bestQualities and bestResult.quality < cur_result.quality: bestResult = cur_result elif bestResult.quality == cur_result.quality: + if any(ext in cur_result.name.lower() for ext in prefered_words): + logger.log(u"Preferring " + cur_result.name + " (preferred words)") + bestResult = cur_result if "proper" in cur_result.name.lower() or "real" in cur_result.name.lower() or "repack" in cur_result.name.lower(): logger.log(u"Preferring " + cur_result.name + " (repack/proper/real over nuked)") bestResult = cur_result @@ -263,6 +272,9 @@ def pickBestResult(results, show): # pylint: disable=too-many-branches elif "xvid" in bestResult.name.lower() and "x264" in cur_result.name.lower(): logger.log(u"Preferring " + cur_result.name + " (x264 over xvid)") bestResult = cur_result + if any(ext in bestResult.name.lower() and ext not in cur_result.name.lower() for ext in undesired_words): + logger.log(u"Dont want this release " + cur_result.name + " (contains undesired word(s))") + bestResult = cur_result if bestResult: logger.log(u"Picked " + bestResult.name + " as the best", logger.DEBUG) @@ -744,4 +756,4 @@ def searchProviders(show, episodes, manualSearch=False, downCurQuality=False): # Remove provider from thread name before return results threading.currentThread().name = origThreadName - return finalResults + return finalResults \ No newline at end of file diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index ebbc322db3..0f4800ef13 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -88,12 +88,8 @@ from tornado.web import RequestHandler, HTTPError, authenticated from tornado.gen import coroutine from tornado.ioloop import IOLoop - -from concurrent.futures import ThreadPoolExecutor -from tornado.process import cpu_count - from tornado.concurrent import run_on_executor - +from concurrent.futures import ThreadPoolExecutor from mako.runtime import UNDEFINED mako_lookup = None @@ -259,7 +255,7 @@ def __init__(self, *args, **kwargs): super(WebHandler, self).__init__(*args, **kwargs) self.io_loop = IOLoop.current() - executor = ThreadPoolExecutor(cpu_count()) + executor = ThreadPoolExecutor(50) @authenticated @coroutine @@ -2238,7 +2234,7 @@ def __init__(self, *args, **kwargs): def index(self): try: - changes = helpers.getURL('https://api.pymedusa.com/changelog.md', session=requests.Session(), returns='text') + changes = helpers.getURL('https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/CHANGES.md', session=requests.Session(), returns='text') except Exception: logger.log(u'Could not load changes from repo, giving a link!', logger.DEBUG) changes = 'Could not load changes from the repo. [Click here for CHANGES.md](https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/CHANGES.md)' @@ -4061,7 +4057,7 @@ def saveSearch(self, use_nzbs=None, use_torrents=None, nzb_dir=None, sab_usernam torrent_dir=None, torrent_username=None, torrent_password=None, torrent_host=None, torrent_label=None, torrent_label_anime=None, torrent_path=None, torrent_verify_cert=None, torrent_seed_time=None, torrent_paused=None, torrent_high_bandwidth=None, - torrent_rpcurl=None, torrent_auth_type=None, ignore_words=None, trackers_list=None, require_words=None, ignored_subs_list=None): + torrent_rpcurl=None, torrent_auth_type=None, ignore_words=None, prefered_words=None, undesired_words=None, trackers_list=None, require_words=None, ignored_subs_list=None): results = [] @@ -4084,6 +4080,8 @@ def saveSearch(self, use_nzbs=None, use_torrents=None, nzb_dir=None, sab_usernam sickbeard.USENET_RETENTION = try_int(usenet_retention, 500) sickbeard.IGNORE_WORDS = ignore_words if ignore_words else "" + sickbeard.PREFERED_WORDS = prefered_words if prefered_words else "" + sickbeard.UNDESIRED_WORDS = undesired_words if undesired_words else "" sickbeard.TRACKERS_LIST = trackers_list if trackers_list else "" sickbeard.REQUIRE_WORDS = require_words if require_words else "" sickbeard.IGNORED_SUBS_LIST = ignored_subs_list if ignored_subs_list else "" @@ -5229,7 +5227,7 @@ def clearerrors(self, level=logger.ERROR): return self.redirect("/errorlogs/viewlog/") - def viewlog(self, minLevel=logger.INFO, logFilter="", logSearch=None, maxLines=1000): + def viewlog(self, minLevel=logger.INFO, logFilter="", logSearch=None, maxLines=500): def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): @@ -5245,6 +5243,8 @@ def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): if match: level = match.group(7) logName = match.group(8) + if not sickbeard.DEBUG and (level == 'DEBUG' or level == 'DB'): + continue if level not in logger.LOGGING_LEVELS: lastLine = False continue @@ -5287,10 +5287,6 @@ def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): 'SEARCHQUEUE-MANUAL': u'Search Queue (Manual)', 'SEARCHQUEUE-RETRY': u'Search Queue (Retry/Failed)', 'SEARCHQUEUE-RSS': u'Search Queue (RSS)', - 'SHOWQUEUE-FORCE-UPDATE': u'Search Queue (Forced Update)', - 'SHOWQUEUE-UPDATE': u'Search Queue (Update)', - 'SHOWQUEUE-REFRESH': u'Search Queue (Refresh)', - 'SHOWQUEUE-FORCE-REFRESH': u'Search Queue (Forced Refresh)', 'FINDPROPERS': u'Find Propers', 'POSTPROCESSER': u'Postprocesser', 'FINDSUBTITLES': u'Find Subtitles', @@ -5330,4 +5326,4 @@ def submit_errors(self): submitter_notification = ui.notifications.error if issue_id is None else ui.notifications.message submitter_notification(submitter_result) - return self.redirect("/errorlogs/") + return self.redirect("/errorlogs/") \ No newline at end of file From eaa08ac290bfaf0e2e012cb29637274e39863487 Mon Sep 17 00:00:00 2001 From: Labrys Date: Fri, 26 Feb 2016 14:56:26 -0500 Subject: [PATCH 28/34] Add to remove words list --- sickbeard/helpers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 910298180d..1d99be40c1 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -113,14 +113,19 @@ def remove_non_release_groups(name): r'\[eztv\]$': 'searchre', r'\[ettv\]$': 'searchre', r'\[cttv\]$': 'searchre', + r'\.\[vtv\]$': 'searchre', r'\[vtv\]$': 'searchre', r'\[EtHD\]$': 'searchre', r'\[GloDLS\]$': 'searchre', r'\[silv4\]$': 'searchre', r'\[Seedbox\]$': 'searchre', r'\[PublicHD\]$': 'searchre', + r'-\=\{SPARROW\}\=-$': 'searchre', + r'\=\{SPARR$': 'searchre', + r'\.\[720P\]\[HEVC\]$': 'searchre', r'\[AndroidTwoU\]$': 'searchre', - r'\[brassetv]\]$': 'searchre', + r'\[brassetv\]]$': 'searchre', + r'\[Talamasca32\]]$': 'searchre', r'\(musicbolt\.com\)$': 'searchre', r'\.\[BT\]$': 'searchre', r' \[1044\]$': 'searchre', @@ -128,6 +133,8 @@ def remove_non_release_groups(name): r'\.GiuseppeTnT$': 'searchre', r'\.Renc$': 'searchre', r'\.gz$': 'searchre', + r'\.English$': 'searchre', + r'\.German$': 'searchre', r'(? Date: Sat, 27 Feb 2016 12:57:20 +0100 Subject: [PATCH 29/34] Bring back some commits --- gui/slick/views/config_search.mako | 3 +-- sickbeard/__init__.py | 16 +++++++--------- sickbeard/webserve.py | 20 +++++++++++++------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/gui/slick/views/config_search.mako b/gui/slick/views/config_search.mako index 8d98414a50..f7f6ffdb82 100644 --- a/gui/slick/views/config_search.mako +++ b/gui/slick/views/config_search.mako @@ -149,7 +149,6 @@ -
- \ No newline at end of file + diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index 33404844b3..3324212c52 100644 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -138,7 +138,7 @@ GIT_PATH = None DEVELOPER = False -NEWS_URL = 'https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/news.md' +NEWS_URL = 'https://api.pymedusa.com/news.md' NEWS_LAST_READ = None NEWS_LATEST = None NEWS_UNREAD = 0 @@ -351,7 +351,6 @@ PLEX_CLIENT_HOST = None PLEX_SERVER_USERNAME = None PLEX_SERVER_PASSWORD = None -PLEX_SERVER_NO_AUTH = False USE_PLEX_CLIENT = False PLEX_CLIENT_USERNAME = None @@ -605,7 +604,7 @@ def initialize(consoleLogging=True): # pylint: disable=too-many-locals, too-man USE_KODI, KODI_ALWAYS_ON, KODI_NOTIFY_ONSNATCH, KODI_NOTIFY_ONDOWNLOAD, KODI_NOTIFY_ONSUBTITLEDOWNLOAD, KODI_UPDATE_FULL, KODI_UPDATE_ONLYFIRST, \ KODI_UPDATE_LIBRARY, KODI_HOST, KODI_USERNAME, KODI_PASSWORD, BACKLOG_FREQUENCY, \ USE_TRAKT, TRAKT_USERNAME, TRAKT_ACCESS_TOKEN, TRAKT_REFRESH_TOKEN, TRAKT_REMOVE_WATCHLIST, TRAKT_SYNC_WATCHLIST, TRAKT_REMOVE_SHOW_FROM_SICKRAGE, TRAKT_METHOD_ADD, TRAKT_START_PAUSED, traktCheckerScheduler, TRAKT_USE_RECOMMENDED, TRAKT_SYNC, TRAKT_SYNC_REMOVE, TRAKT_DEFAULT_INDEXER, TRAKT_REMOVE_SERIESLIST, TRAKT_TIMEOUT, TRAKT_BLACKLIST_NAME, \ - USE_PLEX_SERVER, PLEX_SERVER_NO_AUTH, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \ + USE_PLEX_SERVER, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \ PLEX_SERVER_HOST, PLEX_SERVER_TOKEN, PLEX_CLIENT_HOST, PLEX_SERVER_USERNAME, PLEX_SERVER_PASSWORD, PLEX_SERVER_HTTPS, MIN_BACKLOG_FREQUENCY, SKIP_REMOVED_FILES, ALLOWED_EXTENSIONS, \ USE_EMBY, EMBY_HOST, EMBY_APIKEY, \ showUpdateScheduler, __INITIALIZED__, INDEXER_DEFAULT_LANGUAGE, EP_DEFAULT_DELETED_STATUS, LAUNCH_BROWSER, TRASH_REMOVE_SHOW, TRASH_ROTATE_LOGS, SORT_ARTICLE, \ @@ -1028,7 +1027,6 @@ def path_leaf(path): PLEX_CLIENT_USERNAME = check_setting_str(CFG, 'Plex', 'plex_client_username', '', censor_log=True) PLEX_CLIENT_PASSWORD = check_setting_str(CFG, 'Plex', 'plex_client_password', '', censor_log=True) PLEX_SERVER_HTTPS = bool(check_setting_int(CFG, 'Plex', 'plex_server_https', 0)) - PLEX_SERVER_NO_AUTH = bool(check_setting_int(CFG, 'Plex', 'plex_server_no_auth', 0)) USE_EMBY = bool(check_setting_int(CFG, 'Emby', 'use_emby', 0)) EMBY_HOST = check_setting_str(CFG, 'Emby', 'emby_host', '') @@ -1473,11 +1471,12 @@ def path_leaf(path): run_delay=update_interval) # processors + update_interval = datetime.timedelta(minutes=AUTOPOSTPROCESSER_FREQUENCY) autoPostProcesserScheduler = scheduler.Scheduler(auto_postprocessor.PostProcessor(), - cycleTime=datetime.timedelta( - minutes=AUTOPOSTPROCESSER_FREQUENCY), + cycleTime=update_interval, threadName="POSTPROCESSER", - silent=not PROCESS_AUTOMATICALLY) + silent=not PROCESS_AUTOMATICALLY, + run_delay=update_interval) traktCheckerScheduler = scheduler.Scheduler(traktChecker.TraktChecker(), cycleTime=datetime.timedelta(hours=1), @@ -1944,7 +1943,6 @@ def save_config(): # pylint: disable=too-many-statements, too-many-branches new_config['Plex']['plex_client_host'] = PLEX_CLIENT_HOST new_config['Plex']['plex_server_username'] = PLEX_SERVER_USERNAME new_config['Plex']['plex_server_password'] = helpers.encrypt(PLEX_SERVER_PASSWORD, ENCRYPTION_VERSION) - new_config['Plex']['plex_server_no_auth'] = int(PLEX_SERVER_NO_AUTH) new_config['Plex']['use_plex_client'] = int(USE_PLEX_CLIENT) new_config['Plex']['plex_client_username'] = PLEX_CLIENT_USERNAME @@ -2187,4 +2185,4 @@ def launchBrowser(protocol='http', startPort=None, web_root='/'): try: webbrowser.open(browserURL, 1, 1) except Exception: - logger.log(u"Unable to launch a browser", logger.ERROR) \ No newline at end of file + logger.log(u"Unable to launch a browser", logger.ERROR) diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 0f4800ef13..fcbabd5315 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -88,8 +88,12 @@ from tornado.web import RequestHandler, HTTPError, authenticated from tornado.gen import coroutine from tornado.ioloop import IOLoop -from tornado.concurrent import run_on_executor + from concurrent.futures import ThreadPoolExecutor +from tornado.process import cpu_count + +from tornado.concurrent import run_on_executor + from mako.runtime import UNDEFINED mako_lookup = None @@ -255,7 +259,7 @@ def __init__(self, *args, **kwargs): super(WebHandler, self).__init__(*args, **kwargs) self.io_loop = IOLoop.current() - executor = ThreadPoolExecutor(50) + executor = ThreadPoolExecutor(cpu_count()) @authenticated @coroutine @@ -2234,7 +2238,7 @@ def __init__(self, *args, **kwargs): def index(self): try: - changes = helpers.getURL('https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/CHANGES.md', session=requests.Session(), returns='text') + changes = helpers.getURL('https://api.pymedusa.com/changelog.md', session=requests.Session(), returns='text') except Exception: logger.log(u'Could not load changes from repo, giving a link!', logger.DEBUG) changes = 'Could not load changes from the repo. [Click here for CHANGES.md](https://raw.githubusercontent.com/pymedusa/sickrage.github.io/master/sickrage-news/CHANGES.md)' @@ -5227,7 +5231,7 @@ def clearerrors(self, level=logger.ERROR): return self.redirect("/errorlogs/viewlog/") - def viewlog(self, minLevel=logger.INFO, logFilter="", logSearch=None, maxLines=500): + def viewlog(self, minLevel=logger.INFO, logFilter="", logSearch=None, maxLines=1000): def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): @@ -5243,8 +5247,6 @@ def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): if match: level = match.group(7) logName = match.group(8) - if not sickbeard.DEBUG and (level == 'DEBUG' or level == 'DB'): - continue if level not in logger.LOGGING_LEVELS: lastLine = False continue @@ -5287,6 +5289,10 @@ def Get_Data(Levelmin, data_in, lines_in, regex, Filter, Search, mlines): 'SEARCHQUEUE-MANUAL': u'Search Queue (Manual)', 'SEARCHQUEUE-RETRY': u'Search Queue (Retry/Failed)', 'SEARCHQUEUE-RSS': u'Search Queue (RSS)', + 'SHOWQUEUE-FORCE-UPDATE': u'Search Queue (Forced Update)', + 'SHOWQUEUE-UPDATE': u'Search Queue (Update)', + 'SHOWQUEUE-REFRESH': u'Search Queue (Refresh)', + 'SHOWQUEUE-FORCE-REFRESH': u'Search Queue (Forced Refresh)', 'FINDPROPERS': u'Find Propers', 'POSTPROCESSER': u'Postprocesser', 'FINDSUBTITLES': u'Find Subtitles', @@ -5326,4 +5332,4 @@ def submit_errors(self): submitter_notification = ui.notifications.error if issue_id is None else ui.notifications.message submitter_notification(submitter_result) - return self.redirect("/errorlogs/") \ No newline at end of file + return self.redirect("/errorlogs/") From cd6c71775c50fc6de474acade2c5223935768d56 Mon Sep 17 00:00:00 2001 From: medariox Date: Sat, 27 Feb 2016 14:00:21 +0100 Subject: [PATCH 30/34] Change preferred words vars --- gui/slick/views/config_search.mako | 4 ++-- sickbeard/__init__.py | 8 ++++---- sickbeard/search.py | 10 +++++----- sickbeard/webserve.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/gui/slick/views/config_search.mako b/gui/slick/views/config_search.mako index f7f6ffdb82..fbe181e164 100644 --- a/gui/slick/views/config_search.mako +++ b/gui/slick/views/config_search.mako @@ -129,7 +129,7 @@