Updated subliminal (ceb641d) and itasa subs provider

lib/subliminal/cli.py

@@ -19,7 +19,7 @@
 from six.moves import configparser
 
 from subliminal import (AsyncProviderPool, Episode, Movie, Video, __version__, check_video, compute_score, get_scores,
-                        provider_manager, refine, region, save_subtitles, scan_video, scan_videos)
+                        provider_manager, refine, refiner_manager, region, save_subtitles, scan_video, scan_videos)
 from subliminal.core import ARCHIVE_EXTENSIONS, search_external_subtitles
 
 logger = logging.getLogger(__name__)
@@ -62,6 +62,7 @@ def __init__(self, path):
         self.config.add_section('general')
         self.config.set('general', 'languages', json.dumps(['en']))
         self.config.set('general', 'providers', json.dumps(sorted([p.name for p in provider_manager])))
+        self.config.set('general', 'refiners', json.dumps(sorted([r.name for r in refiner_manager])))
         self.config.set('general', 'single', str(0))
         self.config.set('general', 'embedded_subtitles', str(1))
         self.config.set('general', 'age', str(int(timedelta(weeks=2).total_seconds())))
@@ -93,6 +94,14 @@ def providers(self):
     def providers(self, value):
         self.config.set('general', 'providers', json.dumps(sorted([p.lower() for p in value])))
 
+    @property
+    def refiners(self):
+        return json.loads(self.config.get('general', 'refiners'))
+
+    @refiners.setter
+    def refiners(self, value):
+        self.config.set('general', 'refiners', json.dumps([r.lower() for r in value]))
+
     @property
     def single(self):
         return self.config.getboolean('general', 'single')
@@ -196,6 +205,8 @@ def convert(self, value, param, ctx):
 
 PROVIDER = click.Choice(sorted(provider_manager.names()))
 
+REFINER = click.Choice(sorted(refiner_manager.names()))
+
 dirs = AppDirs('subliminal')
 cache_file = 'subliminal.dbm'
 config_file = 'config.ini'
@@ -230,7 +241,6 @@ def subliminal(ctx, addic7ed, itasa, legendastv, opensubtitles, subscenter, cach
     # configure logging
     if debug:
         handler = logging.StreamHandler()
-        # TODO: change format to something nicer (use colorlogs + funcName)
         handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
         logging.getLogger('subliminal').addHandler(handler)
         logging.getLogger('subliminal').setLevel(logging.DEBUG)
@@ -266,6 +276,7 @@ def cache(ctx, clear_subliminal):
 @click.option('-l', '--language', type=LANGUAGE, required=True, multiple=True, help='Language as IETF code, '
               'e.g. en, pt-BR (can be used multiple times).')
 @click.option('-p', '--provider', type=PROVIDER, multiple=True, help='Provider to use (can be used multiple times).')
+@click.option('-r', '--refiner', type=REFINER, multiple=True, help='Refiner to use (can be used multiple times).')
 @click.option('-a', '--age', type=AGE, help='Filter videos newer than AGE, e.g. 12h, 1w2d.')
 @click.option('-d', '--directory', type=click.STRING, metavar='DIR', help='Directory where to save subtitles, '
               'default is next to the video file.')
@@ -281,8 +292,8 @@ def cache(ctx, clear_subliminal):
 @click.option('-v', '--verbose', count=True, help='Increase verbosity.')
 @click.argument('path', type=click.Path(), required=True, nargs=-1)
 @click.pass_obj
-def download(obj, provider, language, age, directory, encoding, single, force, hearing_impaired,
-             min_score, max_workers, archives, verbose, path):
+def download(obj, provider, refiner, language, age, directory, encoding, single, force, hearing_impaired, min_score,
+             max_workers, archives, verbose, path):
     """Download best subtitles.
 
     PATH can be an directory containing videos, a video file path or a video file name. It can be used multiple times.
@@ -312,15 +323,14 @@ def download(obj, provider, language, age, directory, encoding, single, force, h
                     continue
                 if not force:
                     video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values())
-                refine(video, embedded_subtitles=not force)
+                refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force)
                 videos.append(video)
                 continue
 
             # directories
             if os.path.isdir(p):
                 try:
-                    scanned_videos = scan_videos(p, age=age, archives=archives, subtitles=not force,
-                                                 subtitles_dir=directory)
+                    scanned_videos = scan_videos(p, age=age, archives=archives)
                 except:
                     logger.exception('Unexpected error while collecting directory path %s', p)
                     errored_paths.append(p)
@@ -330,7 +340,7 @@ def download(obj, provider, language, age, directory, encoding, single, force, h
                         if not force:
                             video.subtitle_languages |= set(search_external_subtitles(video.name,
                                                                                       directory=directory).values())
-                        refine(video, embedded_subtitles=not force)
+                        refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force)
                         videos.append(video)
                     else:
                         ignored_videos.append(video)
@@ -346,7 +356,7 @@ def download(obj, provider, language, age, directory, encoding, single, force, h
             if check_video(video, languages=language, age=age, undefined=single):
                 if not force:
                     video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values())
-                refine(video, embedded_subtitles=not force)
+                refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force)
                 videos.append(video)
             else:
                 ignored_videos.append(video)
@@ -392,7 +402,10 @@ def download(obj, provider, language, age, directory, encoding, single, force, h
                                                       hearing_impaired=hearing_impaired, only_one=single)
                 downloaded_subtitles[v] = subtitles
 
-    # TODO: warn about discarded providers
+    if p.discarded_providers:
+                click.secho('Some providers have been discarded due to unexpected errors: %s' %
+                            ', '.join(p.discarded_providers), fg='yellow')
+
     # save subtitles
     total_subtitles = 0
     for v, subtitles in downloaded_subtitles.items():

lib/subliminal/core.py

@@ -17,7 +17,7 @@
 from .extensions import provider_manager, refiner_manager
 from .score import compute_score as default_compute_score
 from .subtitle import SUBTITLE_EXTENSIONS, get_subtitle_path
-from .utils import hash_napiprojekt, hash_opensubtitles, hash_thesubdb
+from .utils import hash_itasa, hash_napiprojekt, hash_opensubtitles, hash_thesubdb
 from .video import VIDEO_EXTENSIONS, Episode, Movie, Video
 
 #: Supported archive extensions
@@ -383,6 +383,7 @@ def scan_video(path):
     video.size = os.path.getsize(path)
     if video.size > 10485760:
         logger.debug('Size is %d', video.size)
+        video.hashes['itasa'] = hash_itasa(path)
         video.hashes['opensubtitles'] = hash_opensubtitles(path)
         video.hashes['thesubdb'] = hash_thesubdb(path)
         video.hashes['napiprojekt'] = hash_napiprojekt(path)
@@ -440,7 +441,7 @@ def scan_archive(path):
     return video
 
 
-def scan_videos(path, age=None, archives=True, **kwargs):
+def scan_videos(path, age=None, archives=True):
     """Scan `path` for videos and their subtitles.
 
     See :func:`refine` to find additional information for the video.
@@ -516,7 +517,7 @@ def scan_videos(path, age=None, archives=True, **kwargs):
     return videos
 
 
-def refine(video, episode_refiners=('metadata', 'tvdb', 'omdb'), movie_refiners=('metadata', 'omdb'), **kwargs):
+def refine(video, episode_refiners=None, movie_refiners=None, **kwargs):
     """Refine a video using :ref:`refiners`.
 
     .. note::
@@ -527,14 +528,14 @@ def refine(video, episode_refiners=('metadata', 'tvdb', 'omdb'), movie_refiners=
     :type video: :class:`~subliminal.video.Video`
     :param tuple episode_refiners: refiners to use for episodes.
     :param tuple movie_refiners: refiners to use for movies.
-    :param \*\*kwargs: parameters for refiners.
+    :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions.
 
     """
     refiners = ()
     if isinstance(video, Episode):
-        refiners = episode_refiners or ()
+        refiners = episode_refiners or ('metadata', 'tvdb', 'omdb')
     elif isinstance(video, Movie):
-        refiners = movie_refiners or ()
+        refiners = movie_refiners or ('metadata', 'omdb')
     for refiner in refiners:
         logger.info('Refining video with %s', refiner)
         try:
@@ -548,14 +549,13 @@ def list_subtitles(videos, languages, pool_class=ProviderPool, **kwargs):
 
     The `videos` must pass the `languages` check of :func:`check_video`.
 
-    All other parameters are passed onwards to the provided `pool_class` constructor.
-
     :param videos: videos to list subtitles for.
     :type videos: set of :class:`~subliminal.video.Video`
     :param languages: languages to search for.
     :type languages: set of :class:`~babelfish.language.Language`
     :param pool_class: class to use as provider pool.
     :type: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :param \*\*kwargs: additional parameters for the provided `pool_class` constructor.
     :return: found subtitles per video.
     :rtype: dict of :class:`~subliminal.video.Video` to list of :class:`~subliminal.subtitle.Subtitle`
 
@@ -588,12 +588,11 @@ def list_subtitles(videos, languages, pool_class=ProviderPool, **kwargs):
 def download_subtitles(subtitles, pool_class=ProviderPool, **kwargs):
     """Download :attr:`~subliminal.subtitle.Subtitle.content` of `subtitles`.
 
-    All other parameters are passed onwards to the `pool_class` constructor.
-
     :param subtitles: subtitles to download.
     :type subtitles: list of :class:`~subliminal.subtitle.Subtitle`
     :param pool_class: class to use as provider pool.
     :type: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :param \*\*kwargs: additional parameters for the provided `pool_class` constructor.
 
     """
     with pool_class(**kwargs) as pool:
@@ -608,8 +607,6 @@ def download_best_subtitles(videos, languages, min_score=0, hearing_impaired=Fal
 
     The `videos` must pass the `languages` and `undefined` (`only_one`) checks of :func:`check_video`.
 
-    All other parameters are passed onwards to the `pool_class` constructor.
-
     :param videos: videos to download subtitles for.
     :type videos: set of :class:`~subliminal.video.Video`
     :param languages: languages to download.
@@ -621,6 +618,7 @@ def download_best_subtitles(videos, languages, min_score=0, hearing_impaired=Fal
         `hearing_impaired` as keyword argument and returns the score.
     :param pool_class: class to use as provider pool.
     :type: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :param \*\*kwargs: additional parameters for the provided `pool_class` constructor.
     :return: downloaded subtitles per video.
     :rtype: dict of :class:`~subliminal.video.Video` to list of :class:`~subliminal.subtitle.Subtitle`
 

lib/subliminal/providers/itasa.py

@@ -28,14 +28,15 @@
 class ItaSASubtitle(Subtitle):
     provider_name = 'itasa'
 
-    def __init__(self, sub_id, series, season, episode, format, full_data):
+    def __init__(self, sub_id, series, season, episode, format, full_data, hash=None):
         super(ItaSASubtitle, self).__init__(Language('ita'))
         self.sub_id = sub_id
         self.series = series
         self.season = season
         self.episode = episode
         self.format = format
         self.full_data = full_data
+        self.hash = hash
 
     @property
     def id(self):
@@ -58,6 +59,12 @@ def get_matches(self, video, hearing_impaired=False):
             matches.add('format')
         if not video.format and not self.format:
             matches.add('format')
+        # hash
+        if 'itasa' in video.hashes and self.hash == video.hashes['itasa']:
+            print('Hash %s' % video.hashes['itasa'])
+            if 'series' in matches and 'season' in matches and 'episode' in matches:
+                matches.add('hash')
+
         # other properties
         matches |= guess_matches(video, guessit(self.full_data), partial=True)
 
@@ -69,6 +76,8 @@ class ItaSAProvider(Provider):
 
     video_types = (Episode,)
 
+    required_hash = 'itasa'
+
     server_url = 'https://api.italiansubs.net/api/rest/'
 
     apikey = 'd86ad6ec041b334fac1e512174ee04d5'
@@ -247,7 +256,7 @@ def _download_zip(self, sub_id):
 
         return r.content
 
-    def query(self, series, season, episode, format, country=None):
+    def query(self, series, season, episode, format, country=None, hash=None):
 
         # To make queries you need to be logged in
         if not self.logged_in:
@@ -301,7 +310,8 @@ def query(self, series, season, episode, format, country=None):
                         season,
                         episode,
                         format,
-                        subtitle.find('name').text)
+                        subtitle.find('name').text,
+                        hash)
 
                 subtitles.append(sub)
 
@@ -330,7 +340,8 @@ def query(self, series, season, episode, format, country=None):
                         season,
                         episode,
                         format,
-                        subtitle.find('name').text)
+                        subtitle.find('name').text,
+                        hash)
 
                     subtitles.append(sub)
 
@@ -369,7 +380,7 @@ def query(self, series, season, episode, format, country=None):
         return subtitles + additional_subs
 
     def list_subtitles(self, video, languages):
-        return self.query(video.series, video.season, video.episode, video.format)
+        return self.query(video.series, video.season, video.episode, video.format, hash=video.hashes.get('itasa'))
 
     def download_subtitle(self, subtitle):
         pass

lib/subliminal/refiners/__init__.py

@@ -0,0 +1,12 @@
+"""
+Refiners enrich a :class:`~subliminal.video.Video` object by adding information to it.
+
+A refiner is a simple function:
+
+.. py:function:: refine(video, **kwargs)
+
+    :param video: the video to refine.
+    :type video: :class:`~subliminal.video.Video`
+    :param \*\*kwargs: additional parameters for refiners.
+
+"""

lib/subliminal/refiners/metadata.py

@@ -18,7 +18,6 @@ def refine(video, embedded_subtitles=True, **kwargs):
       * :attr:`~subliminal.video.Video.audio_codec`
       * :attr:`~subliminal.video.Video.subtitle_languages`
 
-    :param video: the video to refine.
     :param bool embedded_subtitles: search for embedded subtitles.
 
     """

lib/subliminal/refiners/omdb.py

@@ -104,8 +104,6 @@ def refine(video, **kwargs):
       * :attr:`~subliminal.video.Movie.year`
       * :attr:`~subliminal.video.Video.imdb_id`
 
-    :param video: the video to refine.
-
     """
     if isinstance(video, Episode):
         # exit if the information is complete

lib/subliminal/refiners/tvdb.py

@@ -192,14 +192,25 @@ def get_episode(self, id):
 
 @region.cache_on_arguments(expiration_time=REFINER_EXPIRATION_TIME)
 def search_series(name):
-    """Search series.
+    """Search series and sort the results by likelihood.
+
+     Prefer series with the same name and with continuing status.
 
     :param str name: name of the series.
     :return: the search results.
     :rtype: list
 
     """
-    return tvdb_client.search_series(name)
+    def match(series):
+        key = 0
+        if series['status'] != 'Continuing':
+            key += 1
+        if series['seriesName'] != name:
+            key += 2
+
+        return key
+
+    return sorted(tvdb_client.search_series(name), key=match)
 
 
 @region.cache_on_arguments(expiration_time=REFINER_EXPIRATION_TIME)
@@ -247,8 +258,6 @@ def refine(video, **kwargs):
       * :attr:`~subliminal.video.Video.imdb_id`
       * :attr:`~subliminal.video.Episode.tvdb_id`
 
-    :param video: the video to refine.
-
     """
     # only deal with Episode videos
     if not isinstance(video, Episode):

lib/subliminal/utils.py

@@ -6,6 +6,20 @@
 import struct
 
 
+def hash_itasa(video_path):
+    """Compute a hash using ItaSA's algorithm.
+
+    :param str video_path: path of the video.
+    :return: the hash.
+    :rtype: str
+
+    """
+    readsize = 1024 * 1024 * 10
+    with open(video_path, 'rb') as f:
+        data = f.read(readsize)
+    return hashlib.md5(data).hexdigest()
+
+
 def hash_opensubtitles(video_path):
     """Compute a hash using OpenSubtitles' algorithm.