Changed the way torrent files get their names. Fixes #24

Cleaned code. Closes #21 Added page_start value to sites config. Fixes #17
arivarton · May 6, 2018 · 5f03f2b · 5f03f2b
1 parent 4ba1fba
commit 5f03f2b
Show file tree

Hide file tree

Showing 6 changed files with 114 additions and 78 deletions.
diff --git a/README.md b/README.md
@@ -6,6 +6,8 @@ Note that this doesn't actually download the files that the torrent contains, on
 
 Supports both magnet links and regular .torrent files.
 
+Python2 is not supported.
+
 
 # Usage
 ### Download single torrents
@@ -79,7 +81,7 @@ optional arguments:
 #### Pretend download
 The -x parameter is set so the torrents doesn't actually download, it will only print out information about the torrents that were found with the search criteria.
 ##### Examples
-`mtorrentd deildu 'Mr Robot s02' username password -x`
+`mtorrentd deildu 'Mr Robot s02' --username <username> --password <password> -x`
 `mtorrentd thepiratebay 'Mr Robot s02' -x`
 
 #### Download
@@ -90,9 +92,9 @@ The -p parameter overrides the default maximum page count of 100.
 ###### Examples
 `mtorrentd thepiratebay 'Mr Robot ' -x -p 5`
 ##### -r
-The -r parameter is for regex and will restrict the found torrents based on it.
+The -r parameter is for regex and will restrict the found torrents based on it. Regex defaults to ignore case in mtorrentd.
 ###### Examples
-`mtorrentd thepiratebay 'Mr Robot' -x -r '.*[sS]02.*'`
+`mtorrentd thepiratebay 'Mr Robot' -x -r '.*s02.*'`
 ##### -d
 Override the download directory.
 ###### Examples
@@ -108,6 +110,7 @@ username
 password
 login_path
 page_path (required)
+page_start
 search_path (required)
 append_path
 url (required)
@@ -123,7 +126,12 @@ watch_dir
 ##### Packaged
 Get it from the AUR in Archlinux.
 ##### Manual
+python-setuptools required.
 `python3 setup.py install`
+setup.py does not install libtorrent which means it must be installed manually with your package manager.
+Arch Linux:
+`pacman: pacman -S libtorrent-rasterbar`
+Other distributions should be similar.
 ##### Directly from directory
 It's also possible to run directly from ./mtorrentd.py just make sure dependencies are installed.
 

diff --git a/mtorrentd/config.py b/mtorrentd/config.py
@@ -1,64 +1,80 @@
+"""Handling of config settings and files."""
 import os
 import sys
+import copy
 import yaml
 
 from .paths import CONFIG_PATHS, CONFIG_NAMES
 from .default_config_values import CONFIG, SITES
-from .core import validate_url
+from .validators import integer_0_or_1, true_or_false, validate_url
 
-CONFIG_SETTINGS = {
-    'sites': {
-    'required_values': ['login_required', 'page_path', 'search_path', 'url'],
-    'login_required_values': ['username', 'password'],
-    },
-    'config': {
-    }
+possible_site_values = {
+    'login_required': {'required': True, 'validate': true_or_false},
+    'page_path': {'required': True, 'validate': lambda x: validate_url(x, path=True)},
+    'search_path': {'required': True, 'validate': lambda x: validate_url(x, path=True)},
+    'url': {'required': True, 'validate': validate_url},
+    'login_path': {'required': False, 'default_value': None, 'validate': lambda x: validate_url(x, path=True)},
+    'page_start': {'required': False, 'default_value': 0, 'validate': integer_0_or_1},
+    'append_path': {'required': False, 'default_value': '', 'validate': lambda x: validate_url(x, path=True)},
+    'username': {'required': False, 'default_value': None, 'validate': False},
+    'password': {'required': False, 'default_value': None, 'validate': False}
 }
 
 
 def handle_undefined_values(selected_config, config_selection) -> dict:
+    """Handle undefined values in config files.
+
+    Will exit if value is not specified and required. Otherwise value will be
+    set to it's default that is specified in possible_site_values.
+    """
     if config_selection is 'sites':
         for site, site_values in selected_config.items():
             # Check required values
-            for value in CONFIG_SETTINGS[config_selection]['required_values']:
+            for value in possible_site_values:
                 if value not in site_values.keys():
-                    print(value, 'must be specified under configuration for', site + '.')
-                    exit(78)
-            # Set username and password to None if not defined in a site with
-            # login_required: True
-            if site_values['login_required']:
-                for value in CONFIG_SETTINGS[config_selection]['login_required_values']:
-                    if value not in site_values.keys():
-                        selected_config[site][value] = None
+                    # If value is required then exit if not specified.
+                    if possible_site_values[value]['required']:
+                        print(value, 'must be specified under configuration for',
+                              site + '.')
+                        exit(78)
+                    # If value is not required then set it to its default value
+                    # if it's not specified.
+                    else:
+                        selected_config[site][value] = possible_site_values[value]['default_value']
     elif config_selection is 'config':
         pass
 
     return selected_config
 
 
 def validate_config_values(selected_config, config_selection) -> None:
+    """Validate config values."""
     if config_selection is 'sites':
-        for key, values in selected_config.items():
-            try:
-                validate_url(values['url'])
-                validate_url(values['page_path'], path=True)
-                validate_url(values['search_path'], path=True)
-                validate_url(values.get('append_path', 'Not required'), path=True)
-            except ValueError as err:
-                print('Error when validating config value for %s: %s' % (key, err))
+        for site, values in selected_config.items():
+            for key, value in values.items():
+                try:
+                    if possible_site_values[key]['validate']:
+                        possible_site_values[key]['validate'](value)
+                except ValueError as err:
+                    raise ValueError('Error when validating config value for the option ' +
+                                     '%s under %s: %s' % (key, site, err))
     elif config_selection is 'config':
         try:
             validate_url(selected_config.get('watch_dir', 'Not required'), path=True)
         except ValueError as err:
-            print('Error when validating config value for %s: %s' % (key, err))
+            print('Error when validating config value for %s: %s' % (site, err))
 
 
 def load_config(config_selection) -> dict:
-    # Load default config
+    """Load config.
+
+    User defined config files will take precedence over the built in config.
+    """
+    selected_config = dict()
     if config_selection == 'config':
-        selected_config = CONFIG
+        selected_config = copy.deepcopy(CONFIG)
     elif config_selection == 'sites':
-        selected_config = SITES
+        selected_config = copy.deepcopy(SITES)
     else:
         raise ValueError('Config selection value is not supported.')
 
@@ -83,7 +99,7 @@ def load_config(config_selection) -> dict:
     except FileNotFoundError:
         pass
 
-    selected_config = handle_undefined_values(selected_config, config_selection)
     validate_config_values(selected_config, config_selection)
+    selected_config = handle_undefined_values(selected_config, config_selection)
 
     return selected_config
diff --git a/mtorrentd/core.py b/mtorrentd/core.py
@@ -1,31 +1,21 @@
+"""Core functions of mtorrentd."""
 import os
 import importlib.util
 import libtorrent
 import tempfile
 import shutil
 import requests
+from urllib import parse
 
 from sys import exit
 from time import sleep
-from urllib import parse
 
 from .paths import SITE_MODULES
 
-def validate_url(url, path=False):
-    if path:
-        _value = parse.urlparse(url).path
-    else:
-        _value = parse.urlparse(url).netloc
-    try:
-        if _value:
-            return True
-        else:
-            raise ValueError('Invalid value:', _value)
-    except:
-        raise ValueError('Invalid value:', _value)
 
 
 def session_login(site, username, password, session):
+    """When login is needed for session."""
     payload = {
         'username': username,
         'password': password
@@ -37,23 +27,24 @@ def session_login(site, username, password, session):
 
 
 def load_site_module(site):
+    """Load module to use it as a function."""
     user_module = os.path.join(SITE_MODULES['user'], site + '.py')
     system_module = os.path.join(SITE_MODULES['system'], site + '.py')
     if os.path.isfile(user_module):
         spec = importlib.util.spec_from_file_location(site, user_module)
-        site_module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(site_module)
     elif os.path.isfile(system_module):
         spec = importlib.util.spec_from_file_location(site, system_module)
-        site_module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(site_module)
     else:
         print('Site module not found. Check github for documentation and create a new site module here: %s' %(SITE_MODULES['user']))
         exit(73)
+    site_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(site_module)
+
     return site_module
 
 
 def download_torrent(torrent_link, download_dir, torrent_name=None, session=None) -> None:
+    """Download .torrent from link."""
     if torrent_name:
         full_torrent_path = os.path.join(download_dir, torrent_name + '.torrent')
     else:
@@ -81,6 +72,7 @@ def download_torrent(torrent_link, download_dir, torrent_name=None, session=None
 
 
 def download_magnet2torrent(torrent_link, download_dir, torrent_name=None) -> None:
+    """Download magnet link."""
     tempdir = tempfile.mkdtemp()
     libtorrent_session = libtorrent.session()
     params = {
@@ -116,7 +108,6 @@ def download_magnet2torrent(torrent_link, download_dir, torrent_name=None) -> No
     torrent_file = libtorrent.create_torrent(torrent_info)
 
     print("Saving torrent file here : " + full_torrent_path + " ...")
-    torrent_content = libtorrent.bencode(torrent_file.generate())
     with open(full_torrent_path, "wb") as f:
         f.write(libtorrent.bencode(torrent_file.generate()))
     print("Saved! Cleaning up dir: " + tempdir)

diff --git a/mtorrentd/default_config_values.py b/mtorrentd/default_config_values.py
@@ -1,20 +1,26 @@
+"""Default config values.
+
+Can be replaced by creating a config.yaml or sites.yaml in user config
+directory.
+"""
 CONFIG = {'watch_dir': '~/.mtorrentd/watch'}
 
 # For each value that is required to be set in SITES, it must be defined in
 # config.py to be handled properly.
 SITES = {
     'thepiratebay': {
-            'login_required': False,
-            'page_path': '/',
-            'search_path': 'search/',
-            'url': 'https://thepiratebay.org'
+        'login_required': False,
+        'page_path': '/',
+        'search_path': 'search/',
+        'url': 'https://thepiratebay.org'
     },
     'linuxtracker': {
-            'login_required': False,
-            'page_path': '&pages=',
-            'search_path': 'index.php?&page=torrents&search=',
-            'append_path': '&active=1',
-            'url': 'http://linuxtracker.org'
+        'login_required': False,
+        'page_path': '&pages=',
+        'page_start': 1,
+        'search_path': 'index.php?&page=torrents&search=',
+        'append_path': '&active=1',
+        'url': 'http://linuxtracker.org'
     },
     'deildu': {
         'login_required': True,

diff --git a/mtorrentd/main.py b/mtorrentd/main.py
@@ -11,28 +11,30 @@
 import os
 import argparse
 import re
-from sys import argv, exit
+import sys
 from urllib import parse
 from collections import defaultdict
 import requests
 
+
 from bs4 import BeautifulSoup
 
 from .config import load_config
-from .core import (load_site_module, download_torrent, download_magnet2torrent,
-                   session_login)
+from .core import load_site_module, download_torrent, download_magnet2torrent, session_login
 from .helpers import catch_undefined_credentials
 
 
+
 def search(site, args):
     """Search for torrents and return a dictionary with name and links."""
-    site_module = load_site_module(argv[1])
+    site_module = load_site_module(sys.argv[1])
 
     with requests.Session() as session:
         if site['login_required']:
             session = session_login(site, args.username, args.password, session)
         crawled_content = list()
-        for page in range(args.pages):
+        print('Searching %i pages.' % args.pages)
+        for page in range(site['page_start'], args.pages + site['page_start']):
             search_url = parse.urljoin(site['url'],
                                        site['search_path'] +
                                        args.search_string +
@@ -49,7 +51,10 @@ def search(site, args):
     search_results = defaultdict(str)
     regex = re.compile(args.regex_string, re.IGNORECASE)
     for name_list, download_links in crawled_content:
-        search_results.update({name.get_text().strip(): parse.urljoin(site['url'], link.get('href')) for name, link in zip(name_list, download_links) if regex.match(name.get_text().strip(), re.IGNORECASE)})
+        search_results.update({name.get_text().strip(): parse.urljoin(site['url'],
+                                                                      link.get('href'))
+                               for name, link in zip(name_list, download_links)
+                               if regex.match(name.get_text().strip(), re.IGNORECASE)})
     return search_results
 
 
@@ -73,11 +78,11 @@ def download(site, args):
                     except PermissionError:
                         print('[ERROR] Denied permission to create ' \
                                 'watch folder here: %s' % args.download_dir)
-                        exit(77)
+                        sys.exit(77)
                 if link.startswith('magnet:?xt'):
                     download_magnet2torrent(link, args.download_dir, name)
                 elif link.endswith('.torrent'):
-                    download_torrent(link, args.download_dir, name, session)
+                    download_torrent(link, args.download_dir, session=session)
                 else:
                     print('[ERROR] Download failed. Not a magnet or torrent link.')
 
@@ -115,24 +120,24 @@ def run():
             search_parser.set_defaults(func=download)
 
 
-    if len(argv) > 1:
-        if argv[1].startswith('magnet:?xt') or argv[1].endswith('.torrent'):
+    if len(sys.argv) > 1:
+        if sys.argv[1].startswith('magnet:?xt') or sys.argv[1].endswith('.torrent'):
             single_torrent_parser = argparse.ArgumentParser(add_help=True)
             single_torrent_parser.add_argument('torrent', type=str)
             single_torrent_parser.add_argument('-d', '--download_dir', type=lambda path: os.path.expanduser(path),
                                                default=os.path.expanduser(load_config('config')['watch_dir']))
             args = single_torrent_parser.parse_args()
-            if argv[1].startswith('magnet:?xt'):
+            if sys.argv[1].startswith('magnet:?xt'):
                 download_magnet2torrent(args.torrent, args.download_dir)
-            elif argv[1].endswith('.torrent'):
+            elif sys.argv[1].endswith('.torrent'):
                 download_torrent(args.torrent, args.download_dir)
         else:
             args = parser.parse_args()
 
-            if sites_config[argv[1]]['login_required']:
-                catch_undefined_credentials(argv[1], args)
+            if sites_config[sys.argv[1]]['login_required']:
+                catch_undefined_credentials(sys.argv[1], args)
 
-            args.func(argv[1], args)
+            args.func(sys.argv[1], args)
     else:
         parser.print_help()