diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1256d7038..707d6d2bc4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -105,7 +105,7 @@ services: release:build:master: stage: release_build image: - name: nikolaik/python-nodejs:python3.7-nodejs10-alpine + name: nikolaik/python-nodejs:python3.8-nodejs10-alpine variables: NODE_ENV: "development" script: @@ -118,6 +118,7 @@ release:build:master: - bumpversion --allow-dirty release package.json sickrage/version.txt - git checkout -b release-$(cat sickrage/version.txt) - yarn run build + - python checksum-generator.py - git add --all - git commit -m "[TASK] Releasing v$(cat sickrage/version.txt)" - git fetch . release-$(cat sickrage/version.txt):master @@ -126,6 +127,7 @@ release:build:master: - git push https://$GIT_ACCESS_USER:$GIT_ACCESS_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:master --follow-tags - git checkout develop - bumpversion --allow-dirty patch package.json sickrage/version.txt + - python checksum-generator.py - git add --all - git commit -m "[TASK] Bump develop branch to v$(cat sickrage/version.txt)" - git push https://$GIT_ACCESS_USER:$GIT_ACCESS_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:develop --follow-tags @@ -142,7 +144,7 @@ release:build:master: release:build:develop: stage: release_build image: - name: nikolaik/python-nodejs:python3.7-nodejs10-alpine + name: nikolaik/python-nodejs:python3.8-nodejs10-alpine variables: NODE_ENV: "development" script: @@ -153,6 +155,7 @@ release:build:develop: - npx git-changelog -t $(git describe --abbrev=0) - bumpversion --allow-dirty dev package.json sickrage/version.txt - yarn run build + - python checksum-generator.py # - python setup.py extract_messages # - crowdin-cli-py upload sources # - crowdin-cli-py download @@ -215,7 +218,7 @@ release:sentry:develop: deploy:pypi: stage: release_deploy - image: python:3.7-alpine3.9 + image: python:3.8-alpine3.9 script: - apk add --no-cache py-pip gcc libffi-dev python3-dev musl-dev openssl-dev - pip install -U twine diff --git a/checksum-generator.py b/checksum-generator.py new file mode 100644 index 0000000000..09071f5916 --- /dev/null +++ b/checksum-generator.py @@ -0,0 +1,47 @@ +# ############################################################################## +# Author: echel0n +# URL: https://sickrage.ca/ +# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git +# - +# This file is part of SiCKRAGE. +# - +# SiCKRAGE is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# - +# SiCKRAGE is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# - +# You should have received a copy of the GNU General Public License +# along with SiCKRAGE. If not, see . +# ############################################################################## + + +import hashlib +import os +from pathlib import Path + +main_dir = Path(__file__).parent +prog_dir = main_dir.joinpath('sickrage') +checksum_file = prog_dir.joinpath('checksums.md5') + + +def md5(filename): + with open(filename, "rb") as f: + file_hash = hashlib.md5() + while chunk := f.read(8192): + file_hash.update(chunk) + return file_hash.hexdigest() + + +with open(checksum_file, "wb") as fp: + for root, dirs, files in os.walk(prog_dir): + for file in files: + full_filename = Path(str(root).replace(str(prog_dir), 'sickrage')).joinpath(file) + if full_filename != checksum_file: + fp.write('{} = {}\n'.format(full_filename, md5(full_filename)).encode()) + + print('Finished generating {}'.format(checksum_file)) diff --git a/checksum-validator.py b/checksum-validator.py new file mode 100644 index 0000000000..145240ea61 --- /dev/null +++ b/checksum-validator.py @@ -0,0 +1,52 @@ +# ############################################################################## +# Author: echel0n +# URL: https://sickrage.ca/ +# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git +# - +# This file is part of SiCKRAGE. +# - +# SiCKRAGE is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# - +# SiCKRAGE is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# - +# You should have received a copy of the GNU General Public License +# along with SiCKRAGE. If not, see . +# ############################################################################## + + +import hashlib +import os +from pathlib import Path + +main_dir = Path(__file__).parent +prog_dir = main_dir.joinpath('sickrage') +checksum_file = prog_dir.joinpath('checksums.md5') + + +def md5(filename): + with open(filename, "rb") as f: + file_hash = hashlib.md5() + while chunk := f.read(8192): + file_hash.update(chunk) + return file_hash.hexdigest() + + +with open(checksum_file, "rb") as fp: + failed = False + + for line in fp.readlines(): + file, checksum = line.decode().strip().split(' = ') + full_filename = main_dir.joinpath(file) + if full_filename != checksum_file: + if not os.path.exists(full_filename) or md5(full_filename) != checksum: + print('SiCKRAGE file {} integrity check failed'.format(full_filename)) + failed = True + + if not failed: + print('SiCKRAGE file integrity check passed') diff --git a/requirements.txt b/requirements.txt index afff729af7..201a0bc76b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,6 +45,7 @@ sqlalchemy==1.3.18 sqlalchemy-migrate==0.13.0 mutagen==1.44.0 deluge-client==1.9.0 +dirsync==2.2.4 PyMySQL certifi pyasn1 diff --git a/sickrage/__init__.py b/sickrage/__init__.py index 9e4b8ecc42..9fa64787ab 100644 --- a/sickrage/__init__.py +++ b/sickrage/__init__.py @@ -43,6 +43,7 @@ VERSION_FILE = os.path.join(PROG_DIR, 'version.txt') CHANGELOG_FILE = os.path.join(MAIN_DIR, 'CHANGELOG.md') REQS_FILE = os.path.join(MAIN_DIR, 'requirements.txt') +CHECKSUM_FILE = os.path.join(PROG_DIR, 'checksums.md5') class Daemon(object): @@ -177,18 +178,19 @@ def check_requirements(): sys.exit("Sorry, SiCKRAGE requires Python 3.5+") # install/update requirements - with open(REQS_FILE) as f: - for line in f.readlines(): - try: - req_name, req_version = line.strip().split('==') - if not pkg_resources.get_distribution(req_name).version == req_version: - print('Updating requirement {} to {}'.format(req_name, req_version)) + if os.path.exists(REQS_FILE): + with open(REQS_FILE) as f: + for line in f.readlines(): + try: + req_name, req_version = line.strip().split('==') + if not pkg_resources.get_distribution(req_name).version == req_version: + print('Updating requirement {} to {}'.format(req_name, req_version)) + subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", line.strip()]) + except pkg_resources.DistributionNotFound: + print('Installing requirement {}'.format(line.strip())) subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", line.strip()]) - except pkg_resources.DistributionNotFound: - print('Installing requirement {}'.format(line.strip())) - subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", line.strip()]) - except ValueError: - continue + except ValueError: + continue try: import OpenSSL @@ -202,6 +204,31 @@ def check_requirements(): print('OpenSSL not available, please install for better requests validation: `https://pyopenssl.readthedocs.org/en/latest/install.html`') +def file_cleanup(remove=False): + valid_files = [] + + if not os.path.exists(CHECKSUM_FILE): + return + + with open(CHECKSUM_FILE, "rb") as fp: + for line in fp.readlines(): + file, checksum = line.strip().split(b' = ') + full_filename = os.path.join(MAIN_DIR, file.decode()) + valid_files.append(full_filename) + + for root, dirs, files in os.walk(PROG_DIR): + for file in files: + full_filename = os.path.join(root, file) + if full_filename != CHECKSUM_FILE and full_filename not in valid_files and PROG_DIR in full_filename: + try: + if remove: + os.remove(full_filename) + else: + print('Found unwanted file {}, you should delete this file manually!'.format(full_filename)) + except OSError: + print('Unable to delete filename {} during cleanup, you should delete this file manually!'.format(full_filename)) + + def version(): # Get the version number with open(VERSION_FILE) as f: @@ -232,85 +259,86 @@ def main(): # set system default language gettext.install('messages', LOCALE_DIR, codeset='UTF-8', names=["ngettext"]) + # sickrage startup options + parser = argparse.ArgumentParser(prog='sickrage') + parser.add_argument('-v', '--version', + action='version', + version='%(prog)s {}'.format(version())) + parser.add_argument('-d', '--daemon', + action='store_true', + help='Run as a daemon (*NIX ONLY)') + parser.add_argument('-q', '--quiet', + action='store_true', + help='Disables logging to CONSOLE') + parser.add_argument('-p', '--port', + default=0, + type=int, + help='Override default/configured port to listen on') + parser.add_argument('-H', '--host', + default='', + help='Override default/configured host to listen on') + parser.add_argument('--dev', + action='store_true', + help='Enable developer mode') + parser.add_argument('--debug', + action='store_true', + help='Enable debugging') + parser.add_argument('--datadir', + default=os.path.abspath(os.path.join(os.path.expanduser("~"), '.sickrage')), + help='Overrides data folder for database, config, cache and logs (specify full path)') + parser.add_argument('--config', + default='config.ini', + help='Overrides config filename (specify full path and filename if outside datadir path)') + parser.add_argument('--pidfile', + default='sickrage.pid', + help='Creates a PID file (specify full path and filename if outside datadir path)') + parser.add_argument('--no_clean', + action='store_true', + help='Suppress cleanup of files not present in checksum.md5') + parser.add_argument('--nolaunch', + action='store_true', + help='Suppress launching web browser on startup') + parser.add_argument('--disable_updates', + action='store_true', + help='Disable application updates') + parser.add_argument('--web_root', + default='', + type=str, + help='Overrides URL web root') + parser.add_argument('--db_type', + default='sqlite', + help='Database type: sqlite or mysql') + parser.add_argument('--db_prefix', + default='sickrage', + help='Database prefix you want prepended to database table names') + parser.add_argument('--db_host', + default='localhost', + help='Database hostname (not used for sqlite)') + parser.add_argument('--db_port', + default='3306', + help='Database port number (not used for sqlite)') + parser.add_argument('--db_username', + default='sickrage', + help='Database username (not used for sqlite)') + parser.add_argument('--db_password', + default='sickrage', + help='Database password (not used for sqlite)') + + # Parse startup args + args = parser.parse_args() + # check lib requirements check_requirements() + # cleanup unwanted files + file_cleanup(remove=not args.no_clean) + try: - try: - from sickrage.core import Core - except ImportError: - print('Attempting to install SiCKRAGE missing requirements using pip') - subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", "-U", "pip"]) - subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", "-r", REQS_FILE]) - from sickrage.core import Core + from sickrage.core import Core # main app instance app = Core() - # sickrage startup options - parser = argparse.ArgumentParser(prog='sickrage') - parser.add_argument('-v', '--version', - action='version', - version='%(prog)s {}'.format(version())) - parser.add_argument('-d', '--daemon', - action='store_true', - help='Run as a daemon (*NIX ONLY)') - parser.add_argument('-q', '--quiet', - action='store_true', - help='Disables logging to CONSOLE') - parser.add_argument('-p', '--port', - default=0, - type=int, - help='Override default/configured port to listen on') - parser.add_argument('-H', '--host', - default='', - help='Override default/configured host to listen on') - parser.add_argument('--dev', - action='store_true', - help='Enable developer mode') - parser.add_argument('--debug', - action='store_true', - help='Enable debugging') - parser.add_argument('--datadir', - default=os.path.abspath(os.path.join(os.path.expanduser("~"), '.sickrage')), - help='Overrides data folder for database, config, cache and logs (specify full path)') - parser.add_argument('--config', - default='config.ini', - help='Overrides config filename (specify full path and filename if outside datadir path)') - parser.add_argument('--pidfile', - default='sickrage.pid', - help='Creates a PID file (specify full path and filename if outside datadir path)') - parser.add_argument('--nolaunch', - action='store_true', - help='Suppress launching web browser on startup') - parser.add_argument('--disable_updates', - action='store_true', - help='Disable application updates') - parser.add_argument('--web_root', - default='', - type=str, - help='Overrides URL web root') - parser.add_argument('--db_type', - default='sqlite', - help='Database type: sqlite or mysql') - parser.add_argument('--db_prefix', - default='sickrage', - help='Database prefix you want prepended to database table names') - parser.add_argument('--db_host', - default='localhost', - help='Database hostname (not used for sqlite)') - parser.add_argument('--db_port', - default='3306', - help='Database port number (not used for sqlite)') - parser.add_argument('--db_username', - default='sickrage', - help='Database username (not used for sqlite)') - parser.add_argument('--db_password', - default='sickrage', - help='Database password (not used for sqlite)') - - # Parse startup args - args = parser.parse_args() app.quiet = args.quiet app.host = args.host app.web_port = int(args.port) diff --git a/sickrage/core/__init__.py b/sickrage/core/__init__.py index 9c80704105..cda8c64fe4 100644 --- a/sickrage/core/__init__.py +++ b/sickrage/core/__init__.py @@ -49,7 +49,7 @@ from sickrage.core.databases.cache import CacheDB from sickrage.core.databases.main import MainDB from sickrage.core.helpers import generate_secret, make_dir, get_lan_ip, restore_app_data, get_disk_space_usage, get_free_space, launch_browser, \ - torrent_webui_url, encryption + torrent_webui_url, encryption, md5_file_hash from sickrage.core.logger import Logger from sickrage.core.nameparser.validator import check_force_season_folders from sickrage.core.processors import auto_postprocessor @@ -70,7 +70,7 @@ from sickrage.core.updaters.show_updater import ShowUpdater from sickrage.core.updaters.tz_updater import TimeZoneUpdater from sickrage.core.upnp import UPNPClient -from sickrage.core.version_updater import VersionUpdater +from sickrage.core.version_updater import VersionUpdater, SourceUpdateManager from sickrage.core.webserver import WebServer from sickrage.metadata import MetadataProviders from sickrage.notifiers import NotifierProviders @@ -522,7 +522,7 @@ def start(self): # launch browser window if all([not sickrage.app.no_launch, sickrage.app.config.launch_browser]): self.scheduler.add_job(launch_browser, args=[('http', 'https')[sickrage.app.config.enable_https], - sickrage.app.config.web_host, sickrage.app.config.web_port]) + sickrage.app.config.web_host, sickrage.app.config.web_port]) self.log.info("SiCKRAGE :: STARTED") self.log.info("SiCKRAGE :: APP VERSION:[{}]".format(sickrage.version())) @@ -530,9 +530,9 @@ def start(self): self.log.info("SiCKRAGE :: DATABASE VERSION:[v{}]".format(self.main_db.version)) self.log.info("SiCKRAGE :: DATABASE TYPE:[{}]".format(self.db_type)) self.log.info("SiCKRAGE :: URL:[{}://{}:{}/{}]".format(('http', 'https')[self.config.enable_https], - (self.config.web_host, get_lan_ip())[self.config.web_host == '0.0.0.0'], - self.config.web_port, - self.config.web_root)) + (self.config.web_host, get_lan_ip())[self.config.web_host == '0.0.0.0'], + self.config.web_port, + self.config.web_root)) def load_shows(self): threading.currentThread().setName('CORE') diff --git a/sickrage/core/databases/main/__init__.py b/sickrage/core/databases/main/__init__.py index 203a9ad30b..dcfc6bdeb1 100644 --- a/sickrage/core/databases/main/__init__.py +++ b/sickrage/core/databases/main/__init__.py @@ -135,6 +135,34 @@ def fix_duplicate_episode_scene_numbering(): result.scene_episode = -1 session.commit() + def fix_duplicate_episode_scene_absolute_numbering(): + session = self.session() + + duplicates = session.query( + self.TVEpisode.showid, + self.TVEpisode.scene_absolute_number, + func.count(self.TVEpisode.showid).label('count') + ).group_by( + self.TVEpisode.showid, + self.TVEpisode.scene_absolute_number + ).filter( + self.TVEpisode.scene_absolute_number != -1 + ).having(literal_column('count') > 1) + + for cur_duplicate in duplicates: + sickrage.app.log.debug("Duplicate episode scene absolute numbering detected! " + "showid: {dupe_id} " + "scene absolute number: {dupe_scene_absolute_number} " + "count: {dupe_count}".format(dupe_id=cur_duplicate.showid, + dupe_scene_absolute_number=cur_duplicate.scene_absolute_number, + dupe_count=cur_duplicate.count)) + + for result in session.query(self.TVEpisode).filter_by(showid=cur_duplicate.showid, + scene_absolute_number=cur_duplicate.scene_absolute_number).\ + limit(cur_duplicate.count - 1): + result.scene_absolute_number = -1 + session.commit() + def remove_invalid_episodes(): session = self.session() @@ -189,6 +217,7 @@ def fix_tvshow_table_columns(): remove_invalid_episodes() fix_invalid_scene_numbering() fix_duplicate_episode_scene_numbering() + fix_duplicate_episode_scene_absolute_numbering() fix_tvshow_table_columns() class TVShow(MainDBBase): diff --git a/sickrage/core/helpers/__init__.py b/sickrage/core/helpers/__init__.py index e415d6a17f..36c8bb94c9 100644 --- a/sickrage/core/helpers/__init__.py +++ b/sickrage/core/helpers/__init__.py @@ -1744,10 +1744,11 @@ def strip_accents(name): def md5_file_hash(filename): - hasher = hashlib.md5() - with open(filename, 'rb') as fd: - hasher.update(fd.read()) - return hasher.hexdigest() + with open(filename, "rb") as f: + file_hash = hashlib.md5() + while chunk := f.read(8192): + file_hash.update(chunk) + return file_hash.hexdigest() def get_extension(filename): diff --git a/sickrage/core/version_updater.py b/sickrage/core/version_updater.py index a15149da34..cfa814623e 100644 --- a/sickrage/core/version_updater.py +++ b/sickrage/core/version_updater.py @@ -30,6 +30,8 @@ import threading from time import sleep +import dirsync as dirsync + import sickrage from sickrage.core.helpers import backup_app_data from sickrage.core.websession import WebSession @@ -626,7 +628,12 @@ def update(self): with tempfile.TemporaryFile() as update_tarfile: sickrage.app.log.info("Downloading update from " + repr(tar_download_url)) - update_tarfile.write(WebSession().get(tar_download_url).content) + resp = WebSession().get(tar_download_url) + if not resp or not resp.content: + sickrage.app.log.warning('Failed to download SiCKRAGE update') + return False + + update_tarfile.write(resp.content) update_tarfile.seek(0) with tempfile.TemporaryDirectory(prefix='sr_update_', dir=sickrage.app.data_dir) as unpack_dir: @@ -639,29 +646,13 @@ def update(self): sickrage.app.log.warning("Invalid update data, update failed: not a gzip file") return False - # find update dir name - update_dir_contents = [x for x in os.listdir(unpack_dir) if os.path.isdir(os.path.join(unpack_dir, x))] - if len(update_dir_contents) != 1: - sickrage.app.log.warning("Invalid update data, update failed: " + str(update_dir_contents)) + if len(os.listdir(unpack_dir)) != 1: + sickrage.app.log.warning("Invalid update data, update failed") return False - # walk temp folder and move files to main folder - content_dir = os.path.join(unpack_dir, update_dir_contents[0]) - sickrage.app.log.info("Moving files from " + content_dir + " to " + sickrage.MAIN_DIR) - for dirname, __, filenames in os.walk(content_dir): - dirname = dirname[len(content_dir) + 1:] - for curfile in filenames: - old_path = os.path.join(content_dir, dirname, curfile) - new_path = os.path.join(sickrage.MAIN_DIR, dirname, curfile) - - if os.path.isfile(new_path) and os.path.exists(new_path): - os.remove(new_path) - - try: - shutil.move(old_path, new_path) - except IOError: - os.makedirs(os.path.dirname(new_path)) - shutil.move(old_path, new_path) + update_dir = os.path.join(*[unpack_dir, os.listdir(unpack_dir)[0], 'sickrage']) + sickrage.app.log.info("Sync folder {} to {}".format(update_dir, sickrage.PROG_DIR)) + dirsync.sync(update_dir, sickrage.PROG_DIR, 'sync', purge=True) except Exception as e: sickrage.app.log.error("Error while trying to update: {}".format(e)) return False diff --git a/sickrage/core/webserver/handlers/base.py b/sickrage/core/webserver/handlers/base.py index 7e28de744d..a9cdf8f975 100644 --- a/sickrage/core/webserver/handlers/base.py +++ b/sickrage/core/webserver/handlers/base.py @@ -53,7 +53,7 @@ def write_error(self, status_code, **kwargs): request_info = ''.join(["%s: %s
" % (k, self.request.__dict__[k]) for k in self.request.__dict__.keys()]) error = exc_info[1] - sickrage.app.log.error(error) + sickrage.app.log.debug(error) self.set_header('Content-Type', 'text/html') return self.write(""" diff --git a/sickrage/providers/torrent/extratorrent.py b/sickrage/providers/torrent/extratorrent.py index ec38927927..3c3a80ddc0 100644 --- a/sickrage/providers/torrent/extratorrent.py +++ b/sickrage/providers/torrent/extratorrent.py @@ -85,9 +85,12 @@ def parse(self, data, mode, **kwargs): with bs4_parser(data) as html: torrent_table = html.find('table', class_='tl') - torrent_rows = torrent_table.find_all('tr') + if not torrent_table: + sickrage.app.log.debug('Data returned from provider does not contain any torrents') + return results # Continue only if at least one Release is found + torrent_rows = torrent_table.find_all('tr') if len(torrent_rows) < 2: sickrage.app.log.debug('Data returned from provider does not contain any torrents') return results diff --git a/sickrage/providers/torrent/limetorrents.py b/sickrage/providers/torrent/limetorrents.py index dc370d135f..5c738b5d70 100644 --- a/sickrage/providers/torrent/limetorrents.py +++ b/sickrage/providers/torrent/limetorrents.py @@ -82,7 +82,6 @@ def process_column_header(th): with bs4_parser(data) as html: torrent_table = html.find('table', class_='table2') - if not torrent_table: sickrage.app.log.debug('Data returned from provider does not contain any torrents') return results