Skip to content

Commit

Permalink
Added dirsync to requirements.txt
Browse files Browse the repository at this point in the history 
Refactored source update's to sync and purge files that no longer exist
Added checksum generator and validator to help maintain file integrity of project
Fixed #SICKRAGE-APP-5Q6 - AttributeError in sickrage.core.version_updater in update method, 'NoneType' object has no attribute 'content'
Added fix for dupe scene absolute numbering
Fixed #SICKRAGE-APP-5R0 - AttributeError in sickrage.providers.torrent.extratorrent
  • Loading branch information
@echel0n
echel0n committed Jul 31, 2020
1 parent 0659172 commit 973d96d
Show file tree
Hide file tree
Showing 12 changed files with 274 additions and 120 deletions.
9 changes: 6 additions & 3 deletions .gitlab-ci.yml
View file
Expand Up @@ -105,7 +105,7 @@ services:
release:build:master:
stage: release_build
image:
name: nikolaik/python-nodejs:python3.7-nodejs10-alpine
name: nikolaik/python-nodejs:python3.8-nodejs10-alpine
variables:
NODE_ENV: "development"
script:
Expand All @@ -118,6 +118,7 @@ release:build:master:
- bumpversion --allow-dirty release package.json sickrage/version.txt
- git checkout -b release-$(cat sickrage/version.txt)
- yarn run build
- python checksum-generator.py
- git add --all
- git commit -m "[TASK] Releasing v$(cat sickrage/version.txt)"
- git fetch . release-$(cat sickrage/version.txt):master
Expand All @@ -126,6 +127,7 @@ release:build:master:
- git push https://$GIT_ACCESS_USER:$GIT_ACCESS_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:master --follow-tags
- git checkout develop
- bumpversion --allow-dirty patch package.json sickrage/version.txt
- python checksum-generator.py
- git add --all
- git commit -m "[TASK] Bump develop branch to v$(cat sickrage/version.txt)"
- git push https://$GIT_ACCESS_USER:$GIT_ACCESS_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:develop --follow-tags
Expand All @@ -142,7 +144,7 @@ release:build:master:
release:build:develop:
stage: release_build
image:
name: nikolaik/python-nodejs:python3.7-nodejs10-alpine
name: nikolaik/python-nodejs:python3.8-nodejs10-alpine
variables:
NODE_ENV: "development"
script:
Expand All @@ -153,6 +155,7 @@ release:build:develop:
- npx git-changelog -t $(git describe --abbrev=0)
- bumpversion --allow-dirty dev package.json sickrage/version.txt
- yarn run build
- python checksum-generator.py
# - python setup.py extract_messages
# - crowdin-cli-py upload sources
# - crowdin-cli-py download
Expand Down Expand Up @@ -215,7 +218,7 @@ release:sentry:develop:

deploy:pypi:
stage: release_deploy
image: python:3.7-alpine3.9
image: python:3.8-alpine3.9
script:
- apk add --no-cache py-pip gcc libffi-dev python3-dev musl-dev openssl-dev
- pip install -U twine
Expand Down
47 changes: 47 additions & 0 deletions checksum-generator.py
View file
@@ -0,0 +1,47 @@
# ##############################################################################
# Author: echel0n <echel0n@sickrage.ca>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################


import hashlib
import os
from pathlib import Path

main_dir = Path(__file__).parent
prog_dir = main_dir.joinpath('sickrage')
checksum_file = prog_dir.joinpath('checksums.md5')


def md5(filename):
with open(filename, "rb") as f:
file_hash = hashlib.md5()
while chunk := f.read(8192):
file_hash.update(chunk)
return file_hash.hexdigest()


with open(checksum_file, "wb") as fp:
for root, dirs, files in os.walk(prog_dir):
for file in files:
full_filename = Path(str(root).replace(str(prog_dir), 'sickrage')).joinpath(file)
if full_filename != checksum_file:
fp.write('{} = {}\n'.format(full_filename, md5(full_filename)).encode())

print('Finished generating {}'.format(checksum_file))
52 changes: 52 additions & 0 deletions checksum-validator.py
View file
@@ -0,0 +1,52 @@
# ##############################################################################
# Author: echel0n <echel0n@sickrage.ca>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################


import hashlib
import os
from pathlib import Path

main_dir = Path(__file__).parent
prog_dir = main_dir.joinpath('sickrage')
checksum_file = prog_dir.joinpath('checksums.md5')


def md5(filename):
with open(filename, "rb") as f:
file_hash = hashlib.md5()
while chunk := f.read(8192):
file_hash.update(chunk)
return file_hash.hexdigest()


with open(checksum_file, "rb") as fp:
failed = False

for line in fp.readlines():
file, checksum = line.decode().strip().split(' = ')
full_filename = main_dir.joinpath(file)
if full_filename != checksum_file:
if not os.path.exists(full_filename) or md5(full_filename) != checksum:
print('SiCKRAGE file {} integrity check failed'.format(full_filename))
failed = True

if not failed:
print('SiCKRAGE file integrity check passed')
1 change: 1 addition & 0 deletions requirements.txt
View file
Expand Up @@ -45,6 +45,7 @@ sqlalchemy==1.3.18
sqlalchemy-migrate==0.13.0
mutagen==1.44.0
deluge-client==1.9.0
dirsync==2.2.4
PyMySQL
certifi
pyasn1
Expand Down
192 changes: 110 additions & 82 deletions sickrage/__init__.py
View file
Expand Up @@ -43,6 +43,7 @@
VERSION_FILE = os.path.join(PROG_DIR, 'version.txt')
CHANGELOG_FILE = os.path.join(MAIN_DIR, 'CHANGELOG.md')
REQS_FILE = os.path.join(MAIN_DIR, 'requirements.txt')
CHECKSUM_FILE = os.path.join(PROG_DIR, 'checksums.md5')


class Daemon(object):
Expand Down Expand Up @@ -177,18 +178,19 @@ def check_requirements():
sys.exit("Sorry, SiCKRAGE requires Python 3.5+")

# install/update requirements
with open(REQS_FILE) as f:
for line in f.readlines():
try:
req_name, req_version = line.strip().split('==')
if not pkg_resources.get_distribution(req_name).version == req_version:
print('Updating requirement {} to {}'.format(req_name, req_version))
if os.path.exists(REQS_FILE):
with open(REQS_FILE) as f:
for line in f.readlines():
try:
req_name, req_version = line.strip().split('==')
if not pkg_resources.get_distribution(req_name).version == req_version:
print('Updating requirement {} to {}'.format(req_name, req_version))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", line.strip()])
except pkg_resources.DistributionNotFound:
print('Installing requirement {}'.format(line.strip()))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", line.strip()])
except pkg_resources.DistributionNotFound:
print('Installing requirement {}'.format(line.strip()))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", line.strip()])
except ValueError:
continue
except ValueError:
continue

try:
import OpenSSL
Expand All @@ -202,6 +204,31 @@ def check_requirements():
print('OpenSSL not available, please install for better requests validation: `https://pyopenssl.readthedocs.org/en/latest/install.html`')


def file_cleanup(remove=False):
valid_files = []

if not os.path.exists(CHECKSUM_FILE):
return

with open(CHECKSUM_FILE, "rb") as fp:
for line in fp.readlines():
file, checksum = line.strip().split(b' = ')
full_filename = os.path.join(MAIN_DIR, file.decode())
valid_files.append(full_filename)

for root, dirs, files in os.walk(PROG_DIR):
for file in files:
full_filename = os.path.join(root, file)
if full_filename != CHECKSUM_FILE and full_filename not in valid_files and PROG_DIR in full_filename:
try:
if remove:
os.remove(full_filename)
else:
print('Found unwanted file {}, you should delete this file manually!'.format(full_filename))
except OSError:
print('Unable to delete filename {} during cleanup, you should delete this file manually!'.format(full_filename))


def version():
# Get the version number
with open(VERSION_FILE) as f:
Expand Down Expand Up @@ -232,85 +259,86 @@ def main():
# set system default language
gettext.install('messages', LOCALE_DIR, codeset='UTF-8', names=["ngettext"])

# sickrage startup options
parser = argparse.ArgumentParser(prog='sickrage')
parser.add_argument('-v', '--version',
action='version',
version='%(prog)s {}'.format(version()))
parser.add_argument('-d', '--daemon',
action='store_true',
help='Run as a daemon (*NIX ONLY)')
parser.add_argument('-q', '--quiet',
action='store_true',
help='Disables logging to CONSOLE')
parser.add_argument('-p', '--port',
default=0,
type=int,
help='Override default/configured port to listen on')
parser.add_argument('-H', '--host',
default='',
help='Override default/configured host to listen on')
parser.add_argument('--dev',
action='store_true',
help='Enable developer mode')
parser.add_argument('--debug',
action='store_true',
help='Enable debugging')
parser.add_argument('--datadir',
default=os.path.abspath(os.path.join(os.path.expanduser("~"), '.sickrage')),
help='Overrides data folder for database, config, cache and logs (specify full path)')
parser.add_argument('--config',
default='config.ini',
help='Overrides config filename (specify full path and filename if outside datadir path)')
parser.add_argument('--pidfile',
default='sickrage.pid',
help='Creates a PID file (specify full path and filename if outside datadir path)')
parser.add_argument('--no_clean',
action='store_true',
help='Suppress cleanup of files not present in checksum.md5')
parser.add_argument('--nolaunch',
action='store_true',
help='Suppress launching web browser on startup')
parser.add_argument('--disable_updates',
action='store_true',
help='Disable application updates')
parser.add_argument('--web_root',
default='',
type=str,
help='Overrides URL web root')
parser.add_argument('--db_type',
default='sqlite',
help='Database type: sqlite or mysql')
parser.add_argument('--db_prefix',
default='sickrage',
help='Database prefix you want prepended to database table names')
parser.add_argument('--db_host',
default='localhost',
help='Database hostname (not used for sqlite)')
parser.add_argument('--db_port',
default='3306',
help='Database port number (not used for sqlite)')
parser.add_argument('--db_username',
default='sickrage',
help='Database username (not used for sqlite)')
parser.add_argument('--db_password',
default='sickrage',
help='Database password (not used for sqlite)')

# Parse startup args
args = parser.parse_args()

# check lib requirements
check_requirements()

# cleanup unwanted files
file_cleanup(remove=not args.no_clean)

try:
try:
from sickrage.core import Core
except ImportError:
print('Attempting to install SiCKRAGE missing requirements using pip')
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", "-U", "pip"])
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", "-r", REQS_FILE])
from sickrage.core import Core
from sickrage.core import Core

# main app instance
app = Core()

# sickrage startup options
parser = argparse.ArgumentParser(prog='sickrage')
parser.add_argument('-v', '--version',
action='version',
version='%(prog)s {}'.format(version()))
parser.add_argument('-d', '--daemon',
action='store_true',
help='Run as a daemon (*NIX ONLY)')
parser.add_argument('-q', '--quiet',
action='store_true',
help='Disables logging to CONSOLE')
parser.add_argument('-p', '--port',
default=0,
type=int,
help='Override default/configured port to listen on')
parser.add_argument('-H', '--host',
default='',
help='Override default/configured host to listen on')
parser.add_argument('--dev',
action='store_true',
help='Enable developer mode')
parser.add_argument('--debug',
action='store_true',
help='Enable debugging')
parser.add_argument('--datadir',
default=os.path.abspath(os.path.join(os.path.expanduser("~"), '.sickrage')),
help='Overrides data folder for database, config, cache and logs (specify full path)')
parser.add_argument('--config',
default='config.ini',
help='Overrides config filename (specify full path and filename if outside datadir path)')
parser.add_argument('--pidfile',
default='sickrage.pid',
help='Creates a PID file (specify full path and filename if outside datadir path)')
parser.add_argument('--nolaunch',
action='store_true',
help='Suppress launching web browser on startup')
parser.add_argument('--disable_updates',
action='store_true',
help='Disable application updates')
parser.add_argument('--web_root',
default='',
type=str,
help='Overrides URL web root')
parser.add_argument('--db_type',
default='sqlite',
help='Database type: sqlite or mysql')
parser.add_argument('--db_prefix',
default='sickrage',
help='Database prefix you want prepended to database table names')
parser.add_argument('--db_host',
default='localhost',
help='Database hostname (not used for sqlite)')
parser.add_argument('--db_port',
default='3306',
help='Database port number (not used for sqlite)')
parser.add_argument('--db_username',
default='sickrage',
help='Database username (not used for sqlite)')
parser.add_argument('--db_password',
default='sickrage',
help='Database password (not used for sqlite)')

# Parse startup args
args = parser.parse_args()
app.quiet = args.quiet
app.host = args.host
app.web_port = int(args.port)
Expand Down

0 comments on commit 973d96d

Please sign in to comment.