models.py

import flask
from enum import Enum, IntEnum
from datetime import datetime, timezone
from nyaa import app, db
from nyaa.torrents import create_magnet

from sqlalchemy import func, ForeignKeyConstraint, Index
from sqlalchemy.ext import declarative
from sqlalchemy_utils import ChoiceType, EmailType, PasswordType
from sqlalchemy_fulltext import FullText

from werkzeug.security import generate_password_hash, check_password_hash
from ipaddress import ip_address

import re
import base64
from markupsafe import escape as escape_markup
from urllib.parse import urlencode, unquote as unquote_url
from hashlib import md5

if app.config['USE_MYSQL']:
    from sqlalchemy.dialects import mysql
    BinaryType = mysql.BINARY
    DescriptionTextType = mysql.TEXT
    MediumBlobType = mysql.MEDIUMBLOB
    COL_UTF8_GENERAL_CI = 'utf8_general_ci'
    COL_UTF8MB4_BIN = 'utf8mb4_bin'
    COL_ASCII_GENERAL_CI = 'ascii_general_ci'
else:
    BinaryType = db.Binary
    DescriptionTextType = db.String
    MediumBlobType = db.BLOB
    COL_UTF8_GENERAL_CI = 'NOCASE'
    COL_UTF8MB4_BIN = None
    COL_ASCII_GENERAL_CI = 'NOCASE'


# For property timestamps
UTC_EPOCH = datetime.utcfromtimestamp(0)


class DeclarativeHelperBase(object):
    ''' This class eases our nyaa-sukebei shenanigans by automatically adjusting
        __tablename__ and providing class methods for renaming references. '''
    # See http://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/api.html

    __tablename_base__ = None
    __flavor__ = None

    @classmethod
    def _table_prefix_string(cls):
        return cls.__flavor__.lower() + '_'

    @classmethod
    def _table_prefix(cls, table_name):
        return cls._table_prefix_string() + table_name

    @classmethod
    def _flavor_prefix(cls, table_name):
        return cls.__flavor__ + table_name

    @declarative.declared_attr
    def __tablename__(cls):
        return cls._table_prefix(cls.__tablename_base__)


class FlagProperty(object):
    ''' This class will act as a wrapper between the given flag and the class's
        flag collection. '''

    def __init__(self, flag, flags_attr='flags'):
        self._flag = flag
        self._flags_attr_name = flags_attr

    def _get_flags(self, instance):
        return getattr(instance, self._flags_attr_name)

    def _set_flags(self, instance, value):
        return setattr(instance, self._flags_attr_name, value)

    def __get__(self, instance, owner_class):
        if instance is None:
            raise AttributeError()
        return bool(self._get_flags(instance) & self._flag)

    def __set__(self, instance, value):
        new_flags = (self._get_flags(instance) & ~self._flag) | (bool(value) and self._flag)
        self._set_flags(instance, new_flags)


class TorrentFlags(IntEnum):
    NONE = 0
    ANONYMOUS = 1
    HIDDEN = 2
    TRUSTED = 4
    REMAKE = 8
    COMPLETE = 16
    DELETED = 32


class TorrentBase(DeclarativeHelperBase):
    __tablename_base__ = 'torrents'

    id = db.Column(db.Integer, primary_key=True)
    info_hash = db.Column(BinaryType(length=20), unique=True, nullable=False, index=True)
    display_name = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI),
                             nullable=False, index=True)
    torrent_name = db.Column(db.String(length=255), nullable=False)
    information = db.Column(db.String(length=255), nullable=False)
    description = db.Column(DescriptionTextType(collation=COL_UTF8MB4_BIN), nullable=False)

    filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True)
    encoding = db.Column(db.String(length=32), nullable=False)
    flags = db.Column(db.Integer, default=0, nullable=False, index=True)

    @declarative.declared_attr
    def uploader_id(cls):
        # Even though this is same for both tables, declarative requires this
        return db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)

    uploader_ip = db.Column(db.Binary(length=16), default=None, nullable=True)
    has_torrent = db.Column(db.Boolean, nullable=False, default=False)

    comment_count = db.Column(db.Integer, default=0, nullable=False, index=True)

    created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False)
    updated_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow,
                             onupdate=datetime.utcnow, nullable=False)

    @declarative.declared_attr
    def main_category_id(cls):
        fk = db.ForeignKey(cls._table_prefix('main_categories.id'))
        return db.Column(db.Integer, fk, nullable=False)

    sub_category_id = db.Column(db.Integer, nullable=False)

    @declarative.declared_attr
    def redirect(cls):
        fk = db.ForeignKey(cls._table_prefix('torrents.id'))
        return db.Column(db.Integer, fk, nullable=True)

    @declarative.declared_attr
    def __table_args__(cls):
        return (
            Index(cls._table_prefix('uploader_flag_idx'), 'uploader_id', 'flags'),
            ForeignKeyConstraint(
                ['main_category_id', 'sub_category_id'],
                [cls._table_prefix('sub_categories.main_category_id'),
                 cls._table_prefix('sub_categories.id')]
            ), {}
        )

    @declarative.declared_attr
    def user(cls):
        return db.relationship('User', uselist=False, back_populates=cls._table_prefix('torrents'))

    @declarative.declared_attr
    def main_category(cls):
        return db.relationship(cls._flavor_prefix('MainCategory'), uselist=False,
                               back_populates='torrents', lazy="joined")

    @declarative.declared_attr
    def sub_category(cls):
        join_sql = ("and_({0}SubCategory.id == foreign({0}Torrent.sub_category_id), "
                    "{0}SubCategory.main_category_id == {0}Torrent.main_category_id)")
        return db.relationship(cls._flavor_prefix('SubCategory'), uselist=False,
                               backref='torrents', lazy="joined",
                               primaryjoin=join_sql.format(cls.__flavor__))

    @declarative.declared_attr
    def info(cls):
        return db.relationship(cls._flavor_prefix('TorrentInfo'), uselist=False,
                               cascade="all, delete-orphan", back_populates='torrent')

    @declarative.declared_attr
    def filelist(cls):
        return db.relationship(cls._flavor_prefix('TorrentFilelist'), uselist=False,
                               cascade="all, delete-orphan", back_populates='torrent')

    @declarative.declared_attr
    def stats(cls):
        return db.relationship(cls._flavor_prefix('Statistic'), uselist=False,
                               cascade="all, delete-orphan", back_populates='torrent',
                               lazy='joined')

    @declarative.declared_attr
    def trackers(cls):
        return db.relationship(cls._flavor_prefix('TorrentTrackers'), uselist=True,
                               cascade="all, delete-orphan", lazy='select',
                               order_by=cls._flavor_prefix('TorrentTrackers.order'))

    @declarative.declared_attr
    def comments(cls):
        return db.relationship(cls._flavor_prefix('Comment'), uselist=True,
                               cascade="all, delete-orphan")

    def __repr__(self):
        return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self)

    def update_comment_count(self):
        self.comment_count = Comment.query.filter_by(torrent_id=self.id).count()
        return self.comment_count

    @property
    def created_utc_timestamp(self):
        ''' Returns a UTC POSIX timestamp, as seconds '''
        return (self.created_time - UTC_EPOCH).total_seconds()

    @property
    def information_as_link(self):
        ''' Formats the .information into an IRC or HTTP(S) <a> if possible,
            otherwise escapes it. '''
        irc_match = re.match(r'^#([a-zA-Z0-9-_]+)@([a-zA-Z0-9-_.:]+)$', self.information)
        if irc_match:
            # Return a formatted IRC uri
            return '<a href="irc://{1}/{0}">#{0}@{1}</a>'.format(*irc_match.groups())

        url_match = re.match(r'^(https?:\/\/.+?)$', self.information)
        if url_match:
            url = url_match.group(1)

            invalid_url_characters = '<>"'
            # Check if url contains invalid characters
            if not any(c in url for c in invalid_url_characters):
                return '<a href="{0}">{1}</a>'.format(url, escape_markup(unquote_url(url)))
        # Escaped
        return escape_markup(self.information)

    @property
    def info_hash_as_b32(self):
        return base64.b32encode(self.info_hash).decode('utf-8')

    @property
    def info_hash_as_hex(self):
        return self.info_hash.hex()

    @property
    def magnet_uri(self):
        return create_magnet(self)

    @property
    def uploader_ip_string(self):
        if self.uploader_ip:
            return str(ip_address(self.uploader_ip))

    # Flag properties below

    anonymous = FlagProperty(TorrentFlags.ANONYMOUS)
    hidden = FlagProperty(TorrentFlags.HIDDEN)
    deleted = FlagProperty(TorrentFlags.DELETED)
    trusted = FlagProperty(TorrentFlags.TRUSTED)
    remake = FlagProperty(TorrentFlags.REMAKE)
    complete = FlagProperty(TorrentFlags.COMPLETE)

    # Class methods

    @classmethod
    def by_id(cls, id):
        return cls.query.get(id)

    @classmethod
    def by_info_hash(cls, info_hash):
        return cls.query.filter_by(info_hash=info_hash).first()

    @classmethod
    def by_info_hash_hex(cls, info_hash_hex):
        info_hash_bytes = bytearray.fromhex(info_hash_hex)
        return cls.by_info_hash(info_hash_bytes)


class TorrentFilelistBase(DeclarativeHelperBase):
    __tablename_base__ = 'torrents_filelist'

    __table_args__ = {'mysql_row_format': 'COMPRESSED'}

    @declarative.declared_attr
    def torrent_id(cls):
        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
        return db.Column(db.Integer, fk, primary_key=True)

    filelist_blob = db.Column(MediumBlobType, nullable=True)

    @declarative.declared_attr
    def torrent(cls):
        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False,
                               back_populates='filelist')


class TorrentInfoBase(DeclarativeHelperBase):
    __tablename_base__ = 'torrents_info'

    __table_args__ = {'mysql_row_format': 'COMPRESSED'}

    @declarative.declared_attr
    def torrent_id(cls):
        return db.Column(db.Integer, db.ForeignKey(
            cls._table_prefix('torrents.id'), ondelete="CASCADE"), primary_key=True)
    info_dict = db.Column(MediumBlobType, nullable=True)

    @declarative.declared_attr
    def torrent(cls):
        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False, back_populates='info')


class StatisticBase(DeclarativeHelperBase):
    __tablename_base__ = 'statistics'

    @declarative.declared_attr
    def torrent_id(cls):
        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
        return db.Column(db.Integer, fk, primary_key=True)

    seed_count = db.Column(db.Integer, default=0, nullable=False, index=True)
    leech_count = db.Column(db.Integer, default=0, nullable=False, index=True)
    download_count = db.Column(db.Integer, default=0, nullable=False, index=True)
    last_updated = db.Column(db.DateTime(timezone=False))

    @declarative.declared_attr
    def torrent(cls):
        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False,
                               back_populates='stats')


class Trackers(db.Model):
    __tablename__ = 'trackers'

    id = db.Column(db.Integer, primary_key=True)
    uri = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI),
                    nullable=False, unique=True)
    disabled = db.Column(db.Boolean, nullable=False, default=False)

    @classmethod
    def by_uri(cls, uri):
        return cls.query.filter_by(uri=uri).first()


class TorrentTrackersBase(DeclarativeHelperBase):
    __tablename_base__ = 'torrent_trackers'

    @declarative.declared_attr
    def torrent_id(cls):
        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
        return db.Column(db.Integer, fk, primary_key=True)

    @declarative.declared_attr
    def tracker_id(cls):
        fk = db.ForeignKey('trackers.id', ondelete="CASCADE")
        return db.Column(db.Integer, fk, primary_key=True)

    order = db.Column(db.Integer, nullable=False, index=True)

    @declarative.declared_attr
    def tracker(cls):
        return db.relationship('Trackers', uselist=False, lazy='joined')

    @classmethod
    def by_torrent_id(cls, torrent_id):
        return cls.query.filter_by(torrent_id=torrent_id).order_by(cls.order.desc())


class MainCategoryBase(DeclarativeHelperBase):
    __tablename_base__ = 'main_categories'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(length=64), nullable=False)

    @declarative.declared_attr
    def sub_categories(cls):
        return db.relationship(cls._flavor_prefix('SubCategory'), back_populates='main_category')

    @declarative.declared_attr
    def torrents(cls):
        return db.relationship(cls._flavor_prefix('Torrent'), back_populates='main_category')

    def get_category_ids(self):
        return (self.id, 0)

    @property
    def id_as_string(self):
        return '_'.join(str(x) for x in self.get_category_ids())

    @classmethod
    def by_id(cls, id):
        return cls.query.get(id)


class SubCategoryBase(DeclarativeHelperBase):
    __tablename_base__ = 'sub_categories'

    id = db.Column(db.Integer, primary_key=True)

    @declarative.declared_attr
    def main_category_id(cls):
        fk = db.ForeignKey(cls._table_prefix('main_categories.id'))
        return db.Column(db.Integer, fk, primary_key=True)

    name = db.Column(db.String(length=64), nullable=False)

    @declarative.declared_attr
    def main_category(cls):
        return db.relationship(cls._flavor_prefix('MainCategory'), uselist=False,
                               back_populates='sub_categories')

    def get_category_ids(self):
        return (self.main_category_id, self.id)

    @property
    def id_as_string(self):
        return '_'.join(str(x) for x in self.get_category_ids())

    @classmethod
    def by_category_ids(cls, main_cat_id, sub_cat_id):
        return cls.query.get((sub_cat_id, main_cat_id))


class CommentBase(DeclarativeHelperBase):
    __tablename_base__ = 'comments'

    id = db.Column(db.Integer, primary_key=True)

    @declarative.declared_attr
    def torrent_id(cls):
        return db.Column(db.Integer, db.ForeignKey(
            cls._table_prefix('torrents.id'), ondelete='CASCADE'), nullable=False)

    @declarative.declared_attr
    def user_id(cls):
        return db.Column(db.Integer, db.ForeignKey('users.id', ondelete='CASCADE'))

    created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow)
    text = db.Column(db.String(length=255, collation=COL_UTF8MB4_BIN), nullable=False)

    @declarative.declared_attr
    def user(cls):
        return db.relationship('User', uselist=False,
                               back_populates=cls._table_prefix('comments'), lazy="joined")

    def __repr__(self):
        return '<Comment %r>' % self.id

    @property
    def created_utc_timestamp(self):
        ''' Returns a UTC POSIX timestamp, as seconds '''
        return (self.created_time - UTC_EPOCH).total_seconds()


class UserLevelType(IntEnum):
    REGULAR = 0
    TRUSTED = 1
    MODERATOR = 2
    SUPERADMIN = 3


class UserStatusType(Enum):
    INACTIVE = 0
    ACTIVE = 1
    BANNED = 2


class User(db.Model):
    __tablename__ = 'users'

    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String(length=32, collation=COL_ASCII_GENERAL_CI),
                         unique=True, nullable=False)
    email = db.Column(EmailType(length=255, collation=COL_ASCII_GENERAL_CI),
                      unique=True, nullable=True)
    password_hash = db.Column(PasswordType(max_length=255, schemes=['argon2']), nullable=False)
    status = db.Column(ChoiceType(UserStatusType, impl=db.Integer()), nullable=False)
    level = db.Column(ChoiceType(UserLevelType, impl=db.Integer()), nullable=False)

    created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow)
    last_login_date = db.Column(db.DateTime(timezone=False), default=None, nullable=True)
    last_login_ip = db.Column(db.Binary(length=16), default=None, nullable=True)

    nyaa_torrents = db.relationship('NyaaTorrent', back_populates='user', lazy='dynamic')
    nyaa_comments = db.relationship('NyaaComment', back_populates='user', lazy='dynamic')

    sukebei_torrents = db.relationship('SukebeiTorrent', back_populates='user', lazy='dynamic')
    sukebei_comments = db.relationship('SukebeiComment', back_populates='user', lazy='dynamic')

    def __init__(self, username, email, password):
        self.username = username
        self.email = email
        self.password_hash = password
        self.status = UserStatusType.INACTIVE
        self.level = UserLevelType.REGULAR

    def __repr__(self):
        return '<User %r>' % self.username

    def validate_authorization(self, password):
        ''' Returns a boolean for whether the user can be logged in '''
        checks = [
            # Password must match
            password == self.password_hash,
            # Reject inactive and banned users
            self.status == UserStatusType.ACTIVE
        ]
        return all(checks)

    def gravatar_url(self):
        # from http://en.gravatar.com/site/implement/images/python/
        size = 120
        # construct the url
        default_avatar = flask.url_for('static', filename='img/avatar/default.png', _external=True)
        gravatar_url = 'https://www.gravatar.com/avatar/{}?{}'.format(
            md5(self.email.encode('utf-8').lower()).hexdigest(),
            urlencode({'d': default_avatar, 's': str(size)}))
        return gravatar_url

    @property
    def userlevel_str(self):
        if self.level == UserLevelType.REGULAR:
            return 'User'
        elif self.level == UserLevelType.TRUSTED:
            return 'Trusted'
        elif self.level >= UserLevelType.MODERATOR:
            return 'Moderator'

    @property
    def userlevel_color(self):
        if self.level == UserLevelType.REGULAR:
            return 'default'
        elif self.level == UserLevelType.TRUSTED:
            return 'success'
        elif self.level >= UserLevelType.MODERATOR:
            return 'purple'

    @property
    def ip_string(self):
        if self.last_login_ip:
            return str(ip_address(self.last_login_ip))

    @classmethod
    def by_id(cls, id):
        return cls.query.get(id)

    @classmethod
    def by_username(cls, username):
        user = cls.query.filter_by(username=username).first()
        return user

    @classmethod
    def by_email(cls, email):
        user = cls.query.filter_by(email=email).first()
        return user

    @classmethod
    def by_username_or_email(cls, username_or_email):
        return cls.by_username(username_or_email) or cls.by_email(username_or_email)

    @property
    def is_moderator(self):
        return self.level >= UserLevelType.MODERATOR

    @property
    def is_superadmin(self):
        return self.level == UserLevelType.SUPERADMIN

    @property
    def is_trusted(self):
        return self.level >= UserLevelType.TRUSTED


class ReportStatus(IntEnum):
    IN_REVIEW = 0
    VALID = 1
    INVALID = 2


class ReportBase(DeclarativeHelperBase):
    __tablename_base__ = 'reports'

    id = db.Column(db.Integer, primary_key=True)
    created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow)
    reason = db.Column(db.String(length=255), nullable=False)
    status = db.Column(ChoiceType(ReportStatus, impl=db.Integer()), nullable=False)

    @declarative.declared_attr
    def torrent_id(cls):
        return db.Column(db.Integer, db.ForeignKey(
            cls._table_prefix('torrents.id'), ondelete='CASCADE'), nullable=False)

    @declarative.declared_attr
    def user_id(cls):
        return db.Column(db.Integer, db.ForeignKey('users.id'))

    @declarative.declared_attr
    def user(cls):
        return db.relationship('User', uselist=False, lazy="joined")

    @declarative.declared_attr
    def torrent(cls):
        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False, lazy="joined")

    def __init__(self, torrent_id, user_id, reason):
        self.torrent_id = torrent_id
        self.user_id = user_id
        self.reason = reason
        self.status = ReportStatus.IN_REVIEW

    def __repr__(self):
        return '<Report %r>' % self.id

    @property
    def created_utc_timestamp(self):
        ''' Returns a UTC POSIX timestamp, as seconds '''
        return (self.created_time - UTC_EPOCH).total_seconds()

    @classmethod
    def by_id(cls, id):
        return cls.query.get(id)

    @classmethod
    def not_reviewed(cls, page):
        reports = cls.query.filter_by(status=0).paginate(page=page, per_page=20)
        return reports

    @classmethod
    def remove_reviewed(cls, id):
        return cls.query.filter(cls.torrent_id == id, cls.status == 0).delete()


# Actually declare our site-specific classes

# Torrent
class NyaaTorrent(TorrentBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiTorrent(TorrentBase, db.Model):
    __flavor__ = 'Sukebei'


# Fulltext models for MySQL
if app.config['USE_MYSQL']:
    class NyaaTorrentNameSearch(FullText, NyaaTorrent):
        __fulltext_columns__ = ('display_name',)
        __table_args__ = {'extend_existing': True}

    class SukebeiTorrentNameSearch(FullText, SukebeiTorrent):
        __fulltext_columns__ = ('display_name',)
        __table_args__ = {'extend_existing': True}
else:
    # Bogus classes for Sqlite
    class NyaaTorrentNameSearch(object):
        pass

    class SukebeiTorrentNameSearch(object):
        pass


# TorrentFilelist
class NyaaTorrentFilelist(TorrentFilelistBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiTorrentFilelist(TorrentFilelistBase, db.Model):
    __flavor__ = 'Sukebei'


# TorrentInfo
class NyaaTorrentInfo(TorrentInfoBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiTorrentInfo(TorrentInfoBase, db.Model):
    __flavor__ = 'Sukebei'


# Statistic
class NyaaStatistic(StatisticBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiStatistic(StatisticBase, db.Model):
    __flavor__ = 'Sukebei'


# TorrentTrackers
class NyaaTorrentTrackers(TorrentTrackersBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiTorrentTrackers(TorrentTrackersBase, db.Model):
    __flavor__ = 'Sukebei'


# MainCategory
class NyaaMainCategory(MainCategoryBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiMainCategory(MainCategoryBase, db.Model):
    __flavor__ = 'Sukebei'


# SubCategory
class NyaaSubCategory(SubCategoryBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiSubCategory(SubCategoryBase, db.Model):
    __flavor__ = 'Sukebei'


# Comment
class NyaaComment(CommentBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiComment(CommentBase, db.Model):
    __flavor__ = 'Sukebei'


# Report
class NyaaReport(ReportBase, db.Model):
    __flavor__ = 'Nyaa'


class SukebeiReport(ReportBase, db.Model):
    __flavor__ = 'Sukebei'


# Choose our defaults for models.Torrent etc
if app.config['SITE_FLAVOR'] == 'nyaa':
    Torrent = NyaaTorrent
    TorrentFilelist = NyaaTorrentFilelist
    TorrentInfo = NyaaTorrentInfo
    Statistic = NyaaStatistic
    TorrentTrackers = NyaaTorrentTrackers
    MainCategory = NyaaMainCategory
    SubCategory = NyaaSubCategory
    Comment = NyaaComment
    Report = NyaaReport

    TorrentNameSearch = NyaaTorrentNameSearch
elif app.config['SITE_FLAVOR'] == 'sukebei':
    Torrent = SukebeiTorrent
    TorrentFilelist = SukebeiTorrentFilelist
    TorrentInfo = SukebeiTorrentInfo
    Statistic = SukebeiStatistic
    TorrentTrackers = SukebeiTorrentTrackers
    MainCategory = SukebeiMainCategory
    SubCategory = SukebeiSubCategory
    Comment = SukebeiComment
    Report = SukebeiReport

    TorrentNameSearch = SukebeiTorrentNameSearch