diff --git a/mlp_api/mlp_api/__init__.py b/mlp_api/mlp_api/__init__.py index e69de29..fb989c4 100644 --- a/mlp_api/mlp_api/__init__.py +++ b/mlp_api/mlp_api/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ('celery_app',) diff --git a/mlp_api/mlp_api/celery.py b/mlp_api/mlp_api/celery.py new file mode 100644 index 0000000..86d75ab --- /dev/null +++ b/mlp_api/mlp_api/celery.py @@ -0,0 +1,18 @@ +import os +from celery import Celery + +# set default environment variable to the 'celery' program: +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mlp_api.settings') + +# create celery app and set related module's name ('mlp_api') +app = Celery('mlp_api') + +# Configure celery app using django's settings +# 'namespace' argument needs for refference to celery's settings in settings.py +# for example: 'CELERY_BROKER_URL' for message broker +app.config_from_object('django.conf:settings', namespace='CELERY') + +# If we need specify task for some django app, we can create 'tasks.py' in app's +# directory, create there required task, and say to celery app find all +# these django-app-specified tasks automatically +app.autodiscover_tasks() diff --git a/mlp_api/mlp_api/settings.py b/mlp_api/mlp_api/settings.py index bff4a79..58e3ed9 100644 --- a/mlp_api/mlp_api/settings.py +++ b/mlp_api/mlp_api/settings.py @@ -190,6 +190,11 @@ ), } +# Celery configuration: +CELERY_TASK_SERIALIZER = 'pickle' +CELERY_RESULT_SERIALIZER = 'pickle' +CELERY_ACCEPT_CONTENT = ['json', 'pickle'] + # Include settings for develop # File named settings_dev.py diff --git a/mlp_api/projectApp/models.py b/mlp_api/projectApp/models.py index ef7c0c6..4046657 100644 --- a/mlp_api/projectApp/models.py +++ b/mlp_api/projectApp/models.py @@ -1,3 +1,4 @@ +from io import BytesIO from django.db import models from django.utils.translation import ugettext_lazy as _ from django.core.validators import ( @@ -6,10 +7,10 @@ FileExtensionValidator ) from django.utils.text import slugify - from tinymce.models import HTMLField -from projectApp.validators import FileSizeValidator - +from projectApp.tasks import process_project_preview +from projectApp.utils import get_project_preview_path +from utils.validators import FileSizeValidator class Project(models.Model): """Represents project, published on the web-site.""" @@ -30,7 +31,7 @@ class Project(models.Model): ) preview = models.ImageField( _('preview'), - upload_to='projectApp/previews/', + upload_to=get_project_preview_path, validators=[ FileExtensionValidator(allowed_extensions=['jpg', 'jpeg']), FileSizeValidator(2.5 * 1024 * 1024) @@ -75,21 +76,44 @@ class Meta: verbose_name = _("enrolled project") verbose_name_plural = _("enrolled projects") + def _prepare_preview_update(self): + """ + Checks if there is some difference between present and old images, like: + old was replaced or removed, or old doesn't exist and a new was passed. + Returns dictionary with parameters for image processing task. + (dictionary will be empty if nothing of told above will happen). + """ + + old_qs = Project.objects.filter(pk=self.id) + old_preview = old_qs.first().preview if old_qs.exists() else None + preview_process_params = {} + if self.preview != old_preview: + if old_preview: + preview_process_params.update({'img_old_path': old_preview.name}) + if self.preview: + img_file = self.preview.file.file + # At the moment we don't need to save the whole uploaded image + # cause it will do asynchronous celery worker + # so let's put an empty byte stream instead actual image file: + self.preview.save(self.preview.name, BytesIO(), save=False) + preview_process_params.update({ + 'img_path': self.preview.name, + 'img_file': img_file + }) + + return preview_process_params + def save(self, *args, **kwargs): # Update slug field, if it's empty - write slugified title there: self.slug = self.slug.lower() if self.slug else slugify(self.title) - - # Check preview, if it has updated, remove old image: - try: - old_self = Project.objects.get(id=self.id) - if (not self.preview) or self.preview != old_self.preview: - old_self.preview.delete(save=False) - except: - # If it's a new project or a first preview, just don't do nothing: - pass + # Prepare preview update: + preview_process_params = self._prepare_preview_update() super().save(*args, **kwargs) + if preview_process_params: + process_project_preview.delay(**preview_process_params) + def __str__(self): return "'%s' project." % self.title diff --git a/mlp_api/projectApp/tasks.py b/mlp_api/projectApp/tasks.py new file mode 100644 index 0000000..d49b3ed --- /dev/null +++ b/mlp_api/projectApp/tasks.py @@ -0,0 +1,22 @@ +import os +from celery import shared_task +from django.conf import settings +from PIL import Image + +@shared_task +def process_project_preview(img_path=None, img_file=None, img_old_path=None): + """ + The task takes next arguments: + img_path, img_old_path - path, relative to MEDIA_ROOT, + to corresponding new and old preview images. + img_file - a new profile's preview image file, uploaded by user. + """ + if img_old_path is not None: + old_preview_abs_path = os.path.join(settings.MEDIA_ROOT, img_old_path) + try: + os.remove(old_preview_abs_path) + except FileNotFoundError: + pass + if img_path is not None and img_file is not None: + preview_abs_path = os.path.join(settings.MEDIA_ROOT, img_path) + Image.open(img_file).save(preview_abs_path, 'JPEG') diff --git a/mlp_api/projectApp/utils.py b/mlp_api/projectApp/utils.py new file mode 100644 index 0000000..23ecfa9 --- /dev/null +++ b/mlp_api/projectApp/utils.py @@ -0,0 +1,8 @@ +from os import path + +def get_project_preview_path(instance, filename): + """ + Takes an instance of Project model and name of preview's picture. + Returns a path to project's picture relative to MEDIA_ROOT. + """ + return path.join('projectApp', 'previews', instance.slug, filename) diff --git a/mlp_api/projectApp/validators.py b/mlp_api/projectApp/validators.py deleted file mode 100644 index a11a72e..0000000 --- a/mlp_api/projectApp/validators.py +++ /dev/null @@ -1,59 +0,0 @@ -from django.core.exceptions import ValidationError -from django.utils.translation import gettext_lazy as _ -from django.utils.deconstruct import deconstructible - -@deconstructible -class FileSizeValidator: - """ - Takes maximum size, allowed for a file, in bytes. - Creates validator for a FileField. - """ - message = _( - "Ensure that file size are less than %(max_size)s %(prefix)sBytes." - ) - code = 'limit_file_size' - - _decimal_prefixes = ('k', 'M', 'G') - - - def __init__(self, max_size, message=None, code=None): - self.max_size = max_size - if message is not None: - self.message = message - if code is not None: - self.code = code - - def __call__(self, value): - if value.size > self.max_size: - raise ValidationError( - self.message, - code=self.code, - params=self._get_pretty_size_params() - ) - - def __eq__(self, other): - return ( - isinstance(other, self.__class__) and - self.max_size == other.max_size and - self.message == other.message and - self.code == other.code - ) - - def _get_pretty_size_params(self): - """ - Returns a dictionary with message parameters - (max_size, prefix) as keys, and values, depends on - maximum allowed bytes for a file. - """ - max_size = self.max_size - prefix = '' - - options_quantity = len(FileSizeValidator._decimal_prefixes) - for i in range(options_quantity, 0, -1): - base = 1024 ** i - if self.max_size >= base: - max_size /= base - prefix = FileSizeValidator._decimal_prefixes[i - 1] - break - - return {'max_size':round(max_size, 2), 'prefix':prefix} diff --git a/mlp_api/projectApp/views.py b/mlp_api/projectApp/views.py index d2581d0..709f416 100644 --- a/mlp_api/projectApp/views.py +++ b/mlp_api/projectApp/views.py @@ -1,7 +1,7 @@ from rest_framework import viewsets from projectApp.models import Project from projectApp.serializers import ProjectSerializer -from projectApp.permissions import IsAdminUserOrReadOnly +from utils.permissions import IsAdminUserOrReadOnly class ProjectViewSet(viewsets.ModelViewSet): diff --git a/mlp_api/requirements.txt b/mlp_api/requirements.txt index 3bb3dee..9dd6f38 100644 --- a/mlp_api/requirements.txt +++ b/mlp_api/requirements.txt @@ -1,7 +1,7 @@ -amqp==1.4.9 +amqp==2.4.2 anyjson==0.3.3 -billiard==3.3.0.23 -celery==3.1.26.post2 +billiard==3.6.0.0 +celery==4.3.0 certifi==2019.3.9 cffi==1.12.3 chardet==3.0.4 @@ -20,7 +20,7 @@ gunicorn==19.9.0 idna==2.8 itypes==1.1.0 Jinja2==2.10.1 -kombu==3.0.37 +kombu==4.5.0 MarkupSafe==1.1.1 openapi-codec==1.3.2 Pillow==6.0.0 diff --git a/mlp_api/userApp/models.py b/mlp_api/userApp/models.py index 2c6314d..2f08e3b 100644 --- a/mlp_api/userApp/models.py +++ b/mlp_api/userApp/models.py @@ -4,18 +4,17 @@ from django.db import models from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from django.core.mail import send_mail +from django.utils.crypto import get_random_string from django.core.validators import MinLengthValidator, FileExtensionValidator from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin from django.conf import settings from PIL import Image from userApp.manager import UserManager -from userApp.utils import ( - get_userpic_path, - get_thumbnail_name, - generate_random_string -) -from userApp.validators import MinImageSizeValidator +from userApp.utils import get_userpic_path +from userApp.tasks import UserAvatarProcessing +from utils.tasks import send_email +from utils.validators import MinImageSizeValidator + class User(AbstractBaseUser, PermissionsMixin): login = models.CharField( @@ -57,7 +56,7 @@ def __str__(self): return self.login + ": " + self.email def email_user(self, subject, message, from_email=None, **kwargs): - send_mail(subject, message, from_email, [self.email], **kwargs) + send_email.delay(subject, message, from_email, [self.email], **kwargs) class UserPersonalData(models.Model): @@ -100,6 +99,11 @@ class UserPersonalData(models.Model): verbose_name=_('user\'s authorizing data') ) + process_avatar = UserAvatarProcessing.initialize( + avatar_size=(460, 460), + avatar_thumbnail_size=(150, 150) + ) + class Meta: verbose_name = _("personal data of a user") verbose_name_plural = _("personal data of users") @@ -108,62 +112,41 @@ def __str__(self): names = (self.first_name, self.last_name, self.auth_data.login) return "%s %s (%s)" % names - # Handling user's profile picture: - avatar_size = (460, 460) # size in pixels - avatar_thumbnail_size = (150, 150) # size in pixels - - def _get_avatar_dir(self): - """ Returns path to directory for user's avatar. """ - user_dir = "user_%s" % self.auth_data.id - return os.path.join(settings.MEDIA_ROOT, 'userApp', 'avatar', user_dir) - - def _make_avatar_from_field(self): - if not self.avatar: return None - - # Verify if there user picture's directory exists: - image_dir = self._get_avatar_dir() - if not os.path.exists(image_dir): - os.makedirs(image_dir) - - # Resizing given user's picture: - correct_image = Image.open(self.avatar.file).resize(self.avatar_size) - image_bytestream = io.BytesIO() - correct_image.save(image_bytestream, 'JPEG') - self.avatar.file = image_bytestream - - # Creating thumbnail: - correct_image.thumbnail(self.avatar_thumbnail_size) - thumbnail_name = get_thumbnail_name(self.avatar.name) - thumbnail_path = os.path.join(image_dir, thumbnail_name) - correct_image.save(thumbnail_path, 'JPEG') - - def _update_avatar(self): - # Check avatar, if it is updated or removed, delete old image: + def _prepare_avatar_update(self): + """ + Checks if there is some difference between present and old images, like: + old was replaced or removed, or old doesn't exist and a new was passed. + Returns dictionary with parameters for image processing task. + (dictionary will be empty if nothing of told above will happen). + """ + try: old_avatar = UserPersonalData.objects.get(id=self.id).avatar - # In case of changes of avatar: - if self.avatar != old_avatar: - # If old avatar is present, it should be removed: - if old_avatar: - # Remove avatar thumbnail: - thumbnail_path = os.path.join( - settings.MEDIA_ROOT, - get_thumbnail_name(old_avatar.name) - ) - os.remove(thumbnail_path) - # Remove avatar: - old_avatar.delete(save=False) - - # If there is a new avatar, it should be stored: - self._make_avatar_from_field() - - except (UserPersonalData.DoesNotExist, FileNotFoundError): - self._make_avatar_from_field() + except UserPersonalData.DoesNotExist: + old_avatar = None + + avatar_processing_params = {} + if self.avatar != old_avatar: + if old_avatar: + avatar_processing_params.update({'img_old_path': old_avatar.name}) + if self.avatar: + img_file = self.avatar.file.file + # At the moment we don't need to save the whole uploaded image + # cause it will do asynchronous celery worker + # so let's put an empty byte stream instead actual image file: + self.avatar.save(self.avatar.name, io.BytesIO(), save=False) + avatar_processing_params.update({ + 'img_path': self.avatar.name, + 'img_file': img_file + }) + + return avatar_processing_params def save(self, *args, **kwargs): - self._update_avatar() + avatar_update_params = self._prepare_avatar_update() super().save(*args, **kwargs) - + if avatar_update_params: + self.process_avatar.delay(**avatar_update_params) class UserProgress(models.Model): level = models.PositiveSmallIntegerField( @@ -257,6 +240,6 @@ def notificate_user(self): ) def save(self, *args, **kwargs): - self.code = generate_random_string(30, 40) + self.code = get_random_string(length=30) self.notificate_user() super().save(*args, **kwargs) diff --git a/mlp_api/userApp/tasks.py b/mlp_api/userApp/tasks.py new file mode 100644 index 0000000..f12385f --- /dev/null +++ b/mlp_api/userApp/tasks.py @@ -0,0 +1,53 @@ +import os +from celery import current_app +from django.conf import settings +from django.utils.text import get_valid_filename +from PIL import Image +from userApp.utils import get_thumbnail_name +from utils.filesystem import remove_directory_files + +class UserAvatarProcessing(current_app.Task): + """ + Class-based task for handling user's avatar download event. + Initializes with two arguments: avatar size and avatar's thumbnail size, + both arguments are tuples with two elements: width, height . + """ + name = 'userApp.tasks.process_user_avatar' + + def __init__(self, avatar_size, avatar_thumbnail_size): + self.avatar_size = avatar_size + self.avatar_thumbnail_size = avatar_thumbnail_size + + def _make_user_avatar(self, avatar, avatar_file): + # Resizing given user's picture: + correct_image = Image.open(avatar_file).resize(self.avatar_size) + correct_image.save(avatar, 'JPEG') + # Creating thumbnail: + thumbnail = get_thumbnail_name(avatar) + correct_image.thumbnail(self.avatar_thumbnail_size) + correct_image.save(thumbnail, 'JPEG') + + def run(self, img_path=None, img_file=None, img_old_path=None): + """ + The task takes next arguments: + path, old_path - path, relative to MEDIA_ROOT, + to corresponding new and old avatar images. + file - a new avatar image file, uploaded by user. + """ + + if img_old_path is not None: + old_avatar = os.path.join(settings.MEDIA_ROOT, img_old_path) + avar_dir = os.path.dirname(old_avatar) + if os.path.isdir(avar_dir): + remove_directory_files(avar_dir) + if img_path is not None and img_file is not None: + avatar = os.path.join(settings.MEDIA_ROOT, img_path) + self._make_user_avatar(avatar, img_file) + + @classmethod + def initialize(cls, avatar_size, avatar_thumbnail_size): + current_app.tasks.register(cls( + avatar_size=avatar_size, + avatar_thumbnail_size=avatar_thumbnail_size + )) + return current_app.tasks[cls.name] diff --git a/mlp_api/userApp/utils.py b/mlp_api/userApp/utils.py index 3172eb8..32c5cd4 100644 --- a/mlp_api/userApp/utils.py +++ b/mlp_api/userApp/utils.py @@ -1,8 +1,5 @@ from os import path import re -import random -from string import ascii_letters, digits - def get_userpic_path(instance, filename): """ @@ -18,11 +15,3 @@ def get_thumbnail_name(avatar_name): Return name of related thumbnail file. """ return re.sub(r'\.jpe?g$', lambda fnd: "_150x150"+fnd.group(0), avatar_name) - -def generate_random_string(min_len, max_len, chars=ascii_letters + digits): - """ - Takes minimum and maximum length of output string and one optional argument - - sequence of characters, on which a new random string will be based. - """ - randrange = range(random.randrange(min_len, max_len)) - return ''.join(random.choice(chars) for _ in randrange) diff --git a/mlp_api/utils/__init__.py b/mlp_api/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mlp_api/utils/filesystem.py b/mlp_api/utils/filesystem.py new file mode 100644 index 0000000..f38fddc --- /dev/null +++ b/mlp_api/utils/filesystem.py @@ -0,0 +1,7 @@ +import os + +def remove_directory_files(directory): + """Removes all files in given directory.""" + paths = (os.path.join(directory, item) for item in os.listdir(directory)) + files = (path for path in paths if os.path.isfile(path)) + for file in files: os.remove(file) diff --git a/mlp_api/projectApp/permissions.py b/mlp_api/utils/permissions.py similarity index 100% rename from mlp_api/projectApp/permissions.py rename to mlp_api/utils/permissions.py diff --git a/mlp_api/utils/tasks.py b/mlp_api/utils/tasks.py new file mode 100644 index 0000000..aa42741 --- /dev/null +++ b/mlp_api/utils/tasks.py @@ -0,0 +1,6 @@ +from celery import shared_task +from django.core.mail import send_mail + +@shared_task +def send_email(subject, message, from_email, to, **kwargs): + send_mail(subject, message, from_email, to, **kwargs) diff --git a/mlp_api/userApp/validators.py b/mlp_api/utils/validators.py similarity index 61% rename from mlp_api/userApp/validators.py rename to mlp_api/utils/validators.py index 7868f0b..762256a 100644 --- a/mlp_api/userApp/validators.py +++ b/mlp_api/utils/validators.py @@ -1,8 +1,70 @@ -from django.utils.deconstruct import deconstructible -from django.utils.translation import gettext_lazy as _ from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ +from django.utils.deconstruct import deconstructible from PIL import Image +@deconstructible +class FileSizeValidator: + """ + Takes maximum size, allowed for a file, in bytes. + Creates validator for a FileField. + """ + message = _( + "Ensure that file size are less than %(max_size)s %(prefix)sBytes." + ) + code = 'limit_file_size' + + _decimal_prefixes = ('k', 'M', 'G') + + + def __init__(self, max_size, message=None, code=None): + self.max_size = max_size + if message is not None: + self.message = message + if code is not None: + self.code = code + + def __call__(self, value): + try: + size = value.size + except FileNotFoundError: + # In case if image was removed manually from user's directory: + raise ValidationError("File was removed from project's directory.") + + if size > self.max_size: + raise ValidationError( + self.message, + code=self.code, + params=self._get_pretty_size_params() + ) + + def __eq__(self, other): + return ( + isinstance(other, self.__class__) and + self.max_size == other.max_size and + self.message == other.message and + self.code == other.code + ) + + def _get_pretty_size_params(self): + """ + Returns a dictionary with message parameters + (max_size, prefix) as keys, and values, depends on + maximum allowed bytes for a file. + """ + max_size = self.max_size + prefix = '' + + options_quantity = len(FileSizeValidator._decimal_prefixes) + for i in range(options_quantity, 0, -1): + base = 1024 ** i + if self.max_size >= base: + max_size /= base + prefix = FileSizeValidator._decimal_prefixes[i - 1] + break + + return {'max_size':round(max_size, 2), 'prefix':prefix} + @deconstructible class ImageSizeValidator: """