Include Celery

mlp_api/mlp_api/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

mlp_api/mlp_api/celery.py

@@ -0,0 +1,18 @@
+import os
+from celery import Celery
+
+# set default environment variable to the 'celery' program:
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mlp_api.settings')
+
+# create celery app and set related module's name ('mlp_api')
+app = Celery('mlp_api')
+
+# Configure celery app using django's settings
+# 'namespace' argument needs for refference to celery's settings in settings.py
+# for example: 'CELERY_BROKER_URL' for message broker
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# If we need specify task for some django app, we can create 'tasks.py' in app's
+# directory, create there required task, and say to celery app find all
+# these django-app-specified tasks automatically
+app.autodiscover_tasks()

mlp_api/mlp_api/settings.py

@@ -190,6 +190,11 @@
     ),
 }
 
+# Celery configuration:
+CELERY_TASK_SERIALIZER = 'pickle'
+CELERY_RESULT_SERIALIZER = 'pickle'
+CELERY_ACCEPT_CONTENT = ['json', 'pickle']
+
 
 # Include settings for develop
 # File named settings_dev.py

mlp_api/projectApp/models.py

@@ -1,3 +1,4 @@
+from io import BytesIO
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django.core.validators import (
@@ -6,10 +7,10 @@
     FileExtensionValidator
 )
 from django.utils.text import slugify
-
 from tinymce.models import HTMLField
-from projectApp.validators import FileSizeValidator
-
+from projectApp.tasks import process_project_preview
+from projectApp.utils import get_project_preview_path
+from utils.validators import FileSizeValidator
 
 class Project(models.Model):
     """Represents project, published on the web-site."""
@@ -30,7 +31,7 @@ class Project(models.Model):
     )
     preview = models.ImageField(
         _('preview'),
-        upload_to='projectApp/previews/',
+        upload_to=get_project_preview_path,
         validators=[
             FileExtensionValidator(allowed_extensions=['jpg', 'jpeg']),
             FileSizeValidator(2.5 * 1024 * 1024)
@@ -75,21 +76,44 @@ class Meta:
         verbose_name = _("enrolled project")
         verbose_name_plural = _("enrolled projects")
 
+    def _prepare_preview_update(self):
+        """
+        Checks if there is some difference between present and old images, like:
+        old was replaced or removed, or old doesn't exist and a new was passed.
+        Returns dictionary with parameters for image processing task.
+        (dictionary will be empty if nothing of told above will happen).
+        """
+
+        old_qs = Project.objects.filter(pk=self.id)
+        old_preview = old_qs.first().preview if old_qs.exists() else None
+        preview_process_params = {}
+        if self.preview != old_preview:
+            if old_preview:
+                preview_process_params.update({'img_old_path': old_preview.name})
+            if self.preview:
+                img_file = self.preview.file.file
+                # At the moment we don't need to save the whole uploaded image
+                # cause it will do asynchronous celery worker
+                # so let's put an empty byte stream instead actual image file:
+                self.preview.save(self.preview.name, BytesIO(), save=False)
+                preview_process_params.update({
+                    'img_path': self.preview.name,
+                    'img_file': img_file
+                })
+
+        return preview_process_params
+
     def save(self, *args, **kwargs):
         # Update slug field, if it's empty - write slugified title there:
         self.slug = self.slug.lower() if self.slug else slugify(self.title)
-
-        # Check preview, if it has updated, remove old image:
-        try:
-            old_self = Project.objects.get(id=self.id)
-            if (not self.preview) or self.preview != old_self.preview:
-                old_self.preview.delete(save=False)
-        except:
-            # If it's a new project or a first preview, just don't do nothing:
-            pass
+        # Prepare preview update:
+        preview_process_params = self._prepare_preview_update()
 
         super().save(*args, **kwargs)
 
+        if preview_process_params:
+            process_project_preview.delay(**preview_process_params)
+
     def __str__(self):
         return "'%s' project." % self.title
 

mlp_api/projectApp/tasks.py

@@ -0,0 +1,22 @@
+import os
+from celery import shared_task
+from django.conf import settings
+from PIL import Image
+
+@shared_task
+def process_project_preview(img_path=None, img_file=None, img_old_path=None):
+    """
+    The task takes next arguments:
+    img_path, img_old_path - path, relative to MEDIA_ROOT,
+        to corresponding new and old preview images.
+    img_file - a new profile's preview image file, uploaded by user.
+    """
+    if img_old_path is not None:
+        old_preview_abs_path = os.path.join(settings.MEDIA_ROOT, img_old_path)
+        try:
+            os.remove(old_preview_abs_path)
+        except FileNotFoundError:
+            pass
+    if img_path is not None and img_file is not None:
+        preview_abs_path = os.path.join(settings.MEDIA_ROOT, img_path)
+        Image.open(img_file).save(preview_abs_path, 'JPEG')

mlp_api/projectApp/utils.py

@@ -0,0 +1,8 @@
+from os import path
+
+def get_project_preview_path(instance, filename):
+    """
+    Takes an instance of Project model and name of preview's picture.
+    Returns a path to project's picture relative to MEDIA_ROOT.
+    """
+    return path.join('projectApp', 'previews', instance.slug, filename)

mlp_api/projectApp/views.py

@@ -1,7 +1,7 @@
 from rest_framework import viewsets
 from projectApp.models import Project
 from projectApp.serializers import ProjectSerializer
-from projectApp.permissions import IsAdminUserOrReadOnly
+from utils.permissions import IsAdminUserOrReadOnly
 
 
 class ProjectViewSet(viewsets.ModelViewSet):

mlp_api/requirements.txt

@@ -1,7 +1,7 @@
-amqp==1.4.9
+amqp==2.4.2
 anyjson==0.3.3
-billiard==3.3.0.23
-celery==3.1.26.post2
+billiard==3.6.0.0
+celery==4.3.0
 certifi==2019.3.9
 cffi==1.12.3
 chardet==3.0.4
@@ -20,7 +20,7 @@ gunicorn==19.9.0
 idna==2.8
 itypes==1.1.0
 Jinja2==2.10.1
-kombu==3.0.37
+kombu==4.5.0
 MarkupSafe==1.1.1
 openapi-codec==1.3.2
 Pillow==6.0.0

mlp_api/userApp/models.py

@@ -4,18 +4,17 @@
 from django.db import models
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
-from django.core.mail import send_mail
+from django.utils.crypto import get_random_string
 from django.core.validators import MinLengthValidator, FileExtensionValidator
 from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin
 from django.conf import settings
 from PIL import Image
 from userApp.manager import UserManager
-from userApp.utils import (
-    get_userpic_path,
-    get_thumbnail_name,
-    generate_random_string
-)
-from userApp.validators import MinImageSizeValidator
+from userApp.utils import get_userpic_path
+from userApp.tasks import UserAvatarProcessing
+from utils.tasks import send_email
+from utils.validators import MinImageSizeValidator
+
 
 class User(AbstractBaseUser, PermissionsMixin):
     login = models.CharField(
@@ -57,7 +56,7 @@ def __str__(self):
         return self.login + ": " + self.email
 
     def email_user(self, subject, message, from_email=None, **kwargs):
-        send_mail(subject, message, from_email, [self.email], **kwargs)
+        send_email.delay(subject, message, from_email, [self.email], **kwargs)
 
 
 class UserPersonalData(models.Model):
@@ -100,6 +99,11 @@ class UserPersonalData(models.Model):
         verbose_name=_('user\'s authorizing data')
     )
 
+    process_avatar = UserAvatarProcessing.initialize(
+        avatar_size=(460, 460),
+        avatar_thumbnail_size=(150, 150)
+    )
+
     class Meta:
         verbose_name = _("personal data of a user")
         verbose_name_plural = _("personal data of users")
@@ -108,62 +112,41 @@ def __str__(self):
         names = (self.first_name, self.last_name, self.auth_data.login)
         return "%s %s (%s)" % names
 
-    # Handling user's profile picture:
-    avatar_size = (460, 460) # size in pixels
-    avatar_thumbnail_size = (150, 150) # size in pixels
-
-    def _get_avatar_dir(self):
-        """ Returns path to directory for user's avatar. """
-        user_dir = "user_%s" % self.auth_data.id
-        return os.path.join(settings.MEDIA_ROOT, 'userApp', 'avatar', user_dir)
-
-    def _make_avatar_from_field(self):
-        if not self.avatar: return None
-
-        # Verify if there user picture's directory exists:
-        image_dir = self._get_avatar_dir()
-        if not os.path.exists(image_dir):
-            os.makedirs(image_dir)
-
-        # Resizing given user's picture:
-        correct_image = Image.open(self.avatar.file).resize(self.avatar_size)
-        image_bytestream = io.BytesIO()
-        correct_image.save(image_bytestream, 'JPEG')
-        self.avatar.file = image_bytestream
-
-        # Creating thumbnail:
-        correct_image.thumbnail(self.avatar_thumbnail_size)
-        thumbnail_name = get_thumbnail_name(self.avatar.name)
-        thumbnail_path = os.path.join(image_dir, thumbnail_name)
-        correct_image.save(thumbnail_path, 'JPEG')
-
-    def _update_avatar(self):
-        # Check avatar, if it is updated or removed, delete old image:
+    def _prepare_avatar_update(self):
+        """
+        Checks if there is some difference between present and old images, like:
+        old was replaced or removed, or old doesn't exist and a new was passed.
+        Returns dictionary with parameters for image processing task.
+        (dictionary will be empty if nothing of told above will happen).
+        """
+
         try:
             old_avatar = UserPersonalData.objects.get(id=self.id).avatar
-            # In case of changes of avatar:
-            if self.avatar != old_avatar:
-                # If old avatar is present, it should be removed:
-                if old_avatar:
-                    # Remove avatar thumbnail:
-                    thumbnail_path = os.path.join(
-                        settings.MEDIA_ROOT,
-                        get_thumbnail_name(old_avatar.name)
-                    )
-                    os.remove(thumbnail_path)
-                    # Remove avatar:
-                    old_avatar.delete(save=False)
-
-                # If there is a new avatar, it should be stored:
-                self._make_avatar_from_field()
-
-        except (UserPersonalData.DoesNotExist, FileNotFoundError):
-            self._make_avatar_from_field()
+        except UserPersonalData.DoesNotExist:
+            old_avatar = None
+
+        avatar_processing_params = {}
+        if self.avatar != old_avatar:
+            if old_avatar:
+                avatar_processing_params.update({'img_old_path': old_avatar.name})
+            if self.avatar:
+                img_file = self.avatar.file.file
+                # At the moment we don't need to save the whole uploaded image
+                # cause it will do asynchronous celery worker
+                # so let's put an empty byte stream instead actual image file:
+                self.avatar.save(self.avatar.name, io.BytesIO(), save=False)
+                avatar_processing_params.update({
+                    'img_path': self.avatar.name,
+                    'img_file': img_file
+                })
+
+        return avatar_processing_params
 
     def save(self, *args, **kwargs):
-        self._update_avatar()
+        avatar_update_params = self._prepare_avatar_update()
         super().save(*args, **kwargs)
-
+        if avatar_update_params:
+            self.process_avatar.delay(**avatar_update_params)
 
 class UserProgress(models.Model):
     level = models.PositiveSmallIntegerField(
@@ -257,6 +240,6 @@ def notificate_user(self):
         )
 
     def save(self, *args, **kwargs):
-        self.code = generate_random_string(30, 40)
+        self.code = get_random_string(length=30)
         self.notificate_user()
         super().save(*args, **kwargs)