Feature/async tasks (#286)

.travis.yml

@@ -10,6 +10,8 @@ env:
 install:
   - pip install --upgrade pip
   - pip install .[files]
+  - pip install .[huey]
+  - pip install .[celery]
   - pip uninstall django-spirit -y
   - pip install -q Django==$DJANGO
   - pip install coveralls pep8==1.5.7 flake8
@@ -36,3 +38,19 @@ jobs:
       install: []
       env: []
       after_success: []
+    - name: "No opt deps"
+      language: python
+      python: "3.8"
+      install:
+        - pip install --upgrade pip
+        - pip install .
+        - pip uninstall django-spirit -y
+        - pip install -q Django==3.0.3
+      script:
+        - python ./spirit/extra/bin/spirit.py startproject project
+        - export PYTHONWARNINGS="default"
+        - export ST_UPLOAD_FILE_ENABLED=0
+        - export ST_INSTALL_HUEY=0
+        - python runtests.py
+      env: []
+      after_success: []

HISTORY.md

@@ -2,7 +2,12 @@
 ==================
 
 * Added Kyrgyz translation (thanks to @jumasheff)
-* Added `django.contrib.humanize` to `INSTALLED_APPS`
+* Added `django.contrib.humanize` to `INSTALLED_APPS` settings
+* Added `HAYSTACK_SIGNAL_PROCESSOR` to settings
+* Added support for `Huey` and `Celery` task managers;
+  added `ST_TASK_MANAGER` and `ST_SEARCH_INDEX_UPDATE_HOURS` settings
+* Added realtime search indexation, full search index (periodic) update,
+  and send email tasks
 * Fixed user comment reply button
 * Fixed paginator current page button (issue #168)
 * Fixed ordered list style for the comments (issue #134)

runtests.py

@@ -14,6 +14,16 @@
 os.environ['DJANGO_SETTINGS_MODULE'] = 'project.project.settings.test'
 
 
+def setup_celery():
+    try:
+        from celery import Celery
+    except ImportError:
+        return
+    app = Celery('test')
+    app.config_from_object('django.conf:settings', namespace='CELERY')
+    app.autodiscover_tasks()
+
+
 def log_warnings():
     logger = logging.getLogger('py.warnings')
     handler = logging.StreamHandler()
@@ -30,6 +40,7 @@ def run_tests(reverse=False):
 def start():
     django.setup()
     log_warnings()
+    setup_celery()
     if run_tests() or run_tests(reverse=True):
         sys.exit(1)
 

setup.py

@@ -45,7 +45,9 @@
     zip_safe=False,
     install_requires=REQUIREMENTS,
     extras_require={
-        'files': PYTHON_MAGIC_DEP},
+        'files': PYTHON_MAGIC_DEP,
+        'huey': 'huey == 2.3.0',
+        'celery': 'celery == 4.4.7'},
     license='MIT License',
     classifiers=[
         'Development Status :: 5 - Production/Stable',

spirit/comment/tests.py

@@ -5,6 +5,7 @@
 import shutil
 import hashlib
 import io
+from unittest import skipIf
 
 from django.test import TestCase, RequestFactory
 from django.urls import reverse
@@ -628,6 +629,7 @@ def test_comment_image_upload_invalid(self):
         self.assertIn('error', res.keys())
         self.assertIn('image', res['error'].keys())
 
+    @skipIf(not settings.ST_UPLOAD_FILE_ENABLED, 'No magic file support')
     @override_settings(
         MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'media_test'),
         FILE_UPLOAD_MAX_MEMORY_SIZE=2621440,
@@ -665,6 +667,7 @@ def test_comment_file_upload(self):
 
         shutil.rmtree(settings.MEDIA_ROOT)  # cleanup
 
+    @skipIf(not settings.ST_UPLOAD_FILE_ENABLED, 'No magic file support')
     @override_settings(
         MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'media_test'),
         FILE_UPLOAD_MAX_MEMORY_SIZE=1,
@@ -702,6 +705,7 @@ def test_comment_file_upload_tmp_file(self):
 
         shutil.rmtree(settings.MEDIA_ROOT)  # cleanup
 
+    @skipIf(not settings.ST_UPLOAD_FILE_ENABLED, 'No magic file support')
     @override_settings(MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'media_test'))
     def test_comment_file_upload_unique(self):
         user_files_parts = ('spirit', 'files', str(self.user.pk))
@@ -738,6 +742,7 @@ def test_comment_file_upload_unique(self):
             user_media, os.listdir(user_media)[0], file_name))
         shutil.rmtree(settings.MEDIA_ROOT)  # cleanup
 
+    @skipIf(not settings.ST_UPLOAD_FILE_ENABLED, 'No magic file support')
     @override_settings(MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'media_test'))
     def test_comment_file_upload_unique_no_duplication(self):
         utils.login(self)
@@ -768,6 +773,7 @@ def test_comment_file_upload_unique_no_duplication(self):
 
         self.assertNotEqual(first_url, second_url)
 
+    @skipIf(not settings.ST_UPLOAD_FILE_ENABLED, 'No magic file support')
     def test_comment_file_upload_invalid_ext(self):
         """
         comment file upload, invalid file extension
@@ -791,6 +797,7 @@ def test_comment_file_upload_invalid_ext(self):
             res['error']['file'],
             ['Unsupported file extension gif. Supported extensions are doc, docx, pdf.'])
 
+    @skipIf(not settings.ST_UPLOAD_FILE_ENABLED, 'No magic file support')
     def test_comment_file_upload_invalid_mime(self):
         """
         comment file upload, invalid mime type

spirit/comment/utils.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 
-from ..topic.notification.models import TopicNotification, UNDEFINED
-from ..topic.unread.models import TopicUnread
+from spirit.core import tasks
+from spirit.topic.notification.models import TopicNotification, UNDEFINED
+from spirit.topic.unread.models import TopicUnread
 from .history.models import CommentHistory
 from .poll.utils.render_static import post_render_static_polls
 
@@ -12,6 +13,7 @@ def comment_posted(comment, mentions):
     TopicNotification.notify_new_mentions(comment=comment, mentions=mentions)
     TopicUnread.unread_new_comment(comment=comment)
     comment.topic.increase_comment_count()
+    tasks.search_index_update(topic_pk=comment.topic.pk)
 
 
 def pre_comment_update(comment):

spirit/core/conf/defaults.py

@@ -7,6 +7,20 @@
 
 import os
 
+#: The task manager to run delayed and periodic tasks
+#: such as send emails, update the search index, clean up django
+#: sessions, etc. Valid values are: ``'celery'``, ``'huey'``, and
+#: ``None``
+ST_TASK_MANAGER = None
+
+#: The age in hours of the items
+#: to index into the search index on each update.
+#: The update runs every this amount of time
+#: when ``ST_TASK_MANAGER`` is set to ``'huey'``.
+#: Other task managers will need to sync their
+#: configuration to this value
+ST_SEARCH_INDEX_UPDATE_HOURS = 24
+
 #: The category's PK containing all of the private topics.
 #: The category is auto-created and so this value should not change
 ST_TOPIC_PRIVATE_CATEGORY_PK = 1

spirit/core/signals.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+import django.dispatch
+
+search_index_update = django.dispatch.Signal(
+    providing_args=['sender', 'instance'])

spirit/core/tasks.py

@@ -1,35 +1,103 @@
 # -*- coding: utf-8 -*-
 
+import logging
+
+from django.db import transaction
+from django.core.mail import send_mail
+from django.apps import apps
+from django.core.management import call_command
+
 from .conf import settings
+from . import signals
 
-try:
-    # TODO: remove this try block.
-    from celery.decorators import task
-except ImportError:
-    task = None
+logger = logging.getLogger(__name__)
 
 
-if not hasattr(settings, 'BROKER_URL'):
-    def task(f):
-        f.delay = f
-        return f
+# XXX support custom task manager __import__('foo.task')?
+def task_manager(tm):
+    if tm == 'celery':
+        from celery import shared_task
+        def task(t):
+            t = shared_task(t)
+            def _task(*args, **kwargs):
+                return t.delay(*args, **kwargs)
+            return _task
+        return task
+    if tm == 'huey':
+        from huey.contrib.djhuey import db_task
+        return db_task()
+    assert tm is None
+    return lambda t: t
 
+task = task_manager(settings.ST_TASK_MANAGER)
 
-@task
-def send_notification():
-    pass
 
+def periodic_task_manager(tm):
+    if tm == 'huey':
+        from huey import crontab
+        from huey.contrib.djhuey import db_periodic_task
+        def periodic_task(hours):
+            return db_periodic_task(crontab(
+                minute='0', hour='*/{}'.format(hours)))
+        return periodic_task
+    assert tm in ('celery', None)
+    def fake_periodic_task(*args, **kwargs):
+        return task_manager(tm)
+    return fake_periodic_task
 
-@task
-def backup_database():
-    pass
+periodic_task = periodic_task_manager(settings.ST_TASK_MANAGER)
 
 
-@task
-def search_index_update():
-    pass
+def delayed_task(t):
+    t = task(t)  # wrap at import time
+    def delayed_task_inner(*args, **kwargs):
+        transaction.on_commit(lambda: t(*args, **kwargs))
+    return delayed_task_inner
+
+
+@delayed_task
+def send_email(subject, message, from_email, recipients):
+    # Avoid retrying this task. It's better to log the exception
+    # here instead of possibly spamming users on retry
+    # We send to one recipient at the time, because otherwise
+    # it'll likely get flagged as spam, or it won't reach the
+    # the recipient at all
+    for recipient in recipients:
+        try:
+            send_mail(
+                subject=subject,
+                message=message,
+                from_email=from_email,
+                recipient_list=[recipient])
+        except OSError as err:
+            logger.exception(err)
+            return  # bail out
+
 
+@delayed_task
+def search_index_update(topic_pk):
+    # Indexing is too expensive; bail if
+    # there's no dedicated task manager
+    if settings.ST_TASK_MANAGER is None:
+        return
+    Topic = apps.get_model('spirit_topic.Topic')
+    signals.search_index_update.send(
+        sender=Topic,
+        instance=Topic.objects.get(pk=topic_pk))
 
-@task
+
+@periodic_task(hours=settings.ST_SEARCH_INDEX_UPDATE_HOURS)
+def full_search_index_update():
+    age = settings.ST_SEARCH_INDEX_UPDATE_HOURS
+    call_command("update_index", age=age)
+
+
+@delayed_task
 def clean_sessions():
     pass
+
+
+@delayed_task
+def backup_database():
+    pass
+

spirit/core/templatetags/spirit_tags.py

@@ -20,7 +20,6 @@
 __all__ = [
     'comment',
     'comment_like',
-    'comment_poll',
     'search',
     'topic_favorite',
     'topic_notification',

spirit/core/tests/migrations/0002_auto_20200911_1759.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.0.7 on 2020-09-11 17:59
+
+from django.db import migrations, models
+import spirit.core.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tests', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TaskResultModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('result', models.CharField(max_length=255)),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='autoslugbadpopulatefrommodel',
+            name='slug',
+            field=spirit.core.utils.models.AutoSlugField(allow_unicode=True, populate_from='bad'),
+        ),
+        migrations.AlterField(
+            model_name='autoslugdefaultmodel',
+            name='slug',
+            field=spirit.core.utils.models.AutoSlugField(allow_unicode=True, default='foo'),
+        ),
+        migrations.AlterField(
+            model_name='autoslugpopulatefrommodel',
+            name='slug',
+            field=spirit.core.utils.models.AutoSlugField(allow_unicode=True, populate_from='title'),
+        ),
+    ]

spirit/core/tests/models/__init__.py

@@ -4,8 +4,10 @@
     AutoSlugPopulateFromModel, AutoSlugModel,
     AutoSlugDefaultModel, AutoSlugBadPopulateFromModel
 )
+from .task_result import TaskResultModel
 
 __all__ = [
     'AutoSlugPopulateFromModel', 'AutoSlugModel',
-    'AutoSlugDefaultModel', 'AutoSlugBadPopulateFromModel'
+    'AutoSlugDefaultModel', 'AutoSlugBadPopulateFromModel',
+    'TaskResultModel'
 ]

spirit/core/tests/models/task_result.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+
+from django.db import models
+
+
+class TaskResultModel(models.Model):
+
+    result = models.CharField(max_length=255)