celery: Use django-celery-beat for storing schedule

The beat file is problematic and it is better to have it in the database
for distributed environments.

This removes one dependency on shared filesystem, see WeblateOrg#732 and WeblateOrg#2984.

ci/pip-install

@@ -28,9 +28,11 @@ else
     pip install --upgrade-strategy eager -U -r requirements-test.txt
     if [ ${1:-latest} = edge ] ; then
         # Install from git / pre-release
-        pip install --upgrade https://github.com/translate/translate/archive/master.zip
-        pip install --upgrade https://github.com/WeblateOrg/language-data/archive/main.zip
-        pip install --upgrade https://github.com/WeblateOrg/translation-finder/archive/main.zip
+        pip install --no-deps --ignore-installed https://github.com/translate/translate/archive/master.zip
+        pip install --no-deps --ignore-installed https://github.com/WeblateOrg/language-data/archive/main.zip
+        pip install --no-deps --ignore-installed https://github.com/WeblateOrg/translation-finder/archive/main.zip
+        # Needed for Django 4.2 support
+        pip install --no-deps --ignore-installed https://github.com/celery/django-celery-beat/archive/main.zip
         pip install --upgrade --pre Django
     fi
 fi

docs/admin/admin.rst

@@ -25,8 +25,8 @@ The Django admin interface
 
 .. warning::
 
-   Will be removed in the future,
-   as its use is discouraged—most features can be managed directly in Weblate.
+   Use with caution as this is the low level interface. You should not need it
+   in most cases.
 
 Here you can manage objects stored in the database, such as users, translations
 and other settings:

docs/admin/config.rst

@@ -420,8 +420,6 @@ The following subdirectories usually exist:
     Version control repositories for translations.
 :file:`backups`
     Daily backup data, please check :ref:`backup-dumps` for details.
-:file:`celery`
-    Celery scheduler data, see :ref:`celery`.
 :file:`fonts`:
     User-uploaded  fonts, see :ref:`fonts`.
 :file:`cache`

docs/admin/continuous.rst

@@ -377,10 +377,11 @@ fulfilled:
    :ref:`addon-weblate.git.squash` add-on in that case.
 
 If you want to commit changes more frequently and without checking of age, you
-can schedule a regular task to perform a commit:
-
-.. literalinclude:: ../../weblate/examples/beat-settings.py
-    :language: python
+can schedule a regular task to perform a commit. This can be done using
+:guilabel:`Periodic Tasks` in :ref:`admin-interface`. First create desired
+:guilabel:`Iterval` (for example 120 seconds). Then add new periodic task and
+choose ``weblate.trans.tasks.commit_pending`` as :guilabel:`Task` with
+``{"hours": 0}`` as :guilabel:`Keyword Arguments` and desired interval.
 
 .. _processing:
 

docs/admin/upgrade.rst

@@ -365,6 +365,17 @@ Please follow :ref:`generic-upgrade-instructions` in order to perform update.
 
 .. seealso:: :ref:`generic-upgrade-instructions`
 
+Upgrade from 4.15 to 4.16
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Please follow :ref:`generic-upgrade-instructions` in order to perform update.
+
+* Celery beat is now stroring the tasks schedule in the database,
+  ``CELERY_BEAT_SCHEDULER`` and :setting:`django:INSTALLED_APPS` need to be
+  changed for that.
+
+.. seealso:: :ref:`generic-upgrade-instructions`
+
 .. _py3:
 
 Upgrading from Python 2 to Python 3

docs/changes.rst

@@ -4,6 +4,7 @@ Weblate 4.16
 Not yet released.
 
 * Format string checks now also detects duplicated formats.
+* Celery beat is now stroring the tasks schedule in the database.
 
 `All changes in detail <https://github.com/WeblateOrg/weblate/milestone/89?closed=1>`__.
 

requirements.txt

@@ -1,10 +1,11 @@
 borgbackup>=1.1.11,<1.3,!=1.2.0
-celery[redis]>=5.0.3,<5.3
+celery[redis]>=5.2.3,<5.3
 charset-normalizer>=2.0.12,<4.0
 cssselect>=1.2,<1.3
 Cython>=0.29.14,<0.30
 diff-match-patch==20200713
 django-appconf>=1.0.3,<1.1
+django-celery-beat>=2.4.0,<2.5
 django-compressor>=2.4,<5
 django-crispy-forms>=1.9.0,<1.15
 django-filter>=2.4.0,<23.0

weblate/settings_docker.py

@@ -695,6 +695,7 @@
     "rest_framework",
     "rest_framework.authtoken",
     "django_filters",
+    "django_celery_beat",
 ]
 
 modify_env_list(INSTALLED_APPS, "APPS")
@@ -1280,7 +1281,7 @@
 
 # Celery settings, it is not recommended to change these
 CELERY_WORKER_MAX_MEMORY_PER_CHILD = 200000
-CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule")
+CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
 CELERY_TASK_ROUTES = {
     "weblate.trans.tasks.auto_translate*": {"queue": "translate"},
     "weblate.accounts.tasks.notify_*": {"queue": "notify"},

weblate/settings_example.py

@@ -396,6 +396,7 @@
     "rest_framework",
     "rest_framework.authtoken",
     "django_filters",
+    "django_celery_beat",
 ]
 
 # Custom exception reporter to include some details
@@ -897,7 +898,7 @@
 
 # Celery settings, it is not recommended to change these
 CELERY_WORKER_MAX_MEMORY_PER_CHILD = 200000
-CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule")
+CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
 CELERY_TASK_ROUTES = {
     "weblate.trans.tasks.auto_translate*": {"queue": "translate"},
     "weblate.accounts.tasks.notify_*": {"queue": "notify"},

weblate/settings_test.py

@@ -57,7 +57,6 @@
 CACHE_DIR = os.path.join(DATA_DIR, "cache")
 MEDIA_ROOT = os.path.join(DATA_DIR, "media")
 STATIC_ROOT = os.path.join(DATA_DIR, "static")
-CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule")
 CELERY_TASK_ALWAYS_EAGER = True
 CELERY_BROKER_URL = "memory://"
 CELERY_TASK_EAGER_PROPAGATES = True

weblate/utils/checks.py

@@ -383,7 +383,6 @@ def check_data_writable(app_configs=None, **kwargs):
         data_dir("home"),
         data_dir("ssh"),
         data_dir("vcs"),
-        data_dir("celery"),
         data_dir("backups"),
         data_dir("fonts"),
         data_dir("cache", "fonts"),

weblate/utils/management/commands/cleanup_celery.py

@@ -2,35 +2,15 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-import os
+# TODO: Drop in Weblate 4.17
 
-from celery.beat import Service
-from django.conf import settings
-
-from weblate.utils.celery import app
 from weblate.utils.management.base import BaseCommand
 
 
 class Command(BaseCommand):
-    help = "removes incompatible celery schedule file"
-
-    @staticmethod
-    def try_remove(filename):
-        if os.path.exists(filename):
-            os.remove(filename)
-
-    @staticmethod
-    def setup_schedule():
-        service = Service(app=app)
-        scheduler = service.get_scheduler()
-        scheduler.setup_schedule()
+    help = "deprecated, will be removed in Weblate 4.17"
 
     def handle(self, *args, **options):
-        try:
-            self.setup_schedule()
-        except Exception as error:
-            if os.path.exists(settings.CELERY_BEAT_SCHEDULE_FILENAME):
-                self.stderr.write(f"Removing corrupted schedule file: {error!r}")
-                self.try_remove(settings.CELERY_BEAT_SCHEDULE_FILENAME)
-                self.try_remove(settings.CELERY_BEAT_SCHEDULE_FILENAME + ".db")
-                self.setup_schedule()
+        self.stderr.write(
+            "This command does nothing, it will be removed in Weblate 4.17!"
+        )

weblate/utils/requirements.py

@@ -41,6 +41,7 @@
     "rapidfuzz",
     "openpyxl",
     "celery",
+    "django-celery-beat",
     "kombu",
     "translation-finder",
     "weblate-language-data",

weblate/utils/tests/test_commands.py

@@ -2,41 +2,19 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-import os
-from glob import glob
 from io import StringIO
 
 from django.core.management import call_command
 from django.test import SimpleTestCase, TestCase
-from django.test.utils import override_settings
 
 from weblate.trans.tests.utils import TempDirMixin
 
 
 class CommandTests(SimpleTestCase, TempDirMixin):
-    def setUp(self):
-        self.create_temp()
-        self.beat = os.path.join(self.tempdir, "beat")
-        self.beat_db = os.path.join(self.tempdir, "beat.db")
-
-    def tearDown(self):
-        self.remove_temp()
-
-    def check_beat(self):
-        self.assertTrue(glob(self.beat + "*"))
-
-    def test_none(self):
-        with override_settings(CELERY_BEAT_SCHEDULE_FILENAME=self.beat):
-            call_command("cleanup_celery")
-        self.check_beat()
-
-    def test_broken(self):
-        for name in (self.beat, self.beat_db):
-            with open(name, "wb") as handle:
-                handle.write(b"\x00")
-        with override_settings(CELERY_BEAT_SCHEDULE_FILENAME=self.beat):
-            call_command("cleanup_celery")
-        self.check_beat()
+    def test_cleanup(self):
+        output = StringIO()
+        call_command("cleanup_celery", stderr=output)
+        self.assertIn(" it will be removed in Weblate", output.getvalue())
 
     def test_queues(self):
         output = StringIO()

weblate/wladmin/sites.py

@@ -11,6 +11,15 @@
 from django.utils.decorators import method_decorator
 from django.utils.translation import gettext_lazy as _
 from django.views.decorators.cache import never_cache
+from django_celery_beat.admin import (
+    ClockedSchedule,
+    ClockedScheduleAdmin,
+    CrontabSchedule,
+    IntervalSchedule,
+    PeriodicTask,
+    PeriodicTaskAdmin,
+    SolarSchedule,
+)
 from rest_framework.authtoken.admin import TokenAdmin
 from rest_framework.authtoken.models import Token
 from social_django.admin import AssociationOption, NonceOption, UserSocialAuthOption
@@ -160,6 +169,13 @@ def discover(self):
         # Django REST Framework
         self.register(Token, TokenAdmin)
 
+        # Django Celery Beat
+        self.register(IntervalSchedule)
+        self.register(CrontabSchedule)
+        self.register(SolarSchedule)
+        self.register(ClockedSchedule, ClockedScheduleAdmin)
+        self.register(PeriodicTask, PeriodicTaskAdmin)
+
         # Simple SSO
         if "simple_sso.sso_server" in settings.INSTALLED_APPS:
             from simple_sso.sso_server.models import Consumer