From 5ace4af66aac8beaa10267378fe1d1b3cc2b3197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20=C4=8Ciha=C5=99?= Date: Thu, 19 Jan 2023 12:54:17 +0100 Subject: [PATCH] celery: Use django-celery-beat for storing schedule The beat file is problematic and it is better to have it in the database for distributed environments. This removes one dependency on shared filesystem, see #732 and #2984. --- ci/pip-install | 8 +++-- docs/admin/admin.rst | 4 +-- docs/admin/config.rst | 2 -- docs/admin/continuous.rst | 9 +++--- docs/admin/upgrade.rst | 11 +++++++ docs/changes.rst | 1 + requirements.txt | 3 +- weblate/examples/beat-settings.py | 12 -------- weblate/settings_docker.py | 3 +- weblate/settings_example.py | 3 +- weblate/settings_test.py | 1 - weblate/utils/checks.py | 1 - .../management/commands/cleanup_celery.py | 30 ++++--------------- weblate/utils/requirements.py | 1 + weblate/utils/tests/test_commands.py | 30 +++---------------- weblate/wladmin/sites.py | 16 ++++++++++ 16 files changed, 56 insertions(+), 79 deletions(-) delete mode 100644 weblate/examples/beat-settings.py diff --git a/ci/pip-install b/ci/pip-install index 39e17d4e804d..a14d2592e1e7 100755 --- a/ci/pip-install +++ b/ci/pip-install @@ -28,9 +28,11 @@ else pip install --upgrade-strategy eager -U -r requirements-test.txt if [ ${1:-latest} = edge ] ; then # Install from git / pre-release - pip install --upgrade https://github.com/translate/translate/archive/master.zip - pip install --upgrade https://github.com/WeblateOrg/language-data/archive/main.zip - pip install --upgrade https://github.com/WeblateOrg/translation-finder/archive/main.zip + pip install --no-deps --ignore-installed https://github.com/translate/translate/archive/master.zip + pip install --no-deps --ignore-installed https://github.com/WeblateOrg/language-data/archive/main.zip + pip install --no-deps --ignore-installed https://github.com/WeblateOrg/translation-finder/archive/main.zip + # Needed for Django 4.2 support + pip install --no-deps --ignore-installed https://github.com/celery/django-celery-beat/archive/main.zip pip install --upgrade --pre Django fi fi diff --git a/docs/admin/admin.rst b/docs/admin/admin.rst index 14d486a09700..d09feaaa908c 100644 --- a/docs/admin/admin.rst +++ b/docs/admin/admin.rst @@ -25,8 +25,8 @@ The Django admin interface .. warning:: - Will be removed in the future, - as its use is discouraged—most features can be managed directly in Weblate. + Use with caution as this is the low level interface. You should not need it + in most cases. Here you can manage objects stored in the database, such as users, translations and other settings: diff --git a/docs/admin/config.rst b/docs/admin/config.rst index 5a5b93c1a75f..8a846ebd1ed4 100644 --- a/docs/admin/config.rst +++ b/docs/admin/config.rst @@ -420,8 +420,6 @@ The following subdirectories usually exist: Version control repositories for translations. :file:`backups` Daily backup data, please check :ref:`backup-dumps` for details. -:file:`celery` - Celery scheduler data, see :ref:`celery`. :file:`fonts`: User-uploaded fonts, see :ref:`fonts`. :file:`cache` diff --git a/docs/admin/continuous.rst b/docs/admin/continuous.rst index 8794470415d2..1dbac88a32c6 100644 --- a/docs/admin/continuous.rst +++ b/docs/admin/continuous.rst @@ -377,10 +377,11 @@ fulfilled: :ref:`addon-weblate.git.squash` add-on in that case. If you want to commit changes more frequently and without checking of age, you -can schedule a regular task to perform a commit: - -.. literalinclude:: ../../weblate/examples/beat-settings.py - :language: python +can schedule a regular task to perform a commit. This can be done using +:guilabel:`Periodic Tasks` in :ref:`admin-interface`. First create desired +:guilabel:`Iterval` (for example 120 seconds). Then add new periodic task and +choose ``weblate.trans.tasks.commit_pending`` as :guilabel:`Task` with +``{"hours": 0}`` as :guilabel:`Keyword Arguments` and desired interval. .. _processing: diff --git a/docs/admin/upgrade.rst b/docs/admin/upgrade.rst index fa0ce56e990b..042579a4e889 100644 --- a/docs/admin/upgrade.rst +++ b/docs/admin/upgrade.rst @@ -365,6 +365,17 @@ Please follow :ref:`generic-upgrade-instructions` in order to perform update. .. seealso:: :ref:`generic-upgrade-instructions` +Upgrade from 4.15 to 4.16 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Please follow :ref:`generic-upgrade-instructions` in order to perform update. + +* Celery beat is now stroring the tasks schedule in the database, + ``CELERY_BEAT_SCHEDULER`` and :setting:`django:INSTALLED_APPS` need to be + changed for that. + +.. seealso:: :ref:`generic-upgrade-instructions` + .. _py3: Upgrading from Python 2 to Python 3 diff --git a/docs/changes.rst b/docs/changes.rst index f808492b5730..ccda4b3f3046 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -4,6 +4,7 @@ Weblate 4.16 Not yet released. * Format string checks now also detects duplicated formats. +* Celery beat is now stroring the tasks schedule in the database. `All changes in detail `__. diff --git a/requirements.txt b/requirements.txt index 71071160c58b..c8cb5020c22a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ borgbackup>=1.1.11,<1.3,!=1.2.0 -celery[redis]>=5.0.3,<5.3 +celery[redis]>=5.2.3,<5.3 charset-normalizer>=2.0.12,<4.0 cssselect>=1.2,<1.3 Cython>=0.29.14,<0.30 diff-match-patch==20200713 django-appconf>=1.0.3,<1.1 +django-celery-beat>=2.4.0,<2.5 django-compressor>=2.4,<5 django-crispy-forms>=1.9.0,<1.15 django-filter>=2.4.0,<23.0 diff --git a/weblate/examples/beat-settings.py b/weblate/examples/beat-settings.py deleted file mode 100644 index a5bcca13f9f3..000000000000 --- a/weblate/examples/beat-settings.py +++ /dev/null @@ -1,12 +0,0 @@ -CELERY_BEAT_SCHEDULE = { - # Unconditionally commit all changes every 2 minutes - "commit": { - "task": "weblate.trans.tasks.commit_pending", - # Omitting hours will honor per component settings, - # otherwise components with no changes older than this - # won't be committed - "kwargs": {"hours": 0}, - # How frequently to execute the job in seconds - "schedule": 120, - } -} diff --git a/weblate/settings_docker.py b/weblate/settings_docker.py index c7407ed706e5..9798449be454 100644 --- a/weblate/settings_docker.py +++ b/weblate/settings_docker.py @@ -695,6 +695,7 @@ "rest_framework", "rest_framework.authtoken", "django_filters", + "django_celery_beat", ] modify_env_list(INSTALLED_APPS, "APPS") @@ -1280,7 +1281,7 @@ # Celery settings, it is not recommended to change these CELERY_WORKER_MAX_MEMORY_PER_CHILD = 200000 -CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule") +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" CELERY_TASK_ROUTES = { "weblate.trans.tasks.auto_translate*": {"queue": "translate"}, "weblate.accounts.tasks.notify_*": {"queue": "notify"}, diff --git a/weblate/settings_example.py b/weblate/settings_example.py index bef732543e90..ab5b03e0f7e8 100644 --- a/weblate/settings_example.py +++ b/weblate/settings_example.py @@ -396,6 +396,7 @@ "rest_framework", "rest_framework.authtoken", "django_filters", + "django_celery_beat", ] # Custom exception reporter to include some details @@ -897,7 +898,7 @@ # Celery settings, it is not recommended to change these CELERY_WORKER_MAX_MEMORY_PER_CHILD = 200000 -CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule") +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" CELERY_TASK_ROUTES = { "weblate.trans.tasks.auto_translate*": {"queue": "translate"}, "weblate.accounts.tasks.notify_*": {"queue": "notify"}, diff --git a/weblate/settings_test.py b/weblate/settings_test.py index 21b7de7bf8a5..96198be76a4e 100644 --- a/weblate/settings_test.py +++ b/weblate/settings_test.py @@ -57,7 +57,6 @@ CACHE_DIR = os.path.join(DATA_DIR, "cache") MEDIA_ROOT = os.path.join(DATA_DIR, "media") STATIC_ROOT = os.path.join(DATA_DIR, "static") -CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule") CELERY_TASK_ALWAYS_EAGER = True CELERY_BROKER_URL = "memory://" CELERY_TASK_EAGER_PROPAGATES = True diff --git a/weblate/utils/checks.py b/weblate/utils/checks.py index 18bc27abe29e..6d535606a3b6 100644 --- a/weblate/utils/checks.py +++ b/weblate/utils/checks.py @@ -383,7 +383,6 @@ def check_data_writable(app_configs=None, **kwargs): data_dir("home"), data_dir("ssh"), data_dir("vcs"), - data_dir("celery"), data_dir("backups"), data_dir("fonts"), data_dir("cache", "fonts"), diff --git a/weblate/utils/management/commands/cleanup_celery.py b/weblate/utils/management/commands/cleanup_celery.py index c48cb3e0792c..29e6d8903fdb 100644 --- a/weblate/utils/management/commands/cleanup_celery.py +++ b/weblate/utils/management/commands/cleanup_celery.py @@ -2,35 +2,15 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import os +# TODO: Drop in Weblate 4.17 -from celery.beat import Service -from django.conf import settings - -from weblate.utils.celery import app from weblate.utils.management.base import BaseCommand class Command(BaseCommand): - help = "removes incompatible celery schedule file" - - @staticmethod - def try_remove(filename): - if os.path.exists(filename): - os.remove(filename) - - @staticmethod - def setup_schedule(): - service = Service(app=app) - scheduler = service.get_scheduler() - scheduler.setup_schedule() + help = "deprecated, will be removed in Weblate 4.17" def handle(self, *args, **options): - try: - self.setup_schedule() - except Exception as error: - if os.path.exists(settings.CELERY_BEAT_SCHEDULE_FILENAME): - self.stderr.write(f"Removing corrupted schedule file: {error!r}") - self.try_remove(settings.CELERY_BEAT_SCHEDULE_FILENAME) - self.try_remove(settings.CELERY_BEAT_SCHEDULE_FILENAME + ".db") - self.setup_schedule() + self.stderr.write( + "This command does nothing, it will be removed in Weblate 4.17!" + ) diff --git a/weblate/utils/requirements.py b/weblate/utils/requirements.py index 2b572bf3b330..b5c0793e1094 100644 --- a/weblate/utils/requirements.py +++ b/weblate/utils/requirements.py @@ -41,6 +41,7 @@ "rapidfuzz", "openpyxl", "celery", + "django-celery-beat", "kombu", "translation-finder", "weblate-language-data", diff --git a/weblate/utils/tests/test_commands.py b/weblate/utils/tests/test_commands.py index 79c24cb28052..13da0585f1f0 100644 --- a/weblate/utils/tests/test_commands.py +++ b/weblate/utils/tests/test_commands.py @@ -2,41 +2,19 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import os -from glob import glob from io import StringIO from django.core.management import call_command from django.test import SimpleTestCase, TestCase -from django.test.utils import override_settings from weblate.trans.tests.utils import TempDirMixin class CommandTests(SimpleTestCase, TempDirMixin): - def setUp(self): - self.create_temp() - self.beat = os.path.join(self.tempdir, "beat") - self.beat_db = os.path.join(self.tempdir, "beat.db") - - def tearDown(self): - self.remove_temp() - - def check_beat(self): - self.assertTrue(glob(self.beat + "*")) - - def test_none(self): - with override_settings(CELERY_BEAT_SCHEDULE_FILENAME=self.beat): - call_command("cleanup_celery") - self.check_beat() - - def test_broken(self): - for name in (self.beat, self.beat_db): - with open(name, "wb") as handle: - handle.write(b"\x00") - with override_settings(CELERY_BEAT_SCHEDULE_FILENAME=self.beat): - call_command("cleanup_celery") - self.check_beat() + def test_cleanup(self): + output = StringIO() + call_command("cleanup_celery", stderr=output) + self.assertIn(" it will be removed in Weblate", output.getvalue()) def test_queues(self): output = StringIO() diff --git a/weblate/wladmin/sites.py b/weblate/wladmin/sites.py index e4eba8c978ea..7a626da9dea1 100644 --- a/weblate/wladmin/sites.py +++ b/weblate/wladmin/sites.py @@ -11,6 +11,15 @@ from django.utils.decorators import method_decorator from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache +from django_celery_beat.admin import ( + ClockedSchedule, + ClockedScheduleAdmin, + CrontabSchedule, + IntervalSchedule, + PeriodicTask, + PeriodicTaskAdmin, + SolarSchedule, +) from rest_framework.authtoken.admin import TokenAdmin from rest_framework.authtoken.models import Token from social_django.admin import AssociationOption, NonceOption, UserSocialAuthOption @@ -160,6 +169,13 @@ def discover(self): # Django REST Framework self.register(Token, TokenAdmin) + # Django Celery Beat + self.register(IntervalSchedule) + self.register(CrontabSchedule) + self.register(SolarSchedule) + self.register(ClockedSchedule, ClockedScheduleAdmin) + self.register(PeriodicTask, PeriodicTaskAdmin) + # Simple SSO if "simple_sso.sso_server" in settings.INSTALLED_APPS: from simple_sso.sso_server.models import Consumer