feat(daemon): allows daemonization options to be fetched from app set…

…tings (#8553)

* feat(daemon): allows daemonization options to be fetched from app settings

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Update docs/userguide/configuration.rst

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* Apply suggestions from code review

Co-authored-by: Omer Katz <omer.katz@kcg.tech>

* doc(configuration): add version added markers to the new daemonization settings

---------

Co-authored-by: Asif Saif Uddin <auvipy@gmail.com>
Co-authored-by: Omer Katz <omer.katz@kcg.tech>
Co-authored-by: Tomer Nosrati <tomer.nosrati@kcg.tech>

celery/bin/base.py

@@ -4,9 +4,10 @@
 from collections import OrderedDict
 from functools import update_wrapper
 from pprint import pformat
+from typing import Any
 
 import click
-from click import ParamType
+from click import Context, ParamType
 from kombu.utils.objects import cached_property
 
 from celery._state import get_current_app
@@ -170,19 +171,36 @@ def format_options(self, ctx, formatter):
                 formatter.write_dl(opts_group)
 
 
+class DaemonOption(CeleryOption):
+    """Common daemonization option"""
+    def __init__(self, *args, **kwargs):
+        super().__init__(args,
+                         help_group=kwargs.pop("help_group", "Daemonization Options"),
+                         callback=kwargs.pop("callback", self.daemon_setting),
+                         **kwargs)
+
+    def daemon_setting(self, ctx: Context, opt: CeleryOption, value: Any) -> Any:
+        """
+        Try to fetch deamonization option from applications settings.
+        Use the daemon command name as prefix (eg. `worker` -> `worker_pidfile`)
+        """
+        return value or getattr(ctx.obj.app.conf, f"{ctx.command.name}_{self.name}", None)
+
+
 class CeleryDaemonCommand(CeleryCommand):
     """Daemon commands."""
 
     def __init__(self, *args, **kwargs):
         """Initialize a Celery command with common daemon options."""
         super().__init__(*args, **kwargs)
-        self.params.append(CeleryOption(('-f', '--logfile'), help_group="Daemonization Options",
-                           help="Log destination; defaults to stderr"))
-        self.params.append(CeleryOption(('--pidfile',), help_group="Daemonization Options"))
-        self.params.append(CeleryOption(('--uid',), help_group="Daemonization Options"))
-        self.params.append(CeleryOption(('--gid',), help_group="Daemonization Options"))
-        self.params.append(CeleryOption(('--umask',), help_group="Daemonization Options"))
-        self.params.append(CeleryOption(('--executable',), help_group="Daemonization Options"))
+        self.params.extend((
+            DaemonOption("--logfile", "-f", help="Log destination; defaults to stderr"),
+            DaemonOption("--pidfile", help="PID file path; defaults to no PID file"),
+            DaemonOption("--uid", help="Drops privileges to this user ID"),
+            DaemonOption("--gid", help="Drops privileges to this group ID"),
+            DaemonOption("--umask", help="Create files and directories with this umask"),
+            DaemonOption("--executable", help="Override path to the Python executable"),
+        ))
 
 
 class CommaSeparatedList(ParamType):

docs/userguide/configuration.rst

@@ -3219,6 +3219,71 @@ Message serialization format used when sending event messages.
     :ref:`calling-serializers`.
 
 
+.. setting:: events_logfile
+
+``events_logfile``
+~~~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional file path for :program:`celery events` to log into (defaults to `stdout`).
+
+.. versionadded:: 5.4
+
+.. setting:: events_pidfile
+
+``events_pidfile``
+~~~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional file path for :program:`celery events` to create/store its PID file (default to no PID file created).
+
+.. versionadded:: 5.4
+
+.. setting:: events_uid
+
+``events_uid``
+~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional user ID to use when events :program:`celery events` drops its privileges (defaults to no UID change).
+
+.. versionadded:: 5.4
+
+.. setting:: events_gid
+
+``events_gid``
+~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional group ID to use when :program:`celery events` daemon drops its privileges (defaults to no GID change).
+
+.. versionadded:: 5.4
+
+.. setting:: events_umask
+
+``events_umask``
+~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional `umask` to use when :program:`celery events` creates files (log, pid...) when daemonizing.
+
+.. versionadded:: 5.4
+
+.. setting:: events_executable
+
+``events_executable``
+~~~~~~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional `python` executable path for :program:`celery events` to use when deaemonizing (defaults to :data:`sys.executable`).
+
+
 .. _conf-control:
 
 Remote Control Commands
@@ -3487,6 +3552,74 @@ Default: ``"kombu.asynchronous.hub.timer:Timer"``.
 Name of the ETA scheduler class used by the worker.
 Default is or set by the pool implementation.
 
+.. setting:: worker_logfile
+
+``worker_logfile``
+~~~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional file path for :program:`celery worker` to log into (defaults to `stdout`).
+
+.. versionadded:: 5.4
+
+.. setting:: worker_pidfile
+
+``worker_pidfile``
+~~~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional file path for :program:`celery worker` to create/store its PID file (defaults to no PID file created).
+
+.. versionadded:: 5.4
+
+.. setting:: worker_uid
+
+``worker_uid``
+~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional user ID to use when :program:`celery worker` daemon drops its privileges (defaults to no UID change).
+
+.. versionadded:: 5.4
+
+.. setting:: worker_gid
+
+``worker_gid``
+~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional group ID to use when :program:`celery worker` daemon drops its privileges (defaults to no GID change).
+
+.. versionadded:: 5.4
+
+.. setting:: worker_umask
+
+``worker_umask``
+~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional `umask` to use when :program:`celery worker` creates files (log, pid...) when daemonizing.
+
+.. versionadded:: 5.4
+
+.. setting:: worker_executable
+
+``worker_executable``
+~~~~~~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional `python` executable path for :program:`celery worker` to use when deaemonizing (defaults to :data:`sys.executable`).
+
+.. versionadded:: 5.4
+
+
+
 .. _conf-celerybeat:
 
 Beat Settings (:program:`celery beat`)
@@ -3573,3 +3706,69 @@ Default: None.
 When using cron, the number of seconds :mod:`~celery.bin.beat` can look back
 when deciding whether a cron schedule is due. When set to `None`, cronjobs that
 are past due will always run immediately.
+
+.. setting:: beat_logfile
+
+``beat_logfile``
+~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional file path for :program:`celery beat` to log into (defaults to `stdout`).
+
+.. versionadded:: 5.4
+
+.. setting:: beat_pidfile
+
+``beat_pidfile``
+~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional file path for :program:`celery beat` to create/store it PID file (defaults to no PID file created).
+
+.. versionadded:: 5.4
+
+.. setting:: beat_uid
+
+``beat_uid``
+~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional user ID to use when beat :program:`celery beat` drops its privileges (defaults to no UID change).
+
+.. versionadded:: 5.4
+
+.. setting:: beat_gid
+
+``beat_gid``
+~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional group ID to use when :program:`celery beat` daemon drops its privileges (defaults to no GID change).
+
+.. versionadded:: 5.4
+
+.. setting:: beat_umask
+
+``beat_umask``
+~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional `umask` to use when :program:`celery beat` creates files (log, pid...) when daemonizing.
+
+.. versionadded:: 5.4
+
+.. setting:: beat_executable
+
+``beat_executable``
+~~~~~~~~~~~~~~~~~~~
+
+Default: :const:`None`
+
+An optional `python` executable path for :program:`celery beat` to use when deaemonizing (defaults to :data:`sys.executable`).
+
+.. versionadded:: 5.4

t/unit/bin/proj/daemon.py

@@ -0,0 +1,4 @@
+from celery import Celery
+
+app = Celery(set_as_current=False)
+app.config_from_object("t.unit.bin.proj.daemon_config")

t/unit/bin/proj/daemon_config.py

@@ -0,0 +1,22 @@
+# Test config for t/unit/bin/test_deamonization.py
+
+beat_pidfile = "/tmp/beat.test.pid"
+beat_logfile = "/tmp/beat.test.log"
+beat_uid = 42
+beat_gid = 4242
+beat_umask = 0o777
+beat_executable = "/beat/bin/python"
+
+events_pidfile = "/tmp/events.test.pid"
+events_logfile = "/tmp/events.test.log"
+events_uid = 42
+events_gid = 4242
+events_umask = 0o777
+events_executable = "/events/bin/python"
+
+worker_pidfile = "/tmp/worker.test.pid"
+worker_logfile = "/tmp/worker.test.log"
+worker_uid = 42
+worker_gid = 4242
+worker_umask = 0o777
+worker_executable = "/worker/bin/python"

t/unit/bin/test_daemonization.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+from click.testing import CliRunner
+
+from celery.bin.celery import celery
+
+from .proj import daemon_config as config
+
+
+@pytest.mark.usefixtures('depends_on_current_app')
+@pytest.mark.parametrize("daemon", ["worker", "beat", "events"])
+def test_daemon_options_from_config(daemon: str, cli_runner: CliRunner):
+
+    with patch(f"celery.bin.{daemon}.{daemon}.callback") as mock:
+        cli_runner.invoke(celery, f"-A t.unit.bin.proj.daemon {daemon}")
+
+    mock.assert_called_once()
+    for param in "logfile", "pidfile", "uid", "gid", "umask", "executable":
+        assert mock.call_args.kwargs[param] == getattr(config, f"{daemon}_{param}")