From 55043f31b0adf00353a8c7b64c03c0a93106287b Mon Sep 17 00:00:00 2001 From: Stephen Herr Date: Wed, 10 Jan 2024 15:06:53 -0500 Subject: [PATCH] Load gunicorn defaults before overriding with app settings fixes: #4917 --- CHANGES/4917.feature | 1 + pulpcore/app/entrypoint.py | 23 +++--- pulpcore/app/pulpcore_gunicorn_application.py | 71 +++++++++++++++++++ pulpcore/content/entrypoint.py | 19 ++--- 4 files changed, 86 insertions(+), 28 deletions(-) create mode 100644 CHANGES/4917.feature create mode 100644 pulpcore/app/pulpcore_gunicorn_application.py diff --git a/CHANGES/4917.feature b/CHANGES/4917.feature new file mode 100644 index 0000000000..a12c5a0326 --- /dev/null +++ b/CHANGES/4917.feature @@ -0,0 +1 @@ +Enabled the gunicorn applications for ``pulpcore-api`` and ``pulpcore-content`` to load configurations from the "gunicorn.conf.py" file. diff --git a/pulpcore/app/entrypoint.py b/pulpcore/app/entrypoint.py index d99ea26827..a4d8994fc5 100644 --- a/pulpcore/app/entrypoint.py +++ b/pulpcore/app/entrypoint.py @@ -9,9 +9,9 @@ from django.db import connection from django.db.utils import InterfaceError, OperationalError from gunicorn.workers.sync import SyncWorker -from gunicorn.app.base import BaseApplication from pulpcore.app.apps import pulp_plugin_configs +from pulpcore.app.pulpcore_gunicorn_application import PulpcoreGunicornApplication logger = getLogger(__name__) @@ -80,19 +80,14 @@ def run(self): self.api_app_status.delete() -class PulpcoreApiApplication(BaseApplication): - def __init__(self, options): - self.options = options or {} - super().__init__() - - def load_config(self): - [ - self.cfg.set(key.lower(), value) - for key, value in self.options.items() - if value is not None - ] - self.cfg.set("default_proc_name", "pulpcore-api") - self.cfg.set("worker_class", PulpApiWorker.__module__ + "." + PulpApiWorker.__qualname__) +class PulpcoreApiApplication(PulpcoreGunicornApplication): + def load_app_specific_config(self): + self.set_option("default_proc_name", "pulpcore-api", enforced=True) + self.set_option( + "worker_class", + PulpApiWorker.__module__ + "." + PulpApiWorker.__qualname__, + enforced=True, + ) def load(self): using_pulp_api_worker.set(True) diff --git a/pulpcore/app/pulpcore_gunicorn_application.py b/pulpcore/app/pulpcore_gunicorn_application.py new file mode 100644 index 0000000000..a66e676f83 --- /dev/null +++ b/pulpcore/app/pulpcore_gunicorn_application.py @@ -0,0 +1,71 @@ +import sys + +from gunicorn.app.base import Application + + +class PulpcoreGunicornApplication(Application): + """ + A common class for the api and content applications to inherit from that loads the default + gunicorn configs (including from a config file if one exists) and then overrides with the values + specified in the init scripts. With warnings / errors if the user overrides something that won't + take effect or cannot be changed. + """ + + def __init__(self, options): + self.options = options or {} + super().__init__() + + def init(self, *args, **kwargs): + """ + A hook for setting application-specific configs, which we instead do below in load_config + where it's non-overridable. + """ + pass + + def set_option(self, key, value, enforced=False): + if value is None: # not specified by init script + return + + def _is_default(key, value): + if value is None: + return True + defaults = { + "default_proc_name": "gunicorn", + "reload_extra_files": [], + "access_log_format": '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"', + "bind": ["127.0.0.1:8000"], + "threads": 1, + "worker_class": "", + } + return key in defaults and str(defaults[key]) == str(value) + + current_value = getattr(self.cfg, key, None) + if not _is_default(key, current_value) and str(current_value) != str(value): + if enforced: + sys.stderr.write( + f"Error: {key} is set to {current_value} in gunicorn.conf.py but must not be " + "changed!\n" + ) + exit(1) + else: + sys.stderr.write( + f"Warning: {key} is set to {current_value} in gunicorn.conf.py but is " + f"overridden to {value} by init script!\n" + ) + + self.cfg.set(key, value) + + def load_config(self): + # Load default gunicorn configs, including reading from the default config file. + super().load_config() + # Override with settings that we've specified in the startup script. + for key, value in self.options.items(): + self.set_option(key, value) + self.set_option("threads", "1", enforced=True) + self.load_app_specific_config() + + def load_app_specific_config(self): + raise NotImplementedError + + def load(self): + raise NotImplementedError diff --git a/pulpcore/content/entrypoint.py b/pulpcore/content/entrypoint.py index e158d11652..5ae79cfe22 100644 --- a/pulpcore/content/entrypoint.py +++ b/pulpcore/content/entrypoint.py @@ -1,20 +1,11 @@ import click -from gunicorn.app.base import BaseApplication +from pulpcore.app.pulpcore_gunicorn_application import PulpcoreGunicornApplication -class PulpcoreContentApplication(BaseApplication): - def __init__(self, options): - self.options = options or {} - super().__init__() - - def load_config(self): - [ - self.cfg.set(key.lower(), value) - for key, value in self.options.items() - if value is not None - ] - self.cfg.set("default_proc_name", "pulpcore-content") - self.cfg.set("worker_class", "aiohttp.GunicornWebWorker") +class PulpcoreContentApplication(PulpcoreGunicornApplication): + def load_app_specific_config(self): + self.set_option("default_proc_name", "pulpcore-content", enforced=True) + self.set_option("worker_class", "aiohttp.GunicornWebWorker", enforced=True) def load(self): import pulpcore.content