From 51c1ca14d6015d3bdfbd8b8d7f26b7671d1a40e6 Mon Sep 17 00:00:00 2001 From: Jason Joyce Date: Thu, 28 Mar 2024 12:24:53 -0400 Subject: [PATCH] Adding celery scaffold to the project. Adding celery_scaffold for use with flask based projects. It provides a way to configure celery for use, using the flask configuration files. It also provides a celery_app and a flask_app that can be used in your project. A base_scaffold was pulled out due to the celery worker assuming any 'app' attribute is of type Celery. The original app_scaffold is available to provide backward compatibility. It leverages the new base_scaffold and sets the 'app' attribute to flask_app to ensure existing use cases are handled. Signed-off-by: Jason Joyce --- README.md | 10 ++ setup.cfg | 1 + src/flask_container_scaffold/app_scaffold.py | 82 ++------------ src/flask_container_scaffold/base_scaffold.py | 102 ++++++++++++++++++ .../celery_scaffold.py | 42 ++++++++ tests/unit/test_celery.py | 51 +++++++++ 6 files changed, 217 insertions(+), 71 deletions(-) create mode 100644 src/flask_container_scaffold/base_scaffold.py create mode 100644 src/flask_container_scaffold/celery_scaffold.py create mode 100644 tests/unit/test_celery.py diff --git a/README.md b/README.md index d702c68..8ff2c8f 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,16 @@ example: }, }) +### Using CeleryScaffold + +This class has all of the same support as the above AppScaffold and takes +the same parameters. Each CeleryScaffold instance has a flask_app and celery_app +attribute that can be used in your project + + celery_scaffold = CeleryScaffold(name=__name__, config=config) + flask_app = celery_scaffold.flask_app + celery_app = celery_scaffold.celery_app + ### Using the parse_input method This method is used to validate incoming data against a pydantic model. A diff --git a/setup.cfg b/setup.cfg index 4ad70ee..95784b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,7 @@ package_dir = = src packages = find: install_requires = + celery flask pydantic toolchest diff --git a/src/flask_container_scaffold/app_scaffold.py b/src/flask_container_scaffold/app_scaffold.py index 2c6c5b2..8cf4ee6 100644 --- a/src/flask_container_scaffold/app_scaffold.py +++ b/src/flask_container_scaffold/app_scaffold.py @@ -1,11 +1,7 @@ -import os +from flask_container_scaffold.base_scaffold import BaseScaffold -from flask import Flask -from flask_container_scaffold.app_configurator import AppConfigurator - - -class AppScaffold(object): +class AppScaffold(BaseScaffold): def __init__(self, app=None, name=__name__, config=None, @@ -13,16 +9,18 @@ def __init__(self, app=None, instance_path=None, instance_relative_config=True): """ - This class provides a way to dynamically configure a Flask application. + This class provides compatibility with older versions of scaffold that + expect an instance with an 'app' attribute. :param obj app: An existing Flask application, if passed, otherwise we will create a new one :param str name: The name of the application, defaults to __name__. :param dict config: A dict of configuration details. This can include standard Flask configuration keys, like 'TESTING', or - 'CUSTOM_SETTINGS' (which can be a string referencing a file with custom - configuration, or a dictionary containing any values your application - may need) to make them available to the application during runtime + 'CUSTOM_SETTINGS' (which can be a string referencing a file with + custom configuration, or a dictionary containing any values your + application may need) to make them available to the application + during runtime :param bool settings_required: Whether your app requires certain settings be specified in a settings.cfg file :param str instance_path: Passthrough parameter to flask. An @@ -35,64 +33,6 @@ def __init__(self, app=None, the application root. """ - # TODO: Consider taking **kwargs here, so we can automatically support - # all params the flask object takes, and just pass them through. Keep - # the ones we already have, as they are needed for the current code to - # work. - Flask.jinja_options = dict(Flask.jinja_options, trim_blocks=True, - lstrip_blocks=True) - self.app = (app or - Flask(name, - instance_relative_config=instance_relative_config, - instance_path=instance_path)) - self.config = config - self.silent = not settings_required - self.relative = instance_relative_config - self._init_app() - - def _init_app(self): - self._load_flask_settings() - self._load_custom_settings() - - def _load_flask_settings(self): - """ - This loads the 'core' settings, ie, anything you could set directly - on a Flask app. These can be specified in the following order, each - overriding the last, if specified: - - via config mapping - - via Flask settings.cfg file - - via environment variable 'FLASK_SETTINGS' - """ - config_not_loaded = True - if self.config is not None: - # load the config if passed in - self.app.config.from_mapping(self.config) - config_not_loaded = False - # load the instance config, if it exists and/or is required - try: - self.app.config.from_pyfile('settings.cfg', silent=self.silent) - config_not_loaded = False - except Exception: - config_not_loaded = True - # Load any additional config specified in the FLASK_SETTINGS file, - # if it exists. We only want to fail in the case where settings are - # required by the app. - if ((config_not_loaded and not self.silent) or - os.environ.get('FLASK_SETTINGS')): - self.app.config.from_envvar('FLASK_SETTINGS') - - def _load_custom_settings(self): - """ - Load any custom configuration for the app from: - - app.config['CUSTOM_SETTINGS'] - - environment variable 'CUSTOM_SETTINGS' - """ - configurator = AppConfigurator(self.app, self.relative) - if self.app.config.get('CUSTOM_SETTINGS') is not None: - # load the config if passed in - custom = self.app.config.get('CUSTOM_SETTINGS') - configurator.parse(custom) - # Next, load from override file, if specified - if os.environ.get('CUSTOM_SETTINGS') is not None: - custom = os.environ.get('CUSTOM_SETTINGS') - configurator.parse(custom) + super().__init__(app, name, config, settings_required, + instance_path, instance_relative_config) + self.app = app or self.flask_app diff --git a/src/flask_container_scaffold/base_scaffold.py b/src/flask_container_scaffold/base_scaffold.py new file mode 100644 index 0000000..208aec7 --- /dev/null +++ b/src/flask_container_scaffold/base_scaffold.py @@ -0,0 +1,102 @@ +import os + +from flask import Flask + +from flask_container_scaffold.app_configurator import AppConfigurator + + +class BaseScaffold(object): + + def __init__(self, app=None, + name=__name__, config=None, + settings_required=False, + instance_path=None, + instance_relative_config=True): + """ + This base class provides a way to dynamically configure a Flask + application. + + :param obj app: An existing Flask application, if passed, otherwise we + will create a new one + :param str name: The name of the application, defaults to __name__. + :param dict config: A dict of configuration details. This can include + standard Flask configuration keys, like 'TESTING', or + 'CUSTOM_SETTINGS' (which can be a string referencing a file with + custom configuration, or a dictionary containing any values your + application may need) to make them available to the application + during runtime + :param bool settings_required: Whether your app requires certain + settings be specified in a settings.cfg file + :param str instance_path: Passthrough parameter to flask. An + alternative instance path for the application. By default + the folder 'instance' next to the package or module is + assumed to be the instance path. + :param bool instance_relative_config: Passthrough parameter to flask. + If set to True relative filenames for loading the config + are assumed to be relative to the instance path instead of + the application root. + + """ + # TODO: Consider taking **kwargs here, so we can automatically support + # all params the flask object takes, and just pass them through. Keep + # the ones we already have, as they are needed for the current code to + # work. + Flask.jinja_options = dict(Flask.jinja_options, trim_blocks=True, + lstrip_blocks=True) + self.flask_app = app or Flask( + name, + instance_relative_config=instance_relative_config, + instance_path=instance_path, + ) + self.config = config + self.silent = not settings_required + self.relative = instance_relative_config + self._init_app() + + def _init_app(self): + self._load_flask_settings() + self._load_custom_settings() + + def _load_flask_settings(self): + """ + This loads the 'core' settings, ie, anything you could set directly + on a Flask app. These can be specified in the following order, each + overriding the last, if specified: + - via config mapping + - via Flask settings.cfg file + - via environment variable 'FLASK_SETTINGS' + """ + config_not_loaded = True + if self.config is not None: + # load the config if passed in + self.flask_app.config.from_mapping(self.config) + config_not_loaded = False + # load the instance config, if it exists and/or is required + try: + self.flask_app.config.from_pyfile('settings.cfg', + silent=self.silent) + config_not_loaded = False + except Exception: + config_not_loaded = True + # Load any additional config specified in the FLASK_SETTINGS file, + # if it exists. We only want to fail in the case where settings are + # required by the app. + if ((config_not_loaded and not self.silent) or + os.environ.get('FLASK_SETTINGS')): + self.flask_app.config.from_envvar('FLASK_SETTINGS') + + def _load_custom_settings(self): + """ + Load any custom configuration for the app from: + - app.config['CUSTOM_SETTINGS'] + - environment variable 'CUSTOM_SETTINGS' + """ + configurator = AppConfigurator(self.flask_app, self.relative) + if self.flask_app.config.get('CUSTOM_SETTINGS') is not None: + # load the config if passed in + custom = self.flask_app.config.get('CUSTOM_SETTINGS') + configurator.parse(custom) + # Next, load from override file, if specified + if os.environ.get('CUSTOM_SETTINGS') is not None: + custom = os.environ.get('CUSTOM_SETTINGS') + configurator.parse(custom) diff --git a/src/flask_container_scaffold/celery_scaffold.py b/src/flask_container_scaffold/celery_scaffold.py new file mode 100644 index 0000000..9615660 --- /dev/null +++ b/src/flask_container_scaffold/celery_scaffold.py @@ -0,0 +1,42 @@ +from celery import Celery + +from flask_container_scaffold.base_scaffold import BaseScaffold + + +class CeleryScaffold(BaseScaffold): + + def __init__(self, flask_app=None, name=__name__, config=None, + settings_required=False, + instance_path=None, + instance_relative_config=True): + """ + This class provides both a flask 'app' and a celery 'app' that has been + configured via flask. + + :param obj flask_app: An existing Flask application, if passed, + otherwise we will create a new one using BaseScaffold. + :param str name: The name of the application, defaults to __name__. + :param dict config: A dict of configuration details. This can include + standard Flask configuration keys, like 'TESTING', or + 'CUSTOM_SETTINGS' (which can be a string referencing a file with + custom configuration, or a dictionary containing any values your + application may need) to make them available to the application + during runtime + :param bool settings_required: Whether your app requires certain + settings be specified in a settings.cfg file + :param str instance_path: Passthrough parameter to flask. An + alternative instance path for the application. By default + the folder 'instance' next to the package or module is + assumed to be the instance path. + :param bool instance_relative_config: Passthrough parameter to flask. + If set to True relative filenames for loading the config + are assumed to be relative to the instance path instead of + the application root. + + """ + super().__init__(flask_app, name, config, settings_required, + instance_path, instance_relative_config) + self.flask_app = flask_app or self.flask_app + self.celery_app = Celery(self.flask_app.name) + self.celery_app.config_from_object(self.flask_app.config.get("CELERY")) + self.celery_app.set_default() diff --git a/tests/unit/test_celery.py b/tests/unit/test_celery.py new file mode 100644 index 0000000..2a5e550 --- /dev/null +++ b/tests/unit/test_celery.py @@ -0,0 +1,51 @@ +import pytest + +from celery import Celery +from flask import Flask + +from flask_container_scaffold.celery_scaffold import CeleryScaffold + + +def test_celery_flask_empty_config(): + """ + GIVEN an instance of CeleryScaffold with an empty config + WHEN we try to create the app + THEN we get a celery app and a flask app + """ + scaffold = CeleryScaffold() + assert scaffold.flask_app is not None + assert isinstance(scaffold.flask_app, Flask) + assert scaffold.celery_app is not None + assert isinstance(scaffold.celery_app, Celery) + + +def test_celery_broker_set(): + """ + GIVEN an instance of CeleryScaffold + AND a config with a broker url + WHEN we create the app + THEN we get a celery app with a broker url matching the config + """ + config = {'CELERY': {'broker': 'pyamqp://'}} + scaffold = CeleryScaffold(config=config) + app = scaffold.celery_app + assert app is not None + assert isinstance(app, Celery) + assert config['CELERY']['broker'] == app.conf.find_value_for_key('broker') + + +def test_celery_bad_config(): + """ + GIVEN an instance of CeleryScaffold + AND a config with a bad config item + WHEN we create the app + THEN we get a celery app + AND the config doesn't have the bad item + """ + config = {'CELERY': {'bad_config_item': 'my_bad_config'}} + scaffold = CeleryScaffold(config=config) + app = scaffold.celery_app + assert app is not None + assert isinstance(app, Celery) + with pytest.raises(KeyError): + app.conf.find_value_for_key('bad_config_item')