From 0610b638d8fb7fa6505d2f0243237c0ac31e0f2f Mon Sep 17 00:00:00 2001 From: Kennedy Kori Date: Wed, 6 Jul 2022 17:03:26 +0300 Subject: [PATCH] feat: Implement config loading bootstrap --- app/__init__.py | 66 ++++++++++++++++++------ app/lib/__init__.py | 3 +- app/lib/module_loading.py | 50 +++++++++++++++++- tests/lib/test_module_loading.py | 31 ++++++++++-- tests/test_app.py | 87 +++++++++++++++++++++++++++++++- 5 files changed, 216 insertions(+), 21 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 0c95f92..9388fdb 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,6 +1,5 @@ -import inspect from logging.config import dictConfig -from typing import Any, Dict, List, Mapping, Optional, Sequence, Type, cast +from typing import Any, Dict, List, Mapping, Optional, Sequence, Type import yaml from yaml import Loader @@ -11,7 +10,7 @@ Config, ImproperlyConfiguredError, SettingInitializer, - import_string, + import_string_as_klass, ) # ============================================================================= @@ -20,6 +19,8 @@ _LOGGING_CONFIG_KEY = "LOGGING" +_SETTINGS_INITIALIZERS_CONFIG_KEY = "SETTINGS_INITIALIZERS" + _SUPPORTED_DATA_SOURCE_TYPES_CONFIG_KEY = "SUPPORTED_DATA_SOURCE_TYPES" _DEFAULT_CONFIG: Dict[str, Any] = { @@ -43,9 +44,8 @@ }, "root": {"level": "INFO", "handlers": ["console"]}, }, - _SUPPORTED_DATA_SOURCE_TYPES_CONFIG_KEY: [ - "app.imp.sql_data.SQLDataSourceType" - ], + _SETTINGS_INITIALIZERS_CONFIG_KEY: [], + _SUPPORTED_DATA_SOURCE_TYPES_CONFIG_KEY: [], } @@ -73,13 +73,43 @@ # ============================================================================= -def _load_config_file(config_file_path: str) -> Mapping[str, Any]: +def _load_config_file( + config_file_path: str, +) -> Mapping[str, Any]: # pragma: no cover # TODO: Ensure that a valid config file path was given and if not raise an # appropriate Exception. with open(config_file_path, "rb") as config_file: return yaml.load(config_file, Loader=Loader) +def _load_settings_initializers( + initializers_dotted_paths: Sequence[str], +) -> Sequence[SettingInitializer]: + initializers: List[SettingInitializer] = list() + for _initializer_dotted_path in initializers_dotted_paths: + try: + initializer_klass: Type[SettingInitializer] + initializer_klass = import_string_as_klass( + _initializer_dotted_path, SettingInitializer + ) + initializers.append(initializer_klass()) + except ImportError as exp: + raise ImproperlyConfiguredError( + message='"%s" does not seem to be a valid path.' + % _initializer_dotted_path + ) from exp + except TypeError as exp: + raise ImproperlyConfiguredError( + message=( + 'Invalid value, "%s" is either not class or is not a ' + 'subclass of "app.lib.SettingInitializer".' + % _initializer_dotted_path + ) + ) from exp + + return initializers + + # ============================================================================= # DEFAULT SETTINGS INITIALIZERS # ============================================================================= @@ -130,22 +160,22 @@ def _dotted_path_to_data_source_type_klass( dotted_path: str, ) -> Type[DataSourceType]: try: - _module = import_string(dotted_path) + data_source_type_klass: Type[DataSourceType] + data_source_type_klass = import_string_as_klass( + dotted_path, DataSourceType + ) + return data_source_type_klass except ImportError as exp: raise ImproperlyConfiguredError( message='"%s" does not seem to be a valid path.' % dotted_path ) from exp - - if not inspect.isclass(_module) or not issubclass( - _module, DataSourceType - ): # type: ignore + except TypeError as exp: raise ImproperlyConfiguredError( message=( 'Invalid value, "%s" is either not class or is not a ' 'subclass of "app.core.DataSourceType".' % dotted_path ) - ) - return cast(Type[DataSourceType], _module) + ) from exp # ============================================================================= @@ -174,11 +204,17 @@ def setup( # Load the application settings _settings_dict: Dict[str, Any] = dict(initial_settings or _DEFAULT_CONFIG) - if config_file_path: # load config from a file when provided + # Load config from a file when provided + if config_file_path: # pragma: no branch _settings_dict.update(_load_config_file(config_file_path)) # Load initializers _initializers: List[Any] = list(settings_initializers or []) + _initializers.extend( + _load_settings_initializers( + _settings_dict.get(_SETTINGS_INITIALIZERS_CONFIG_KEY, tuple()) + ) + ) _initializers.insert(0, _LoggingInitializer()) _initializers.insert(1, _SupportedDataSourceTypesInitializer()) diff --git a/app/lib/__init__.py b/app/lib/__init__.py index 557b842..f7890ad 100644 --- a/app/lib/__init__.py +++ b/app/lib/__init__.py @@ -2,7 +2,7 @@ from .checkers import ensure_not_none, ensure_not_none_nor_empty from .config import * # noqa: F401,F403 from .config import __all__ as _all_config -from .module_loading import import_string +from .module_loading import import_string, import_string_as_klass from .tasks import * # noqa: F401,F403 from .tasks import __all__ as _all_tasks from .transports import * # noqa: F401,F403 @@ -13,6 +13,7 @@ "ensure_not_none", "ensure_not_none_nor_empty", "import_string", + "import_string_as_klass", ] __all__ += _all_config # type: ignore __all__ += _all_tasks # type: ignore diff --git a/app/lib/module_loading.py b/app/lib/module_loading.py index 8225efc..4b3c09a 100644 --- a/app/lib/module_loading.py +++ b/app/lib/module_loading.py @@ -1,7 +1,20 @@ # The contents of this module are copied from Django sources. +import inspect import sys from importlib import import_module from types import ModuleType +from typing import Type, TypeVar, cast + +# ============================================================================= +# TYPES +# ============================================================================= + +_T = TypeVar("_T") + + +# ============================================================================= +# HELPERS +# ============================================================================= def _cached_import(module_path: str, class_name: str) -> ModuleType: @@ -16,10 +29,15 @@ def _cached_import(module_path: str, class_name: str) -> ModuleType: return getattr(modules[module_path], class_name) +# ============================================================================= +# IMPORT UTILITIES +# ============================================================================= + + def import_string(dotted_path: str) -> ModuleType: """ Import a dotted module path and return the attribute/class designated by - the last name in the path. Raise ImportError if the import failed. + the last name in the path. Raise ``ImportError`` if the import failed. :param dotted_path: A dotted path to an attribute or class. @@ -41,3 +59,33 @@ def import_string(dotted_path: str) -> ModuleType: 'Module "%s" does not define a "%s" attribute/class' % (module_path, class_name) ) from err + + +def import_string_as_klass( + dotted_path: str, target_klass: Type[_T] +) -> Type[_T]: + """ + Import a dotted module as the given class type. Raise ``ImportError`` if + the import failed and a ``TypeError`` if the imported module is not of the + given class type or derived from it. + + :param dotted_path: A dotted path to a class. + :param target_klass: The class type that the imported module should have or + be derived from. + + :return: The class designated by the last name in the path. + + :raise ImportError: If the import fails for some reason. + :raise TypeError: If the imported module is not of the given class type or + derived from the given class. + """ + _module = import_string(dotted_path) + if not inspect.isclass(_module) or not issubclass( # noqa + _module, target_klass + ): + raise TypeError( + 'Invalid value, "%s" is either not a class or a subclass of "%s".' + % (dotted_path, target_klass.__qualname__) + ) + + return cast(Type[target_klass], _module) diff --git a/tests/lib/test_module_loading.py b/tests/lib/test_module_loading.py index a627db2..d20c902 100644 --- a/tests/lib/test_module_loading.py +++ b/tests/lib/test_module_loading.py @@ -1,9 +1,10 @@ import pytest -from app.lib import Config, import_string +from app.core import Task +from app.lib import Config, Consumer, import_string, import_string_as_klass -def test_correct_return_value_on_valid_dotted_path() -> None: +def test_correct_return_value_on_valid_dotted_path1() -> None: """ Assert that when given a valid dotted path as input, ``import_string`` returns the correct class or attribute. @@ -14,7 +15,7 @@ def test_correct_return_value_on_valid_dotted_path() -> None: ) # noqa -def test_correct_expected_behavior_on_invalid_dotted_path() -> None: +def test_correct_expected_behavior_on_invalid_dotted_path1() -> None: """ Assert that an ``ImportError`` is raised when an invalid dotted path is given as input to ``import_string``. @@ -23,3 +24,27 @@ def test_correct_expected_behavior_on_invalid_dotted_path() -> None: import_string("aninvalidpath") with pytest.raises(ImportError): import_string("app.no_such") + + +def test_correct_return_value_on_valid_dotted_path2() -> None: + """ + Assert that when given a valid dotted path as input, + ``import_string_import_string_as_klass`` returns the correct class. + """ + assert import_string_as_klass("app.lib.Config", Config) is Config + assert import_string_as_klass("app.lib.Consumer", Task) is Consumer + + +def test_correct_expected_behavior_on_invalid_dotted_path2() -> None: + """ + Assert that an ``ImportError`` or ``TypeError`` is raised when an invalid + dotted path is given as input to ``import_import_string_as_klass``. + """ + with pytest.raises(ImportError): + import_string_as_klass("aninvalidpath", Config) + with pytest.raises(TypeError): + import_string_as_klass( + "app.lib.module_loading.import_string", import_string + ) + with pytest.raises(TypeError): + import_string_as_klass("app.lib.Config", Task) diff --git a/tests/test_app.py b/tests/test_app.py index ecf54f2..6d6ffff 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,4 +1,40 @@ +from typing import Any, Dict, Mapping +from unittest import TestCase + +import pytest + import app +from app.lib import ImproperlyConfiguredError, SettingInitializer + +# ============================================================================= +# HELPERS +# ============================================================================= + + +def _test_config_factory() -> Mapping[str, Any]: + return { + "SETTINGS_INITIALIZERS": [ + "tests.test_app._FakeDataSourceTypesConfigInitializer" + ], + "SUPPORTED_DATA_SOURCE_TYPES": ["app.imp.sql_data.SQLDataSourceType"], + } + + +class _FakeDataSourceTypesConfigInitializer(SettingInitializer): + """A fake settings initializers for testing.""" + + @property + def setting(self) -> str: + return "SUPPORTED_DATA_SOURCE_TYPES" + + def execute(self, an_input: Any) -> Any: + # Do nothing + return an_input + + +# ============================================================================= +# TESTS +# ============================================================================= def test_app_globals_are_set_after_successful_setup() -> None: @@ -6,7 +42,56 @@ def test_app_globals_are_set_after_successful_setup() -> None: assert app.registry is None # After setup, globals should be not None. - app.setup() + app.setup(initial_settings=_test_config_factory()) assert app.settings is not None assert app.registry is not None + + +class TestAppModule(TestCase): + """Tests for the app module.""" + + def setUp(self) -> None: + super().setUp() + self._default_config: Dict[str, Any] = dict() + self._some_state: int = 0 + + def test_valid_config_is_successful(self) -> None: + """Assert that a valid config will be executed successfully.""" + app.setup(initial_settings=self._default_config) + + def test_invalid_settings_initializers_config_causes_error(self) -> None: + """ + Assert that invalid dotted paths or dotted paths to non + :config:`setting initializers ` on the + *SETTINGS_INITIALIZERS* setting results in errors. + """ + config1: Dict[str, Any] = dict(self._default_config) + config2: Dict[str, Any] = dict(self._default_config) + + config1["SETTINGS_INITIALIZERS"] = ["invalid_dotted_path"] + config2["SETTINGS_INITIALIZERS"] = ["app.core.Task"] + with pytest.raises(ImproperlyConfiguredError): + app.setup(initial_settings=config1) + + # Not an initializer + with pytest.raises(ImproperlyConfiguredError): + app.setup(initial_settings=config2) + + def test_invalid_data_source_types_config_causes_error(self) -> None: + """ + Assert that invalid dotted paths or dotted paths to non + :config:`data source types ` on the + *SUPPORTED_DATA_SOURCE_TYPES* setting result in errors. + """ + config1: Dict[str, Any] = dict(self._default_config) + config2: Dict[str, Any] = dict(self._default_config) + + config1["SUPPORTED_DATA_SOURCE_TYPES"] = ["invalid_dotted_path"] + config2["SUPPORTED_DATA_SOURCE_TYPES"] = ["app.core.Task"] + with pytest.raises(ImproperlyConfiguredError): + app.setup(initial_settings=config1) + + # Not a data source type + with pytest.raises(ImproperlyConfiguredError): + app.setup(initial_settings=config2)