feat: Add app configuration capabilities (#17)

.gitignore

@@ -305,6 +305,9 @@ dist/*
 .env
 .envs/*
 !.envs/.local/
+config.yaml
 local.sh
+logs/*
+!logs/.gitkeep
 secrets/*
 cloud_sql_proxy

app/__init__.py

@@ -11,8 +11,10 @@
 # CONSTANTS
 # =============================================================================
 
+_LOGGING_CONFIG_KEY = "LOGGING"
+
 _DEFAULT_CONFIG: Dict[str, Any] = {
-    "logging": {
+    _LOGGING_CONFIG_KEY: {
         "version": 1,
         "disable_existing_loggers": False,
         "formatters": {
@@ -53,7 +55,7 @@ class _LoggingInitializer(Task[Any, Any]):
 
     def execute(self, an_input: Optional[Mapping[str, Any]]) -> Any:
         logging_config: Dict[str, Any] = dict(
-            an_input or _DEFAULT_CONFIG["logging"]
+            an_input or _DEFAULT_CONFIG[_LOGGING_CONFIG_KEY]
         )
         dictConfig(logging_config)
         return logging_config
@@ -85,7 +87,7 @@ def setup(
     if config_file_path:  # load config from a file when provided
         _settings_dict.update(_load_config_file(config_file_path))
     _initializers_dict: Dict[str, Any] = dict(settings_initializers or {})
-    _initializers_dict.update({"logging": _LoggingInitializer()})
+    _initializers_dict.update({_LOGGING_CONFIG_KEY: _LoggingInitializer()})
 
     global settings
     settings = Config(

app/__main__.py

@@ -97,9 +97,9 @@ def main() -> None:  # pragma: no cover
     """
 
     parser = argparse_factory()
-    parser.parse_args()
+    args = parser.parse_args()
 
-    app.setup()
+    app.setup(config_file_path=args.config)
     main_pipeline: Pipeline[None, Any] = main_pipeline_factory()
     main_pipeline.execute(None)
     print("Done ...")

app/core/__init__.py

@@ -1,4 +1,5 @@
 from .domain import AbstractDomainObject, DataSource, ExtractMetadata
+from .exceptions import IDRClientException
 from .mixins import InitFromMapping, ToMapping, ToTask
 from .task import Task
 from .transport import Transport, TransportOptions
@@ -7,6 +8,7 @@
     "AbstractDomainObject",
     "DataSource",
     "ExtractMetadata",
+    "IDRClientException",
     "InitFromMapping",
     "Task",
     "ToMapping",

app/core/exceptions.py

@@ -0,0 +1,25 @@
+from typing import Optional
+
+
+class IDRClientException(Exception):
+    """Base exception for most explicit exceptions raised by this app."""
+
+    def __init__(self, message: Optional[str] = None, *args):
+        """Initialize an ``IDRClientException`` with the given parameters.
+
+        :param message: An optional error message.
+        :param args: args to pass to forward to the base exception.
+        """
+        self._message: Optional[str] = message
+        super(IDRClientException, self).__init__(self._message, *args)
+
+    @property
+    def message(self) -> Optional[str]:
+        """
+        Return the error message passed to this exception at initialization
+        or ``None`` if one was not given.
+
+        :return: The error message passed to this exception at initialization
+            or None if one wasn't given.
+        """
+        return self._message

app/lib/__init__.py

@@ -1,10 +1,11 @@
-from .config import Config, SettingInitializer
+from .config import *  # noqa: F401,F403
+from .config import __all__ as _all_config
 from .tasks import *  # noqa: F401,F403
 from .tasks import __all__ as _all_tasks
 from .transports import *  # noqa: F401,F403
 from .transports import __all__ as _all_transports
 
-__all__ = ["Config", "SettingInitializer"]
-
+__all__ = []
+__all__ += _all_config  # type: ignore
 __all__ += _all_tasks  # type: ignore
 __all__ += _all_transports  # type: ignore

app/lib/config/__init__.py

@@ -0,0 +1,4 @@
+from .config import Config, SettingInitializer
+from .exceptions import MissingSettingError
+
+__all__ = ["Config", "MissingSettingError", "SettingInitializer"]

app/lib/config.py → app/lib/config/config.py

@@ -1,21 +1,39 @@
+import logging
 from typing import Any, Dict, Mapping, Optional
 
 from app.core import Task
 
+from .exceptions import MissingSettingError
+
 # =============================================================================
 # TYPES
 # =============================================================================
 
 SettingInitializer = Task[Any, Any]
 
 
+# =============================================================================
+# CONSTANTS
+# =============================================================================
+
+_LOGGER = logging.getLogger(__name__)
+
+
 # =============================================================================
 # CONFIG
 # =============================================================================
 
 
 class Config:
-    """An object that holds the app settings."""
+    """An object that holds the app settings.
+
+    Only read only access to the settings is available post initialization. Any
+    required modifications to the settings should be done at initialization
+    time by passing a mapping of setting names and ``SettingInitializer``
+    instances to this class's constructor. Setting names are encouraged to be
+    uppercase to convey that they are read only. Setting names that are also
+    valid python identifiers can also be accessed using the dot notation.
+    """
 
     def __init__(
         self,
@@ -50,7 +68,26 @@ def __init__(
 
     def __getattr__(self, setting: str) -> Any:
         """Make settings available using the dot operator."""
-        return self._settings[setting]
+        try:
+            return self._settings[setting]
+        except KeyError:
+            raise MissingSettingError(setting=setting)
+
+    def get(self, setting: str, default: Any = None) -> Any:
+        """
+        Retrieve the value of the given setting or return the given default if
+        no such setting exists in this ``Config`` instance. This method can
+        also be used for retrieval of settings with invalid python identifier
+        names.
+
+        :param setting: The name of the setting value to retrieve.
+        :param default: A value to return when no setting with the given name
+            exists in this config.
+
+        :return: The value of the given setting if it is present in this config
+            or the given default otherwise.
+        """
+        return self._settings.get(setting, default)
 
     def _run_initializers(self) -> None:
         """
@@ -63,5 +100,11 @@ def _run_initializers(self) -> None:
         :return: None.
         """
         for setting, initializer in self._initializers.items():
-            setting_val: Any = initializer(self._settings.get(setting))
+            raw_setting_val: Any = self._settings.get(setting)
+            setting_val: Any = initializer(raw_setting_val)
+            _LOGGER.debug(
+                'Ran initializer for the setting "%s" with raw value "%s".',
+                str(setting),
+                str(raw_setting_val),
+            )
             self._settings[setting] = setting_val

app/lib/config/exceptions.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from app.core import IDRClientException
+
+
+class MissingSettingError(IDRClientException, LookupError):
+    """Non existing setting access error."""
+
+    def __init__(self, setting: str, message: Optional[str] = None):
+        """Initialize a ``MissingSettingError`` with the given properties.
+
+        :param setting: The missing setting.
+        :param message: An optional message for the resulting exception. If
+            none is provided, then a generic one is automatically generated.
+        """
+        self._setting: str = setting
+        self._message: str = (
+            message or 'Setting "%s" does not exist.' % self._setting
+        )
+        IDRClientException.__init__(self, message=self._message)
+
+    @property
+    def setting(self) -> str:
+        """
+        Return the missing setting that resulted in this exception being
+        raised.
+
+        :return: The missing setting.
+        """
+        return self._setting

pyproject.toml

@@ -54,7 +54,7 @@ strictSetInference = true
 typeCheckingMode = "basic"
 
 [tool.pytest.ini_options]
-addopts = "--cov=app --cov-fail-under=74 --cov-report=html --cov-report=term-missing -n auto --junitxml='junitxml_report/report.xml' -v --durations=10 --cache-clear -p no:sugar"
+addopts = "--cov=app --cov-fail-under=77 --cov-report=html --cov-report=term-missing -n auto --junitxml='junitxml_report/report.xml' -v --durations=10 --cache-clear -p no:sugar"
 console_output_style = "progress"
 log_cli = 1
 log_cli_date_format = "%Y-%m-%d %H:%M:%S"

tests/lib/config/test_config.py

@@ -0,0 +1,125 @@
+from typing import Any, Mapping
+from unittest import TestCase
+
+import pytest
+
+from app.core import Task
+from app.lib import Config, MissingSettingError, SettingInitializer
+
+# =============================================================================
+# HELPERS
+# =============================================================================
+
+
+class _SettingToStringInitializer(Task[Any, str]):
+    """
+    A simple initializer that takes a setting raw value and returns it's string
+    representation.
+    """
+
+    def execute(self, an_input: Any) -> str:
+        return str(an_input)
+
+
+class _AddOneInitializer(Task[int, int]):
+    """
+    A simple initializer that takes an integer setting value and returns the
+    value + 1 or 0 if the the value is None.
+    """
+
+    def execute(self, an_input: int) -> int:
+        return an_input + 1 if an_input is not None else 0
+
+
+# =============================================================================
+# TEST CASES
+# =============================================================================
+
+
+class TestConfig(TestCase):
+    """Tests for the :class:`Config` class."""
+
+    def setUp(self) -> None:
+        super().setUp()
+        # Settings
+        self._setting_1_value: int = 0
+        self._setting_2_value: int = 0
+        self._settings: Mapping[str, Any] = {
+            "SETTING_1": self._setting_1_value,
+            "SETTING_2": self._setting_2_value,
+        }
+
+        # Setting Initializers
+        self._setting_1_initializer = _SettingToStringInitializer()
+        self._setting_2_initializer = _AddOneInitializer()
+        self._initializers: Mapping[str, SettingInitializer] = {
+            "SETTING_1": self._setting_1_initializer,
+            "SETTING_2": self._setting_2_initializer,
+        }
+
+    def test_initializers_are_run(self) -> None:
+        """
+        Assert that initializers are run and the settings value are updated.
+        """
+        config = Config(
+            settings=self._settings, settings_initializers=self._initializers
+        )
+
+        assert config.SETTING_1 == "0"
+        assert config.SETTING_2 == 1
+
+    def test_initializers_can_set_default_values(self) -> None:
+        """
+        Assert that when a setting is not provided but an initializer for the
+        setting exists, the value returned after running the initializer
+        becomes the new value of the setting.
+        """
+        self._settings = {"SETTING_1": self._setting_1_value}
+        config = Config(
+            settings=self._settings, settings_initializers=self._initializers
+        )
+
+        assert config.SETTING_1 == "0"
+        assert config.SETTING_2 == 0  # SETTING_2 now has a default
+
+    def test_missing_settings_retrieval_using_dot_notation(self) -> None:
+        """
+        Assert that retrieval of missing settings using the dot notation
+        results in a :class:`MissingSettingError` being raised.
+        """
+        config = Config(settings={})
+        with pytest.raises(MissingSettingError) as exp:
+            _: Any = config.INVALID_SETTING
+
+        assert exp.value.setting == "INVALID_SETTING"
+
+    def test_settings_retrieval_using_get_method(self) -> None:
+        """
+        Assert that retrieval of settings using the method returns the
+        expected setting or the default value when the setting is missing.
+        """
+        self._settings = {
+            **self._settings,
+            "weird::setting::name": "some value",
+        }
+        config = Config(
+            settings=self._settings, settings_initializers=self._initializers
+        )
+
+        assert config.get("SETTING_1") == "0"
+        assert config.get("weird::setting::name") == "some value"
+        assert config.get("MISSING", default="default") == "default"
+        assert config.get("MISSING") is None
+
+    def test_settings_without_initializers_are_not_modified(self) -> None:
+        """
+        Assert that settings without initializers are not modified when
+        initializers run and are returned as is.
+        """
+        setting_3_value: str = "Do not modify!"
+        self._settings = {**self._settings, "SETTING_3": setting_3_value}
+        config = Config(
+            settings=self._settings, settings_initializers=self._initializers
+        )
+
+        assert config.SETTING_3 == setting_3_value