feat: validate custom config (#128)

README.md

@@ -118,7 +118,7 @@ it adds additional features that allow easy modification of specifications on
 the fly. In particular, links to routers and security definitions can be added
 to each specified endpoint.
 
-*Example:*
+_Example:_
 
 ```yaml
 api:
@@ -167,7 +167,7 @@ collections for you. To use that functionality, simply include the top-level
 `db` keyword section in your configuration file and tune its behavior through
 the available parameters.
 
-*Example:*
+_Example:_
 
 ```yaml
 db:
@@ -212,7 +212,7 @@ FOCA provides a convenient, configurable exception handler and a simple way
 of adding new exceptions to be used with that handler. To use it, specify a
 top-level `exceptions` section in the app configuration file.
 
-*Example:*
+_Example:_
 
 ```yaml
 exceptions:
@@ -242,7 +242,7 @@ FOCA offers limited support for running asynchronous tasks via the
 [RabbitMQ][res-rabbitmq] broker and [Celery][res-celery]. To make use of it,
 include the `jobs` top-level section in the app configuration file.
 
-*Example:*
+_Example:_
 
 ```yaml
 jobs:
@@ -270,7 +270,7 @@ application's logging behavior in an effort to provide a single configuration
 file for every application. To use it, simply add a `log` top-level section in
 your app configuration file.
 
-*Example:*
+_Example:_
 
 ```yaml
 log:
@@ -350,15 +350,62 @@ server:
 
 ### Custom configuration
 
+If you would like FOCA to validate your custom app configuration (e.g.,
+parameters required for individual controllers, you can provide a path, in
+dot notation, to a [`pydantic`][res-pydantic] `BaseModel`-derived model. FOCA
+then tries to instantiate the model class with any custom parameters listed
+under keyword section `custom`.
+
+Suppose you have a model like the following defined in module
+`my_app.custom_config`:
+
+```py
+from pydantic import BaseModel
+
+
+class CustomConfig(BaseModel):
+    my_param: int = 5
+```
+
+And you have, in your app configuration file `my_app/config.yaml`, the
+following section:
+
+```console
+custom:
+  my_param: 10
+```
+
+You can then have FOCA validate your custom configuration against the
+`CustomConfig` class by including it in the `foca()` call like so:
+
+```py
+from foca.foca import foca
+
+my_app = foca(
+  config="my_app/config.yaml",
+  custom_config_model="my_app.custom_config.CustomConfig",
+)
+```
+
+We recommend that, when defining your `pydantic` model, that you supply
+default values wherever possible. In this way, the custom configuration
+parameters will always be available, even if not explicitly listed in the app
+configuration (like with the FOCA-specific parameters).
+
+> Note that there is tooling available to automatically generate `pydantic`
+> models from different file formats like JSON Schema etc. See here for the
+> [datamodel-code-generator][res-datamodel-code-generator] project.
+
 Apart from the reserved keyword sections listed above, you are free to include
 any other sections and parameters in your app configuration file. FOCA will
 simply attach these to your application instance as described
 [above](#configuration-file) and shown
 [below](#accessing-configuration-parameters). Note, however, that any
-such parameters need to be **MANUALLY VALIDATED**, as unfortunately (or
-fortunately!) we can't read your mind just yet.
+such parameters need to be _manually_ validated. The same is true if you
+include a `custom` section but do _not_ provide a validation model class via
+the `custom_config_model` parameter when calling `foca()`.
 
-*Example:*
+_Example:_
 
 ```yaml
 my_custom_param: 'some_value'
@@ -518,7 +565,7 @@ question etc.
 [org-elixir-cloud]: <https://github.com/elixir-cloud-aai/elixir-cloud-aai>
 [res-celery]: <http://docs.celeryproject.org/>
 [res-connexion]: <https://github.com/zalando/connexion>
-[res-cors]: <https://flask-cors.readthedocs.io/en/latest/>
+[res-datamodel-code-generator]: <https://github.com/koxudaxi/datamodel-code-generator/>res-cors]: <https://flask-cors.readthedocs.io/en/latest/>
 [res-elixir-cloud-coc]: <https://github.com/elixir-cloud-aai/elixir-cloud-aai/blob/dev/CODE_OF_CONDUCT.md>
 [res-elixir-cloud-contributing]: <https://github.com/elixir-cloud-aai/elixir-cloud-aai/blob/dev/CONTRIBUTING.md>
 [res-flask]: <http://flask.pocoo.org/>
@@ -529,6 +576,7 @@ question etc.
 [res-python-logging]: <https://docs.python.org/3/library/logging.html>
 [res-python-logging-how-to]: <https://docs.python.org/3/howto/logging.html?highlight=yaml#configuring-logging>
 [res-openapi]: <https://www.openapis.org/>
+[res-pydantic]: <https://pydantic-docs.helpmanual.io/>
 [res-rabbitmq]: <https://www.rabbitmq.com/>
 [res-semver]: <https://semver.org/>
 [res-swagger]: <https://swagger.io/tools/swagger-ui/>

foca/config/config_parser.py

@@ -1,10 +1,13 @@
 """Parser for YAML-based app configuration."""
 
+from importlib import import_module
 import logging
 from logging.config import dictConfig
+from pathlib import Path
 from typing import (Dict, Optional)
 
 from addict import Dict as Addict
+from pydantic import BaseModel
 import yaml
 
 from foca.models.config import (Config, LogConfig)
@@ -17,23 +20,52 @@ class ConfigParser():
 
     Args:
         config_file: Path to config file in YAML format.
+        custom_config_model: Path to model to be used for custom config
+            parameter validation, supplied in "dot notation", e.g.,
+            ``myapp.config.models.CustomConfig`, where ``CustomConfig`` is the
+            actual importable name of a `pydantic` model for your custom
+            configuration, deriving from ``BaseModel``. FOCA will attempt to
+            instantiate the model with the values passed to the ``custom``
+            section in the application's configuration, if present. Wherever
+            possible, make sure that default values are supplied for each
+            config parameters, so as to make it easier for others to
+            write/modify their app configuration.
         format_logs: Whether log formatting should be configured.
 
     Attributes:
         config_file: Path to config file in YAML format.
+        custom_config_model: Path to model to be used for custom config
+            parameter validation, supplied in "dot notation", e.g.,
+            ``myapp.config.models.CustomConfig`, where ``CustomConfig`` is the
+            actual importable name of a `pydantic` model for your custom
+            configuration, deriving from ``BaseModel``. FOCA will attempt to
+            instantiate the model with the values passed to the ``custom``
+            section in the application's configuration, if present. Wherever
+            possible, make sure that default values are supplied for each
+            config parameters, so as to make it easier for others to
+            write/modify their app configuration.
         format_logs: Whether log formatting should be configured.
     """
 
     def __init__(
         self,
         config_file: Optional[str] = None,
+        custom_config_model: Optional[str] = None,
         format_logs: bool = True
     ) -> None:
         """Constructor method."""
-        if config_file:
+        if config_file is not None:
             self.config = Config(**self.parse_yaml(config_file))
         else:
             self.config = Config()
+        if custom_config_model is not None:
+            setattr(
+                self.config,
+                'custom',
+                self.parse_custom_config(
+                    model=custom_config_model,
+                )
+            )
         if format_logs:
             self._configure_logging()
         logger.debug(f"Parsed config: {self.config.dict(by_alias=True)}")
@@ -61,20 +93,20 @@ def parse_yaml(conf: str) -> Dict:
 
         Raises:
             OSError: File cannot be accessed.
-            yaml.YAMLError: File contents cannot be parsed.
+            ValueError: File contents cannot be parsed.
         """
         try:
             with open(conf) as config_file:
                 try:
                     return yaml.safe_load(config_file)
-                except yaml.YAMLError:
-                    raise yaml.YAMLError(
+                except yaml.YAMLError as exc:
+                    raise ValueError(
                         f"file '{conf}' is not valid YAML"
-                    )
-        except OSError as e:
+                    ) from exc
+        except OSError as exc:
             raise OSError(
                 f"file '{conf}' could not be read"
-            ) from e
+            ) from exc
 
     @staticmethod
     def merge_yaml(*args: str) -> Optional[Dict]:
@@ -101,3 +133,46 @@ def merge_yaml(*args: str) -> Optional[Dict]:
             yaml_dict.update(Addict(ConfigParser.parse_yaml(arg)))
 
         return yaml_dict.to_dict()
+
+    def parse_custom_config(self, model: str) -> BaseModel:
+        """Parse custom configuration and validate against a model.
+
+        The method will attempt to instantiate the model with the parameters
+        provided in the application configuration's ``custom`` section, if
+        provided. Any required configuration parameters for which no defaults
+        are provided in the model indeed will have to be listed in such a
+        section.
+
+        Args:
+            model: Path to model to be used for configuration validation,
+            supplied in "dot notation", e.g.,
+            ``myapp.config.models.CustomConfig`, where ``CustomConfig`` is the
+            actual importable name of a `pydantic` model for your custom
+            configuration, deriving from ``BaseModel``.
+
+        Returns:
+            Custom configuration model instantiated with the parameters listed
+                in the app configuration's ``custom``.
+        """
+        module = Path(model).stem
+        model_class = Path(model).suffix[1:]
+        try:
+            model_class = getattr(import_module(module), model_class)
+        except ModuleNotFoundError:
+            raise ValueError(
+                f"failed validating custom configuration: module '{module}' "
+                "not available"
+            )
+        except (AttributeError, ImportError):
+            raise ValueError(
+                f"failed validating custom configuration: module '{module}' "
+                f"has no class {model_class} or could not be imported"
+            )
+        try:
+            custom_config = model_class(**self.config.custom)
+        except Exception as exc:
+            raise ValueError(
+                "failed validating custom configuration: provided custom "
+                f"configuration does not match model class in '{model}'"
+            ) from exc
+        return custom_config

foca/foca.py

@@ -17,19 +17,36 @@
 logger = logging.getLogger(__name__)
 
 
-def foca(config: Optional[str] = None) -> App:
+def foca(
+    config: Optional[str] = None,
+    custom_config_model: Optional[str] = None,
+) -> App:
     """Set up and initialize FOCA-based microservice.
 
     Args:
         config: Path to application configuration file in YAML format. Cf.
             :py:class:`foca.models.config.Config` for required file structure.
+        custom_config_model: Path to model to be used for custom config
+            parameter validation, supplied in "dot notation", e.g.,
+            ``myapp.config.models.CustomConfig`, where ``CustomConfig`` is the
+            actual importable name of a `pydantic` model for your custom
+            configuration, deriving from ``BaseModel``. FOCA will attempt to
+            instantiate the model with the values passed to the ``custom``
+            section in the application's configuration, if present. Wherever
+            possible, make sure that default values are supplied for each
+            config parameters, so as to make it easier for others to
+            write/modify their app configuration.
 
     Returns:
         Connexion application instance.
     """
 
     # Parse config parameters and format logging
-    conf = ConfigParser(config, format_logs=True).config
+    conf = ConfigParser(
+        config_file=config,
+        custom_config_model=custom_config_model,
+        format_logs=True,
+    ).config
     logger.info("Log formatting configured.")
     if config:
         logger.info(f"Configuration file '{config}' parsed.")

foca/models/config.py

@@ -212,7 +212,7 @@ class ExceptionConfig(FOCABaseConfig):
             as keys and a dictionary of JSON members (as per `required_members`
             and `extension_members`) as values. Path should be a dot-separated
             path to the module containing the dictionary (which needs to also
-            contain imports for all listed expcetions), followed by the name of
+            contain imports for all listed exceptions), followed by the name of
             the dictionary itself. For example, for ``myapp.errors.exc_dict``,
             the dictionary ``exc_dict`` would be attempted to be imported from
             module ``myapp.errors`` (must be available in the Pythonpath). To
@@ -253,7 +253,7 @@ class ExceptionConfig(FOCABaseConfig):
             as keys and a dictionary of JSON members (as per `required_members`
             and `extension_members`) as values. Path should be a dot-separated
             path to the module containing the dictionary (which needs to also
-            contain imports for all listed expcetions), followed by the name of
+            contain imports for all listed exceptions), followed by the name of
             the dictionary itself. For example, for ``myapp.errors.exc_dict``,
             the dictionary ``exc_dict`` would be attempted to be imported from
             module ``myapp.errors`` (must be available in the Pythonpath). To

tests/api/test_register_openapi.py

@@ -122,7 +122,7 @@ def test_openapi_2_json_invalid(self):
         """Registration failing because of invalid JSON spec file."""
         app = App(__name__)
         spec_configs = [SpecConfig(path=PATH_SPECS_INVALID_JSON)]
-        with pytest.raises(YAMLError):
+        with pytest.raises(ValueError):
             register_openapi(app=app, specs=spec_configs)
 
     def test_openapi_not_found(self):