Merge pull request #326 from crocs-muni/pydantic-settings-management

switch to pydantic in settings management

docs/configuration.md

@@ -6,21 +6,49 @@ mystnb:
 ---
 # Configuration
 
-The configuration is stored in yaml file `settings.yaml` at `sec_certs.config` package. Below are the supported options, descriptions and default values.
+The configuration class is defined in [configuration.py](https://github.com/crocs-muni/sec-certs/tree/main/src/sec_certs/configuration.py). From CLI, you can load custom configuration yaml with `-c` or `--config` argument. From Python, you can replace the default configuration with
+
+```python
+from pathlib import Path
+import sec_certs.configuration as config_module
+
+config_module.config.load_from_yaml("/path/to/your/config.yaml")
+
+# or just set the individual key
+config_module.config.log_filepath = Path("/some/path/where/log/will/be/stored.txt")
+```
+
+The configuration yaml is a simple flat dictionary of keys and values. The configuration file can specify only *some* of the fields. For the content of unspecified fields, environment variable with `seccerts_` prefix (case insensitive) will be checked. If such variable is not set, default value will be used. Content in the yaml always beats the environment variable.
+
+For instance, when user provides the following yaml
+
+```yaml
+log_filepath: my_own_log_file.txt
+n_threads: 7
+```
+
+and sets `SECCERTS_MINIMAL_TOKEN_LENGTH=4` as environment variable, only these 3 keys will be loaded with `config.load_from_yaml()`, others will be untouched.
+
+```{tip}
+You can load settings even without providing yaml configuration. Simply set the corresponding environment variables or use `.env` file.
+```
+
+## Configuration keys, types, default values and descriptions
 
 
 ```{code-cell} python
-from sec_certs.config import configuration
+from sec_certs.configuration import config, Configuration
 from myst_nb import glue
 from IPython.display import Markdown
+import typing
 
-cfg = configuration.config
+type_hints = typing.get_type_hints(Configuration)
 text = ""
-for key in cfg.__dict__:
-    text += f"`{key}`\n\n- Description: {cfg.get_desription(key)}\n"
-    text += f"- Default value: `{cfg.__getattribute__(key)}`\n\n"
+for field, value in config.__fields__.items():
+    text += f"`{field}`\n\n"
+    text += f"- type: `{type_hints[field]}`\n"
+    text += f"- default: `{value.default}`\n"
+    text += f"- description: {value.field_info.description}\n"
+    text += f"- env name: `{list(value.field_info.extra['env_names'])[0]}`\n\n"
 glue("text", Markdown(text))
 ```
-```{glue:md} text
-:format: myst
-```

pyproject.toml

@@ -58,6 +58,7 @@
     "pySankeyBeta",
     "scipy>=1.9.0",
     "networkx",
+    "pydantic",
   ]
 
   [project.optional-dependencies]

src/sec_certs/cli.py

@@ -9,8 +9,9 @@
 from typing import Callable
 
 import click
+from pydantic import ValidationError
 
-from sec_certs.config.configuration import config
+from sec_certs.configuration import config
 from sec_certs.dataset import CCDataset, FIPSDataset
 from sec_certs.dataset.dataset import Dataset
 from sec_certs.utils.helpers import warn_if_missing_poppler, warn_if_missing_tesseract
@@ -157,7 +158,7 @@ def build_or_load_dataset(
     "configpath",
     default=None,
     type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True),
-    help="Path to your own config yaml file that will override the default one.",
+    help="Path to your own config yaml file that will override the default config.",
 )
 @click.option(
     "-i",
@@ -176,6 +177,16 @@ def main(
     quiet: bool,
 ):
     try:
+        if configpath:
+            try:
+                config.load_from_yaml(configpath)
+            except FileNotFoundError:
+                click.echo("Error: Bad path to configuration file", err=True)
+                sys.exit(EXIT_CODE_NOK)
+            except (ValueError, ValidationError) as e:
+                click.echo(f"Error: Bad format of configuration file: {e}", err=True)
+                sys.exit(EXIT_CODE_NOK)
+
         file_handler = logging.FileHandler(config.log_filepath)
         stream_handler = logging.StreamHandler(sys.stderr)
         formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
@@ -185,16 +196,6 @@ def main(
         logging.basicConfig(level=logging.INFO, handlers=handlers)
         start = datetime.now()
 
-        if configpath:
-            try:
-                config.load(configpath)
-            except FileNotFoundError:
-                click.echo("Error: Bad path to configuration file", err=True)
-                sys.exit(EXIT_CODE_NOK)
-            except ValueError as e:
-                click.echo(f"Error: Bad format of configuration file: {e}", err=True)
-                sys.exit(EXIT_CODE_NOK)
-
         actions_set = (
             {"build", "process-aux-dsets", "download", "convert", "analyze"} if "all" in actions else set(actions)
         )