From 97fb3f38242c594773ac3767c7258a6db9ca1952 Mon Sep 17 00:00:00 2001 From: Batkhuu Date: Thu, 7 Sep 2023 19:02:29 +0900 Subject: [PATCH] :bug: Fixed error occur when loading empty config files. --- .github/workflows/2.build-publish.yml | 5 +- README.md | 205 ++++++++++++++------ examples/.env | 6 - examples/advanced/app.py | 23 +++ examples/advanced/config.py | 25 +++ examples/{ => advanced}/configs/config.yml | 1 + examples/{ => advanced}/configs/logger.json | 0 examples/advanced/logger.py | 8 + examples/advanced/schema.py | 37 ++++ examples/main.py | 78 -------- examples/requirements.txt | 1 + examples/simple/configs/1.base.yml | 7 + examples/simple/configs/2.extra.yml | 8 + examples/simple/main.py | 23 +++ onion_config/_base.py | 6 +- requirements.build.txt | 4 + requirements.dev.txt | 10 +- requirements.test.txt | 5 + setup.py | 2 +- 19 files changed, 297 insertions(+), 157 deletions(-) delete mode 100644 examples/.env create mode 100755 examples/advanced/app.py create mode 100644 examples/advanced/config.py rename examples/{ => advanced}/configs/config.yml (91%) rename examples/{ => advanced}/configs/logger.json (100%) create mode 100644 examples/advanced/logger.py create mode 100644 examples/advanced/schema.py delete mode 100755 examples/main.py create mode 100644 examples/requirements.txt create mode 100644 examples/simple/configs/1.base.yml create mode 100644 examples/simple/configs/2.extra.yml create mode 100755 examples/simple/main.py create mode 100644 requirements.build.txt create mode 100644 requirements.test.txt diff --git a/.github/workflows/2.build-publish.yml b/.github/workflows/2.build-publish.yml index 671b49c..165bc5f 100644 --- a/.github/workflows/2.build-publish.yml +++ b/.github/workflows/2.build-publish.yml @@ -28,8 +28,7 @@ jobs: - name: Install dependencies run: | python -m pip install -U pip - python -m pip install -r ./requirements.txt - python -m pip install pytest==7.4.1 pytest-cov==4.1.0 + python -m pip install -r ./requirements.test.txt - name: Test with pytest run: python -m pytest -sv # run: python -m pytest -sv -o log_cli=true @@ -50,7 +49,7 @@ jobs: - name: Install dependencies run: | python -m pip install -U pip - python -m pip install build==0.10.0 twine==4.0.2 + python -m pip install -r ./requirements.build.txt - name: Build and publish package # run: | # echo -e "[testpypi]\nusername = __token__\npassword = ${{ secrets.TEST_PYPI_API_TOKEN }}" > ~/.pypirc diff --git a/README.md b/README.md index 472bec4..9bc7e95 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![PyPI](https://img.shields.io/pypi/v/onion-config?logo=PyPi)](https://pypi.org/project/onion-config) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/onion-config?logo=Python)](https://docs.conda.io/en/latest/miniconda.html) -`onion_config` is a Python package that allows for easy configuration management. It allows for loading and validating configuration data from environment variables and config files in JSON and YAML formats. +`onion_config` is a python package that allows for easy configuration management. It allows for loading and validating configuration data from environment variables and config files in JSON and YAML formats. `Pydantic` based custom config package for python projects. @@ -109,44 +109,94 @@ export PYTHONPATH="${PWD}:${PYTHONPATH}" ## Usage/Examples -### **Simple** - To use `onion_config`, import the `ConfigLoader` class from the package: ```python -from onion_config import ConfigLoader -``` - -You can then create an instance of `ConfigLoader`: - -```python -config_loader = ConfigLoader(auto_load=True) +from onion_config import ConfigLoader, BaseConfig ``` -This will automatically load configuration data from environment variables and config files located in the default directory (`'./configs'`). The configuration data can then be accessed via the `config` property of the `ConfigLoader` instance: +You can create an instance of `ConfigLoader` with `auto_load` flag. This will automatically load configuration data from environment variables and config files located in the default directory (`'./configs'`). The configuration data can then be accessed via the `config` property of the `ConfigLoader` instance: ```python -config = config_loader.config +config: BaseConfig = ConfigLoader(auto_load=True).config ``` -### **Advanced** +### **Simple** -[**`configs/config.yml`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/configs/config.yml): +[**`configs/1.base.yml`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/simple/configs/1.base.yml): ```yaml -env: production +env: test app: name: "My App" - bind_host: "0.0.0.0" version: "0.0.1" - ignore_val: "Ignore me" + nested: + key: "value" +``` -logger: - output: "stdout" +[**`configs/2.extra.yml`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/simple/configs/2.extra.yml): + +```yaml +app: + name: "New App" + nested: + some: "value" + description: "Description of my app." + +another_val: + extra: 1 ``` -[**`configs/logger.json`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/configs/logger.json): +[**`main.py`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/simple/main.py) + +```python +import sys +import pprint +import logging + +from onion_config import ConfigLoader, BaseConfig + + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +logger = logging.getLogger(__name__) + + +try: + config: BaseConfig = ConfigLoader().load() +except Exception: + logger.exception("Failed to load config:") + exit(2) + +if __name__ == "__main__": + logger.info(f" App name: {config.app['name']}") + logger.info(f" Config:\n{pprint.pformat(config.model_dump())}\n") +``` + +Run the [**`examples/simple`**](https://github.com/bybatkhuu/mod.python-config/tree/main/examples/simple): + +```sh +cd ./examples/simple + +python ./main.py +``` + +Output: + +```txt +INFO:__main__: App name: New App +INFO:__main__: Config: +{'another_val': {'extra': 1}, + 'app': {'description': 'Description of my app.', + 'name': 'New App', + 'nested': {'key': 'value', 'some': 'value'}, + 'version': '0.0.1'}, + 'env': 'test'} +``` + +### **Advanced** + +[**`configs/logger.json`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/advanced/configs/logger.json): ```json { @@ -158,7 +208,23 @@ logger: } ``` -[**`.env`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/.env): +[**`configs/config.yml`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/advanced/configs/config.yml): + +```yaml +env: production + +app: + name: "My App" + port: 9000 + bind_host: "0.0.0.0" + version: "0.0.1" + ignore_val: "Ignore me" + +logger: + output: "stdout" +``` + +[**`.env`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/advanced/.env): ```sh ENV=development @@ -169,31 +235,27 @@ APP_NAME="New App" APP_SECRET="my_secret" ``` -[**`main.py`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/main.py): +[**`logger.py`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/advanced/logger.py): ```python import sys -import pprint import logging + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +logger = logging.getLogger(__name__) +``` + +[**`schema.py`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/advanced/schema.py): + +```python from enum import Enum from typing import Union from pydantic import Field, SecretStr from pydantic_settings import SettingsConfigDict -from onion_config import ConfigLoader, BaseConfig - - -logging.basicConfig(stream=sys.stdout, level=logging.INFO) -logger = logging.getLogger(__name__) - - -# Pre-load function to modify config data before loading and validation: -def _pre_load_hook(config_data: dict) -> dict: - config_data["app"]["port"] = "80" - config_data["extra_val"] = "Something extra!" +from onion_config import BaseConfig - return config_data # Environments as Enum: class EnvEnum(str, Enum): @@ -219,18 +281,45 @@ class ConfigSchema(BaseConfig): env: EnvEnum = Field(EnvEnum.LOCAL) debug: bool = Field(False) app: AppConfig = Field(...) +``` +[**`config.py`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/advanced/config.py): -if __name__ == "__main__": - try: - # Main 'config' object for usage: - config: ConfigSchema = ConfigLoader( - config_schema=ConfigSchema, pre_load_hook=_pre_load_hook - ).load() - except Exception: - logger.exception("Failed to load config:") - exit(2) +```python +from onion_config import ConfigLoader + +from logger import logger +from schema import ConfigSchema + + +# Pre-load function to modify config data before loading and validation: +def _pre_load_hook(config_data: dict) -> dict: + config_data["app"]["port"] = "80" + config_data["extra_val"] = "Something extra!" + return config_data + +config = None +try: + _config_loader = ConfigLoader( + config_schema=ConfigSchema, pre_load_hook=_pre_load_hook + ) + # Main config object: + config: ConfigSchema = _config_loader.load() +except Exception: + logger.exception("Failed to load config:") + exit(2) +``` + +[**`app.py`**](https://github.com/bybatkhuu/mod.python-config/blob/main/examples/advanced/app.py): + +```python +import pprint + +from config import config +from logger import logger + +if __name__ == "__main__": logger.info(f" ENV: {config.env}") logger.info(f" DEBUG: {config.debug}") logger.info(f" Extra: {config.extra_val}") @@ -246,25 +335,25 @@ if __name__ == "__main__": logger.error(f" {e}\n") ``` -Run the [**`example`**](https://github.com/bybatkhuu/mod.python-config/tree/main/examples): +Run the [**`examples/advanced`**](https://github.com/bybatkhuu/mod.python-config/tree/main/examples/advanced): ```sh -cd ./examples +cd ./examples/advanced -python ./main.py +python ./app.py ``` Output: ```txt -INFO:__main__: ENV: development -INFO:__main__: DEBUG: True -INFO:__main__: Extra: Something extra! -INFO:__main__: Logger: {'level': 'info', 'output': 'stdout'} -INFO:__main__: App: name='New App' bind_host='0.0.0.0' port=80 secret=SecretStr('**********') version='0.0.1' description=None -INFO:__main__: Secret: 'my_secret' - -INFO:__main__: Config: +INFO:logger: ENV: development +INFO:logger: DEBUG: True +INFO:logger: Extra: Something extra! +INFO:logger: Logger: {'level': 'info', 'output': 'stdout'} +INFO:logger: App: name='New App' bind_host='0.0.0.0' port=80 secret=SecretStr('**********') version='0.0.1' description=None +INFO:logger: Secret: 'my_secret' + +INFO:logger: Config: {'app': {'bind_host': '0.0.0.0', 'description': None, 'name': 'New App', @@ -276,7 +365,7 @@ INFO:__main__: Config: 'extra_val': 'Something extra!', 'logger': {'level': 'info', 'output': 'stdout'}} -ERROR:__main__: 1 validation error for AppConfig +ERROR:logger: 1 validation error for AppConfig port Instance is frozen [type=frozen_instance, input_value=8443, input_type=int] ``` @@ -288,8 +377,8 @@ port To run tests, run the following command: ```sh -# Install python development dependencies: -pip install -r ./requirements.dev.txt +# Install python test dependencies: +pip install -r ./requirements.test.txt # Run tests: python -m pytest -sv diff --git a/examples/.env b/examples/.env deleted file mode 100644 index c00c39d..0000000 --- a/examples/.env +++ /dev/null @@ -1,6 +0,0 @@ -ENV=development - -DEBUG=true - -APP_NAME="New App" -APP_SECRET="my_secret" diff --git a/examples/advanced/app.py b/examples/advanced/app.py new file mode 100755 index 0000000..94bc0af --- /dev/null +++ b/examples/advanced/app.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import pprint + +from config import config +from logger import logger + + +if __name__ == "__main__": + logger.info(f" ENV: {config.env}") + logger.info(f" DEBUG: {config.debug}") + logger.info(f" Extra: {config.extra_val}") + logger.info(f" Logger: {config.logger}") + logger.info(f" App: {config.app}") + logger.info(f" Secret: '{config.app.secret.get_secret_value()}'\n") + logger.info(f" Config:\n{pprint.pformat(config.model_dump())}\n") + + try: + # This will raise ValidationError + config.app.port = 8443 + except Exception as e: + logger.error(f" {e}\n") diff --git a/examples/advanced/config.py b/examples/advanced/config.py new file mode 100644 index 0000000..a091ba1 --- /dev/null +++ b/examples/advanced/config.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +from onion_config import ConfigLoader + +from logger import logger +from schema import ConfigSchema + + +# Pre-load function to modify config data before loading and validation: +def _pre_load_hook(config_data: dict) -> dict: + config_data["app"]["port"] = "80" + config_data["extra_val"] = "Something extra!" + return config_data + + +config = None +try: + _config_loader = ConfigLoader( + config_schema=ConfigSchema, pre_load_hook=_pre_load_hook + ) + # Main config object: + config: ConfigSchema = _config_loader.load() +except Exception: + logger.exception("Failed to load config:") + exit(2) diff --git a/examples/configs/config.yml b/examples/advanced/configs/config.yml similarity index 91% rename from examples/configs/config.yml rename to examples/advanced/configs/config.yml index a48adad..147b572 100644 --- a/examples/configs/config.yml +++ b/examples/advanced/configs/config.yml @@ -2,6 +2,7 @@ env: production app: name: "My App" + port: 9000 bind_host: "0.0.0.0" version: "0.0.1" ignore_val: "Ignore me" diff --git a/examples/configs/logger.json b/examples/advanced/configs/logger.json similarity index 100% rename from examples/configs/logger.json rename to examples/advanced/configs/logger.json diff --git a/examples/advanced/logger.py b/examples/advanced/logger.py new file mode 100644 index 0000000..6cac7d7 --- /dev/null +++ b/examples/advanced/logger.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +import sys +import logging + + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +logger = logging.getLogger(__name__) diff --git a/examples/advanced/schema.py b/examples/advanced/schema.py new file mode 100644 index 0000000..8ad9a1a --- /dev/null +++ b/examples/advanced/schema.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from enum import Enum +from typing import Union + +from pydantic import Field, SecretStr +from pydantic_settings import SettingsConfigDict + +from onion_config import BaseConfig + + +# Environments as Enum: +class EnvEnum(str, Enum): + LOCAL = "local" + DEVELOPMENT = "development" + TEST = "test" + STAGING = "staging" + PRODUCTION = "production" + + +# App config schema: +class AppConfig(BaseConfig): + name: str = Field("App", min_length=2, max_length=32) + bind_host: str = Field("localhost", min_length=2, max_length=128) + port: int = Field(8000, ge=80, lt=65536) + secret: SecretStr = Field(..., min_length=8, max_length=64) + version: str = Field(..., min_length=5, max_length=16) + description: Union[str, None] = Field(None, min_length=4, max_length=64) + + model_config = SettingsConfigDict(extra="ignore", env_prefix="APP_") + + +# Main config schema: +class ConfigSchema(BaseConfig): + env: EnvEnum = Field(EnvEnum.LOCAL) + debug: bool = Field(False) + app: AppConfig = Field(...) diff --git a/examples/main.py b/examples/main.py deleted file mode 100755 index 69c8c42..0000000 --- a/examples/main.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys -import pprint -import logging -from enum import Enum -from typing import Union - -from pydantic import Field, SecretStr -from pydantic_settings import SettingsConfigDict - -from onion_config import ConfigLoader, BaseConfig - - -logging.basicConfig(stream=sys.stdout, level=logging.INFO) -logger = logging.getLogger(__name__) - - -# Pre-load function to modify config data before loading and validation: -def _pre_load_hook(config_data: dict) -> dict: - config_data["app"]["port"] = "80" - config_data["extra_val"] = "Something extra!" - - return config_data - - -# Environments as Enum: -class EnvEnum(str, Enum): - LOCAL = "local" - DEVELOPMENT = "development" - TEST = "test" - STAGING = "staging" - PRODUCTION = "production" - - -# App config schema: -class AppConfig(BaseConfig): - name: str = Field("App", min_length=2, max_length=32) - bind_host: str = Field("localhost", min_length=2, max_length=128) - port: int = Field(8000, ge=80, lt=65536) - secret: SecretStr = Field(..., min_length=8, max_length=64) - version: str = Field(..., min_length=5, max_length=16) - description: Union[str, None] = Field(None, min_length=4, max_length=64) - - model_config = SettingsConfigDict(extra="ignore", env_prefix="APP_") - - -# Main config schema: -class ConfigSchema(BaseConfig): - env: EnvEnum = Field(EnvEnum.LOCAL) - debug: bool = Field(False) - app: AppConfig = Field(...) - - -if __name__ == "__main__": - try: - # Main 'config' object for usage: - config: ConfigSchema = ConfigLoader( - config_schema=ConfigSchema, pre_load_hook=_pre_load_hook - ).load() - except Exception: - logger.exception("Failed to load config:") - exit(2) - - logger.info(f" ENV: {config.env}") - logger.info(f" DEBUG: {config.debug}") - logger.info(f" Extra: {config.extra_val}") - logger.info(f" Logger: {config.logger}") - logger.info(f" App: {config.app}") - logger.info(f" Secret: '{config.app.secret.get_secret_value()}'\n") - logger.info(f" Config:\n{pprint.pformat(config.model_dump())}\n") - - try: - # This will raise ValidationError - config.app.port = 8443 - except Exception as e: - logger.error(f" {e}\n") diff --git a/examples/requirements.txt b/examples/requirements.txt new file mode 100644 index 0000000..977c5ca --- /dev/null +++ b/examples/requirements.txt @@ -0,0 +1 @@ +-e .. diff --git a/examples/simple/configs/1.base.yml b/examples/simple/configs/1.base.yml new file mode 100644 index 0000000..0a3e268 --- /dev/null +++ b/examples/simple/configs/1.base.yml @@ -0,0 +1,7 @@ +env: test + +app: + name: "My App" + version: "0.0.1" + nested: + key: "value" diff --git a/examples/simple/configs/2.extra.yml b/examples/simple/configs/2.extra.yml new file mode 100644 index 0000000..f5eee72 --- /dev/null +++ b/examples/simple/configs/2.extra.yml @@ -0,0 +1,8 @@ +app: + name: "New App" + nested: + some: "value" + description: "Description of my app." + +another_val: + extra: 1 diff --git a/examples/simple/main.py b/examples/simple/main.py new file mode 100755 index 0000000..5d33724 --- /dev/null +++ b/examples/simple/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import pprint +import logging + +from onion_config import ConfigLoader, BaseConfig + + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +logger = logging.getLogger(__name__) + + +try: + config: BaseConfig = ConfigLoader().load() +except Exception: + logger.exception("Failed to load config:") + exit(2) + +if __name__ == "__main__": + logger.info(f" App name: {config.app['name']}") + logger.info(f" Config:\n{pprint.pformat(config.model_dump())}\n") diff --git a/onion_config/_base.py b/onion_config/_base.py index 213d59d..be4a975 100644 --- a/onion_config/_base.py +++ b/onion_config/_base.py @@ -14,8 +14,8 @@ from pydantic_settings import BaseSettings ## Internal modules -from ._schema import BaseConfig from ._utils import deep_merge +from ._schema import BaseConfig from .__version__ import __version__ @@ -174,7 +174,7 @@ def _load_config_files(self, configs_dir: Union[str, None] = None): for _json_file_path in _json_file_paths: try: with open(_json_file_path, "r", encoding="utf8") as _json_file: - _new_config_dict = json.load(_json_file) + _new_config_dict = json.load(_json_file) or {} self.config_data = deep_merge( self.config_data, _new_config_dict ) @@ -191,7 +191,7 @@ def _load_config_files(self, configs_dir: Union[str, None] = None): for _yaml_file_path in _yaml_file_paths: try: with open(_yaml_file_path, "r", encoding="utf8") as _yaml_file: - _new_config_dict = yaml.safe_load(_yaml_file) + _new_config_dict = yaml.safe_load(_yaml_file) or {} self.config_data = deep_merge( self.config_data, _new_config_dict ) diff --git a/requirements.build.txt b/requirements.build.txt new file mode 100644 index 0000000..07addaa --- /dev/null +++ b/requirements.build.txt @@ -0,0 +1,4 @@ +setuptools>=67.8.0,<70.0.0 +wheel>=0.40.0,<1.0.0 +build>=0.10.0,<1.0.0 +twine>=4.0.2,<5.0.0 diff --git a/requirements.dev.txt b/requirements.dev.txt index d6986e8..a73a243 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,8 +1,2 @@ -pytest>=7.3.1,<8.0.0 -pytest-cov>=4.1.0,<5.0.0 -pytest-xdist>=3.3.1,<4.0.0 -pytest-benchmark>=4.0.0,<5.0.0 -setuptools>=67.8.0,<70.0.0 -wheel>=0.40.0,<1.0.0 -build>=0.10.0,<1.0.0 -twine>=4.0.2,<5.0.0 +-r requirements.test.txt +-r requirements.build.txt diff --git a/requirements.test.txt b/requirements.test.txt new file mode 100644 index 0000000..e1646f8 --- /dev/null +++ b/requirements.test.txt @@ -0,0 +1,5 @@ +-r ./requirements.txt +pytest>=7.3.1,<8.0.0 +pytest-cov>=4.1.0,<5.0.0 +pytest-xdist>=3.3.1,<4.0.0 +pytest-benchmark>=4.0.0,<5.0.0 diff --git a/setup.py b/setup.py index efb6a45..037e4b5 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ packages=[_package_name], version=f"{_package_version}", license="MIT", - description="onion_config is a Python package that allows for easy configuration management. It allows for loading and validating configuration data from environment variables and config files in JSON and YAML formats.", + description="onion_config is a python package that allows for easy configuration management. It allows for loading and validating configuration data from environment variables and config files in JSON and YAML formats.", long_description=open("README.md", "r").read(), long_description_content_type="text/markdown", author="Batkhuu Byambajav",