From 4c6fcbdd480b844fb9bdc12b01a4cc1f657b56b1 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 30 Oct 2025 23:36:50 +0100 Subject: [PATCH 1/4] add deep merge functionality to config file sources --- pydantic_settings/sources/base.py | 7 ++++--- pydantic_settings/sources/providers/json.py | 3 ++- pydantic_settings/sources/providers/toml.py | 3 ++- pydantic_settings/sources/providers/yaml.py | 3 ++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pydantic_settings/sources/base.py b/pydantic_settings/sources/base.py index 748f75e6..def769c9 100644 --- a/pydantic_settings/sources/base.py +++ b/pydantic_settings/sources/base.py @@ -13,7 +13,7 @@ from pydantic._internal._typing_extra import ( # type: ignore[attr-defined] get_origin, ) -from pydantic._internal._utils import is_model_class +from pydantic._internal._utils import deep_update, is_model_class from pydantic.fields import FieldInfo from typing_extensions import get_args from typing_inspection import typing_objects @@ -193,16 +193,17 @@ def __call__(self) -> dict[str, Any]: class ConfigFileSourceMixin(ABC): - def _read_files(self, files: PathType | None) -> dict[str, Any]: + def _read_files(self, files: PathType | None, deep_merge: bool = False) -> dict[str, Any]: if files is None: return {} if isinstance(files, (str, os.PathLike)): files = [files] vars: dict[str, Any] = {} + update = deep_update if deep_merge else dict.update for file in files: file_path = Path(file).expanduser() if file_path.is_file(): - vars.update(self._read_file(file_path)) + update(vars, self._read_file(file_path)) return vars @abstractmethod diff --git a/pydantic_settings/sources/providers/json.py b/pydantic_settings/sources/providers/json.py index 837601c3..1459f897 100644 --- a/pydantic_settings/sources/providers/json.py +++ b/pydantic_settings/sources/providers/json.py @@ -26,6 +26,7 @@ def __init__( settings_cls: type[BaseSettings], json_file: PathType | None = DEFAULT_PATH, json_file_encoding: str | None = None, + deep_merge: bool = False, ): self.json_file_path = json_file if json_file != DEFAULT_PATH else settings_cls.model_config.get('json_file') self.json_file_encoding = ( @@ -33,7 +34,7 @@ def __init__( if json_file_encoding is not None else settings_cls.model_config.get('json_file_encoding') ) - self.json_data = self._read_files(self.json_file_path) + self.json_data = self._read_files(self.json_file_path, deep_merge=deep_merge) super().__init__(settings_cls, self.json_data) def _read_file(self, file_path: Path) -> dict[str, Any]: diff --git a/pydantic_settings/sources/providers/toml.py b/pydantic_settings/sources/providers/toml.py index eaff41da..67f8c30e 100644 --- a/pydantic_settings/sources/providers/toml.py +++ b/pydantic_settings/sources/providers/toml.py @@ -50,9 +50,10 @@ def __init__( self, settings_cls: type[BaseSettings], toml_file: PathType | None = DEFAULT_PATH, + deep_merge: bool = False, ): self.toml_file_path = toml_file if toml_file != DEFAULT_PATH else settings_cls.model_config.get('toml_file') - self.toml_data = self._read_files(self.toml_file_path) + self.toml_data = self._read_files(self.toml_file_path, deep_merge=deep_merge) super().__init__(settings_cls, self.toml_data) def _read_file(self, file_path: Path) -> dict[str, Any]: diff --git a/pydantic_settings/sources/providers/yaml.py b/pydantic_settings/sources/providers/yaml.py index 82778b4f..849b0426 100644 --- a/pydantic_settings/sources/providers/yaml.py +++ b/pydantic_settings/sources/providers/yaml.py @@ -40,6 +40,7 @@ def __init__( yaml_file: PathType | None = DEFAULT_PATH, yaml_file_encoding: str | None = None, yaml_config_section: str | None = None, + deep_merge: bool = False, ): self.yaml_file_path = yaml_file if yaml_file != DEFAULT_PATH else settings_cls.model_config.get('yaml_file') self.yaml_file_encoding = ( @@ -52,7 +53,7 @@ def __init__( if yaml_config_section is not None else settings_cls.model_config.get('yaml_config_section') ) - self.yaml_data = self._read_files(self.yaml_file_path) + self.yaml_data = self._read_files(self.yaml_file_path, deep_merge=deep_merge) if self.yaml_config_section: try: From 0c52c4870dcdcf8dcefcab10a49ce75625622ab4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 12 Nov 2025 00:23:37 +0100 Subject: [PATCH 2/4] add tests and docs --- docs/index.md | 127 ++++++++++++++++++++++++++++-- pydantic_settings/sources/base.py | 10 ++- tests/test_source_json.py | 34 ++++++++ tests/test_source_toml.py | 44 +++++++++++ tests/test_source_yaml.py | 44 +++++++++++ 5 files changed, 251 insertions(+), 8 deletions(-) diff --git a/docs/index.md b/docs/index.md index 55d1161e..fcb54985 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2401,12 +2401,7 @@ Other settings sources are available for common configuration files: - `TomlConfigSettingsSource` using `toml_file` argument - `YamlConfigSettingsSource` using `yaml_file` and yaml_file_encoding arguments -You can also provide multiple files by providing a list of path: -```py -toml_file = ['config.default.toml', 'config.custom.toml'] -``` -To use them, you can use the same mechanism described [here](#customise-settings-sources) - +To use them, you can use the same mechanism described [here](#customise-settings-sources). ```py from pydantic import BaseModel @@ -2448,6 +2443,126 @@ foobar = "Hello" nested_field = "world!" ``` +You can also provide multiple files by providing a list of paths. + +```py +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, + TomlConfigSettingsSource, +) + +class Nested(BaseModel): + foo: int + bar: int = 0 + + +class Settings(BaseSettings): + hello: str + nested: Nested + model_config = SettingsConfigDict(toml_file=['config.default.toml', 'config.custom.toml']) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return (TomlConfigSettingsSource(settings_cls),) +``` + +The following two configuration files + +```toml +# config.default.toml +hello = "World" + +[nested] +foo = 1 +bar = 2 +``` + +```toml +# config.custom.toml +[nested] +foo = 3 +``` + +are equivalent to + +```toml +hello = "world" + +[nested] +foo = 3 +``` + +The files are merged shallowly in increasing order of priority. To enable deep merging, set `deep_merge=True` on the source directly. + +!!! warning + The `deep_merge` option is **not available** through the `SettingsConfigDict`. + +```py +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, + TomlConfigSettingsSource, +) + +class Nested(BaseModel): + foo: int + bar: int = 0 + + +class Settings(BaseSettings): + hello: str + nested: Nested + model_config = SettingsConfigDict(toml_file=['config.default.toml', 'config.custom.toml']) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return (TomlConfigSettingsSource(settings_cls, deep_merge=True),) +``` + +With deep merge enabled, the following two configuration files + +```toml +# config.default.toml +hello = "World" + +[nested] +foo = 1 +bar = 2 +``` + +```toml +# config.custom.toml +[nested] +foo = 3 +``` + +are equivalent to + +```toml +hello = "world" + +[nested] +foo = 3 +bar = 2 +``` + ### pyproject.toml "pyproject.toml" is a standardized file for providing configuration values in Python projects. diff --git a/pydantic_settings/sources/base.py b/pydantic_settings/sources/base.py index e429256c..f2bc6696 100644 --- a/pydantic_settings/sources/base.py +++ b/pydantic_settings/sources/base.py @@ -198,11 +198,17 @@ def _read_files(self, files: PathType | None, deep_merge: bool = False) -> dict[ if isinstance(files, (str, os.PathLike)): files = [files] vars: dict[str, Any] = {} - update = deep_update if deep_merge else dict.update + update = deep_update if deep_merge else self._shallow_update for file in files: file_path = Path(file).expanduser() if file_path.is_file(): - update(vars, self._read_file(file_path)) + vars = update(vars, self._read_file(file_path)) + return vars + + def _shallow_update(self, vars: dict[str, Any], updating_vars: dict[str, Any]) -> dict[str, Any]: + # this mimics the semantics of pydantic._internal._utils.deep_update + vars = vars.copy() + vars.update(updating_vars) return vars @abstractmethod diff --git a/tests/test_source_json.py b/tests/test_source_json.py index 9d515e53..c609301c 100644 --- a/tests/test_source_json.py +++ b/tests/test_source_json.py @@ -5,6 +5,7 @@ import json from pathlib import Path +import pytest from pydantic import BaseModel from pydantic_settings import ( @@ -98,3 +99,36 @@ def settings_customise_sources( s = Settings() assert s.model_dump() == {'json5': 5, 'json6': 6} + + +@pytest.mark.parametrize('deep_merge', [False, True]) +def test_multiple_file_json_merge(tmp_path, deep_merge): + p5 = tmp_path / '.env.json5' + p6 = tmp_path / '.env.json6' + + with open(p5, 'w') as f5: + json.dump({'hello': 'world', 'nested': {'foo': 1, 'bar': 2}}, f5) + with open(p6, 'w') as f6: + json.dump({'nested': {'foo': 3}}, f6) + + class Nested(BaseModel): + foo: int + bar: int = 0 + + class Settings(BaseSettings): + hello: str + nested: Nested + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return (JsonConfigSettingsSource(settings_cls, json_file=[p5, p6], deep_merge=deep_merge),) + + s = Settings() + assert s.model_dump() == {'hello': 'world', 'nested': {'foo': 3, 'bar': 2 if deep_merge else 0}} diff --git a/tests/test_source_toml.py b/tests/test_source_toml.py index 4f1b648f..8a1c3cb5 100644 --- a/tests/test_source_toml.py +++ b/tests/test_source_toml.py @@ -114,3 +114,47 @@ def settings_customise_sources( s = Settings() assert s.model_dump() == {'toml1': 1, 'toml2': 2} + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +@pytest.mark.parametrize('deep_merge', [False, True]) +def test_multiple_file_toml_merge(tmp_path, deep_merge): + p1 = tmp_path / '.env.toml1' + p2 = tmp_path / '.env.toml2' + p1.write_text( + """ + hello = "world" + + [nested] + foo=1 + bar=2 + """ + ) + p2.write_text( + """ + [nested] + foo=3 + """ + ) + + class Nested(BaseModel): + foo: int + bar: int = 0 + + class Settings(BaseSettings): + hello: str + nested: Nested + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return (TomlConfigSettingsSource(settings_cls, toml_file=[p1, p2], deep_merge=deep_merge),) + + s = Settings() + assert s.model_dump() == {'hello': 'world', 'nested': {'foo': 3, 'bar': 2 if deep_merge else 0}} diff --git a/tests/test_source_yaml.py b/tests/test_source_yaml.py index 44e4c88a..32d5fd0b 100644 --- a/tests/test_source_yaml.py +++ b/tests/test_source_yaml.py @@ -167,6 +167,50 @@ def settings_customise_sources( assert s.model_dump() == {'yaml3': 3, 'yaml4': 4} +@pytest.mark.skipif(yaml is None, reason='pyYAML is not installed') +@pytest.mark.parametrize('deep_merge', [False, True]) +def test_multiple_file_yaml_deep_merge(tmp_path, deep_merge): + p3 = tmp_path / '.env.yaml3' + p4 = tmp_path / '.env.yaml4' + p3.write_text( + """ + hello: world + + nested: + foo: 1 + bar: 2 + """ + ) + p4.write_text( + """ + nested: + foo: 3 + """ + ) + + class Nested(BaseModel): + foo: int + bar: int = 0 + + class Settings(BaseSettings): + hello: str + nested: Nested + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return (YamlConfigSettingsSource(settings_cls, yaml_file=[p3, p4], deep_merge=deep_merge),) + + s = Settings() + assert s.model_dump() == {'hello': 'world', 'nested': {'foo': 3, 'bar': 2 if deep_merge else 0}} + + @pytest.mark.skipif(yaml is None, reason='pyYAML is not installed') def test_yaml_config_section(tmp_path): p = tmp_path / '.env' From 2d469a6e8fd911bc15e6e80616e7e7095ab14f98 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 12 Nov 2025 00:27:53 +0100 Subject: [PATCH 3/4] fix docs --- docs/index.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index fcb54985..f7073848 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2446,6 +2446,8 @@ nested_field = "world!" You can also provide multiple files by providing a list of paths. ```py +from pydantic import BaseModel + from pydantic_settings import ( BaseSettings, PydanticBaseSettingsSource, @@ -2453,6 +2455,7 @@ from pydantic_settings import ( TomlConfigSettingsSource, ) + class Nested(BaseModel): foo: int bar: int = 0 @@ -2461,7 +2464,9 @@ class Nested(BaseModel): class Settings(BaseSettings): hello: str nested: Nested - model_config = SettingsConfigDict(toml_file=['config.default.toml', 'config.custom.toml']) + model_config = SettingsConfigDict( + toml_file=['config.default.toml', 'config.custom.toml'] + ) @classmethod def settings_customise_sources( @@ -2507,6 +2512,8 @@ The files are merged shallowly in increasing order of priority. To enable deep m The `deep_merge` option is **not available** through the `SettingsConfigDict`. ```py +from pydantic import BaseModel + from pydantic_settings import ( BaseSettings, PydanticBaseSettingsSource, @@ -2514,6 +2521,7 @@ from pydantic_settings import ( TomlConfigSettingsSource, ) + class Nested(BaseModel): foo: int bar: int = 0 @@ -2522,7 +2530,9 @@ class Nested(BaseModel): class Settings(BaseSettings): hello: str nested: Nested - model_config = SettingsConfigDict(toml_file=['config.default.toml', 'config.custom.toml']) + model_config = SettingsConfigDict( + toml_file=['config.default.toml', 'config.custom.toml'] + ) @classmethod def settings_customise_sources( From 1eb82edb6c1f082c0a029d60bec33dacb3e26a52 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 13 Nov 2025 14:44:11 +0100 Subject: [PATCH 4/4] simple if/else --- pydantic_settings/sources/base.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pydantic_settings/sources/base.py b/pydantic_settings/sources/base.py index f2bc6696..4e733f0f 100644 --- a/pydantic_settings/sources/base.py +++ b/pydantic_settings/sources/base.py @@ -198,17 +198,16 @@ def _read_files(self, files: PathType | None, deep_merge: bool = False) -> dict[ if isinstance(files, (str, os.PathLike)): files = [files] vars: dict[str, Any] = {} - update = deep_update if deep_merge else self._shallow_update for file in files: file_path = Path(file).expanduser() - if file_path.is_file(): - vars = update(vars, self._read_file(file_path)) - return vars + if not file_path.is_file(): + continue - def _shallow_update(self, vars: dict[str, Any], updating_vars: dict[str, Any]) -> dict[str, Any]: - # this mimics the semantics of pydantic._internal._utils.deep_update - vars = vars.copy() - vars.update(updating_vars) + updating_vars = self._read_file(file_path) + if deep_merge: + vars = deep_update(vars, updating_vars) + else: + vars.update(updating_vars) return vars @abstractmethod