feat: Add the ability to add extra settings sources

changes/2107-kozlek.md

@@ -0,0 +1 @@
+Add the ability to customize settings sources (add / disable / change priority order).

docs/build/exec_examples.py

@@ -9,7 +9,7 @@
 import sys
 import textwrap
 import traceback
-from pathlib import Path
+from pathlib import Path, PosixPath
 from typing import Any, List, Tuple
 from unittest.mock import patch
 
@@ -62,6 +62,17 @@ def __call__(self, *args, file=None, flush=None):
         self.statements.append((frame.f_lineno, s))
 
 
+class MockPath(PosixPath):
+    def __new__(cls, name, *args, **kwargs):
+        if name == 'config.json':
+            return cls._from_parts(name, *args, **kwargs)
+        else:
+            return Path.__new__(cls, name, *args, **kwargs)
+
+    def read_text(self, *args, **kwargs) -> str:
+        return '{"foobar": "spam"}'
+
+
 def build_print_lines(s: str, max_len_reduction: int = 0):
     print_lines = []
     max_len = MAX_LINE_LENGTH - 3 - max_len_reduction
@@ -133,7 +144,11 @@ def exec_examples():
     errors = []
     all_md = all_md_contents()
     new_files = {}
-    os.environ.update({'my_auth_key': 'xxx', 'my_api_key': 'xxx'})
+    os.environ.update({
+        'my_auth_key': 'xxx',
+        'my_api_key': 'xxx',
+        'database_dsn': 'postgres://postgres@localhost:5432/env_db',
+    })
 
     sys.path.append(str(EXAMPLES_DIR))
     for file in sorted(EXAMPLES_DIR.iterdir()):
@@ -173,14 +188,15 @@ def error(desc: str):
                 del sys.modules[file.stem]
             mp = MockPrint(file)
             mod = None
-            with patch('builtins.print') as mock_print:
-                if print_intercept:
-                    mock_print.side_effect = mp
-                try:
-                    mod = importlib.import_module(file.stem)
-                except Exception:
-                    tb = traceback.format_exception(*sys.exc_info())
-                    error(''.join(e for e in tb if '/pydantic/docs/examples/' in e or not e.startswith('  File ')))
+            with patch('pathlib.Path', MockPath):
+                with patch('builtins.print') as patch_print:
+                    if print_intercept:
+                        patch_print.side_effect = mp
+                    try:
+                        mod = importlib.import_module(file.stem)
+                    except Exception:
+                        tb = traceback.format_exception(*sys.exc_info())
+                        error(''.join(e for e in tb if '/pydantic/docs/examples/' in e or not e.startswith('  File ')))
 
             if mod and not mod.__file__.startswith(str(EXAMPLES_DIR)):
                 error(f'module path "{mod.__file__}" not inside "{EXAMPLES_DIR}", name may shadow another module?')

docs/examples/settings_add_custom_source.py

@@ -0,0 +1,41 @@
+import json
+from pathlib import Path
+from typing import Dict, Any
+
+from pydantic import BaseSettings
+
+
+def json_config_settings_source(settings: BaseSettings) -> Dict[str, Any]:
+    """
+    A simple settings source that loads variables from a JSON file
+    at the project's root.
+
+    Here we happen to choose to use the `env_file_encoding` from Config
+    when reading `config.json`
+    """
+    encoding = settings.__config__.env_file_encoding
+    return json.loads(Path('config.json').read_text(encoding))
+
+
+class Settings(BaseSettings):
+    foobar: str
+
+    class Config:
+        env_file_encoding = 'utf-8'
+
+        @classmethod
+        def customise_sources(
+            cls,
+            init_settings,
+            env_settings,
+            file_secret_settings,
+        ):
+            return (
+                init_settings,
+                json_config_settings_source,
+                env_settings,
+                file_secret_settings,
+            )
+
+
+print(Settings())

docs/examples/settings_disable_source.py

@@ -0,0 +1,22 @@
+from typing import Tuple
+
+from pydantic import BaseSettings
+from pydantic.env_settings import SettingsSourceCallable
+
+
+class Settings(BaseSettings):
+    my_api_key: str
+
+    class Config:
+        @classmethod
+        def customise_sources(
+            cls,
+            init_settings: SettingsSourceCallable,
+            env_settings: SettingsSourceCallable,
+            file_secret_settings: SettingsSourceCallable,
+        ) -> Tuple[SettingsSourceCallable, ...]:
+            # here we choose to ignore arguments from init_settings
+            return env_settings, file_secret_settings
+
+
+print(Settings(my_api_key='this is ignored'))

docs/examples/settings_env_priority.py

@@ -0,0 +1,20 @@
+from typing import Tuple
+from pydantic import BaseSettings, PostgresDsn
+from pydantic.env_settings import SettingsSourceCallable
+
+
+class Settings(BaseSettings):
+    database_dsn: PostgresDsn
+
+    class Config:
+        @classmethod
+        def customise_sources(
+            cls,
+            init_settings: SettingsSourceCallable,
+            env_settings: SettingsSourceCallable,
+            file_secret_settings: SettingsSourceCallable,
+        ) -> Tuple[SettingsSourceCallable, ...]:
+            return env_settings, init_settings, file_secret_settings
+
+
+print(Settings(database_dsn='postgres://postgres@localhost:5432/kwargs_db'))

docs/usage/settings.md

@@ -208,3 +208,46 @@ the selected value is determined as follows (in descending order of priority):
 3. Variables loaded from a dotenv (`.env`) file.
 4. Variables loaded from the secrets directory.
 5. The default field values for the `Settings` model.
+
+## Customise settings sources
+
+If the default order of priority doesn't match your needs, it's possible to change it by overriding
+the `customise_sources` method on the `Config` class of your `Settings` .
+
+`customise_sources` takes three callables as arguments and returns any number of callables as a tuple. In turn these
+callables are called to build the inputs to the fields of the settings class.
+
+Each callable should take an instance of the settings class as its sole argument and return a `dict`.
+
+### Changing Priority
+
+The order of the returned callables decides the priority of inputs; first item is the highest priority.
+
+```py
+{!.tmp_examples/settings_env_priority.py!}
+```
+_(This script is complete, it should run "as is")_
+
+By flipping `env_settings` and `init_settings`, environment variables now have precedence over `__init__` kwargs.
+
+### Adding sources
+
+As explained earlier, *pydantic* ships with multiples built-in settings sources. However, you may occationally
+need to add your own custom sources, `customise_sources` makes this very easy:
+
+```py
+{!.tmp_examples/settings_add_custom_source.py!}
+```
+_(This script is complete, it should run "as is")_
+
+### Removing sources
+
+You might also want to disable a source:
+
+```py
+{!.tmp_examples/settings_disable_source.py!}
+```
+_(This script is complete, it should run "as is", here you might need to set the `my_api_key` environment variable)_
+
+Because of the callables approach of `customise_sources`, evaluation of sources is lazy so unused sources don't
+have an adverse effect on performance.