Merge pull request #107 from SmileyChris/better-database-config

Better-database-config

README.md

@@ -79,39 +79,40 @@ The other setting worth thinking about is `SECRET_KEY`. By default, `SECRET_KEY`
 
 ## Database configuration
 
-By defining multiple `DatabaseDsn` attributes of the `DatabaseSettings` sub-class, you can easily configure one or more database connections with environment variables. Google Cloud SQL database connections from within Google Cloud Run are supported; the DatabaseDsn type will detect and automatically escape DSN strings of the form `postgres://username:password@/cloudsql/project:region:instance/database` so that they can be properly handled. The below example is taken from the test project in this repository, and shows a working multi-database configuration file. In this example, the value for `DJANGO_SETTINGS_MODULE` should be set, as below, to `settings_test.database_settings.TestSettings`.
+The default database configuration can be configured by an environment variable named `DATABASE_URL`, containing a DSN (Data Source Name) string.
 
-```python
-class Databases(DatabaseSettings):
-    DEFAULT: DatabaseDsn = Field(env="DATABASE_URL")
-    SECONDARY: DatabaseDsn = Field(env="SECONDARY_DATABASE_URL")
+Google Cloud SQL database connections from within Google Cloud Run are supported; the `DatabaseDsn` type will detect and automatically escape DSN strings of the form `postgres://username:password@/cloudsql/project:region:instance/database` so that they can be properly handled.
 
+Alternatively you can set all your databases at once, by using the `DATABASES` setting (either in a `PydanticSettings` sub-class or via the `DJANGO_DATABASES` environment variable:
 
-class TestSettings(PydanticSettings):
-    DATABASES: Databases = Field({})
+```python
+def MySettings(PydanticSettings):
+    DATABASES = {"default": "sqlite:///db.sqlite3"}  # type: ignore
 ```
 
+It is also possible to configure additional database connections with environment variables in the same way as the default `DATABASE_URL` configuration by using a `Field` that has a `configure_database` argument that points to the database alias in the `DATABASES` dictionary.
+
 ```python
-❯ DJANGO_SETTINGS_MODULE=settings_test.database_settings.TestSettings DATABASE_URL=postgres://username:password@/cloudsql/project:region:instance/database SECONDARY_DATABASE_URL=sqlite:///foo poetry run python manage.py shell
-Python 3.10.2 (main, Feb  2 2022, 06:19:27) [Clang 13.0.0 (clang-1300.0.29.3)] on darwin
-Type "help", "copyright", "credits" or "license" for more information.
-(InteractiveConsole)
->>> from rich import print
->>> from django.conf import settings
->>> print(settings.DATABASES)
-{
-    'default': {
-        'NAME': 'database',
-        'USER': 'username',
-        'PASSWORD': 'password',
-        'HOST': '/cloudsql/project:region:instance',
-        'PORT': '',
-        'CONN_MAX_AGE': 0,
-        'ENGINE': 'django.db.backends.postgresql'
-    },
-    'secondary': {'NAME': 'foo', 'USER': '', 'PASSWORD': '', 'HOST': '', 'PORT': '', 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.sqlite3'}
-}
->>>
+from pydantic_settings import PydanticSettings
+from pydantic_settings.database import DatabaseDsn
+
+
+def MySettings(PydanticSettings):
+    secondary_database_dsn: Optional[DatabaseDsn] = Field(
+        env="SECONDARY_DATABASE_URL", configure_database="secondary"
+    )
+```
+
+For example, the `settings_test/database_settings.py` file is has a settings subclass configured like this and outputs the changes to the `DATABASES` setting when run directly:
+
+```
+❯ DATABASE_URL=postgres://username:password@/cloudsql/project:region:instance/database SECONDARY_DATABASE_URL=sqlite:///foo python settings_test/database_settings.py
+{'default': {'ENGINE': 'django.db.backends.postgresql',
+             'HOST': '/cloudsql/project:region:instance',
+             'NAME': 'database',
+             'PASSWORD': 'password',
+             'USER': 'username'},
+ 'secondary': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': 'foo'}}
 ```
 
 ## Sentry configuration

pydantic_settings/cache.py

@@ -0,0 +1,138 @@
+import re
+from typing import Dict
+from urllib.parse import parse_qs
+
+from django import VERSION
+from pydantic import AnyUrl
+
+from pydantic_settings.models import CacheModel
+
+BUILTIN_DJANGO_BACKEND = "django.core.cache.backends.redis.RedisCache"
+DJANGO_REDIS_BACKEND = (
+    "django_redis.cache.RedisCache" if VERSION[0] < 4 else BUILTIN_DJANGO_BACKEND
+)
+
+CACHE_ENGINES = {
+    "db": "django.core.cache.backends.db.DatabaseCache",
+    "djangopylibmc": "django_pylibmc.memcached.PyLibMCCache",
+    "dummy": "django.core.cache.backends.dummy.DummyCache",
+    "elasticache": "django_elasticache.memcached.ElastiCache",
+    "file": "django.core.cache.backends.filebased.FileBasedCache",
+    "hiredis": DJANGO_REDIS_BACKEND,
+    "locmem": "django.core.cache.backends.locmem.LocMemCache",
+    "memcached": "django.core.cache.backends.memcached.PyLibMCCache",
+    "pymemcache": "django.core.cache.backends.memcached.PyMemcacheCache",
+    "pymemcached": "django.core.cache.backends.memcached.MemcachedCache",
+    "redis-cache": "redis_cache.RedisCache",
+    "redis": DJANGO_REDIS_BACKEND,
+    "rediss": DJANGO_REDIS_BACKEND,
+    "uwsgicache": "uwsgicache.UWSGICache",
+}
+
+REDIS_PARSERS = {
+    "hiredis": "redis.connection.HiredisParser",
+}
+
+FILE_UNIX_PREFIX = (
+    "memcached",
+    "pymemcached",
+    "pymemcache",
+    "djangopylibmc",
+    "redis",
+    "hiredis",
+)
+
+
+class CacheDsn(AnyUrl):
+    __slots__ = AnyUrl.__slots__ + ("query_args",)
+    host_required = False
+
+    query_args: Dict[str, str]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.query:
+            self.query_args = {
+                key.upper(): ";".join(val) for key, val in parse_qs(self.query).items()
+            }
+        else:
+            self.query_args = {}
+
+    allowed_schemes = set(CACHE_ENGINES)
+
+    def to_settings_model(self) -> CacheModel:
+        return CacheModel(**parse(self))
+
+    @property
+    def is_redis_scheme(self) -> bool:
+        return self.scheme in ("redis", "rediss", "hiredis")
+
+
+def parse(dsn: CacheDsn) -> dict:
+    """Parses a cache URL."""
+    backend = CACHE_ENGINES[dsn.scheme]
+    config = {"BACKEND": backend}
+
+    options = {}
+    if dsn.scheme in REDIS_PARSERS:
+        options["PARSER_CLASS"] = REDIS_PARSERS[dsn.scheme]
+
+    cache_args = dsn.query_args.copy()
+
+    # File based
+    if dsn.host is None:
+        path = dsn.path
+
+        if dsn.scheme in FILE_UNIX_PREFIX:
+            path = "unix:" + path
+
+        if dsn.is_redis_scheme:
+            match = re.match(r"(.*)/(\d+)$", path)
+            if match:
+                path, db = match.groups()
+            else:
+                db = "0"
+            path = f"{path}?db={db}"
+
+        config["LOCATION"] = path
+    # Redis URL based
+    elif dsn.is_redis_scheme:
+        # Specifying the database is optional, use db 0 if not specified.
+        db = (dsn.path and dsn.path[1:]) or "0"
+        port = dsn.port if dsn.port else 6379
+        scheme = "rediss" if dsn.scheme == "rediss" else "redis"
+        location = f"{scheme}://{dsn.host}:{port}/{db}"
+        if dsn.password:
+            if backend == BUILTIN_DJANGO_BACKEND or dsn.scheme == "redis-cache":
+                location = location.replace("://", f"://{dsn.password}@", 1)
+            else:
+                options["PASSWORD"] = dsn.password
+        config["LOCATION"] = location
+
+        # Pop redis-cache specific arguments.
+        if dsn.scheme == "redis-cache":
+            for key in ("PARSER_CLASS", "CONNECTION_POOL_CLASS"):
+                if val := cache_args.pop(key, None):
+                    options[key] = val
+
+            pool_class_opts = {}
+            for pool_class_key in ("MAX_CONNECTIONS", "TIMEOUT"):
+                if val := cache_args.pop(pool_class_key, None):
+                    pool_class_opts[pool_class_key] = val
+            if pool_class_opts:
+                options["CONNECTION_POOL_CLASS_KWARGS"] = pool_class_opts
+
+    if dsn.scheme == "uwsgicache":
+        config["LOCATION"] = config.get("LOCATION") or "default"
+
+    # Pop special options from cache_args
+    # https://docs.djangoproject.com/en/4.0/topics/cache/#cache-arguments
+    for key in ["MAX_ENTRIES", "CULL_FREQUENCY"]:
+        if val := dsn.query_args.pop(key, None):
+            options[key] = int(val)
+
+    config.update(cache_args)
+    if options:
+        config["OPTIONS"] = options
+
+    return config

pydantic_settings/database.py

@@ -1,13 +1,31 @@
 import re
+import urllib.parse
 from typing import Dict, Optional, Pattern, Tuple, cast
 from urllib.parse import quote_plus
 
 from pydantic import AnyUrl
 from pydantic.validators import constr_length_validator, str_validator
 
+from pydantic_settings.models import DatabaseModel
+
 _cloud_sql_regex_cache = None
 
 
+DB_ENGINES = {
+    "postgres": "django.db.backends.postgresql",
+    "postgresql": "django.db.backends.postgresql",
+    "postgis": "django.contrib.gis.db.backends.postgis",
+    "mssql": "sql_server.pyodbc",
+    "mysql": "django.db.backends.mysql",
+    "mysqlgis": "django.contrib.gis.db.backends.mysql",
+    "sqlite": "django.db.backends.sqlite3",
+    "spatialite": "django.contrib.gis.db.backends.spatialite",
+    "oracle": "django.db.backends.oracle",
+    "oraclegis": "django.contrib.gis.db.backends.oracle",
+    "redshift": "django_redshift_backend",
+}
+
+
 def cloud_sql_regex() -> Pattern[str]:
     global _cloud_sql_regex_cache
     if _cloud_sql_regex_cache is None:
@@ -21,46 +39,7 @@ def cloud_sql_regex() -> Pattern[str]:
 
 
 class DatabaseDsn(AnyUrl):
-    def __init__(
-        self,
-        url: str,
-        *,
-        scheme: str,
-        user: Optional[str] = None,
-        password: Optional[str] = None,
-        host: Optional[str] = None,
-        tld: Optional[str] = None,
-        host_type: str = "domain",
-        port: Optional[str] = None,
-        path: str = None,
-        query: Optional[str] = None,
-        fragment: Optional[str] = None,
-    ) -> None:
-        str.__init__(url)
-        self.scheme = scheme
-        self.user = user
-        self.password = password
-        self.host = host
-        self.tld = tld
-        self.host_type = host_type
-        self.port = port
-        self.path = path
-        self.query = query
-        self.fragment = fragment
-
-    allowed_schemes = {
-        "postgres",
-        "postgresql",
-        "postgis",
-        "mssql",
-        "mysql",
-        "mysqlgis",
-        "sqlite",
-        "spatialite",
-        "oracle",
-        "oraclegis",
-        "redshift",
-    }
+    allowed_schemes = set(DB_ENGINES)
 
     @classmethod
     def validate(cls, value, field, config):
@@ -101,3 +80,16 @@ def validate_host(
             return host, None, "socket", False
 
         return super().validate_host(parts)
+
+    def to_settings_model(self) -> DatabaseModel:
+        name = self.path
+        if name and name.startswith("/"):
+            name = name[1:]
+        return DatabaseModel(
+            NAME=name or "",
+            USER=self.user or "",
+            PASSWORD=self.password or "",
+            HOST=urllib.parse.unquote(self.host) if self.host else "",
+            PORT=self.port or "",
+            ENGINE=DB_ENGINES[self.scheme],
+        )

pydantic_settings/models.py

@@ -0,0 +1,66 @@
+from typing import List, Optional
+
+from pydantic import DirectoryPath
+from pydantic.main import BaseModel
+from typing_extensions import TypedDict
+
+
+class TemplateBackendModel(BaseModel):
+    BACKEND: str
+    NAME: Optional[str]
+    DIRS: Optional[List[DirectoryPath]]
+    APP_DIRS: Optional[bool]
+    OPTIONS: Optional[dict]
+
+
+class CacheModel(BaseModel):
+    BACKEND: str
+    KEY_FUNCTION: Optional[str] = None
+    KEY_PREFIX: str = ""
+    LOCATION: str = ""
+    OPTIONS: Optional[dict] = None
+    TIMEOUT: Optional[int] = None
+    VERSION: int = 1
+
+
+class DatabateTestDict(TypedDict, total=False):
+    CHARSET: Optional[str]
+    COLLATION: Optional[str]
+    DEPENDENCIES: Optional[List[str]]
+    MIGRATE: bool
+    MIRROR: Optional[str]
+    NAME: Optional[str]
+    SERIALIZE: Optional[bool]  # Deprecated since v4.0
+    TEMPLATE: Optional[str]
+    CREATE_DB: Optional[bool]
+    CREATE_USER: Optional[bool]
+    USER: Optional[str]
+    PASSWORD: Optional[str]
+    ORACLE_MANAGED_FILES: Optional[bool]
+    TBLSPACE: Optional[str]
+    TBLSPACE_TMP: Optional[str]
+    DATAFILE: Optional[str]
+    DATAFILE_TMP: Optional[str]
+    DATAFILE_MAXSIZE: Optional[str]
+    DATAFILE_TMP_MAXSIZE: Optional[str]
+    DATAFILE_SIZE: Optional[str]
+    DATAFILE_TMP_SIZE: Optional[str]
+    DATAFILE_EXTSIZE: Optional[str]
+    DATAFILE_TMP_EXTSIZE: Optional[str]
+
+
+class DatabaseModel(BaseModel):
+    ATOMIC_REQUESTS: bool = False
+    AUTOCOMMIT: bool = True
+    ENGINE: str
+    HOST: str = ""
+    NAME: str = ""
+    CONN_MAX_AGE: int = 0
+    OPTIONS: dict = {}
+    PASSWORD: str = ""
+    PORT: str = ""
+    TIME_ZONE: Optional[str] = None
+    DISABLE_SERVER_SIDE_CURSORS: bool = False
+    USER: str = ""
+    TEST: Optional[DatabateTestDict] = None
+    DATA_UPLOAD_MEMORY_MAX_SIZE: Optional[int] = None