Added tomllib (vendored) as a replacement for toml fix #708

toml kept as a fallback until 4.0.0 to nor break compatibility

- toml follows 0.5.0 spec
- tomlib follows 1.0.0 spec
- toml allows emojis and unicode chars unencoded
- tomllib foolows the spec where only encoded chars are allowed

dynaconf/cli.py

@@ -25,6 +25,7 @@
 from dynaconf.validator import Validator
 from dynaconf.vendor import click
 from dynaconf.vendor import toml
+from dynaconf.vendor import tomllib
 
 os.environ["PYTHONIOENCODING"] = "utf-8"
 
@@ -695,7 +696,16 @@ def validate(path):  # pragma: no cover
         click.echo(click.style(f"{path} not found", fg="white", bg="red"))
         sys.exit(1)
 
-    validation_data = toml.load(open(str(path)))
+    try:  # try tomlib first
+        validation_data = tomllib.load(open(str(path), "rb"))
+    except UnicodeDecodeError:  # fallback to legacy toml (TBR in 4.0.0)
+        warnings.warn(
+            "TOML files should have only UTF-8 encoded characters. "
+            "starting on 4.0.0 dynaconf will stop allowing invalid chars.",
+        )
+        validation_data = toml.load(
+            open(str(path), encoding=default_settings.ENCODING_FOR_DYNACONF),
+        )
 
     success = True
     for env, name_data in validation_data.items():

dynaconf/loaders/base.py

@@ -21,7 +21,14 @@ class BaseLoader:
     """
 
     def __init__(
-        self, obj, env, identifier, extensions, file_reader, string_reader
+        self,
+        obj,
+        env,
+        identifier,
+        extensions,
+        file_reader,
+        string_reader,
+        opener_params=None,
     ):
         """Instantiates a loader for different sources"""
         self.obj = obj
@@ -30,6 +37,10 @@ def __init__(
         self.extensions = extensions
         self.file_reader = file_reader
         self.string_reader = string_reader
+        self.opener_params = opener_params or {
+            "mode": "r",
+            "encoding": obj.get("ENCODING_FOR_DYNACONF", "utf-8"),
+        }
 
     @staticmethod
     def warn_not_installed(obj, identifier):  # pragma: no cover
@@ -77,12 +88,7 @@ def get_source_data(self, files):
         for source_file in files:
             if source_file.endswith(self.extensions):
                 try:
-                    with open(
-                        source_file,
-                        encoding=self.obj.get(
-                            "ENCODING_FOR_DYNACONF", "utf-8"
-                        ),
-                    ) as open_file:
+                    with open(source_file, **self.opener_params) as open_file:
                         content = self.file_reader(open_file)
                         self.obj._loaded_files.append(source_file)
                         if content:

dynaconf/loaders/toml_loader.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
 
-import io
+import warnings
 from pathlib import Path
 
 from dynaconf import default_settings
 from dynaconf.constants import TOML_EXTENSIONS
 from dynaconf.loaders.base import BaseLoader
 from dynaconf.utils import object_merge
-from dynaconf.vendor import toml
+from dynaconf.vendor import toml  # Backwards compatibility with uiri/toml
+from dynaconf.vendor import tomllib  # New tomllib stdlib on py3.11
 
 
 def load(obj, env=None, silent=True, key=None, filename=None):
@@ -22,19 +23,54 @@ def load(obj, env=None, silent=True, key=None, filename=None):
     :return: None
     """
 
-    loader = BaseLoader(
-        obj=obj,
-        env=env,
-        identifier="toml",
-        extensions=TOML_EXTENSIONS,
-        file_reader=toml.load,
-        string_reader=toml.loads,
-    )
-    loader.load(
-        filename=filename,
-        key=key,
-        silent=silent,
-    )
+    try:
+        loader = BaseLoader(
+            obj=obj,
+            env=env,
+            identifier="toml",
+            extensions=TOML_EXTENSIONS,
+            file_reader=tomllib.load,
+            string_reader=tomllib.loads,
+            opener_params={"mode": "rb"},
+        )
+        loader.load(
+            filename=filename,
+            key=key,
+            silent=silent,
+        )
+    except UnicodeDecodeError:  # pragma: no cover
+        """
+        NOTE: Compat functions exists to keep backwards compatibility with
+        the new tomllib library. The old library was called `toml` and
+        the new one is called `tomllib`.
+
+        The old lib uiri/toml allowed unicode characters and readed files
+        as string.
+
+        The new tomllib (stdlib) does not allow unicode characters, only
+        utf-8 encoded, and read files as binary.
+
+        NOTE: In dynaconf 4.0.0 we will drop support for the old library
+        removing the compat functions and calling directly the new lib.
+        """
+        loader = BaseLoader(
+            obj=obj,
+            env=env,
+            identifier="toml",
+            extensions=TOML_EXTENSIONS,
+            file_reader=toml.load,
+            string_reader=toml.loads,
+        )
+        loader.load(
+            filename=filename,
+            key=key,
+            silent=silent,
+        )
+
+        warnings.warn(
+            "TOML files should have only UTF-8 encoded characters. "
+            "starting on 4.0.0 dynaconf will stop allowing invalid chars.",
+        )
 
 
 def write(settings_path, settings_data, merge=True):
@@ -46,17 +82,33 @@ def write(settings_path, settings_data, merge=True):
     """
     settings_path = Path(settings_path)
     if settings_path.exists() and merge:  # pragma: no cover
+        try:  # tomllib first
+            with open(str(settings_path), "rb") as open_file:
+                object_merge(tomllib.load(open_file), settings_data)
+        except UnicodeDecodeError:  # pragma: no cover
+            # uiri/toml fallback (TBR on 4.0.0)
+            with open(
+                str(settings_path),
+                encoding=default_settings.ENCODING_FOR_DYNACONF,
+            ) as open_file:
+                object_merge(toml.load(open_file), settings_data)
+
+    try:  # tomllib first
+        with open(str(settings_path), "wb") as open_file:
+            tomllib.dump(encode_nulls(settings_data), open_file)
+    except UnicodeEncodeError:  # pragma: no cover
+        # uiri/toml fallback (TBR on 4.0.0)
         with open(
-            str(settings_path), encoding=default_settings.ENCODING_FOR_DYNACONF
+            str(settings_path),
+            "w",
+            encoding=default_settings.ENCODING_FOR_DYNACONF,
         ) as open_file:
-            object_merge(toml.load(open_file), settings_data)
-
-    with open(
-        str(settings_path),
-        "w",
-        encoding=default_settings.ENCODING_FOR_DYNACONF,
-    ) as open_file:
-        toml.dump(encode_nulls(settings_data), open_file)
+            toml.dump(encode_nulls(settings_data), open_file)
+
+        warnings.warn(
+            "TOML files should have only UTF-8 encoded characters. "
+            "starting on 4.0.0 dynaconf will stop allowing invalid chars.",
+        )
 
 
 def encode_nulls(data):

dynaconf/utils/parse_conf.py

@@ -13,6 +13,7 @@
 from dynaconf.utils.boxing import DynaBox
 from dynaconf.utils.functional import empty
 from dynaconf.vendor import toml
+from dynaconf.vendor import tomllib
 
 try:
     from jinja2 import Environment
@@ -277,10 +278,22 @@ def get_converter(converter_key, value, box_settings):
 
 def parse_with_toml(data):
     """Uses TOML syntax to parse data"""
-    try:
-        return toml.loads(f"key={data}")["key"]
-    except (toml.TomlDecodeError, KeyError):
-        return data
+    try:  # try tomllib first
+        try:
+            return tomllib.loads(f"key={data}")["key"]
+        except (tomllib.TOMLDecodeError, KeyError):
+            return data
+    except UnicodeDecodeError:  # pragma: no cover
+        # fallback to toml (TBR in 4.0.0)
+        try:
+            return toml.loads(f"key={data}")["key"]
+        except (toml.TomlDecodeError, KeyError):
+            return data
+        warnings.warn(
+            "TOML files should have only UTF-8 encoded characters. "
+            "starting on 4.0.0 dynaconf will stop allowing invalid chars.",
+            DeprecationWarning,
+        )
 
 
 def _parse_conf_data(data, tomlfy=False, box_settings=None):

dynaconf/vendor/box/converters.py

@@ -11,7 +11,7 @@
 
 import dynaconf.vendor.ruamel.yaml as yaml
 from dynaconf.vendor.box.exceptions import BoxError, BoxWarning
-from dynaconf.vendor import toml
+from dynaconf.vendor import tomllib as toml
 
 
 BOX_PARAMETERS = ('default_box', 'default_box_attr', 'conversion_box',

dynaconf/vendor/box/from_file.py

@@ -3,7 +3,7 @@
 from json import JSONDecodeError
 from pathlib import Path
 from typing import Union
-from dynaconf.vendor.toml import TomlDecodeError
+from dynaconf.vendor.tomllib import TOMLDecodeError
 from dynaconf.vendor.ruamel.yaml import YAMLError
 
 
@@ -35,7 +35,7 @@ def _to_yaml(data):
 def _to_toml(data):
     try:
         return Box.from_toml(data)
-    except TomlDecodeError:
+    except TOMLDecodeError:
         raise BoxError('File is not TOML as expected')
 
 

dynaconf/vendor/toml/DEPRECATION.txt

@@ -0,0 +1,3 @@
+This lib will be deprecated on 4.0.0
+toml_loader and all the other places
+will default to tomllib.

dynaconf/vendor/tomllib/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
+
+__all__ = (
+    "loads",
+    "load",
+    "TOMLDecodeError",
+    "dump",
+    "dumps",
+)
+
+from ._parser import TOMLDecodeError, load, loads
+from ._writer import dump, dumps
+
+# Pretend this exception was created here.
+TOMLDecodeError.__module__ = __name__