Removed vendor_src folder (#798)

* Removed vendor_src folder

Now `vendor` is the source
and minification happens during release process.

* Added tomllib (vendored) as a replacement for toml fix #708

toml kept as a fallback until 4.0.0 to nor break compatibility

- toml follows 0.5.0 spec
- tomlib follows 1.0.0 spec
- toml allows emojis and unicode chars unencoded
- tomllib foolows the spec where only encoded chars are allowed

.gitignore

@@ -79,3 +79,8 @@ junit/
 # django example static files
 example/django_example/admin/
 example/django_example/debug_toolbar/
+
+
+# Vendor_src exists only during build process
+dynaconf/vendor_src/
+dynaconf/vendor/source

Makefile

@@ -210,7 +210,9 @@ pep8:
 	flake8 dynaconf --exclude=dynaconf/vendor*
 
 dist: clean
+	@make minify_vendor
 	@python setup.py sdist bdist_wheel
+	@make source_vendor
 
 publish:
 	make run-tox
@@ -236,3 +238,21 @@ docs:
 run-tox:
 	tox --recreate
 	rm -rf .tox
+
+minify_vendor:
+	# Ensure vendor is source and cleanup vendor_src bck folder
+	ls dynaconf/vendor/source && rm -rf dynaconf/vendor_src
+
+	# Backup dynaconf/vendor folder as dynaconf/vendor_src
+	mv dynaconf/vendor dynaconf/vendor_src
+
+	# create a new dynaconf/vendor folder with minified files
+	./minify.sh
+
+
+source_vendor:
+	# Ensure dynaconf/vendor_src is source and cleanup vendor folder
+	ls dynaconf/vendor_src/source && rm -rf dynaconf/vendor
+
+	# Restore dynaconf/vendor_src folder as dynaconf/vendor
+	mv dynaconf/vendor_src dynaconf/vendor

dynaconf/cli.py

@@ -25,6 +25,7 @@
 from dynaconf.validator import Validator
 from dynaconf.vendor import click
 from dynaconf.vendor import toml
+from dynaconf.vendor import tomllib
 
 os.environ["PYTHONIOENCODING"] = "utf-8"
 
@@ -695,7 +696,16 @@ def validate(path):  # pragma: no cover
         click.echo(click.style(f"{path} not found", fg="white", bg="red"))
         sys.exit(1)
 
-    validation_data = toml.load(open(str(path)))
+    try:  # try tomlib first
+        validation_data = tomllib.load(open(str(path), "rb"))
+    except UnicodeDecodeError:  # fallback to legacy toml (TBR in 4.0.0)
+        warnings.warn(
+            "TOML files should have only UTF-8 encoded characters. "
+            "starting on 4.0.0 dynaconf will stop allowing invalid chars.",
+        )
+        validation_data = toml.load(
+            open(str(path), encoding=default_settings.ENCODING_FOR_DYNACONF),
+        )
 
     success = True
     for env, name_data in validation_data.items():

dynaconf/loaders/base.py

@@ -21,7 +21,14 @@ class BaseLoader:
     """
 
     def __init__(
-        self, obj, env, identifier, extensions, file_reader, string_reader
+        self,
+        obj,
+        env,
+        identifier,
+        extensions,
+        file_reader,
+        string_reader,
+        opener_params=None,
     ):
         """Instantiates a loader for different sources"""
         self.obj = obj
@@ -30,6 +37,10 @@ def __init__(
         self.extensions = extensions
         self.file_reader = file_reader
         self.string_reader = string_reader
+        self.opener_params = opener_params or {
+            "mode": "r",
+            "encoding": obj.get("ENCODING_FOR_DYNACONF", "utf-8"),
+        }
 
     @staticmethod
     def warn_not_installed(obj, identifier):  # pragma: no cover
@@ -77,12 +88,7 @@ def get_source_data(self, files):
         for source_file in files:
             if source_file.endswith(self.extensions):
                 try:
-                    with open(
-                        source_file,
-                        encoding=self.obj.get(
-                            "ENCODING_FOR_DYNACONF", "utf-8"
-                        ),
-                    ) as open_file:
+                    with open(source_file, **self.opener_params) as open_file:
                         content = self.file_reader(open_file)
                         self.obj._loaded_files.append(source_file)
                         if content:

dynaconf/loaders/toml_loader.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
 
-import io
+import warnings
 from pathlib import Path
 
 from dynaconf import default_settings
 from dynaconf.constants import TOML_EXTENSIONS
 from dynaconf.loaders.base import BaseLoader
 from dynaconf.utils import object_merge
-from dynaconf.vendor import toml
+from dynaconf.vendor import toml  # Backwards compatibility with uiri/toml
+from dynaconf.vendor import tomllib  # New tomllib stdlib on py3.11
 
 
 def load(obj, env=None, silent=True, key=None, filename=None):
@@ -22,19 +23,54 @@ def load(obj, env=None, silent=True, key=None, filename=None):
     :return: None
     """
 
-    loader = BaseLoader(
-        obj=obj,
-        env=env,
-        identifier="toml",
-        extensions=TOML_EXTENSIONS,
-        file_reader=toml.load,
-        string_reader=toml.loads,
-    )
-    loader.load(
-        filename=filename,
-        key=key,
-        silent=silent,
-    )
+    try:
+        loader = BaseLoader(
+            obj=obj,
+            env=env,
+            identifier="toml",
+            extensions=TOML_EXTENSIONS,
+            file_reader=tomllib.load,
+            string_reader=tomllib.loads,
+            opener_params={"mode": "rb"},
+        )
+        loader.load(
+            filename=filename,
+            key=key,
+            silent=silent,
+        )
+    except UnicodeDecodeError:  # pragma: no cover
+        """
+        NOTE: Compat functions exists to keep backwards compatibility with
+        the new tomllib library. The old library was called `toml` and
+        the new one is called `tomllib`.
+
+        The old lib uiri/toml allowed unicode characters and readed files
+        as string.
+
+        The new tomllib (stdlib) does not allow unicode characters, only
+        utf-8 encoded, and read files as binary.
+
+        NOTE: In dynaconf 4.0.0 we will drop support for the old library
+        removing the compat functions and calling directly the new lib.
+        """
+        loader = BaseLoader(
+            obj=obj,
+            env=env,
+            identifier="toml",
+            extensions=TOML_EXTENSIONS,
+            file_reader=toml.load,
+            string_reader=toml.loads,
+        )
+        loader.load(
+            filename=filename,
+            key=key,
+            silent=silent,
+        )
+
+        warnings.warn(
+            "TOML files should have only UTF-8 encoded characters. "
+            "starting on 4.0.0 dynaconf will stop allowing invalid chars.",
+        )
 
 
 def write(settings_path, settings_data, merge=True):
@@ -46,17 +82,33 @@ def write(settings_path, settings_data, merge=True):
     """
     settings_path = Path(settings_path)
     if settings_path.exists() and merge:  # pragma: no cover
+        try:  # tomllib first
+            with open(str(settings_path), "rb") as open_file:
+                object_merge(tomllib.load(open_file), settings_data)
+        except UnicodeDecodeError:  # pragma: no cover
+            # uiri/toml fallback (TBR on 4.0.0)
+            with open(
+                str(settings_path),
+                encoding=default_settings.ENCODING_FOR_DYNACONF,
+            ) as open_file:
+                object_merge(toml.load(open_file), settings_data)
+
+    try:  # tomllib first
+        with open(str(settings_path), "wb") as open_file:
+            tomllib.dump(encode_nulls(settings_data), open_file)
+    except UnicodeEncodeError:  # pragma: no cover
+        # uiri/toml fallback (TBR on 4.0.0)
         with open(
-            str(settings_path), encoding=default_settings.ENCODING_FOR_DYNACONF
+            str(settings_path),
+            "w",
+            encoding=default_settings.ENCODING_FOR_DYNACONF,
         ) as open_file:
-            object_merge(toml.load(open_file), settings_data)
-
-    with open(
-        str(settings_path),
-        "w",
-        encoding=default_settings.ENCODING_FOR_DYNACONF,
-    ) as open_file:
-        toml.dump(encode_nulls(settings_data), open_file)
+            toml.dump(encode_nulls(settings_data), open_file)
+
+        warnings.warn(
+            "TOML files should have only UTF-8 encoded characters. "
+            "starting on 4.0.0 dynaconf will stop allowing invalid chars.",
+        )
 
 
 def encode_nulls(data):

dynaconf/utils/parse_conf.py

@@ -13,6 +13,7 @@
 from dynaconf.utils.boxing import DynaBox
 from dynaconf.utils.functional import empty
 from dynaconf.vendor import toml
+from dynaconf.vendor import tomllib
 
 try:
     from jinja2 import Environment
@@ -277,10 +278,22 @@ def get_converter(converter_key, value, box_settings):
 
 def parse_with_toml(data):
     """Uses TOML syntax to parse data"""
-    try:
-        return toml.loads(f"key={data}")["key"]
-    except (toml.TomlDecodeError, KeyError):
-        return data
+    try:  # try tomllib first
+        try:
+            return tomllib.loads(f"key={data}")["key"]
+        except (tomllib.TOMLDecodeError, KeyError):
+            return data
+    except UnicodeDecodeError:  # pragma: no cover
+        # fallback to toml (TBR in 4.0.0)
+        try:
+            return toml.loads(f"key={data}")["key"]
+        except (toml.TomlDecodeError, KeyError):
+            return data
+        warnings.warn(
+            "TOML files should have only UTF-8 encoded characters. "
+            "starting on 4.0.0 dynaconf will stop allowing invalid chars.",
+            DeprecationWarning,
+        )
 
 
 def _parse_conf_data(data, tomlfy=False, box_settings=None):

dynaconf/vendor/box/__init__.py

@@ -1,8 +1,15 @@
-__author__='Chris Griffith'
-__version__='4.2.3'
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+
+__author__ = 'Chris Griffith'
+__version__ = '4.2.3'
+
 from .box import Box
 from .box_list import BoxList
 from .config_box import ConfigBox
 from .shorthand_box import SBox
-from .exceptions import BoxError,BoxKeyError
-from .from_file import box_from_file
+from .exceptions import BoxError, BoxKeyError
+from .from_file import box_from_file
+
+
+