From abc5d6836b1052f52a77c2136a2d743c179c8d25 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Thu, 7 Sep 2023 20:43:04 -0500 Subject: [PATCH 1/5] Use stdlib toml provider on Python >= 3.11. Switch to tomli otherwise --- configurator/config.py | 24 ++++++++++++---- configurator/parsers.py | 12 +++++++- setup.py | 2 +- tests/test_toml_loading.py | 57 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 tests/test_toml_loading.py diff --git a/configurator/config.py b/configurator/config.py index ed9f97c..a677591 100644 --- a/configurator/config.py +++ b/configurator/config.py @@ -1,6 +1,6 @@ import os from copy import deepcopy -from io import open, StringIO +from io import open, StringIO, BytesIO from os.path import exists, expanduser from .mapping import load, store, target, convert, if_supplied @@ -30,9 +30,16 @@ def from_text(cls, text, parser, encoding='ascii'): :ref:`parser `. If ``text`` is provided as :class:`bytes`, then the ``encoding`` specified will be used to decode it. """ - if isinstance(text, bytes): - text = text.decode(encoding) - return cls.from_stream(StringIO(text), parser) + if parser == 'toml' or getattr(parser, '__module__', '').startswith('toml'): + if isinstance(text, bytes): + stream = BytesIO(text) + else: + stream = BytesIO(text.encode('utf-8')) + else: + if isinstance(text, bytes): + text = text.decode(encoding) + stream = StringIO(text) + return cls.from_stream(stream, parser) @classmethod def from_stream(cls, stream, parser=None): @@ -70,7 +77,14 @@ def from_path(cls, path, parser=None, encoding=None, optional=False): full_path = expanduser(path) if optional and not exists(full_path): return cls() - with open(full_path, encoding=encoding) as stream: + mode = 'rt' + if parser is None: + if str(path).endswith('.toml'): + mode = 'rb' + else: + if getattr(parser, '__module__', '').startswith('toml'): + mode = 'rb' + with open(full_path, encoding=encoding, mode=mode) as stream: return cls.from_stream(stream, parser) @classmethod diff --git a/configurator/parsers.py b/configurator/parsers.py index 93bc3b3..a469f90 100644 --- a/configurator/parsers.py +++ b/configurator/parsers.py @@ -1,3 +1,5 @@ +import sys + from collections import defaultdict @@ -8,11 +10,19 @@ class ParseError(Exception): """ +if sys.version_info >= (3, 11): + # stdlib + _toml_mod = 'tomllib' +else: + _toml_mod = 'tomli' + + class Parsers(defaultdict): + # file extension: module name, method name supported = { 'json': ('json', 'load'), - 'toml': ('toml', 'load'), + 'toml': (_toml_mod, 'load'), 'yml': ('yaml', 'safe_load'), 'yaml': ('yaml', 'safe_load'), } diff --git a/setup.py b/setup.py index 6e0b155..15cbbb4 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ python_requires=">=3.6", extras_require=dict( yaml=['pyyaml'], - toml=['toml'], + toml=['tomli; python_version < "3.11"'], test=[ 'jinja2', 'mock', diff --git a/tests/test_toml_loading.py b/tests/test_toml_loading.py new file mode 100644 index 0000000..ec5c888 --- /dev/null +++ b/tests/test_toml_loading.py @@ -0,0 +1,57 @@ +import pytest +from testfixtures import compare + +from configurator import Config + + +def test_load_toml_from_path(tmp_path): + path = tmp_path / 'test.toml' + path.write_bytes(b'k = "v"') + config = Config.from_path(path) + compare(config.k, "v") + + +def test_load_toml_from_byte_stream_implicit_parser(tmp_path): + path = tmp_path / 'test.toml' + path.write_bytes(b'k = "v"') + with path.open(mode="rb") as stream: + config = Config.from_stream(stream) + compare(config.k, "v") + + +def test_load_toml_from_byte_stream_explicit_parser(tmp_path): + path = tmp_path / 'test.toml' + path.write_bytes(b'k = "v"') + parser = Config.parsers['toml'] + with path.open(mode="rb") as stream: + config = Config.from_stream(stream, parser) + compare(config.k, "v") + + +def test_load_toml_from_text_stream_implicit_parser(tmp_path): + path = tmp_path / 'test.toml' + path.write_bytes(b'k = "v"') + msg = "must be opened in binary mode" + with path.open(mode="rt") as stream, pytest.raises(TypeError, match=msg): + Config.from_stream(stream) + + +def test_load_toml_from_text_stream_explicit_parser(tmp_path): + path = tmp_path / 'test.toml' + path.write_bytes(b'k = "v"') + parser = Config.parsers['toml'] + msg = "must be opened in binary mode" + with path.open(mode="rt") as stream, pytest.raises(TypeError, match=msg): + Config.from_stream(stream, parser) + + +def test_load_toml_from_text(): + parser = Config.parsers['toml'] + config = Config.from_text('k = "v"', parser) + compare(config.k, "v") + + +def test_load_toml_from_bytes(): + parser = Config.parsers['toml'] + config = Config.from_text(b'k = "v"', parser) + compare(config.k, "v") From 2192c75942ed31936ea77a090a1833b397b9a248 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Thu, 7 Sep 2023 21:01:53 -0500 Subject: [PATCH 2/5] drop 3.6? --- .circleci/config.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59b8939..76fed77 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ common: &common matrix: parameters: image: - - cimg/python:3.6 + - cimg/python:3.7 - cimg/python:3.11 extras: - "[test]" diff --git a/setup.py b/setup.py index 15cbbb4..94dc3c7 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ packages=find_packages(exclude=["tests"]), zip_safe=False, include_package_data=True, - python_requires=">=3.6", + python_requires=">=3.7", extras_require=dict( yaml=['pyyaml'], toml=['tomli; python_version < "3.11"'], From c66336906123c5a56e0e96f9a0037fd40ffb933f Mon Sep 17 00:00:00 2001 From: wim glenn Date: Thu, 7 Sep 2023 21:06:57 -0500 Subject: [PATCH 3/5] address the coverage miss --- tests/test_toml_loading.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_toml_loading.py b/tests/test_toml_loading.py index ec5c888..b10e55a 100644 --- a/tests/test_toml_loading.py +++ b/tests/test_toml_loading.py @@ -4,13 +4,21 @@ from configurator import Config -def test_load_toml_from_path(tmp_path): +def test_load_toml_from_path_implicit_parser(tmp_path): path = tmp_path / 'test.toml' path.write_bytes(b'k = "v"') config = Config.from_path(path) compare(config.k, "v") +def test_load_toml_from_path_explicit_parser(tmp_path): + path = tmp_path / 'test.toml' + path.write_bytes(b'k = "v"') + parser = Config.parsers['toml'] + config = Config.from_path(path, parser) + compare(config.k, "v") + + def test_load_toml_from_byte_stream_implicit_parser(tmp_path): path = tmp_path / 'test.toml' path.write_bytes(b'k = "v"') From b10c0b0ea8a1bad4860aaa687bd2d956e318e1e8 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Mon, 11 Sep 2023 20:00:19 -0500 Subject: [PATCH 4/5] wrap tomllib.load to handle text streams --- configurator/_toml.py | 16 ++++++++++++++++ configurator/config.py | 24 +++++------------------- configurator/parsers.py | 14 +++----------- tests/test_toml_loading.py | 12 ++++++------ 4 files changed, 30 insertions(+), 36 deletions(-) create mode 100644 configurator/_toml.py diff --git a/configurator/_toml.py b/configurator/_toml.py new file mode 100644 index 0000000..f70d18b --- /dev/null +++ b/configurator/_toml.py @@ -0,0 +1,16 @@ +import io + +try: + import tomllib +except ImportError: + import tomli as tomllib + + +def load(f, /, *, parse_float=float): + # wrapper around tomllib.load to be more forgiving of streams opened in text mode + if isinstance(f, io.TextIOWrapper): + return tomllib.load(f.buffer, parse_float=parse_float) + elif isinstance(f, io.StringIO): + return tomllib.loads(f.getvalue(), parse_float=parse_float) + else: + return tomllib.load(f, parse_float=parse_float) diff --git a/configurator/config.py b/configurator/config.py index a677591..ed9f97c 100644 --- a/configurator/config.py +++ b/configurator/config.py @@ -1,6 +1,6 @@ import os from copy import deepcopy -from io import open, StringIO, BytesIO +from io import open, StringIO from os.path import exists, expanduser from .mapping import load, store, target, convert, if_supplied @@ -30,16 +30,9 @@ def from_text(cls, text, parser, encoding='ascii'): :ref:`parser `. If ``text`` is provided as :class:`bytes`, then the ``encoding`` specified will be used to decode it. """ - if parser == 'toml' or getattr(parser, '__module__', '').startswith('toml'): - if isinstance(text, bytes): - stream = BytesIO(text) - else: - stream = BytesIO(text.encode('utf-8')) - else: - if isinstance(text, bytes): - text = text.decode(encoding) - stream = StringIO(text) - return cls.from_stream(stream, parser) + if isinstance(text, bytes): + text = text.decode(encoding) + return cls.from_stream(StringIO(text), parser) @classmethod def from_stream(cls, stream, parser=None): @@ -77,14 +70,7 @@ def from_path(cls, path, parser=None, encoding=None, optional=False): full_path = expanduser(path) if optional and not exists(full_path): return cls() - mode = 'rt' - if parser is None: - if str(path).endswith('.toml'): - mode = 'rb' - else: - if getattr(parser, '__module__', '').startswith('toml'): - mode = 'rb' - with open(full_path, encoding=encoding, mode=mode) as stream: + with open(full_path, encoding=encoding) as stream: return cls.from_stream(stream, parser) @classmethod diff --git a/configurator/parsers.py b/configurator/parsers.py index a469f90..48c1434 100644 --- a/configurator/parsers.py +++ b/configurator/parsers.py @@ -1,6 +1,5 @@ -import sys - from collections import defaultdict +from importlib import import_module class ParseError(Exception): @@ -10,19 +9,12 @@ class ParseError(Exception): """ -if sys.version_info >= (3, 11): - # stdlib - _toml_mod = 'tomllib' -else: - _toml_mod = 'tomli' - - class Parsers(defaultdict): # file extension: module name, method name supported = { 'json': ('json', 'load'), - 'toml': (_toml_mod, 'load'), + 'toml': ('configurator._toml', 'load'), 'yml': ('yaml', 'safe_load'), 'yaml': ('yaml', 'safe_load'), } @@ -33,5 +25,5 @@ def __missing__(self, extension): except KeyError: raise ParseError('No parser found for {!r}'.format(extension)) else: - module = __import__(module_name) + module = import_module(module_name) return getattr(module, parser_name) diff --git a/tests/test_toml_loading.py b/tests/test_toml_loading.py index b10e55a..151eeb0 100644 --- a/tests/test_toml_loading.py +++ b/tests/test_toml_loading.py @@ -39,18 +39,18 @@ def test_load_toml_from_byte_stream_explicit_parser(tmp_path): def test_load_toml_from_text_stream_implicit_parser(tmp_path): path = tmp_path / 'test.toml' path.write_bytes(b'k = "v"') - msg = "must be opened in binary mode" - with path.open(mode="rt") as stream, pytest.raises(TypeError, match=msg): - Config.from_stream(stream) + with path.open(mode="rt") as stream: + config = Config.from_stream(stream) + compare(config.k, "v") def test_load_toml_from_text_stream_explicit_parser(tmp_path): path = tmp_path / 'test.toml' path.write_bytes(b'k = "v"') parser = Config.parsers['toml'] - msg = "must be opened in binary mode" - with path.open(mode="rt") as stream, pytest.raises(TypeError, match=msg): - Config.from_stream(stream, parser) + with path.open(mode="rt") as stream: + config = Config.from_stream(stream, parser) + compare(config.k, "v") def test_load_toml_from_text(): From 195f0bfc23b88f379b17bccc51128af982d9b2f5 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Mon, 11 Sep 2023 20:01:46 -0500 Subject: [PATCH 5/5] positional only args require 3.8 --- .circleci/config.yml | 2 +- configurator/_toml.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 76fed77..59b8939 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ common: &common matrix: parameters: image: - - cimg/python:3.7 + - cimg/python:3.6 - cimg/python:3.11 extras: - "[test]" diff --git a/configurator/_toml.py b/configurator/_toml.py index f70d18b..4e74a4e 100644 --- a/configurator/_toml.py +++ b/configurator/_toml.py @@ -6,7 +6,7 @@ import tomli as tomllib -def load(f, /, *, parse_float=float): +def load(f, *, parse_float=float): # wrapper around tomllib.load to be more forgiving of streams opened in text mode if isinstance(f, io.TextIOWrapper): return tomllib.load(f.buffer, parse_float=parse_float) diff --git a/setup.py b/setup.py index 94dc3c7..15cbbb4 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ packages=find_packages(exclude=["tests"]), zip_safe=False, include_package_data=True, - python_requires=">=3.7", + python_requires=">=3.6", extras_require=dict( yaml=['pyyaml'], toml=['tomli; python_version < "3.11"'],