From 03166bcd5d86426ef055d147697dea1c9a9215e9 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 6 Mar 2024 15:05:39 +0000 Subject: [PATCH 1/6] Add compat.py39.LOCALE_ENCODING --- setuptools/compat/py39.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 setuptools/compat/py39.py diff --git a/setuptools/compat/py39.py b/setuptools/compat/py39.py new file mode 100644 index 0000000000..04a4abe5a9 --- /dev/null +++ b/setuptools/compat/py39.py @@ -0,0 +1,9 @@ +import sys + +# Explicitly use the ``"locale"`` encoding in versions that support it, +# otherwise just rely on the implicit handling of ``encoding=None``. +# Since all platforms that support ``EncodingWarning`` also support +# ``encoding="locale"``, this can be used to suppress the warning. +# However, please try to use UTF-8 when possible +# (.pth files are the notorious exception: python/cpython#77102, pypa/setuptools#3937). +LOCALE_ENCODING = "locale" if sys.version_info >= (3, 10) else None From ff99234609151de8abdfbe1d97e41071f93964ce Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 7 Mar 2024 09:11:21 +0000 Subject: [PATCH 2/6] Re-use compat.py39.LOCALE_ENCODING in editable_wheel --- setuptools/command/editable_wheel.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 9d319398c9..5f08ab53fc 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -14,7 +14,6 @@ import io import os import shutil -import sys import traceback from contextlib import suppress from enum import Enum @@ -44,6 +43,7 @@ namespaces, ) from .._path import StrPath +from ..compat import py39 from ..discovery import find_package_path from ..dist import Distribution from ..warnings import ( @@ -558,9 +558,8 @@ def _encode_pth(content: str) -> bytes: (There seems to be some variety in the way different version of Python handle ``encoding=None``, not all of them use ``locale.getpreferredencoding(False)``). """ - encoding = "locale" if sys.version_info >= (3, 10) else None with io.BytesIO() as buffer: - wrapper = io.TextIOWrapper(buffer, encoding) + wrapper = io.TextIOWrapper(buffer, encoding=py39.LOCALE_ENCODING) wrapper.write(content) wrapper.flush() buffer.seek(0) From 76ac799acfbb4bec9fec0815d282c444eb92f49f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 7 Mar 2024 09:44:05 +0000 Subject: [PATCH 3/6] Explicitly use 'locale' encoding for .pth files in easy_install --- setuptools/command/easy_install.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 402355bd81..c256770239 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -74,7 +74,7 @@ DEVELOP_DIST, ) import pkg_resources -from ..compat import py311 +from ..compat import py39, py311 from .._path import ensure_directory from ..extern.jaraco.text import yield_lines @@ -491,7 +491,7 @@ def check_site_dir(self): # noqa: C901 # is too complex (12) # FIXME try: if test_exists: os.unlink(testfile) - open(testfile, 'w').close() + open(testfile, 'wb').close() os.unlink(testfile) except OSError: self.cant_write_to_target() @@ -576,7 +576,7 @@ def check_pth_processing(self): _one_liner( """ import os - f = open({ok_file!r}, 'w') + f = open({ok_file!r}, 'w', encoding="utf-8") f.write('OK') f.close() """ @@ -588,7 +588,8 @@ def check_pth_processing(self): os.unlink(ok_file) dirname = os.path.dirname(ok_file) os.makedirs(dirname, exist_ok=True) - f = open(pth_file, 'w') + f = open(pth_file, 'w', encoding=py39.LOCALE_ENCODING) + # ^-- Requires encoding="locale" instead of "utf-8" (python/cpython#77102). except OSError: self.cant_write_to_target() else: @@ -872,7 +873,7 @@ def write_script(self, script_name, contents, mode="t", blockers=()): ensure_directory(target) if os.path.exists(target): os.unlink(target) - with open(target, "w" + mode) as f: + with open(target, "w" + mode) as f: # TODO: is it safe to use "utf-8"? f.write(contents) chmod(target, 0o777 - mask) @@ -1016,7 +1017,7 @@ def install_exe(self, dist_filename, tmpdir): # Write EGG-INFO/PKG-INFO if not os.path.exists(pkg_inf): - f = open(pkg_inf, 'w') + f = open(pkg_inf, 'w') # TODO: probably it is safe to use "utf-8" f.write('Metadata-Version: 1.0\n') for k, v in cfg.items('metadata'): if k != 'target_version': @@ -1277,7 +1278,9 @@ def update_pth(self, dist): # noqa: C901 # is too complex (11) # FIXME filename = os.path.join(self.install_dir, 'setuptools.pth') if os.path.islink(filename): os.unlink(filename) - with open(filename, 'wt') as f: + + with open(filename, 'wt', encoding=py39.LOCALE_ENCODING) as f: + # Requires encoding="locale" instead of "utf-8" (python/cpython#77102). f.write(self.pth_file.make_relative(dist.location) + '\n') def unpack_progress(self, src, dst): @@ -1503,9 +1506,9 @@ def expand_paths(inputs): # noqa: C901 # is too complex (11) # FIXME continue # Read the .pth file - f = open(os.path.join(dirname, name)) - lines = list(yield_lines(f)) - f.close() + with open(os.path.join(dirname, name), encoding=py39.LOCALE_ENCODING) as f: + # Requires encoding="locale" instead of "utf-8" (python/cpython#77102). + lines = list(yield_lines(f)) # Yield existing non-dupe, non-import directory lines from it for line in lines: @@ -1619,7 +1622,8 @@ def _load_raw(self): paths = [] dirty = saw_import = False seen = dict.fromkeys(self.sitedirs) - f = open(self.filename, 'rt') + f = open(self.filename, 'rt', encoding=py39.LOCALE_ENCODING) + # ^-- Requires encoding="locale" instead of "utf-8" (python/cpython#77102). for line in f: path = line.rstrip() # still keep imports and empty/commented lines for formatting @@ -1690,7 +1694,8 @@ def save(self): data = '\n'.join(lines) + '\n' if os.path.islink(self.filename): os.unlink(self.filename) - with open(self.filename, 'wt') as f: + with open(self.filename, 'wt', encoding=py39.LOCALE_ENCODING) as f: + # Requires encoding="locale" instead of "utf-8" (python/cpython#77102). f.write(data) elif os.path.exists(self.filename): log.debug("Deleting empty %s", self.filename) From fc93ece16304292e6931f8a5730610098dae40dc Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 7 Mar 2024 09:44:36 +0000 Subject: [PATCH 4/6] Add comments to remind about utf-8 in easy-install --- setuptools/command/easy_install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index c256770239..858fb20f83 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -873,7 +873,7 @@ def write_script(self, script_name, contents, mode="t", blockers=()): ensure_directory(target) if os.path.exists(target): os.unlink(target) - with open(target, "w" + mode) as f: # TODO: is it safe to use "utf-8"? + with open(target, "w" + mode) as f: # TODO: is it safe to use utf-8? f.write(contents) chmod(target, 0o777 - mask) @@ -1017,7 +1017,7 @@ def install_exe(self, dist_filename, tmpdir): # Write EGG-INFO/PKG-INFO if not os.path.exists(pkg_inf): - f = open(pkg_inf, 'w') # TODO: probably it is safe to use "utf-8" + f = open(pkg_inf, 'w') # TODO: probably it is safe to use utf-8 f.write('Metadata-Version: 1.0\n') for k, v in cfg.items('metadata'): if k != 'target_version': @@ -1088,7 +1088,7 @@ def process(src, dst): if locals()[name]: txt = os.path.join(egg_tmp, 'EGG-INFO', name + '.txt') if not os.path.exists(txt): - f = open(txt, 'w') + f = open(txt, 'w') # TODO: probably it is safe to use utf-8 f.write('\n'.join(locals()[name]) + '\n') f.close() From 98c877396b9ecd0e94b6c46a41ea9cef87dc2965 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 7 Mar 2024 09:47:56 +0000 Subject: [PATCH 5/6] Explicitly use 'locale' encoding for .pth files in setuptools.namespaces --- setuptools/namespaces.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setuptools/namespaces.py b/setuptools/namespaces.py index e8f2941d45..0185d55f94 100644 --- a/setuptools/namespaces.py +++ b/setuptools/namespaces.py @@ -2,6 +2,8 @@ from distutils import log import itertools +from .compat import py39 + flatten = itertools.chain.from_iterable @@ -23,7 +25,8 @@ def install_namespaces(self): list(lines) return - with open(filename, 'wt') as f: + with open(filename, 'wt', encoding=py39.LOCALE_ENCODING) as f: + # Requires encoding="locale" instead of "utf-8" (python/cpython#77102). f.writelines(lines) def uninstall_namespaces(self): From 1dd135cba9c40e25b4cd2b650de4b7299aae5e1c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 7 Mar 2024 10:08:57 +0000 Subject: [PATCH 6/6] Add news fragment --- docs/conf.py | 4 ++++ newsfragments/4265.feature.rst | 3 +++ 2 files changed, 7 insertions(+) create mode 100644 newsfragments/4265.feature.rst diff --git a/docs/conf.py b/docs/conf.py index be8856849b..534da15a37 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,6 +55,10 @@ pattern=r'(Python #|bpo-)(?P\d+)', url='https://bugs.python.org/issue{python}', ), + dict( + pattern=r'\bpython/cpython#(?P\d+)', + url='{GH}/python/cpython/issues/{cpython}', + ), dict( pattern=r'Interop #(?P\d+)', url='{GH}/pypa/interoperability-peps/issues/{interop}', diff --git a/newsfragments/4265.feature.rst b/newsfragments/4265.feature.rst new file mode 100644 index 0000000000..bcb0467205 --- /dev/null +++ b/newsfragments/4265.feature.rst @@ -0,0 +1,3 @@ +Explicitly use ``encoding="locale"`` for ``.pth`` files whenever possible, +to reduce ``EncodingWarnings``. +This avoid errors with UTF-8 (see discussion in python/cpython#77102).