From 8b6bc3af6fbe03b3ef186dfff1f99e606c4ea600 Mon Sep 17 00:00:00 2001 From: Dan Yeaw Date: Mon, 20 Nov 2023 21:57:54 -0500 Subject: [PATCH] Fix PEP597 EncodingWarnings PEP 685 makes UTF-8 the default mode for Python. Explicitly add encoding parameters to open calls. - Replace subprocess check_output with the more modern run command and set the default encoding of UTF-8 - compat: remove exec_command_stdout function since no longer needed to maintain compatibility across platforms --- .github/workflows/ci.yml | 2 + PyInstaller/building/api.py | 2 +- PyInstaller/building/splash.py | 2 +- PyInstaller/compat.py | 50 +------------------ PyInstaller/configure.py | 2 +- PyInstaller/depend/bindepend.py | 2 +- .../hooks/hook-gi.repository.GdkPixbuf.py | 8 ++- PyInstaller/utils/conftest.py | 2 +- PyInstaller/utils/hooks/__init__.py | 6 ++- PyInstaller/utils/hooks/gi.py | 2 +- PyInstaller/utils/hooks/qt/__init__.py | 2 +- PyInstaller/utils/osx.py | 10 ++-- news/8117.bugfix.rst | 1 + tests/functional/test_basic.py | 12 ++--- .../test_binary_vs_data_reclassification.py | 2 +- tests/functional/test_hook_utilities.py | 5 +- tests/functional/test_misc.py | 8 +-- tests/functional/test_pkgutil.py | 2 +- tests/functional/test_symlinks.py | 6 +-- tests/unit/test_normalize_icon_type.py | 2 +- 20 files changed, 45 insertions(+), 83 deletions(-) create mode 100644 news/8117.bugfix.rst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c8b83afe..82b784c8ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,8 @@ env: PYINSTALLER_STRICT_BUNDLE_CODESIGN_ERROR: 1 # Enable strict verification of macOS bundles w.r.t. the code-signing requirements. PYINSTALLER_VERIFY_BUNDLE_SIGNATURE: 1 + # Enable PEP 597 EncodingWarnings + PYTHONWARNDEFAULTENCODING: true permissions: contents: read # to fetch code (actions/checkout) diff --git a/PyInstaller/building/api.py b/PyInstaller/building/api.py index 05b456c054..ca22da1656 100644 --- a/PyInstaller/building/api.py +++ b/PyInstaller/building/api.py @@ -787,7 +787,7 @@ def assemble(self): # Linux: append data into custom ELF section using objcopy. logger.info("Appending %s to custom ELF section in EXE", append_type) cmd = ['objcopy', '--add-section', f'pydata={append_file}', build_name] - p = subprocess.run(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, encoding='utf-8') if p.returncode: raise SystemError(f"objcopy Failure: {p.returncode} {p.stdout}") diff --git a/PyInstaller/building/splash.py b/PyInstaller/building/splash.py index 6d6717b378..35fbfe8a02 100644 --- a/PyInstaller/building/splash.py +++ b/PyInstaller/building/splash.py @@ -433,7 +433,7 @@ def generate_script(self): script = re.sub(' +', ' ', script) # Write script to disk, so that it is transparent to the user what script is executed. - with open(self.script_name, "w") as script_file: + with open(self.script_name, "w", encoding="utf-8") as script_file: script_file.write(script) return script diff --git a/PyInstaller/compat.py b/PyInstaller/compat.py index 329f321b59..ee3702766e 100644 --- a/PyInstaller/compat.py +++ b/PyInstaller/compat.py @@ -91,7 +91,7 @@ # Linux distributions such as Alpine or OpenWRT use musl as their libc implementation and resultantly need specially # compiled bootloaders. On musl systems, ldd with no arguments prints 'musl' and its version. -is_musl = is_linux and "musl" in subprocess.getoutput(["ldd"]) +is_musl = is_linux and "musl" in subprocess.run(["ldd"], capture_output=True) # macOS version _macos_ver = tuple(int(x) for x in platform.mac_ver()[0].split('.')) if is_darwin else None @@ -410,54 +410,6 @@ def exec_command_rc(*cmdargs: str, **kwargs: float | bool | list | None): return subprocess.call(cmdargs, **kwargs) -def exec_command_stdout( - *command_args: str, encoding: str | None = None, **kwargs: float | str | bytes | bool | list | None -): - """ - Capture and return the standard output of the command specified by the passed positional arguments, optionally - configured by the passed keyword arguments. - - Unlike the legacy `exec_command()` and `exec_command_all()` functions, this modern function is explicitly designed - for cross-platform portability. The return value may be safely used for any purpose, including string manipulation - and parsing. - - .. NOTE:: - If this command's standard output contains _only_ pathnames, this function does _not_ return the correct - filesystem-encoded string expected by PyInstaller. If this is the case, consider calling the filesystem-specific - `exec_command()` function instead. - - Parameters - ---------- - command_args : List[str] - Variadic list whose: - 1. Mandatory first element is the absolute path, relative path, or basename in the current `${PATH}` of the - command to run. - 2. Optional remaining elements are arguments to pass to this command. - encoding : str, optional - Optional name of the encoding with which to decode this command's standard output (e.g., `utf8`), passed as a - keyword argument. If unpassed , this output will be decoded in a portable manner specific to to the current - platform, shell environment, and system settings with Python's built-in `universal_newlines` functionality. - - All remaining keyword arguments are passed as is to the `subprocess.check_output()` function. - - Returns - ---------- - str - Unicode string of this command's standard output decoded according to the "encoding" keyword argument. - """ - - # If no encoding was specified, the current locale is defaulted to. Else, an encoding was specified. To ensure this - # encoding is respected, the "universal_newlines" option is disabled if also passed. Nice, eh? - kwargs['universal_newlines'] = encoding is None - - # Standard output captured from this command as a decoded Unicode string if "universal_newlines" is enabled or an - # encoded byte array otherwise. - stdout = subprocess.check_output(command_args, **kwargs) - - # Return a Unicode string, decoded from this encoded byte array if needed. - return stdout if encoding is None else stdout.decode(encoding) - - def exec_command_all(*cmdargs: str, encoding: str | None = None, **kwargs: int | bool | list | None): """ Run the command specified by the passed positional arguments, optionally configured by the passed keyword arguments. diff --git a/PyInstaller/configure.py b/PyInstaller/configure.py index 6bb50cad02..46234b8b06 100644 --- a/PyInstaller/configure.py +++ b/PyInstaller/configure.py @@ -34,7 +34,7 @@ def _check_upx_availability(upx_dir): [upx_exe, '-V'], stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - universal_newlines=True, + encoding='utf-8', ) except Exception: logger.debug('UPX is not available.') diff --git a/PyInstaller/depend/bindepend.py b/PyInstaller/depend/bindepend.py index b83bebb2c8..d82bcc713c 100644 --- a/PyInstaller/depend/bindepend.py +++ b/PyInstaller/depend/bindepend.py @@ -327,7 +327,7 @@ def _get_imports_ldd(filename, search_paths): stdin=subprocess.DEVNULL, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True, + encoding='utf-8', ) for line in p.stderr.splitlines(): diff --git a/PyInstaller/hooks/hook-gi.repository.GdkPixbuf.py b/PyInstaller/hooks/hook-gi.repository.GdkPixbuf.py index fb0c3402d9..6c6b7c071c 100644 --- a/PyInstaller/hooks/hook-gi.repository.GdkPixbuf.py +++ b/PyInstaller/hooks/hook-gi.repository.GdkPixbuf.py @@ -12,6 +12,7 @@ import glob import os import shutil +import subprocess from PyInstaller import compat from PyInstaller.config import CONF # workpath @@ -80,7 +81,10 @@ def _generate_loader_cache(gdk_pixbuf_query_loaders, libdir, loader_libs): # # On Windows, the loaders lib directory is relative, starts with 'lib', and uses \\ as path separators # (escaped \). - cachedata = compat.exec_command_stdout(gdk_pixbuf_query_loaders, *loader_libs) + cachedata = subprocess.run([gdk_pixbuf_query_loaders, *loader_libs], + check=True, + stdout=subprocess.PIPE, + encoding='utf-8').stdout output_lines = [] prefix = '"' + os.path.join(libdir, 'gdk-pixbuf-2.0', '2.10.0') @@ -132,7 +136,7 @@ def hook(hook_api): # Generate loader cache; we need to store it to CONF['workpath'] so we can collect it as a data file. cachedata = _generate_loader_cache(gdk_pixbuf_query_loaders, libdir, loader_libs) cachefile = os.path.join(CONF['workpath'], 'loaders.cache') - with open(cachefile, 'w') as fp: + with open(cachefile, 'w', encoding='utf-8') as fp: fp.write(cachedata) datas.append((cachefile, LOADER_CACHE_DEST_PATH)) diff --git a/PyInstaller/utils/conftest.py b/PyInstaller/utils/conftest.py index d2f5dc4f70..3c9bc9b82b 100644 --- a/PyInstaller/utils/conftest.py +++ b/PyInstaller/utils/conftest.py @@ -443,7 +443,7 @@ def _examine_executable(self, exe, toc_log): """ print('EXECUTING MATCHING:', toc_log) fname_list = pkg_archive_contents(exe) - with open(toc_log, 'r') as f: + with open(toc_log, 'r', encoding='utf-8') as f: pattern_list = eval(f.read()) # Alphabetical order of patterns. pattern_list.sort() diff --git a/PyInstaller/utils/hooks/__init__.py b/PyInstaller/utils/hooks/__init__.py index 82bc6a775d..db34646f36 100644 --- a/PyInstaller/utils/hooks/__init__.py +++ b/PyInstaller/utils/hooks/__init__.py @@ -13,6 +13,7 @@ import copy import os +import subprocess import textwrap import fnmatch from pathlib import Path @@ -1026,7 +1027,10 @@ def get_installer(module: str): # Attempt to resolve the module file via macports' port command try: - output = compat.exec_command_stdout('port', 'provides', file_name) + output = subprocess.run(['port', 'provides', file_name], + check=True, + stdout=subprocess.PIPE, + encoding='utf-8').stdout if 'is provided by' in output: return 'macports' except ExecCommandFailed: diff --git a/PyInstaller/utils/hooks/gi.py b/PyInstaller/utils/hooks/gi.py index 6066aa4099..0ac8236265 100644 --- a/PyInstaller/utils/hooks/gi.py +++ b/PyInstaller/utils/hooks/gi.py @@ -394,8 +394,8 @@ def compile_glib_schema_files(datas_toc, workdir, collect_source_files=False): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, - text=True, errors='ignore', + encoding='utf-8', ) logger.debug("Output from glib-compile-schemas:\n%s", p.stdout) except subprocess.CalledProcessError as e: diff --git a/PyInstaller/utils/hooks/qt/__init__.py b/PyInstaller/utils/hooks/qt/__init__.py index 9d19b24127..d473e74235 100644 --- a/PyInstaller/utils/hooks/qt/__init__.py +++ b/PyInstaller/utils/hooks/qt/__init__.py @@ -796,7 +796,7 @@ def collect_qtwebengine_files(self): rel_prefix = rel_prefix.replace(os.sep, '/') # Create temporary file in workpath qt_conf_file = os.path.join(CONF['workpath'], "qt.conf") - with open(qt_conf_file, 'w') as fp: + with open(qt_conf_file, 'w', encoding='utf-8') as fp: print("[Paths]", file=fp) print("Prefix = {}".format(rel_prefix), file=fp) datas.append((qt_conf_file, dest)) diff --git a/PyInstaller/utils/osx.py b/PyInstaller/utils/osx.py index 894dd48a46..624b881968 100644 --- a/PyInstaller/utils/osx.py +++ b/PyInstaller/utils/osx.py @@ -310,7 +310,7 @@ def convert_binary_to_thin_arch(filename, thin_arch, output_filename=None): """ output_filename = output_filename or filename cmd_args = ['lipo', '-thin', thin_arch, filename, '-output', output_filename] - p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') if p.returncode: raise SystemError(f"lipo command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}") @@ -320,7 +320,7 @@ def merge_into_fat_binary(output_filename, *slice_filenames): Merge the given single-arch thin binary files into a fat binary. """ cmd_args = ['lipo', '-create', '-output', output_filename, *slice_filenames] - p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') if p.returncode: raise SystemError(f"lipo command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}") @@ -359,7 +359,7 @@ def remove_signature_from_binary(filename): """ logger.debug("Removing signature from file %r", filename) cmd_args = ['codesign', '--remove', '--all-architectures', filename] - p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') if p.returncode: raise SystemError(f"codesign command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}") @@ -381,7 +381,7 @@ def sign_binary(filename, identity=None, entitlements_file=None, deep=False): logger.debug("Signing file %r", filename) cmd_args = ['codesign', '-s', identity, '--force', '--all-architectures', '--timestamp', *extra_args, filename] - p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') if p.returncode: raise SystemError(f"codesign command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}") @@ -535,7 +535,7 @@ def _set_dylib_dependency_paths(filename, target_rpath): # Run `install_name_tool` cmd_args = ["install_name_tool", *install_name_tool_args, filename] - p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') if p.returncode: raise SystemError( f"install_name_tool command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}" diff --git a/news/8117.bugfix.rst b/news/8117.bugfix.rst new file mode 100644 index 0000000000..b5592790ca --- /dev/null +++ b/news/8117.bugfix.rst @@ -0,0 +1 @@ +Fix PEP 597 EncodingWarnings when `PYTHONWARNDEFAULTENCODING=true`. \ No newline at end of file diff --git a/tests/functional/test_basic.py b/tests/functional/test_basic.py index fb47ca6e7d..2b5ac1d4d1 100644 --- a/tests/functional/test_basic.py +++ b/tests/functional/test_basic.py @@ -369,7 +369,7 @@ def test_stderr_encoding(tmpdir, pyi_builder): # py.test has stdout encoding 'cp1252', which is an ANSI codepage. test fails as they do not match. # with -s: py.test process has stdout encoding from windows terminal, which is an OEM codepage. spawned # subprocess has the same encoding. test passes. - with open(os.path.join(tmpdir.strpath, 'stderr_encoding.build'), 'w') as f: + with open(os.path.join(tmpdir.strpath, 'stderr_encoding.build'), 'w', encoding='utf-8') as f: if sys.stderr.isatty(): enc = str(sys.stderr.encoding) else: @@ -381,7 +381,7 @@ def test_stderr_encoding(tmpdir, pyi_builder): def test_stdout_encoding(tmpdir, pyi_builder): - with open(os.path.join(tmpdir.strpath, 'stdout_encoding.build'), 'w') as f: + with open(os.path.join(tmpdir.strpath, 'stdout_encoding.build'), 'w', encoding='utf-8') as f: if sys.stdout.isatty(): enc = str(sys.stdout.encoding) else: @@ -630,7 +630,7 @@ def test_onefile_longpath(pyi_builder, tmpdir): # Create data file with secret _SECRET = 'LongDataPath' src_filename = tmpdir / 'data.txt' - with open(src_filename, 'w') as fp: + with open(src_filename, 'w', encoding='utf-8') as fp: fp.write(_SECRET) # Generate long target filename/path; eight equivalents of SHA256 strings plus data.txt should push just the # _MEIPASS-relative path beyond 260 characters... @@ -759,12 +759,12 @@ def test_package_entry_point_name_collision(pyi_builder): ] # Include a verification that unfrozen Python does still work. - p = subprocess.run([sys.executable, str(script)], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([sys.executable, str(script)], stdout=subprocess.PIPE, encoding="utf-8") assert re.findall("Running (.*) as (.*)", p.stdout) == expected pyi_builder.test_script(str(script)) exe, = pyi_builder._find_executables("matching_name") - p = subprocess.run([exe], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([exe], stdout=subprocess.PIPE, encoding="utf-8") assert re.findall("Running (.*) as (.*)", p.stdout) == expected @@ -815,7 +815,7 @@ def test_spec_options(pyi_builder, SPEC_DIR, capsys): pyi_args=["--", "--optional-dependency", "email", "--optional-dependency", "gzip"] ) exe, = pyi_builder._find_executables("pyi_spec_options") - p = subprocess.run([exe], stdout=subprocess.PIPE, text=True) + p = subprocess.run([exe], stdout=subprocess.PIPE, encoding="utf-8") assert p.stdout == "Available dependencies: email gzip\n" capsys.readouterr() diff --git a/tests/functional/test_binary_vs_data_reclassification.py b/tests/functional/test_binary_vs_data_reclassification.py index 2bb768c0ae..2102a93ea7 100644 --- a/tests/functional/test_binary_vs_data_reclassification.py +++ b/tests/functional/test_binary_vs_data_reclassification.py @@ -23,7 +23,7 @@ def _create_test_data_file(filename): os.makedirs(os.path.dirname(filename), exist_ok=True) # Create a text file - with open(filename, 'w') as fp: + with open(filename, 'w', encoding='utf-8') as fp: fp.write("Test file") diff --git a/tests/functional/test_hook_utilities.py b/tests/functional/test_hook_utilities.py index 8e9ee07099..2116a29317 100644 --- a/tests/functional/test_hook_utilities.py +++ b/tests/functional/test_hook_utilities.py @@ -9,8 +9,7 @@ # # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) # ----------------------------------------------------------------------------- - -from subprocess import run, PIPE +import subprocess from os.path import join @@ -29,7 +28,7 @@ def test_collect_entry_point(pyi_builder_spec, script_dir, tmpdir): pyi_builder_spec.test_spec('list_pytest11_entry_point.spec') exe = join(tmpdir, "dist", "list_pytest11_entry_point", "list_pytest11_entry_point") - p = run([exe], stdout=PIPE, check=True, universal_newlines=True) + p = subprocess.run([exe], stdout=subprocess.PIPE, check=True, encoding="utf-8") collected_plugins = p.stdout.strip("\n").split("\n") assert collected_plugins == plugins diff --git a/tests/functional/test_misc.py b/tests/functional/test_misc.py index 79dd647ba0..e41806cecb 100644 --- a/tests/functional/test_misc.py +++ b/tests/functional/test_misc.py @@ -49,7 +49,7 @@ def _normalize_module_path(module_path, stdlib_dir): def _load_results(filename): # Read pprint-ed results - with open(filename, 'r') as fp: + with open(filename, 'r', encoding='utf-8') as fp: data = fp.read() data = eval(data) @@ -184,7 +184,7 @@ def test_onefile_cleanup_symlinked_dir(pyi_builder, tmpdir): os.mkdir(output_dir) for idx in range(5): output_file = os.path.join(output_dir, f'preexisting-{idx}.txt') - with open(output_file, 'w') as fp: + with open(output_file, 'w', encoding='utf-8') as fp: fp.write(f'Pre-existing file #{idx}') # Check if OS supports creation of symbolic links @@ -206,7 +206,7 @@ def test_onefile_cleanup_symlinked_dir(pyi_builder, tmpdir): # Create five files for idx in range(5): output_file = os.path.join(output_dir, f'output-{idx}.txt') - with open(output_file, 'w') as fp: + with open(output_file, 'w', encoding='utf-8') as fp: fp.write(f'Output file #{idx}') """, app_args=[output_dir] @@ -317,7 +317,7 @@ def some_interactive_debugger_function(): @pytest.mark.darwin def test_bundled_shell_script(pyi_builder, tmpdir): script_file = tmpdir / "test_script.sh" - with open(script_file, "w") as fp: + with open(script_file, "w", encoding="utf-8") as fp: print('#!/bin/sh', file=fp) print('echo "Hello world!"', file=fp) os.chmod(script_file, 0o755) diff --git a/tests/functional/test_pkgutil.py b/tests/functional/test_pkgutil.py index 0b719ce66d..886e93093a 100644 --- a/tests/functional/test_pkgutil.py +++ b/tests/functional/test_pkgutil.py @@ -36,7 +36,7 @@ # name;ispackage def _read_results_file(filename): output = [] - with open(filename, 'r') as fp: + with open(filename, 'r', encoding='utf-8') as fp: for line in fp: tokens = line.split(';') assert len(tokens) == 2 diff --git a/tests/functional/test_symlinks.py b/tests/functional/test_symlinks.py index 37a3717c1c..63a055a5aa 100644 --- a/tests/functional/test_symlinks.py +++ b/tests/functional/test_symlinks.py @@ -34,7 +34,7 @@ def _create_data(tmpdir, orig_filename, link_filename): # Create original file abs_orig_filename = os.path.join(data_path, orig_filename) os.makedirs(os.path.dirname(abs_orig_filename), exist_ok=True) - with open(abs_orig_filename, 'w') as fp: + with open(abs_orig_filename, 'w', encoding='utf-8') as fp: fp.write("secret") # Create symbolic link @@ -243,7 +243,7 @@ def _prepare_chained_links_example(tmpdir): os.makedirs(data_path) # Create original file: file_a - with open(os.path.join(data_path, "file_a"), 'w') as fp: + with open(os.path.join(data_path, "file_a"), 'w', encoding='utf-8') as fp: fp.write("secret") # Create symbolic link: file_b -> file_a @@ -443,7 +443,7 @@ def _prepare_parent_directory_link_example(tmpdir): os.makedirs(original_dir) # Create original file: file_a - with open(os.path.join(original_dir, "file_a"), 'w') as fp: + with open(os.path.join(original_dir, "file_a"), 'w', encoding='utf-8') as fp: fp.write("secret") # Create symbolic link: file_b -> file_a diff --git a/tests/unit/test_normalize_icon_type.py b/tests/unit/test_normalize_icon_type.py index a8f258a9b8..6dce142aa5 100644 --- a/tests/unit/test_normalize_icon_type.py +++ b/tests/unit/test_normalize_icon_type.py @@ -72,7 +72,7 @@ def test_normalize_icon_pillow(tmp_path): # Some random non-image file: Raises an image conversion error icon = os.path.join(tmp_path, 'pyi_icon.notanicon') - with open(icon, "w") as f: + with open(icon, "w", encoding="utf-8") as f: f.write("this is in fact, not an icon") with pytest.raises(ValueError):