Fix PEP597 EncodingWarnings

PEP 685 makes UTF-8 the default mode for Python. Explicitly add
encoding parameters to open calls.

- Replace subprocess check_output with the more modern run
command and set the default encoding of UTF-8
- compat: remove exec_command_stdout function since no longer
needed to maintain compatibility across platforms

.github/workflows/ci.yml

@@ -25,6 +25,8 @@ env:
   PYINSTALLER_STRICT_BUNDLE_CODESIGN_ERROR: 1
   # Enable strict verification of macOS bundles w.r.t. the code-signing requirements.
   PYINSTALLER_VERIFY_BUNDLE_SIGNATURE: 1
+  # Enable PEP 597 EncodingWarnings
+  PYTHONWARNDEFAULTENCODING: true
 
 permissions:
   contents: read # to fetch code (actions/checkout)

PyInstaller/building/api.py

@@ -787,7 +787,7 @@ def assemble(self):
             # Linux: append data into custom ELF section using objcopy.
             logger.info("Appending %s to custom ELF section in EXE", append_type)
             cmd = ['objcopy', '--add-section', f'pydata={append_file}', build_name]
-            p = subprocess.run(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True)
+            p = subprocess.run(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, encoding='utf-8')
             if p.returncode:
                 raise SystemError(f"objcopy Failure: {p.returncode} {p.stdout}")
 

PyInstaller/building/splash.py

@@ -433,7 +433,7 @@ def generate_script(self):
             script = re.sub(' +', ' ', script)
 
         # Write script to disk, so that it is transparent to the user what script is executed.
-        with open(self.script_name, "w") as script_file:
+        with open(self.script_name, "w", encoding="utf-8") as script_file:
             script_file.write(script)
         return script
 

PyInstaller/compat.py

@@ -91,7 +91,7 @@
 
 # Linux distributions such as Alpine or OpenWRT use musl as their libc implementation and resultantly need specially
 # compiled bootloaders. On musl systems, ldd with no arguments prints 'musl' and its version.
-is_musl = is_linux and "musl" in subprocess.getoutput(["ldd"])
+is_musl = is_linux and "musl" in subprocess.run(["ldd"], capture_output=True)
 
 # macOS version
 _macos_ver = tuple(int(x) for x in platform.mac_ver()[0].split('.')) if is_darwin else None
@@ -410,54 +410,6 @@ def exec_command_rc(*cmdargs: str, **kwargs: float | bool | list | None):
     return subprocess.call(cmdargs, **kwargs)
 
 
-def exec_command_stdout(
-    *command_args: str, encoding: str | None = None, **kwargs: float | str | bytes | bool | list | None
-):
-    """
-    Capture and return the standard output of the command specified by the passed positional arguments, optionally
-    configured by the passed keyword arguments.
-
-    Unlike the legacy `exec_command()` and `exec_command_all()` functions, this modern function is explicitly designed
-    for cross-platform portability. The return value may be safely used for any purpose, including string manipulation
-    and parsing.
-
-    .. NOTE::
-       If this command's standard output contains _only_ pathnames, this function does _not_ return the correct
-       filesystem-encoded string expected by PyInstaller. If this is the case, consider calling the filesystem-specific
-       `exec_command()` function instead.
-
-    Parameters
-    ----------
-    command_args : List[str]
-        Variadic list whose:
-        1. Mandatory first element is the absolute path, relative path, or basename in the current `${PATH}` of the
-           command to run.
-        2. Optional remaining elements are arguments to pass to this command.
-    encoding : str, optional
-        Optional name of the encoding with which to decode this command's standard output (e.g., `utf8`), passed as a
-        keyword argument. If unpassed , this output will be decoded in a portable manner specific to to the current
-        platform, shell environment, and system settings with Python's built-in `universal_newlines` functionality.
-
-    All remaining keyword arguments are passed as is to the `subprocess.check_output()` function.
-
-    Returns
-    ----------
-    str
-        Unicode string of this command's standard output decoded according to the "encoding" keyword argument.
-    """
-
-    # If no encoding was specified, the current locale is defaulted to. Else, an encoding was specified. To ensure this
-    # encoding is respected, the "universal_newlines" option is disabled if also passed. Nice, eh?
-    kwargs['universal_newlines'] = encoding is None
-
-    # Standard output captured from this command as a decoded Unicode string if "universal_newlines" is enabled or an
-    # encoded byte array otherwise.
-    stdout = subprocess.check_output(command_args, **kwargs)
-
-    # Return a Unicode string, decoded from this encoded byte array if needed.
-    return stdout if encoding is None else stdout.decode(encoding)
-
-
 def exec_command_all(*cmdargs: str, encoding: str | None = None, **kwargs: int | bool | list | None):
     """
     Run the command specified by the passed positional arguments, optionally configured by the passed keyword arguments.

PyInstaller/configure.py

@@ -34,7 +34,7 @@ def _check_upx_availability(upx_dir):
             [upx_exe, '-V'],
             stdin=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL,
-            universal_newlines=True,
+            encoding='utf-8',
         )
     except Exception:
         logger.debug('UPX is not available.')

PyInstaller/depend/bindepend.py

@@ -327,7 +327,7 @@ def _get_imports_ldd(filename, search_paths):
         stdin=subprocess.DEVNULL,
         stderr=subprocess.PIPE,
         stdout=subprocess.PIPE,
-        universal_newlines=True,
+        encoding='utf-8',
     )
 
     for line in p.stderr.splitlines():

PyInstaller/hooks/hook-gi.repository.GdkPixbuf.py

@@ -12,6 +12,7 @@
 import glob
 import os
 import shutil
+import subprocess
 
 from PyInstaller import compat
 from PyInstaller.config import CONF  # workpath
@@ -80,7 +81,10 @@ def _generate_loader_cache(gdk_pixbuf_query_loaders, libdir, loader_libs):
     #
     # On Windows, the loaders lib directory is relative, starts with 'lib', and uses \\ as path separators
     # (escaped \).
-    cachedata = compat.exec_command_stdout(gdk_pixbuf_query_loaders, *loader_libs)
+    cachedata = subprocess.run([gdk_pixbuf_query_loaders, *loader_libs],
+                               check=True,
+                               stdout=subprocess.PIPE,
+                               encoding='utf-8').stdout
 
     output_lines = []
     prefix = '"' + os.path.join(libdir, 'gdk-pixbuf-2.0', '2.10.0')
@@ -132,7 +136,7 @@ def hook(hook_api):
         # Generate loader cache; we need to store it to CONF['workpath'] so we can collect it as a data file.
         cachedata = _generate_loader_cache(gdk_pixbuf_query_loaders, libdir, loader_libs)
         cachefile = os.path.join(CONF['workpath'], 'loaders.cache')
-        with open(cachefile, 'w') as fp:
+        with open(cachefile, 'w', encoding='utf-8') as fp:
             fp.write(cachedata)
         datas.append((cachefile, LOADER_CACHE_DEST_PATH))
 

PyInstaller/utils/conftest.py

@@ -443,7 +443,7 @@ def _examine_executable(self, exe, toc_log):
         """
         print('EXECUTING MATCHING:', toc_log)
         fname_list = pkg_archive_contents(exe)
-        with open(toc_log, 'r') as f:
+        with open(toc_log, 'r', encoding='utf-8') as f:
             pattern_list = eval(f.read())
         # Alphabetical order of patterns.
         pattern_list.sort()

PyInstaller/utils/hooks/__init__.py

@@ -13,6 +13,7 @@
 
 import copy
 import os
+import subprocess
 import textwrap
 import fnmatch
 from pathlib import Path
@@ -1026,7 +1027,10 @@ def get_installer(module: str):
 
         # Attempt to resolve the module file via macports' port command
         try:
-            output = compat.exec_command_stdout('port', 'provides', file_name)
+            output = subprocess.run(['port', 'provides', file_name],
+                                    check=True,
+                                    stdout=subprocess.PIPE,
+                                    encoding='utf-8').stdout
             if 'is provided by' in output:
                 return 'macports'
         except ExecCommandFailed:

PyInstaller/utils/hooks/gi.py

@@ -394,8 +394,8 @@ def compile_glib_schema_files(datas_toc, workdir, collect_source_files=False):
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             check=True,
-            text=True,
             errors='ignore',
+            encoding='utf-8',
         )
         logger.debug("Output from glib-compile-schemas:\n%s", p.stdout)
     except subprocess.CalledProcessError as e:

PyInstaller/utils/hooks/qt/__init__.py

@@ -796,7 +796,7 @@ def collect_qtwebengine_files(self):
                     rel_prefix = rel_prefix.replace(os.sep, '/')
                 # Create temporary file in workpath
                 qt_conf_file = os.path.join(CONF['workpath'], "qt.conf")
-                with open(qt_conf_file, 'w') as fp:
+                with open(qt_conf_file, 'w', encoding='utf-8') as fp:
                     print("[Paths]", file=fp)
                     print("Prefix = {}".format(rel_prefix), file=fp)
             datas.append((qt_conf_file, dest))

PyInstaller/utils/osx.py

@@ -310,7 +310,7 @@ def convert_binary_to_thin_arch(filename, thin_arch, output_filename=None):
     """
     output_filename = output_filename or filename
     cmd_args = ['lipo', '-thin', thin_arch, filename, '-output', output_filename]
-    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
     if p.returncode:
         raise SystemError(f"lipo command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}")
 
@@ -320,7 +320,7 @@ def merge_into_fat_binary(output_filename, *slice_filenames):
     Merge the given single-arch thin binary files into a fat binary.
     """
     cmd_args = ['lipo', '-create', '-output', output_filename, *slice_filenames]
-    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
     if p.returncode:
         raise SystemError(f"lipo command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}")
 
@@ -359,7 +359,7 @@ def remove_signature_from_binary(filename):
     """
     logger.debug("Removing signature from file %r", filename)
     cmd_args = ['codesign', '--remove', '--all-architectures', filename]
-    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
     if p.returncode:
         raise SystemError(f"codesign command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}")
 
@@ -381,7 +381,7 @@ def sign_binary(filename, identity=None, entitlements_file=None, deep=False):
 
     logger.debug("Signing file %r", filename)
     cmd_args = ['codesign', '-s', identity, '--force', '--all-architectures', '--timestamp', *extra_args, filename]
-    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
     if p.returncode:
         raise SystemError(f"codesign command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}")
 
@@ -535,7 +535,7 @@ def _set_dylib_dependency_paths(filename, target_rpath):
 
     # Run `install_name_tool`
     cmd_args = ["install_name_tool", *install_name_tool_args, filename]
-    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+    p = subprocess.run(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
     if p.returncode:
         raise SystemError(
             f"install_name_tool command ({cmd_args}) failed with error code {p.returncode}!\noutput: {p.stdout}"

news/8117.bugfix.rst

@@ -0,0 +1 @@
+Fix PEP 597 EncodingWarnings when `PYTHONWARNDEFAULTENCODING=true`.

tests/functional/test_basic.py

@@ -369,7 +369,7 @@ def test_stderr_encoding(tmpdir, pyi_builder):
     #             py.test has stdout encoding 'cp1252', which is an ANSI codepage. test fails as they do not match.
     # with -s:    py.test process has stdout encoding from windows terminal, which is an OEM codepage. spawned
     #             subprocess has the same encoding. test passes.
-    with open(os.path.join(tmpdir.strpath, 'stderr_encoding.build'), 'w') as f:
+    with open(os.path.join(tmpdir.strpath, 'stderr_encoding.build'), 'w', encoding='utf-8') as f:
         if sys.stderr.isatty():
             enc = str(sys.stderr.encoding)
         else:
@@ -381,7 +381,7 @@ def test_stderr_encoding(tmpdir, pyi_builder):
 
 
 def test_stdout_encoding(tmpdir, pyi_builder):
-    with open(os.path.join(tmpdir.strpath, 'stdout_encoding.build'), 'w') as f:
+    with open(os.path.join(tmpdir.strpath, 'stdout_encoding.build'), 'w', encoding='utf-8') as f:
         if sys.stdout.isatty():
             enc = str(sys.stdout.encoding)
         else:
@@ -630,7 +630,7 @@ def test_onefile_longpath(pyi_builder, tmpdir):
     # Create data file with secret
     _SECRET = 'LongDataPath'
     src_filename = tmpdir / 'data.txt'
-    with open(src_filename, 'w') as fp:
+    with open(src_filename, 'w', encoding='utf-8') as fp:
         fp.write(_SECRET)
     # Generate long target filename/path; eight equivalents of SHA256 strings plus data.txt should push just the
     # _MEIPASS-relative path beyond 260 characters...
@@ -759,12 +759,12 @@ def test_package_entry_point_name_collision(pyi_builder):
     ]
 
     # Include a verification that unfrozen Python does still work.
-    p = subprocess.run([sys.executable, str(script)], stdout=subprocess.PIPE, universal_newlines=True)
+    p = subprocess.run([sys.executable, str(script)], stdout=subprocess.PIPE, encoding="utf-8")
     assert re.findall("Running (.*) as (.*)", p.stdout) == expected
 
     pyi_builder.test_script(str(script))
     exe, = pyi_builder._find_executables("matching_name")
-    p = subprocess.run([exe], stdout=subprocess.PIPE, universal_newlines=True)
+    p = subprocess.run([exe], stdout=subprocess.PIPE, encoding="utf-8")
     assert re.findall("Running (.*) as (.*)", p.stdout) == expected
 
 
@@ -815,7 +815,7 @@ def test_spec_options(pyi_builder, SPEC_DIR, capsys):
         pyi_args=["--", "--optional-dependency", "email", "--optional-dependency", "gzip"]
     )
     exe, = pyi_builder._find_executables("pyi_spec_options")
-    p = subprocess.run([exe], stdout=subprocess.PIPE, text=True)
+    p = subprocess.run([exe], stdout=subprocess.PIPE, encoding="utf-8")
     assert p.stdout == "Available dependencies: email gzip\n"
 
     capsys.readouterr()

tests/functional/test_binary_vs_data_reclassification.py

@@ -23,7 +23,7 @@
 def _create_test_data_file(filename):
     os.makedirs(os.path.dirname(filename), exist_ok=True)
     # Create a text file
-    with open(filename, 'w') as fp:
+    with open(filename, 'w', encoding='utf-8') as fp:
         fp.write("Test file")
 
 

tests/functional/test_hook_utilities.py

@@ -9,8 +9,7 @@
 #
 # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception)
 # -----------------------------------------------------------------------------
-
-from subprocess import run, PIPE
+import subprocess
 from os.path import join
 
 
@@ -29,7 +28,7 @@ def test_collect_entry_point(pyi_builder_spec, script_dir, tmpdir):
     pyi_builder_spec.test_spec('list_pytest11_entry_point.spec')
     exe = join(tmpdir, "dist", "list_pytest11_entry_point", "list_pytest11_entry_point")
 
-    p = run([exe], stdout=PIPE, check=True, universal_newlines=True)
+    p = subprocess.run([exe], stdout=subprocess.PIPE, check=True, encoding="utf-8")
     collected_plugins = p.stdout.strip("\n").split("\n")
 
     assert collected_plugins == plugins

tests/functional/test_misc.py

@@ -49,7 +49,7 @@ def _normalize_module_path(module_path, stdlib_dir):
 
     def _load_results(filename):
         # Read pprint-ed results
-        with open(filename, 'r') as fp:
+        with open(filename, 'r', encoding='utf-8') as fp:
             data = fp.read()
         data = eval(data)
 
@@ -184,7 +184,7 @@ def test_onefile_cleanup_symlinked_dir(pyi_builder, tmpdir):
     os.mkdir(output_dir)
     for idx in range(5):
         output_file = os.path.join(output_dir, f'preexisting-{idx}.txt')
-        with open(output_file, 'w') as fp:
+        with open(output_file, 'w', encoding='utf-8') as fp:
             fp.write(f'Pre-existing file #{idx}')
 
     # Check if OS supports creation of symbolic links
@@ -206,7 +206,7 @@ def test_onefile_cleanup_symlinked_dir(pyi_builder, tmpdir):
         # Create five files
         for idx in range(5):
             output_file = os.path.join(output_dir, f'output-{idx}.txt')
-            with open(output_file, 'w') as fp:
+            with open(output_file, 'w', encoding='utf-8') as fp:
                 fp.write(f'Output file #{idx}')
         """,
         app_args=[output_dir]
@@ -317,7 +317,7 @@ def some_interactive_debugger_function():
 @pytest.mark.darwin
 def test_bundled_shell_script(pyi_builder, tmpdir):
     script_file = tmpdir / "test_script.sh"
-    with open(script_file, "w") as fp:
+    with open(script_file, "w", encoding="utf-8") as fp:
         print('#!/bin/sh', file=fp)
         print('echo "Hello world!"', file=fp)
     os.chmod(script_file, 0o755)

tests/functional/test_pkgutil.py

@@ -36,7 +36,7 @@
 # name;ispackage
 def _read_results_file(filename):
     output = []
-    with open(filename, 'r') as fp:
+    with open(filename, 'r', encoding='utf-8') as fp:
         for line in fp:
             tokens = line.split(';')
             assert len(tokens) == 2

tests/functional/test_symlinks.py

@@ -34,7 +34,7 @@ def _create_data(tmpdir, orig_filename, link_filename):
     # Create original file
     abs_orig_filename = os.path.join(data_path, orig_filename)
     os.makedirs(os.path.dirname(abs_orig_filename), exist_ok=True)
-    with open(abs_orig_filename, 'w') as fp:
+    with open(abs_orig_filename, 'w', encoding='utf-8') as fp:
         fp.write("secret")
 
     # Create symbolic link
@@ -243,7 +243,7 @@ def _prepare_chained_links_example(tmpdir):
     os.makedirs(data_path)
 
     # Create original file: file_a
-    with open(os.path.join(data_path, "file_a"), 'w') as fp:
+    with open(os.path.join(data_path, "file_a"), 'w', encoding='utf-8') as fp:
         fp.write("secret")
 
     # Create symbolic link: file_b -> file_a
@@ -443,7 +443,7 @@ def _prepare_parent_directory_link_example(tmpdir):
     os.makedirs(original_dir)
 
     # Create original file: file_a
-    with open(os.path.join(original_dir, "file_a"), 'w') as fp:
+    with open(os.path.join(original_dir, "file_a"), 'w', encoding='utf-8') as fp:
         fp.write("secret")
 
     # Create symbolic link: file_b -> file_a

tests/unit/test_normalize_icon_type.py

@@ -72,7 +72,7 @@ def test_normalize_icon_pillow(tmp_path):
     # Some random non-image file: Raises an image conversion error
 
     icon = os.path.join(tmp_path, 'pyi_icon.notanicon')
-    with open(icon, "w") as f:
+    with open(icon, "w", encoding="utf-8") as f:
         f.write("this is in fact, not an icon")
 
     with pytest.raises(ValueError):