Skip to content

gh-133677: Fix tests when running in non-UTF-8 locale #133865

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 12, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Lib/test/support/strace_helper.py
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ def events(self):

This assumes the program under inspection doesn't print any non-utf8
strings which would mix into the strace output."""
decoded_events = self.event_bytes.decode('utf-8')
decoded_events = self.event_bytes.decode('utf-8', 'surrogateescape')
matches = [
_syscall_regex.match(event)
for event in decoded_events.splitlines()
4 changes: 2 additions & 2 deletions Lib/test/test_argparse.py
Original file line number Diff line number Diff line change
@@ -6972,8 +6972,8 @@ def make_zip_script(self, script_name, name_in_zip=None):
return zip_name

def check_usage(self, expected, *args, **kwargs):
res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs)
self.assertEqual(res.out.splitlines()[0].decode(),
res = script_helper.assert_python_ok(*args, '-h', **kwargs)
self.assertEqual(os.fsdecode(res.out.splitlines()[0]),
f'usage: {expected} [-h]')

def test_script(self, compiled=False):
8 changes: 4 additions & 4 deletions Lib/test/test_asyncio/test_tools.py
Original file line number Diff line number Diff line change
@@ -791,21 +791,21 @@ def test_table_output_format(self):
class TestAsyncioToolsEdgeCases(unittest.TestCase):

def test_task_awaits_self(self):
"""A task directly awaits itself should raise a cycle."""
"""A task directly awaits itself - should raise a cycle."""
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are all unrelated.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is very related. Error happens when non-ASCII "–" is written to stdout with a non-UTF-8 encoding.

input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])]
with self.assertRaises(tools.CycleFoundException) as ctx:
tools.build_async_tree(input_)
self.assertIn([1, 1], ctx.exception.cycles)

def test_task_with_missing_awaiter_id(self):
"""Awaiter ID not in task list should not crash, just show 'Unknown'."""
"""Awaiter ID not in task list - should not crash, just show 'Unknown'."""
input_ = [(1, [(1, "Task-A", [[["coro"], 999]])])] # 999 not defined
table = tools.build_task_table(input_)
self.assertEqual(len(table), 1)
self.assertEqual(table[0][4], "Unknown")

def test_duplicate_coroutine_frames(self):
"""Same coroutine frame repeated under a parent should deduplicate."""
"""Same coroutine frame repeated under a parent - should deduplicate."""
input_ = [
(
1,
@@ -829,7 +829,7 @@ def test_duplicate_coroutine_frames(self):
self.assertIn("Task-1", flat)

def test_task_with_no_name(self):
"""Task with no name in id2name should still render with fallback."""
"""Task with no name in id2name - should still render with fallback."""
input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])]
# If name is None, fallback to string should not crash
tree = tools.build_async_tree(input_)
12 changes: 8 additions & 4 deletions Lib/test/test_pathlib/test_pathlib.py
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@
from test.support import is_emscripten, is_wasi
from test.support import infinite_recursion
from test.support import os_helper
from test.support.os_helper import TESTFN, FakePath
from test.support.os_helper import TESTFN, FS_NONASCII, FakePath
try:
import fcntl
except ImportError:
@@ -770,12 +770,16 @@ def test_as_uri_windows(self):
self.assertEqual(self.make_uri(P('c:/')), 'file:///c:/')
self.assertEqual(self.make_uri(P('c:/a/b.c')), 'file:///c:/a/b.c')
self.assertEqual(self.make_uri(P('c:/a/b%#c')), 'file:///c:/a/b%25%23c')
self.assertEqual(self.make_uri(P('c:/a/b\xe9')), 'file:///c:/a/b%C3%A9')
self.assertEqual(self.make_uri(P('//some/share/')), 'file://some/share/')
self.assertEqual(self.make_uri(P('//some/share/a/b.c')),
'file://some/share/a/b.c')
self.assertEqual(self.make_uri(P('//some/share/a/b%#c\xe9')),
'file://some/share/a/b%25%23c%C3%A9')

from urllib.parse import quote_from_bytes
QUOTED_FS_NONASCII = quote_from_bytes(os.fsencode(FS_NONASCII))
self.assertEqual(self.make_uri(P('c:/a/b' + FS_NONASCII)),
'file:///c:/a/b' + QUOTED_FS_NONASCII)
self.assertEqual(self.make_uri(P('//some/share/a/b%#c' + FS_NONASCII)),
'file://some/share/a/b%25%23c' + QUOTED_FS_NONASCII)

@needs_windows
def test_ordering_windows(self):
2 changes: 1 addition & 1 deletion Lib/test/test_urllib.py
Original file line number Diff line number Diff line change
@@ -109,7 +109,7 @@ def setUp(self):
finally:
f.close()
self.pathname = os_helper.TESTFN
self.quoted_pathname = urllib.parse.quote(self.pathname)
self.quoted_pathname = urllib.parse.quote(os.fsencode(self.pathname))
self.returned_obj = urllib.request.urlopen("file:%s" % self.quoted_pathname)

def tearDown(self):
2 changes: 1 addition & 1 deletion Lib/test/test_zipfile/test_core.py
Original file line number Diff line number Diff line change
@@ -3642,7 +3642,7 @@ def test_cli_with_metadata_encoding_extract(self):
except OSError:
pass
except UnicodeEncodeError:
self.skipTest(f'cannot encode file name {fn!r}')
self.skipTest(f'cannot encode file name {fn!a}')

zipfile.main(["--metadata-encoding=shift_jis", "-e", TESTFN, TESTFN2])
listing = os.listdir(TESTFN2)
Loading
Oops, something went wrong.