From 36693d4ea08b5e2dba3fe7709179f99c9d2582c9 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Fri, 3 Jan 2025 09:51:41 +0100 Subject: [PATCH 1/7] Use `EnvironmentVarGuard` in `test_cmd_line` instead of manually messing with `os.environ` --- Lib/test/test_cmd_line.py | 264 +++++++++++++++++++------------------- 1 file changed, 132 insertions(+), 132 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 634efda354407f..43f1e5bcdd6324 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -238,37 +238,37 @@ def test_coding(self): @unittest.skipIf(sys.platform == 'win32', 'Windows has a native unicode API') def test_undecodable_code(self): - undecodable = b"\xff" - env = os.environ.copy() - # Use C locale to get ascii for the locale encoding - env['LC_ALL'] = 'C' - env['PYTHONCOERCECLOCALE'] = '0' - code = ( - b'import locale; ' - b'print(ascii("' + undecodable + b'"), ' - b'locale.getencoding())') - p = subprocess.Popen( - [sys.executable, "-c", code], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - env=env) - stdout, stderr = p.communicate() - if p.returncode == 1: - # _Py_char2wchar() decoded b'\xff' as '\udcff' (b'\xff' is not - # decodable from ASCII) and run_command() failed on - # PyUnicode_AsUTF8String(). This is the expected behaviour on - # Linux. - pattern = b"Unable to decode the command from the command line:" - elif p.returncode == 0: - # _Py_char2wchar() decoded b'\xff' as '\xff' even if the locale is - # C and the locale encoding is ASCII. It occurs on FreeBSD, Solaris - # and Mac OS X. - pattern = b"'\\xff' " - # The output is followed by the encoding name, an alias to ASCII. - # Examples: "US-ASCII" or "646" (ISO 646, on Solaris). - else: - raise AssertionError("Unknown exit code: %s, output=%a" % (p.returncode, stdout)) - if not stdout.startswith(pattern): - raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) + with os_helper.EnvironmentVarGuard() as env: + undecodable = b"\xff" + # Use C locale to get ascii for the locale encoding + env['LC_ALL'] = 'C' + env['PYTHONCOERCECLOCALE'] = '1' + code = ( + b'import locale; ' + b'print(ascii("' + undecodable + b'"), ' + b'locale.getencoding())') + p = subprocess.Popen( + [sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + stdout, stderr = p.communicate() + if p.returncode == 1: + # _Py_char2wchar() decoded b'\xff' as '\udcff' (b'\xff' is not + # decodable from ASCII) and run_command() failed on + # PyUnicode_AsUTF8String(). This is the expected behaviour on + # Linux. + pattern = b"Unable to decode the command from the command line:" + elif p.returncode == 0: + # _Py_char2wchar() decoded b'\xff' as '\xff' even if the locale is + # C and the locale encoding is ASCII. It occurs on FreeBSD, Solaris + # and Mac OS X. + pattern = b"'\\xff' " + # The output is followed by the encoding name, an alias to ASCII. + # Examples: "US-ASCII" or "646" (ISO 646, on Solaris). + else: + raise AssertionError("Unknown exit code: %s, output=%a" % (p.returncode, stdout)) + if not stdout.startswith(pattern): + raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) @unittest.skipIf(sys.platform == 'win32', 'Windows has a native unicode API') @@ -287,10 +287,10 @@ def run_default(arg): def run_c_locale(arg): cmd = [sys.executable, '-c', code, arg] - env = dict(os.environ) - env['LC_ALL'] = 'C' - return subprocess.run(cmd, stdout=subprocess.PIPE, - text=True, env=env) + with os_helper.EnvironmentVarGuard() as env: + env['LC_ALL'] = 'C' + return subprocess.run(cmd, stdout=subprocess.PIPE, + text=True) def run_utf8_mode(arg): cmd = [sys.executable, '-X', 'utf8', '-c', code, arg] @@ -323,18 +323,18 @@ def test_osx_android_utf8(self): decoded = text.decode('utf-8', 'surrogateescape') expected = ascii(decoded).encode('ascii') + b'\n' - env = os.environ.copy() - # C locale gives ASCII locale encoding, but Python uses UTF-8 - # to parse the command line arguments on Mac OS X and Android. - env['LC_ALL'] = 'C' + with os_helper.EnvironmentVarGuard() as env: + # C locale gives ASCII locale encoding, but Python uses UTF-8 + # to parse the command line arguments on Mac OS X and Android. + env['LC_ALL'] = 'C' - p = subprocess.Popen( - (sys.executable, "-c", code, text), - stdout=subprocess.PIPE, - env=env) - stdout, stderr = p.communicate() - self.assertEqual(stdout, expected) - self.assertEqual(p.returncode, 0) + p = subprocess.Popen( + (sys.executable, "-c", code, text), + stdout=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + self.assertEqual(stdout, expected) + self.assertEqual(p.returncode, 0) def test_non_interactive_output_buffering(self): code = textwrap.dedent(""" @@ -412,22 +412,22 @@ def test_empty_PYTHONPATH_issue16309(self): self.assertEqual(out1, out2) def test_displayhook_unencodable(self): - for encoding in ('ascii', 'latin-1', 'utf-8'): - env = os.environ.copy() - env['PYTHONIOENCODING'] = encoding - p = subprocess.Popen( - [sys.executable, '-i'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=env) - # non-ascii, surrogate, non-BMP printable, non-BMP unprintable - text = "a=\xe9 b=\uDC80 c=\U00010000 d=\U0010FFFF" - p.stdin.write(ascii(text).encode('ascii') + b"\n") - p.stdin.write(b'exit()\n') - data = kill_python(p) - escaped = repr(text).encode(encoding, 'backslashreplace') - self.assertIn(escaped, data) + with os_helper.EnvironmentVarGuard() as env: + for encoding in ('ascii', 'latin-1', 'utf-8'): + env['PYTHONIOENCODING'] = encoding + p = subprocess.Popen( + [sys.executable, '-i'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + # non-ascii, surrogate, non-BMP printable, non-BMP unprintable + text = "a=\xe9 b=\uDC80 c=\U00010000 d=\U0010FFFF" + p.stdin.write(ascii(text).encode('ascii') + b"\n") + p.stdin.write(b'exit()\n') + data = kill_python(p) + escaped = repr(text).encode(encoding, 'backslashreplace') + self.assertIn(escaped, data) def check_input(self, code, expected): with tempfile.NamedTemporaryFile("wb+") as stdin: @@ -684,23 +684,23 @@ def test_set_pycache_prefix(self): assert_python_ok(*args, **env) def run_xdev(self, *args, check_exitcode=True, xdev=True): - env = dict(os.environ) - env.pop('PYTHONWARNINGS', None) - env.pop('PYTHONDEVMODE', None) - env.pop('PYTHONMALLOC', None) + with os_helper.EnvironmentVarGuard() as env: + del env['PYTHONWARNINGS'] + del env['PYTHONDEVMODE'] + del env['PYTHONMALLOC'] - if xdev: - args = (sys.executable, '-X', 'dev', *args) - else: - args = (sys.executable, *args) - proc = subprocess.run(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - env=env) - if check_exitcode: - self.assertEqual(proc.returncode, 0, proc) - return proc.stdout.rstrip() + if xdev: + args = (sys.executable, '-X', 'dev', *args) + else: + args = (sys.executable, *args) + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + if check_exitcode: + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() @support.cpython_only def test_xdev(self): @@ -774,16 +774,16 @@ def check_warnings_filters(self, cmdline_option, envvar, use_pywarning=False): code += ("print(' '.join('%s::%s' % (f[0], f[2].__name__) " "for f in warnings.filters))") args = (sys.executable, '-W', cmdline_option, '-bb', '-c', code) - env = dict(os.environ) - env.pop('PYTHONDEVMODE', None) - env["PYTHONWARNINGS"] = envvar - proc = subprocess.run(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - env=env) - self.assertEqual(proc.returncode, 0, proc) - return proc.stdout.rstrip() + with os_helper.EnvironmentVarGuard() as env: + del env['PYTHONDEVMODE'] + env["PYTHONWARNINGS"] = envvar + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() def test_warnings_filter_precedence(self): expected_filters = ("error::BytesWarning " @@ -808,20 +808,20 @@ def test_warnings_filter_precedence(self): def check_pythonmalloc(self, env_var, name): code = 'import _testinternalcapi; print(_testinternalcapi.pymem_getallocatorsname())' - env = dict(os.environ) - env.pop('PYTHONDEVMODE', None) - if env_var is not None: - env['PYTHONMALLOC'] = env_var - else: - env.pop('PYTHONMALLOC', None) - args = (sys.executable, '-c', code) - proc = subprocess.run(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - env=env) - self.assertEqual(proc.stdout.rstrip(), name) - self.assertEqual(proc.returncode, 0) + with os_helper.EnvironmentVarGuard() as env: + del env['PYTHONDEVMODE'] + if env_var is not None: + env['PYTHONMALLOC'] = env_var + else: + del env['PYTHONMALLOC'] + args = (sys.executable, '-c', code) + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + self.assertEqual(proc.stdout.rstrip(), name) + self.assertEqual(proc.returncode, 0) @support.cpython_only def test_pythonmalloc(self): @@ -866,20 +866,20 @@ def test_pythonmalloc(self): def test_pythondevmode_env(self): # Test the PYTHONDEVMODE environment variable code = "import sys; print(sys.flags.dev_mode)" - env = dict(os.environ) - env.pop('PYTHONDEVMODE', None) - args = (sys.executable, '-c', code) + with os_helper.EnvironmentVarGuard() as env: + del env['PYTHONDEVMODE'] + args = (sys.executable, '-c', code) - proc = subprocess.run(args, stdout=subprocess.PIPE, - universal_newlines=True, env=env) - self.assertEqual(proc.stdout.rstrip(), 'False') - self.assertEqual(proc.returncode, 0, proc) + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'False') + self.assertEqual(proc.returncode, 0, proc) - env['PYTHONDEVMODE'] = '1' - proc = subprocess.run(args, stdout=subprocess.PIPE, - universal_newlines=True, env=env) - self.assertEqual(proc.stdout.rstrip(), 'True') - self.assertEqual(proc.returncode, 0, proc) + env['PYTHONDEVMODE'] = '1' + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True) + self.assertEqual(proc.stdout.rstrip(), 'True') + self.assertEqual(proc.returncode, 0, proc) def test_python_gil(self): cases = [ @@ -905,24 +905,24 @@ def test_python_gil(self): ] ) code = "import sys; print(sys.flags.gil)" - environ = dict(os.environ) - - for env, opt, expected, msg in cases: - with self.subTest(msg, env=env, opt=opt): - environ.pop('PYTHON_GIL', None) - if env is not None: - environ['PYTHON_GIL'] = env - extra_args = [] - if opt is not None: - extra_args = ['-X', f'gil={opt}'] - - proc = subprocess.run([sys.executable, *extra_args, '-c', code], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, env=environ) - self.assertEqual(proc.returncode, 0, proc) - self.assertEqual(proc.stdout.rstrip(), expected) - self.assertEqual(proc.stderr, '') + + with os_helper.EnvironmentVarGuard() as environ: + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + del environ['PYTHON_GIL'] + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') def test_python_asyncio_debug(self): code = "import asyncio; print(asyncio.new_event_loop().get_debug())" From 5b4d3a4fdd22407d9040599d6357d9fc63c536a3 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Fri, 3 Jan 2025 10:34:34 +0100 Subject: [PATCH 2/7] set initial PYTHONCOERCECLOCALE --- Lib/test/test_cmd_line.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 43f1e5bcdd6324..057169e9abbe11 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -242,7 +242,7 @@ def test_undecodable_code(self): undecodable = b"\xff" # Use C locale to get ascii for the locale encoding env['LC_ALL'] = 'C' - env['PYTHONCOERCECLOCALE'] = '1' + env['PYTHONCOERCECLOCALE'] = '0' code = ( b'import locale; ' b'print(ascii("' + undecodable + b'"), ' From 25a1a0995cae0ecfab7a752a05cf819935f98e2d Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sat, 4 Jan 2025 10:30:38 +0100 Subject: [PATCH 3/7] Revert "set initial PYTHONCOERCECLOCALE" This reverts commit 78525d16f842104ab33aa1dac2ef2691a07e18fa. --- Lib/test/test_cmd_line.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 057169e9abbe11..43f1e5bcdd6324 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -242,7 +242,7 @@ def test_undecodable_code(self): undecodable = b"\xff" # Use C locale to get ascii for the locale encoding env['LC_ALL'] = 'C' - env['PYTHONCOERCECLOCALE'] = '0' + env['PYTHONCOERCECLOCALE'] = '1' code = ( b'import locale; ' b'print(ascii("' + undecodable + b'"), ' From 862c169237f4ccfab8e76252858d6aec0b4b9c26 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sat, 4 Jan 2025 10:30:50 +0100 Subject: [PATCH 4/7] Revert "Use `EnvironmentVarGuard` in `test_cmd_line` instead of manually messing with `os.environ`" This reverts commit 52b96f15b3ac2ccfefa9d8245c92145f6413441a. --- Lib/test/test_cmd_line.py | 264 +++++++++++++++++++------------------- 1 file changed, 132 insertions(+), 132 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 43f1e5bcdd6324..634efda354407f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -238,37 +238,37 @@ def test_coding(self): @unittest.skipIf(sys.platform == 'win32', 'Windows has a native unicode API') def test_undecodable_code(self): - with os_helper.EnvironmentVarGuard() as env: - undecodable = b"\xff" - # Use C locale to get ascii for the locale encoding - env['LC_ALL'] = 'C' - env['PYTHONCOERCECLOCALE'] = '1' - code = ( - b'import locale; ' - b'print(ascii("' + undecodable + b'"), ' - b'locale.getencoding())') - p = subprocess.Popen( - [sys.executable, "-c", code], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - stdout, stderr = p.communicate() - if p.returncode == 1: - # _Py_char2wchar() decoded b'\xff' as '\udcff' (b'\xff' is not - # decodable from ASCII) and run_command() failed on - # PyUnicode_AsUTF8String(). This is the expected behaviour on - # Linux. - pattern = b"Unable to decode the command from the command line:" - elif p.returncode == 0: - # _Py_char2wchar() decoded b'\xff' as '\xff' even if the locale is - # C and the locale encoding is ASCII. It occurs on FreeBSD, Solaris - # and Mac OS X. - pattern = b"'\\xff' " - # The output is followed by the encoding name, an alias to ASCII. - # Examples: "US-ASCII" or "646" (ISO 646, on Solaris). - else: - raise AssertionError("Unknown exit code: %s, output=%a" % (p.returncode, stdout)) - if not stdout.startswith(pattern): - raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) + undecodable = b"\xff" + env = os.environ.copy() + # Use C locale to get ascii for the locale encoding + env['LC_ALL'] = 'C' + env['PYTHONCOERCECLOCALE'] = '0' + code = ( + b'import locale; ' + b'print(ascii("' + undecodable + b'"), ' + b'locale.getencoding())') + p = subprocess.Popen( + [sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=env) + stdout, stderr = p.communicate() + if p.returncode == 1: + # _Py_char2wchar() decoded b'\xff' as '\udcff' (b'\xff' is not + # decodable from ASCII) and run_command() failed on + # PyUnicode_AsUTF8String(). This is the expected behaviour on + # Linux. + pattern = b"Unable to decode the command from the command line:" + elif p.returncode == 0: + # _Py_char2wchar() decoded b'\xff' as '\xff' even if the locale is + # C and the locale encoding is ASCII. It occurs on FreeBSD, Solaris + # and Mac OS X. + pattern = b"'\\xff' " + # The output is followed by the encoding name, an alias to ASCII. + # Examples: "US-ASCII" or "646" (ISO 646, on Solaris). + else: + raise AssertionError("Unknown exit code: %s, output=%a" % (p.returncode, stdout)) + if not stdout.startswith(pattern): + raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) @unittest.skipIf(sys.platform == 'win32', 'Windows has a native unicode API') @@ -287,10 +287,10 @@ def run_default(arg): def run_c_locale(arg): cmd = [sys.executable, '-c', code, arg] - with os_helper.EnvironmentVarGuard() as env: - env['LC_ALL'] = 'C' - return subprocess.run(cmd, stdout=subprocess.PIPE, - text=True) + env = dict(os.environ) + env['LC_ALL'] = 'C' + return subprocess.run(cmd, stdout=subprocess.PIPE, + text=True, env=env) def run_utf8_mode(arg): cmd = [sys.executable, '-X', 'utf8', '-c', code, arg] @@ -323,18 +323,18 @@ def test_osx_android_utf8(self): decoded = text.decode('utf-8', 'surrogateescape') expected = ascii(decoded).encode('ascii') + b'\n' - with os_helper.EnvironmentVarGuard() as env: - # C locale gives ASCII locale encoding, but Python uses UTF-8 - # to parse the command line arguments on Mac OS X and Android. - env['LC_ALL'] = 'C' + env = os.environ.copy() + # C locale gives ASCII locale encoding, but Python uses UTF-8 + # to parse the command line arguments on Mac OS X and Android. + env['LC_ALL'] = 'C' - p = subprocess.Popen( - (sys.executable, "-c", code, text), - stdout=subprocess.PIPE, - ) - stdout, stderr = p.communicate() - self.assertEqual(stdout, expected) - self.assertEqual(p.returncode, 0) + p = subprocess.Popen( + (sys.executable, "-c", code, text), + stdout=subprocess.PIPE, + env=env) + stdout, stderr = p.communicate() + self.assertEqual(stdout, expected) + self.assertEqual(p.returncode, 0) def test_non_interactive_output_buffering(self): code = textwrap.dedent(""" @@ -412,22 +412,22 @@ def test_empty_PYTHONPATH_issue16309(self): self.assertEqual(out1, out2) def test_displayhook_unencodable(self): - with os_helper.EnvironmentVarGuard() as env: - for encoding in ('ascii', 'latin-1', 'utf-8'): - env['PYTHONIOENCODING'] = encoding - p = subprocess.Popen( - [sys.executable, '-i'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - # non-ascii, surrogate, non-BMP printable, non-BMP unprintable - text = "a=\xe9 b=\uDC80 c=\U00010000 d=\U0010FFFF" - p.stdin.write(ascii(text).encode('ascii') + b"\n") - p.stdin.write(b'exit()\n') - data = kill_python(p) - escaped = repr(text).encode(encoding, 'backslashreplace') - self.assertIn(escaped, data) + for encoding in ('ascii', 'latin-1', 'utf-8'): + env = os.environ.copy() + env['PYTHONIOENCODING'] = encoding + p = subprocess.Popen( + [sys.executable, '-i'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env) + # non-ascii, surrogate, non-BMP printable, non-BMP unprintable + text = "a=\xe9 b=\uDC80 c=\U00010000 d=\U0010FFFF" + p.stdin.write(ascii(text).encode('ascii') + b"\n") + p.stdin.write(b'exit()\n') + data = kill_python(p) + escaped = repr(text).encode(encoding, 'backslashreplace') + self.assertIn(escaped, data) def check_input(self, code, expected): with tempfile.NamedTemporaryFile("wb+") as stdin: @@ -684,23 +684,23 @@ def test_set_pycache_prefix(self): assert_python_ok(*args, **env) def run_xdev(self, *args, check_exitcode=True, xdev=True): - with os_helper.EnvironmentVarGuard() as env: - del env['PYTHONWARNINGS'] - del env['PYTHONDEVMODE'] - del env['PYTHONMALLOC'] + env = dict(os.environ) + env.pop('PYTHONWARNINGS', None) + env.pop('PYTHONDEVMODE', None) + env.pop('PYTHONMALLOC', None) - if xdev: - args = (sys.executable, '-X', 'dev', *args) - else: - args = (sys.executable, *args) - proc = subprocess.run(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) - if check_exitcode: - self.assertEqual(proc.returncode, 0, proc) - return proc.stdout.rstrip() + if xdev: + args = (sys.executable, '-X', 'dev', *args) + else: + args = (sys.executable, *args) + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + if check_exitcode: + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() @support.cpython_only def test_xdev(self): @@ -774,16 +774,16 @@ def check_warnings_filters(self, cmdline_option, envvar, use_pywarning=False): code += ("print(' '.join('%s::%s' % (f[0], f[2].__name__) " "for f in warnings.filters))") args = (sys.executable, '-W', cmdline_option, '-bb', '-c', code) - with os_helper.EnvironmentVarGuard() as env: - del env['PYTHONDEVMODE'] - env["PYTHONWARNINGS"] = envvar - proc = subprocess.run(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) - self.assertEqual(proc.returncode, 0, proc) - return proc.stdout.rstrip() + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + env["PYTHONWARNINGS"] = envvar + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() def test_warnings_filter_precedence(self): expected_filters = ("error::BytesWarning " @@ -808,20 +808,20 @@ def test_warnings_filter_precedence(self): def check_pythonmalloc(self, env_var, name): code = 'import _testinternalcapi; print(_testinternalcapi.pymem_getallocatorsname())' - with os_helper.EnvironmentVarGuard() as env: - del env['PYTHONDEVMODE'] - if env_var is not None: - env['PYTHONMALLOC'] = env_var - else: - del env['PYTHONMALLOC'] - args = (sys.executable, '-c', code) - proc = subprocess.run(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) - self.assertEqual(proc.stdout.rstrip(), name) - self.assertEqual(proc.returncode, 0) + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + if env_var is not None: + env['PYTHONMALLOC'] = env_var + else: + env.pop('PYTHONMALLOC', None) + args = (sys.executable, '-c', code) + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + self.assertEqual(proc.stdout.rstrip(), name) + self.assertEqual(proc.returncode, 0) @support.cpython_only def test_pythonmalloc(self): @@ -866,20 +866,20 @@ def test_pythonmalloc(self): def test_pythondevmode_env(self): # Test the PYTHONDEVMODE environment variable code = "import sys; print(sys.flags.dev_mode)" - with os_helper.EnvironmentVarGuard() as env: - del env['PYTHONDEVMODE'] - args = (sys.executable, '-c', code) + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + args = (sys.executable, '-c', code) - proc = subprocess.run(args, stdout=subprocess.PIPE, - universal_newlines=True, env=env) - self.assertEqual(proc.stdout.rstrip(), 'False') - self.assertEqual(proc.returncode, 0, proc) + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'False') + self.assertEqual(proc.returncode, 0, proc) - env['PYTHONDEVMODE'] = '1' - proc = subprocess.run(args, stdout=subprocess.PIPE, - universal_newlines=True) - self.assertEqual(proc.stdout.rstrip(), 'True') - self.assertEqual(proc.returncode, 0, proc) + env['PYTHONDEVMODE'] = '1' + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'True') + self.assertEqual(proc.returncode, 0, proc) def test_python_gil(self): cases = [ @@ -905,24 +905,24 @@ def test_python_gil(self): ] ) code = "import sys; print(sys.flags.gil)" - - with os_helper.EnvironmentVarGuard() as environ: - for env, opt, expected, msg in cases: - with self.subTest(msg, env=env, opt=opt): - del environ['PYTHON_GIL'] - if env is not None: - environ['PYTHON_GIL'] = env - extra_args = [] - if opt is not None: - extra_args = ['-X', f'gil={opt}'] - - proc = subprocess.run([sys.executable, *extra_args, '-c', code], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True) - self.assertEqual(proc.returncode, 0, proc) - self.assertEqual(proc.stdout.rstrip(), expected) - self.assertEqual(proc.stderr, '') + environ = dict(os.environ) + + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + environ.pop('PYTHON_GIL', None) + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, env=environ) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') def test_python_asyncio_debug(self): code = "import asyncio; print(asyncio.new_event_loop().get_debug())" From 54f3010471b58f0857fac34a8ffc8af7f365815b Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sat, 4 Jan 2025 10:43:32 +0100 Subject: [PATCH 5/7] Add `EnvironmentVarGuard` for `test_urllib`, `test_urllib2.py`, `urllib2_localnet.py` --- Lib/test/test_urllib.py | 25 ++++++++--------- Lib/test/test_urllib2.py | 46 ++++++++++++++++--------------- Lib/test/test_urllib2_localnet.py | 19 +++++-------- 3 files changed, 42 insertions(+), 48 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 042d3b35b77022..84f3948d233691 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -258,34 +258,31 @@ class ProxyTests_withOrderedEnv(unittest.TestCase): def setUp(self): # We need to test conditions, where variable order _is_ significant - self._saved_env = os.environ - # Monkey patch os.environ, start with empty fake environment - os.environ = collections.OrderedDict() - - def tearDown(self): - os.environ = self._saved_env + self.env = self.enterContext(os_helper.EnvironmentVarGuard()) + # Start with empty fake environment + self.env.clear() def test_getproxies_environment_prefer_lowercase(self): # Test lowercase preference with removal - os.environ['no_proxy'] = '' - os.environ['No_Proxy'] = 'localhost' + self.env['no_proxy'] = '' + self.env['No_Proxy'] = 'localhost' self.assertFalse(urllib.request.proxy_bypass_environment('localhost')) self.assertFalse(urllib.request.proxy_bypass_environment('arbitrary')) - os.environ['http_proxy'] = '' - os.environ['HTTP_PROXY'] = 'http://somewhere:3128' + self.env['http_proxy'] = '' + self.env['HTTP_PROXY'] = 'http://somewhere:3128' proxies = urllib.request.getproxies_environment() self.assertEqual({}, proxies) # Test lowercase preference of proxy bypass and correct matching including ports - os.environ['no_proxy'] = 'localhost, noproxy.com, my.proxy:1234' - os.environ['No_Proxy'] = 'xyz.com' + self.env['no_proxy'] = 'localhost, noproxy.com, my.proxy:1234' + self.env['No_Proxy'] = 'xyz.com' self.assertTrue(urllib.request.proxy_bypass_environment('localhost')) self.assertTrue(urllib.request.proxy_bypass_environment('noproxy.com:5678')) self.assertTrue(urllib.request.proxy_bypass_environment('my.proxy:1234')) self.assertFalse(urllib.request.proxy_bypass_environment('my.proxy')) self.assertFalse(urllib.request.proxy_bypass_environment('arbitrary')) # Test lowercase preference with replacement - os.environ['http_proxy'] = 'http://somewhere:3128' - os.environ['Http_Proxy'] = 'http://somewhereelse:3128' + self.env['http_proxy'] = 'http://somewhere:3128' + self.env['Http_Proxy'] = 'http://somewhereelse:3128' proxies = urllib.request.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http']) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 085b24c25b2daa..e330439b54b61b 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1444,30 +1444,32 @@ def test_proxy(self): [tup[0:2] for tup in o.calls]) def test_proxy_no_proxy(self): - os.environ['no_proxy'] = 'python.org' - o = OpenerDirector() - ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) - o.add_handler(ph) - req = Request("http://www.perl.org/") - self.assertEqual(req.host, "www.perl.org") - o.open(req) - self.assertEqual(req.host, "proxy.example.com") - req = Request("http://www.python.org") - self.assertEqual(req.host, "www.python.org") - o.open(req) - self.assertEqual(req.host, "www.python.org") - del os.environ['no_proxy'] + with os_helper.EnvironmentVarGuard() as env: + env['no_proxy'] = 'python.org' + o = OpenerDirector() + ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) + o.add_handler(ph) + req = Request("http://www.perl.org/") + self.assertEqual(req.host, "www.perl.org") + o.open(req) + self.assertEqual(req.host, "proxy.example.com") + req = Request("http://www.python.org") + self.assertEqual(req.host, "www.python.org") + o.open(req) + self.assertEqual(req.host, "www.python.org") + del env['no_proxy'] def test_proxy_no_proxy_all(self): - os.environ['no_proxy'] = '*' - o = OpenerDirector() - ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) - o.add_handler(ph) - req = Request("http://www.python.org") - self.assertEqual(req.host, "www.python.org") - o.open(req) - self.assertEqual(req.host, "www.python.org") - del os.environ['no_proxy'] + with os_helper.EnvironmentVarGuard() as env: + env['no_proxy'] = '*' + o = OpenerDirector() + ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) + o.add_handler(ph) + req = Request("http://www.python.org") + self.assertEqual(req.host, "www.python.org") + o.open(req) + self.assertEqual(req.host, "www.python.org") + del env['no_proxy'] def test_proxy_https(self): o = OpenerDirector() diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 50c491a3cfd3d0..2b1c7fbde1921b 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -11,6 +11,7 @@ from test import support from test.support import hashlib_helper from test.support import threading_helper +from test.support import os_helper try: import ssl @@ -330,12 +331,9 @@ class ProxyAuthTests(unittest.TestCase): def setUp(self): super(ProxyAuthTests, self).setUp() # Ignore proxy bypass settings in the environment. - def restore_environ(old_environ): - os.environ.clear() - os.environ.update(old_environ) - self.addCleanup(restore_environ, os.environ.copy()) - os.environ['NO_PROXY'] = '' - os.environ['no_proxy'] = '' + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env['NO_PROXY'] = '' + env['no_proxy'] = '' self.digest_auth_handler = DigestAuthHandler() self.digest_auth_handler.set_users({self.USER: self.PASSWD}) @@ -456,12 +454,9 @@ def setUp(self): self.addCleanup(urllib.request.urlcleanup) # Ignore proxies for localhost tests. - def restore_environ(old_environ): - os.environ.clear() - os.environ.update(old_environ) - self.addCleanup(restore_environ, os.environ.copy()) - os.environ['NO_PROXY'] = '*' - os.environ['no_proxy'] = '*' + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env['NO_PROXY'] = '*' + env['no_proxy'] = '*' def urlopen(self, url, data=None, **kwargs): l = [] From 3c56fbf68ea660500e885ce1d3a3a1b97e27426d Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sat, 4 Jan 2025 10:59:11 +0100 Subject: [PATCH 6/7] remove redundant del --- Lib/test/test_urllib2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index e330439b54b61b..39170dc931b4e3 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1457,7 +1457,6 @@ def test_proxy_no_proxy(self): self.assertEqual(req.host, "www.python.org") o.open(req) self.assertEqual(req.host, "www.python.org") - del env['no_proxy'] def test_proxy_no_proxy_all(self): with os_helper.EnvironmentVarGuard() as env: @@ -1469,7 +1468,6 @@ def test_proxy_no_proxy_all(self): self.assertEqual(req.host, "www.python.org") o.open(req) self.assertEqual(req.host, "www.python.org") - del env['no_proxy'] def test_proxy_https(self): o = OpenerDirector() From 3c48465aac79eac15dc904826d256e689833b1ef Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sat, 4 Jan 2025 12:32:49 +0100 Subject: [PATCH 7/7] test windows --- Lib/test/support/os_helper.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 8071c248b9b67e..d7676714594833 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -701,16 +701,23 @@ def __init__(self): self._environ = os.environ self._changed = {} + def _convert(self, envvar): + if os.name == "nt": + return envvar.upper() + return envvar + def __getitem__(self, envvar): - return self._environ[envvar] + return self._environ[self._convert(envvar)] def __setitem__(self, envvar, value): # Remember the initial value on the first access + envvar = self._convert(envvar) if envvar not in self._changed: self._changed[envvar] = self._environ.get(envvar) self._environ[envvar] = value def __delitem__(self, envvar): + envvar = self._convert(envvar) # Remember the initial value on the first access if envvar not in self._changed: self._changed[envvar] = self._environ.get(envvar) @@ -727,10 +734,10 @@ def __len__(self): return len(self._environ) def set(self, envvar, value): - self[envvar] = value + self[self._convert(envvar)] = value def unset(self, envvar): - del self[envvar] + del self[self._convert(envvar)] def copy(self): # We do what os.environ.copy() does.