diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 03b7a8dc378ef3..24fdc3aa23e4f2 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -487,7 +487,14 @@ Available Functions ignored. *module*, if supplied, should be the module name. - If no module is passed, the filename with ``.py`` stripped is used. + If no module is passed, the module regular expression in + :ref:`warnings filter ` will be tested against the module + names constructed from the path components starting from all parent + directories (with ``/__init__.py``, ``.py`` and, on Windows, ``.pyw`` + stripped) and against the filename with ``.py`` stripped. + For example, when the filename is ``'/path/to/package/module.py'``, it will + be tested against ``'path.to.package.module'``, ``'to.package.module'`` + ``'package.module'``, ``'module'``, and ``'/path/to/package/module'``. *registry*, if supplied, should be the ``__warningregistry__`` dictionary of the module. @@ -506,6 +513,11 @@ Available Functions .. versionchanged:: 3.6 Add the *source* parameter. + .. versionchanged:: next + If no module is passed, test the filter regular expression against + module names created from the path, not only the path itself; + strip also ``.pyw`` (on Windows) and ``/__init__.py``. + .. function:: showwarning(message, category, filename, lineno, file=None, line=None) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 8503a4c7f973fc..dcc512a5b0538e 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -601,6 +601,18 @@ unittest (Contributed by Garry Cairns in :gh:`134567`.) +warnings +-------- + +* Improve filtering by module in :func:`warnings.warn_explicit` if no *module* + argument is passed. + It now tests the module regular expression in the warnings filter not only + against the filename with ``.py`` stripped, but also against module names + constructed starting from different parent directories of the filename + (with ``/__init__.py``, ``.py`` and, on Windows, ``.pyw`` stripped). + (Contributed by Serhiy Storchaka in :gh:`135801`.) + + venv ---- diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index 576a17ea7b8501..b7e202c7a8d491 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -520,20 +520,49 @@ def warn(message, category=None, stacklevel=1, source=None, ) +def _match_filename(pattern, filename, *, MS_WINDOWS=(sys.platform == 'win32')): + if not filename: + return pattern.match('') is not None + if filename[0] == '<' and filename[-1] == '>': + return pattern.match(filename) is not None + + is_py = (filename[-3:].lower() == '.py' + if MS_WINDOWS else + filename.endswith('.py')) + if is_py: + filename = filename[:-3] + if pattern.match(filename): # for backward compatibility + return True + if MS_WINDOWS: + if is_py and filename[-9:].lower() in (r'\__init__', '/__init__'): + filename = filename[:-9] + elif not is_py and filename[-4:].lower() == '.pyw': + filename = filename[:-4] + filename = filename.replace('\\', '.') + else: + if is_py and filename.endswith('/__init__'): + filename = filename[:-9] + filename = filename.replace('/', '.') + i = 0 + while True: + if pattern.match(filename, i): + return True + i = filename.find('.', i) + 1 + if not i: + return False + + def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None): lineno = int(lineno) - if module is None: - module = filename or "" - if module[-3:].lower() == ".py": - module = module[:-3] # XXX What about leading pathname? if isinstance(message, Warning): text = str(message) category = message.__class__ else: text = message message = category(message) + modules = None key = (text, category, lineno) with _wm._lock: if registry is None: @@ -549,9 +578,11 @@ def warn_explicit(message, category, filename, lineno, action, msg, cat, mod, ln = item if ((msg is None or msg.match(text)) and issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break + (ln == 0 or lineno == ln) and + (mod is None or (_match_filename(mod, filename) + if module is None else + mod.match(module)))): + break else: action = _wm.defaultaction # Early exit actions diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 1e6f60074308e2..7b47039b08b6e6 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -13,6 +13,7 @@ import textwrap import types import unittest +import warnings import weakref from io import StringIO from pathlib import Path @@ -1124,6 +1125,19 @@ def test_tstring(self): self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + ast.parse(source) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 21]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 034cd5f7f9e89f..2bc3737e0c49cd 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1088,6 +1088,28 @@ def four_freevars(): three_freevars.__globals__, closure=my_closure) + def test_exec_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + exec(source, {}) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + exec(source, {'__name__': 'package.module', '__file__': filename}) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + def test_filter(self): self.assertEqual(list(filter(lambda c: 'a' <= c <= 'z', 'Hello World')), list('elloorld')) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 3ed7a360d64e3c..8233672b8b81a7 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -198,7 +198,7 @@ def test_run_module_bug1764407(self): p.stdin.write(b'Timer\n') p.stdin.write(b'exit()\n') data = kill_python(p) - self.assertTrue(data.find(b'1 loop') != -1) + self.assertIn(b'1 loop', data) self.assertTrue(data.find(b'__main__.Timer') != -1) def test_relativedir_bug46421(self): diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 784c45aa96f8a7..fd9645d734cb01 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -810,6 +810,12 @@ def test_script_as_dev_fd(self): out, err = p.communicate() self.assertEqual(out, b"12345678912345678912345\n") + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + rc, out, err = assert_python_ok('-Werror', '-Walways:::test.test_import.data.syntax_warnings', filename) + self.assertEqual(err.count(b': SyntaxWarning: '), 6) + rc, out, err = assert_python_ok('-Werror', '-Walways:::syntax_warnings', filename) + self.assertEqual(err.count(b': SyntaxWarning: '), 6) def tearDownModule(): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index bc8ef93cb8f9de..8e4f1ddd25e44a 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1745,6 +1745,20 @@ def test_compile_warning_in_finally(self): self.assertEqual(wm.category, SyntaxWarning) self.assertIn("\"is\" with 'int' literal", str(wm.message)) + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + compile(source, filename, 'exec') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class TestBooleanExpression(unittest.TestCase): class Value: diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index b71a36ec2f7aab..e24659ca8e61bd 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -15,6 +15,7 @@ import os import py_compile import random +import re import shutil import stat import subprocess @@ -23,6 +24,7 @@ import threading import time import types +import warnings import unittest from unittest import mock import _imp @@ -51,7 +53,7 @@ TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE) from test.support import script_helper from test.support import threading_helper -from test.test_importlib.util import uncache +from test.test_importlib.util import uncache, temporary_pycache_prefix from types import ModuleType try: import _testsinglephase @@ -412,7 +414,6 @@ def test_from_import_missing_attr_path_is_canonical(self): self.assertIsNotNone(cm.exception) def test_from_import_star_invalid_type(self): - import re with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("__all__ = [b'invalid_type']") @@ -1250,6 +1251,35 @@ class Spec2: origin = "a\x00b" _imp.create_dynamic(Spec2()) + def test_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + unload('test.test_import.data.syntax_warnings') + with (os_helper.temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + import test.test_import.data.syntax_warnings + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = test.test_import.data.syntax_warnings.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + module_re = r'syntax_warnings\z' + unload('test.test_import.data.syntax_warnings') + with (os_helper.temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + import test.test_import.data.syntax_warnings + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = test.test_import.data.syntax_warnings.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): diff --git a/Lib/test/test_import/data/syntax_warnings.py b/Lib/test/test_import/data/syntax_warnings.py new file mode 100644 index 00000000000000..103f07b6187603 --- /dev/null +++ b/Lib/test/test_import/data/syntax_warnings.py @@ -0,0 +1,21 @@ +# Syntax warnings emitted in different parts of the Python compiler. + +# Parser/lexer/lexer.c +x = 1or 0 # line 4 + +# Parser/tokenizer/helpers.c +'\z' # line 7 + +# Parser/string_parser.c +'\400' # line 10 + +# _PyCompile_Warn() in Python/codegen.c +assert(x, 'message') # line 13 +x is 1 # line 14 + +# _PyErr_EmitSyntaxWarning() in Python/ast_preprocess.c +def f(): + try: + pass + finally: + return 42 # line 21 diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 943e63fc13c921..ef2c00e04b820c 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -5,6 +5,7 @@ import re import textwrap import symtable +import warnings import unittest from test import support @@ -586,6 +587,20 @@ def test__symtable_refleak(self): # check error path when 'compile_type' AC conversion failed self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1) + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + symtable.symtable(source, filename, 'exec') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 157852cfa91007..68ba7d4121b36a 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -249,10 +249,21 @@ def test_filter_module(self): self.module.warn_explicit('msg', UserWarning, 'filename', 42, module='package.module') self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + self.assertEqual(len(w), 2) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + self.assertEqual(len(w), 3) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module/__init__.py', 42) + self.assertEqual(len(w), 4) with self.assertRaises(UserWarning): - self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) - with self.assertRaises(UserWarning): - self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module/__init__', 42) + if MS_WINDOWS: + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PY', 42) + self.assertEqual(len(w), 5) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module\__INIT__.PY', 42) + self.assertEqual(len(w), 6) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PYW', 42) + self.assertEqual(len(w), 7) with self.module.catch_warnings(record=True) as w: self.module.simplefilter('error') @@ -276,9 +287,8 @@ def test_filter_module(self): with self.assertRaises(UserWarning): self.module.warn_explicit('msg', UserWarning, '/PATH/TO/PACKAGE/MODULE', 42) if MS_WINDOWS: - if self.module is py_warnings: - self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.PY', 42) - self.assertEqual(len(w), 3) + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.PY', 42) + self.assertEqual(len(w), 3) with self.assertRaises(UserWarning): self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module/__init__.py', 42) with self.assertRaises(UserWarning): @@ -302,9 +312,8 @@ def test_filter_module(self): self.assertEqual(len(w), 1) self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.py', 42) self.assertEqual(len(w), 2) - if self.module is py_warnings: - self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PY', 42) - self.assertEqual(len(w), 3) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PY', 42) + self.assertEqual(len(w), 3) with self.assertRaises(UserWarning): self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.pyw', 42) with self.assertRaises(UserWarning): @@ -399,7 +408,7 @@ def test_message_matching(self): def test_mutate_filter_list(self): class X: - def match(self, a): + def match(self, a, start=0): L[:] = [] L = [("default",X(),UserWarning,X(),0) for i in range(2)] diff --git a/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst b/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst new file mode 100644 index 00000000000000..ad7bec84fe472a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst @@ -0,0 +1,7 @@ +Improve filtering by module in :func:`warnings.warn_explicit` if no *module* +argument is passed. It now tests the module regular expression in the +warnings filter not only against the filename with ``.py`` stripped, but +also against module names constructed starting from different parent +directories of the filename (with ``/__init__.py``, ``.py`` and, on Windows, +``.pyw`` stripped). +``/__init__.py``. diff --git a/Python/_warnings.c b/Python/_warnings.c index 9989b623dbce3a..7af99e06124611 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -171,7 +171,7 @@ _PyWarnings_InitState(PyInterpreterState *interp) /*************************************************************************/ static int -check_matched(PyInterpreterState *interp, PyObject *obj, PyObject *arg) +check_matched(PyInterpreterState *interp, PyObject *obj, PyObject *arg, PyObject *arg2) { PyObject *result; int rc; @@ -180,20 +180,29 @@ check_matched(PyInterpreterState *interp, PyObject *obj, PyObject *arg) if (obj == Py_None) return 1; - /* An internal plain text default filter must match exactly */ - if (PyUnicode_CheckExact(obj)) { - int cmp_result = PyUnicode_Compare(obj, arg); - if (cmp_result == -1 && PyErr_Occurred()) { + if (arg != NULL) { + /* An internal plain text default filter must match exactly */ + if (PyUnicode_CheckExact(obj)) { + int cmp_result = PyUnicode_Compare(obj, arg); + if (cmp_result == -1 && PyErr_Occurred()) { + return -1; + } + return !cmp_result; + } + + /* Otherwise assume a regex filter and call its match() method */ + result = PyObject_CallMethodOneArg(obj, &_Py_ID(match), arg); + } + else { + PyObject *match = PyImport_ImportModuleAttrString("_py_warnings", "_match_filename"); + if (match == NULL) { return -1; } - return !cmp_result; + result = PyObject_CallFunctionObjArgs(match, obj, arg2, NULL); + Py_DECREF(match); } - - /* Otherwise assume a regex filter and call its match() method */ - result = PyObject_CallMethodOneArg(obj, &_Py_ID(match), arg); if (result == NULL) return -1; - rc = PyObject_IsTrue(result); Py_DECREF(result); return rc; @@ -423,7 +432,7 @@ get_default_action(PyInterpreterState *interp) static bool filter_search(PyInterpreterState *interp, PyObject *category, PyObject *text, Py_ssize_t lineno, - PyObject *module, char *list_name, PyObject *filters, + PyObject *module, PyObject *filename, char *list_name, PyObject *filters, PyObject **item, PyObject **matched_action) { bool result = true; *matched_action = NULL; @@ -459,14 +468,14 @@ filter_search(PyInterpreterState *interp, PyObject *category, break; } - good_msg = check_matched(interp, msg, text); + good_msg = check_matched(interp, msg, text, NULL); if (good_msg == -1) { Py_DECREF(tmp_item); result = false; break; } - good_mod = check_matched(interp, mod, module); + good_mod = check_matched(interp, mod, module, filename); if (good_mod == -1) { Py_DECREF(tmp_item); result = false; @@ -504,7 +513,7 @@ filter_search(PyInterpreterState *interp, PyObject *category, static PyObject* get_filter(PyInterpreterState *interp, PyObject *category, PyObject *text, Py_ssize_t lineno, - PyObject *module, PyObject **item) + PyObject *module, PyObject *filename, PyObject **item) { #ifdef Py_DEBUG WarningsState *st = warnings_get_state(interp); @@ -522,7 +531,7 @@ get_filter(PyInterpreterState *interp, PyObject *category, use_global_filters = true; } else { PyObject *context_action = NULL; - if (!filter_search(interp, category, text, lineno, module, "_warnings_context _filters", + if (!filter_search(interp, category, text, lineno, module, filename, "_warnings_context _filters", context_filters, item, &context_action)) { Py_DECREF(context_filters); return NULL; @@ -541,7 +550,7 @@ get_filter(PyInterpreterState *interp, PyObject *category, if (filters == NULL) { return NULL; } - if (!filter_search(interp, category, text, lineno, module, "filters", + if (!filter_search(interp, category, text, lineno, module, filename, "filters", filters, item, &action)) { return NULL; } @@ -612,39 +621,6 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, return 0; } -/* New reference. */ -static PyObject * -normalize_module(PyObject *filename) -{ - PyObject *module; - int kind; - const void *data; - Py_ssize_t len; - - len = PyUnicode_GetLength(filename); - if (len < 0) - return NULL; - - if (len == 0) - return PyUnicode_FromString(""); - - kind = PyUnicode_KIND(filename); - data = PyUnicode_DATA(filename); - - /* if filename.endswith(".py"): */ - if (len >= 3 && - PyUnicode_READ(kind, data, len-3) == '.' && - PyUnicode_READ(kind, data, len-2) == 'p' && - PyUnicode_READ(kind, data, len-1) == 'y') - { - module = PyUnicode_Substring(filename, 0, len-3); - } - else { - module = Py_NewRef(filename); - } - return module; -} - static int update_registry(PyInterpreterState *interp, PyObject *registry, PyObject *text, PyObject *category, int add_zero) @@ -812,15 +788,6 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, return NULL; } - /* Normalize module. */ - if (module == NULL) { - module = normalize_module(filename); - if (module == NULL) - return NULL; - } - else - Py_INCREF(module); - /* Normalize message. */ Py_INCREF(message); /* DECREF'ed in cleanup. */ if (PyObject_TypeCheck(message, (PyTypeObject *)PyExc_Warning)) { @@ -858,7 +825,7 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, /* Else this warning hasn't been generated before. */ } - action = get_filter(interp, category, text, lineno, module, &item); + action = get_filter(interp, category, text, lineno, module, filename, &item); if (action == NULL) goto cleanup; @@ -921,7 +888,6 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, Py_XDECREF(key); Py_XDECREF(text); Py_XDECREF(lineno_obj); - Py_DECREF(module); Py_XDECREF(message); return result; /* Py_None or NULL. */ }