From a8b40f3b9dab8f7ae043ec38453203bd798611c7 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 6 Dec 2017 20:13:49 +1000 Subject: [PATCH 01/11] bpo-32230: Set sys.warnoptions with -X dev Rather than supporting dev mode directly in the warnings module, this instead adjusts the initialisation code to add an extra 'default' entry to sys.warnoptions when dev mode is enabled. This ensures that dev mode behaves *exactly* as if `-Wdefault` had been passed on the command line, including in the way it interacts with `sys.warnoptions`, and with other command line flags like `-bb`. --- Lib/subprocess.py | 3 +- Lib/test/test_cmd_line.py | 32 ++++++++++---- Lib/warnings.py | 20 +++------ .../2017-12-06-20-18-34.bpo-32230.PgGQaB.rst | 3 ++ Modules/main.c | 22 ++++++++++ Python/_warnings.c | 44 ++++++------------- 6 files changed, 70 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 35bfddde4e92f1..05e45d11d7bf4f 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -241,7 +241,7 @@ def _optim_args_from_interpreter_flags(): def _args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current - settings in sys.flags and sys.warnoptions.""" + settings in sys.flags, sys.warnoptions and sys._xoptions.""" flag_opt_map = { 'debug': 'd', # 'inspect': 'i', @@ -268,6 +268,7 @@ def _args_from_interpreter_flags(): # -X options xoptions = getattr(sys, '_xoptions', {}) if 'dev' in xoptions: + args.remove('-Wdefault') args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', 'showalloccount', 'showrefcount'): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 383302bad8b549..817d6173866aa9 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -541,31 +541,47 @@ def test_xdev(self): code = ("import sys, warnings; " "print(' '.join('%s::%s' % (f[0], f[2].__name__) " "for f in warnings.filters))") + if sys.flags.debug: + resource_action = "default" + else: + resource_action = "ignore" out = self.run_xdev("-c", code) self.assertEqual(out, + "default::Warning " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " "ignore::BytesWarning " - "default::ResourceWarning " - "default::Warning") + f"{resource_action}::ResourceWarning") out = self.run_xdev("-b", "-c", code) self.assertEqual(out, + "default::Warning " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " "default::BytesWarning " - "default::ResourceWarning " - "default::Warning") + f"{resource_action}::ResourceWarning") out = self.run_xdev("-bb", "-c", code) self.assertEqual(out, + "default::Warning " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " "error::BytesWarning " - "default::ResourceWarning " - "default::Warning") + f"{resource_action}::ResourceWarning") out = self.run_xdev("-Werror", "-c", code) self.assertEqual(out, "error::Warning " + "default::Warning " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " "ignore::BytesWarning " - "default::ResourceWarning " - "default::Warning") + f"{resource_action}::ResourceWarning") # Memory allocator debug hooks try: diff --git a/Lib/warnings.py b/Lib/warnings.py index 4e7241fe6ca450..600fb26a148a95 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -519,14 +519,11 @@ def _filters_mutated(): # Module initialization _processoptions(sys.warnoptions) if not _warnings_defaults: - dev_mode = ('dev' in getattr(sys, '_xoptions', {})) - py_debug = hasattr(sys, 'gettotalrefcount') - - if not(dev_mode or py_debug): - silence = [ImportWarning, PendingDeprecationWarning] - silence.append(DeprecationWarning) - for cls in silence: - simplefilter("ignore", category=cls) + # TODO: Define a test case that ensures this fallback code always remains + # consistent with init_filters() in _warnings.c + simplefilter("ignore", category=DeprecationWarning, append=1) + simplefilter("ignore", category=PendingDeprecationWarning, append=1) + simplefilter("ignore", category=ImportWarning, append=1) bytes_warning = sys.flags.bytes_warning if bytes_warning > 1: @@ -538,15 +535,10 @@ def _filters_mutated(): simplefilter(bytes_action, category=BytesWarning, append=1) # resource usage warnings are enabled by default in pydebug mode - if dev_mode or py_debug: + if sys.flags.debug: resource_action = "default" else: resource_action = "ignore" simplefilter(resource_action, category=ResourceWarning, append=1) - if dev_mode: - simplefilter("default", category=Warning, append=1) - - del py_debug, dev_mode - del _warnings_defaults diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst new file mode 100644 index 00000000000000..1e815270d178ba --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst @@ -0,0 +1,3 @@ +`-X dev` now injects a ``'default'`` entryinto sys.warnoptions, ensuring +that it behaves identically to actually passing ``-Wdefault`` at the command +line. diff --git a/Modules/main.c b/Modules/main.c index e2ca72c7dc1a46..4b7ceab0d455f5 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -740,11 +740,33 @@ pymain_add_warnings_optlist(_Py_OptList *warnings) return 0; } +static int +pymain_add_warnings_dev_mode(_PyCoreConfig *core_config) +{ + if (core_config->dev_mode) { + PyObject *option = PyUnicode_FromString("default"); + if (option == NULL) { + return -1; + } + if (_PySys_AddWarnOptionWithError(option)) { + Py_DECREF(option); + return -1; + } + Py_DECREF(option); + } + return 0; +} + static int pymain_add_warnings_options(_PyMain *pymain) { PySys_ResetWarnOptions(); + if (pymain_add_warnings_dev_mode(&pymain->core_config) < 0) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; + } + if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; diff --git a/Python/_warnings.c b/Python/_warnings.c index 29370369e25d87..21da0a9320a9ee 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1172,32 +1172,18 @@ create_filter(PyObject *category, _Py_Identifier *id) static PyObject * init_filters(const _PyCoreConfig *config) { - int dev_mode = config->dev_mode; - - Py_ssize_t count = 2; - if (dev_mode) { - count++; - } -#ifndef Py_DEBUG - if (!dev_mode) { - count += 3; - } -#endif + Py_ssize_t count = 5; PyObject *filters = PyList_New(count); if (filters == NULL) return NULL; size_t pos = 0; /* Post-incremented in each use. */ -#ifndef Py_DEBUG - if (!dev_mode) { - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, &PyId_ignore)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ImportWarning, &PyId_ignore)); - } -#endif + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, &PyId_ignore)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ImportWarning, &PyId_ignore)); _Py_Identifier *bytes_action; if (Py_BytesWarningFlag > 1) @@ -1206,23 +1192,19 @@ init_filters(const _PyCoreConfig *config) bytes_action = &PyId_default; else bytes_action = &PyId_ignore; - PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, - bytes_action)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_BytesWarning, bytes_action)); _Py_Identifier *resource_action; + /* resource usage warnings are enabled by default in pydebug mode */ #ifdef Py_DEBUG resource_action = &PyId_default; #else - resource_action = (dev_mode ? &PyId_default: &PyId_ignore); + resource_action = &PyId_ignore; #endif - PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning, - resource_action)); - - if (dev_mode) { - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_Warning, &PyId_default)); - } + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ResourceWarning, resource_action)); for (size_t x = 0; x < pos; x++) { if (PyList_GET_ITEM(filters, x) == NULL) { From 0cb8f29c664f1d315fe0a805a830898cd90aa614 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 6 Dec 2017 20:30:11 +1000 Subject: [PATCH 02/11] Fix typo in NEWS entry --- .../Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst index 1e815270d178ba..b8656e4f25a314 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst @@ -1,3 +1,3 @@ -`-X dev` now injects a ``'default'`` entryinto sys.warnoptions, ensuring +`-X dev` now injects a ``'default'`` entry into sys.warnoptions, ensuring that it behaves identically to actually passing ``-Wdefault`` at the command line. From e1a7d62d9381cf1e49b7ccd1f1c202095ae0c6a3 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 6 Dec 2017 20:44:00 +1000 Subject: [PATCH 03/11] -X dev should take precedence over PYTHONWARNINGS As a command line setting, dev mode should take precedence of the PYTHONWARNINGS environment variable. --- Modules/main.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Modules/main.c b/Modules/main.c index 4b7ceab0d455f5..3ff015f368671c 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -762,12 +762,11 @@ pymain_add_warnings_options(_PyMain *pymain) { PySys_ResetWarnOptions(); - if (pymain_add_warnings_dev_mode(&pymain->core_config) < 0) { + if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; } - - if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { + if (pymain_add_warnings_dev_mode(&pymain->core_config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; } From 89960efb5879c5201fad10592e8a1ec55c2beecf Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 6 Dec 2017 21:03:00 +1000 Subject: [PATCH 04/11] Add explanatory comment about BytesWarning --- Python/_warnings.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 21da0a9320a9ee..9042b328f61e1e 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1186,12 +1186,20 @@ init_filters(const _PyCoreConfig *config) create_filter(PyExc_ImportWarning, &PyId_ignore)); _Py_Identifier *bytes_action; - if (Py_BytesWarningFlag > 1) + if (Py_BytesWarningFlag > 1) { bytes_action = &PyId_error; - else if (Py_BytesWarningFlag) + } else if (Py_BytesWarningFlag) { bytes_action = &PyId_default; - else + } else { + /* We set the 'ignore' action by default, but we don't actually rely + * on the warnings machinery to ignore BytesWarning, as it's too slow. + * + * Instead, the relevant code in bytesobject.c and bytearrayobject.c + * checks Py_BytesWarningFlag directly, and skips the PyErr_WarnEx call + * entirely if it isn't set. + */ bytes_action = &PyId_ignore; + } PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, bytes_action)); From d5c157691dba9a034dbc5ea065b7f99c38cb804a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 6 Dec 2017 21:55:32 +1000 Subject: [PATCH 05/11] Fix Python level checks for a debug build --- Lib/test/test_cmd_line.py | 2 +- Lib/warnings.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 817d6173866aa9..a02db69f969b15 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -541,7 +541,7 @@ def test_xdev(self): code = ("import sys, warnings; " "print(' '.join('%s::%s' % (f[0], f[2].__name__) " "for f in warnings.filters))") - if sys.flags.debug: + if hasattr(sys, 'gettotalrefcount'): resource_action = "default" else: resource_action = "ignore" diff --git a/Lib/warnings.py b/Lib/warnings.py index 600fb26a148a95..9bbc4b5a710dc6 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -534,8 +534,8 @@ def _filters_mutated(): bytes_action = "ignore" simplefilter(bytes_action, category=BytesWarning, append=1) - # resource usage warnings are enabled by default in pydebug mode - if sys.flags.debug: + # resource usage warnings are enabled by default in pydebug builds + if hasattr(sys, 'gettotalrefcount'): resource_action = "default" else: resource_action = "ignore" From 019dbea0b569f2f186688c33b31605c965d8709c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 6 Dec 2017 22:32:54 +1000 Subject: [PATCH 06/11] Add a comment explaining warnings filter priority order --- Modules/main.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Modules/main.c b/Modules/main.c index 3ff015f368671c..0e1c850c27d6d5 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -762,11 +762,28 @@ pymain_add_warnings_options(_PyMain *pymain) { PySys_ResetWarnOptions(); - if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { + /* The priority order for warnings configuration is (highest precedence + * first): + * + * - any '-W' command line options; then + * - the 'PYTHONWARNINGS' environment variable; then + * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then + * - any implicit filters added by _warnings.c/warnings.py + * + * TODO: Move the -b/-bb command line option to the head of the list + * See https://bugs.python.org/issue32231 for discussion + * + * All settings except the last are passed to the warnings module via + * the `sys.warnoptions` list. Since the warnings module works on the basis + * of "the most recently added filter will be checked first", we add + * the lowest precedence entries first so that later entries override them. + */ + + if (pymain_add_warnings_dev_mode(&pymain->core_config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; } - if (pymain_add_warnings_dev_mode(&pymain->core_config) < 0) { + if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; } From 04f7de32e0a5a229bea9f595cac3d2cf000f4656 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 7 Dec 2017 16:05:52 +1000 Subject: [PATCH 07/11] Address PR review feedback - expanded PR to also fix bpo-20361 and have -b & -bb take precedence over any other warnings options - reverted to the short implicit filter list in Py_DEBUG builds (this is now even shorter than it was previously, as there are *no* implicit filters in debug builds) --- Lib/subprocess.py | 8 +++++++- Lib/test/test_cmd_line.py | 41 ++++++++++----------------------------- Lib/warnings.py | 25 ++++++------------------ Modules/main.c | 36 +++++++++++++++++++++++++++++----- Python/_warnings.c | 41 ++++++++++----------------------------- 5 files changed, 64 insertions(+), 87 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 05e45d11d7bf4f..e0e85b76c2a901 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -262,7 +262,13 @@ def _args_from_interpreter_flags(): args.append('-' + opt * v) # -W options - for opt in sys.warnoptions: + warnopts = sys.warnoptions[:] + bytes_warning = sys.flags.bytes_warning + if bytes_warning > 1: + warnopts.remove("error::BytesWarning") + elif bytes_warning: + warnopts.remove("default::BytesWarning") + for opt in warnopts: args.append('-W' + opt) # -X options diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index a02db69f969b15..7bbb88952eec0a 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -542,46 +542,25 @@ def test_xdev(self): "print(' '.join('%s::%s' % (f[0], f[2].__name__) " "for f in warnings.filters))") if hasattr(sys, 'gettotalrefcount'): - resource_action = "default" + expected_filters = "default::Warning" else: - resource_action = "ignore" + expected_filters = ("default::Warning " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " + "ignore::ResourceWarning") out = self.run_xdev("-c", code) - self.assertEqual(out, - "default::Warning " - "ignore::DeprecationWarning " - "ignore::PendingDeprecationWarning " - "ignore::ImportWarning " - "ignore::BytesWarning " - f"{resource_action}::ResourceWarning") + self.assertEqual(out, expected_filters) out = self.run_xdev("-b", "-c", code) - self.assertEqual(out, - "default::Warning " - "ignore::DeprecationWarning " - "ignore::PendingDeprecationWarning " - "ignore::ImportWarning " - "default::BytesWarning " - f"{resource_action}::ResourceWarning") + self.assertEqual(out, f"default::BytesWarning {expected_filters}") out = self.run_xdev("-bb", "-c", code) - self.assertEqual(out, - "default::Warning " - "ignore::DeprecationWarning " - "ignore::PendingDeprecationWarning " - "ignore::ImportWarning " - "error::BytesWarning " - f"{resource_action}::ResourceWarning") + self.assertEqual(out, f"error::BytesWarning {expected_filters}") out = self.run_xdev("-Werror", "-c", code) - self.assertEqual(out, - "error::Warning " - "default::Warning " - "ignore::DeprecationWarning " - "ignore::PendingDeprecationWarning " - "ignore::ImportWarning " - "ignore::BytesWarning " - f"{resource_action}::ResourceWarning") + self.assertEqual(out, f"error::Warning {expected_filters}") # Memory allocator debug hooks try: diff --git a/Lib/warnings.py b/Lib/warnings.py index 9bbc4b5a710dc6..1334fcb6dc3472 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -521,24 +521,11 @@ def _filters_mutated(): if not _warnings_defaults: # TODO: Define a test case that ensures this fallback code always remains # consistent with init_filters() in _warnings.c - simplefilter("ignore", category=DeprecationWarning, append=1) - simplefilter("ignore", category=PendingDeprecationWarning, append=1) - simplefilter("ignore", category=ImportWarning, append=1) - - bytes_warning = sys.flags.bytes_warning - if bytes_warning > 1: - bytes_action = "error" - elif bytes_warning: - bytes_action = "default" - else: - bytes_action = "ignore" - simplefilter(bytes_action, category=BytesWarning, append=1) - - # resource usage warnings are enabled by default in pydebug builds - if hasattr(sys, 'gettotalrefcount'): - resource_action = "default" - else: - resource_action = "ignore" - simplefilter(resource_action, category=ResourceWarning, append=1) + if not hasattr(sys, 'gettotalrefcount'): + # Several warning categories are ignored by default in Py_DEBUG builds + simplefilter("ignore", category=DeprecationWarning, append=1) + simplefilter("ignore", category=PendingDeprecationWarning, append=1) + simplefilter("ignore", category=ImportWarning, append=1) + simplefilter("ignore", category=ResourceWarning, append=1) del _warnings_defaults diff --git a/Modules/main.c b/Modules/main.c index 0e1c850c27d6d5..3df246cb106db1 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -741,7 +741,7 @@ pymain_add_warnings_optlist(_Py_OptList *warnings) } static int -pymain_add_warnings_dev_mode(_PyCoreConfig *core_config) +pymain_add_warning_dev_mode(_PyCoreConfig *core_config) { if (core_config->dev_mode) { PyObject *option = PyUnicode_FromString("default"); @@ -757,6 +757,30 @@ pymain_add_warnings_dev_mode(_PyCoreConfig *core_config) return 0; } +static int +pymain_add_warning_bytes_flag(int bytes_warning_flag) +{ + /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c + * don't even try to emit a warning, so we skip setting the filter in that + * case. + */ + if (bytes_warning_flag) { + const char *filter = (bytes_warning_flag > 1) ? "error::BytesWarning": + "default::BytesWarning"; + PyObject *option = PyUnicode_FromString(filter); + if (option == NULL) { + return -1; + } + if (_PySys_AddWarnOptionWithError(option)) { + Py_DECREF(option); + return -1; + } + Py_DECREF(option); + } + return 0; +} + + static int pymain_add_warnings_options(_PyMain *pymain) { @@ -765,21 +789,19 @@ pymain_add_warnings_options(_PyMain *pymain) /* The priority order for warnings configuration is (highest precedence * first): * + * - the BytesWarning filter, if needed ('-b', '-bb') * - any '-W' command line options; then * - the 'PYTHONWARNINGS' environment variable; then * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then * - any implicit filters added by _warnings.c/warnings.py * - * TODO: Move the -b/-bb command line option to the head of the list - * See https://bugs.python.org/issue32231 for discussion - * * All settings except the last are passed to the warnings module via * the `sys.warnoptions` list. Since the warnings module works on the basis * of "the most recently added filter will be checked first", we add * the lowest precedence entries first so that later entries override them. */ - if (pymain_add_warnings_dev_mode(&pymain->core_config) < 0) { + if (pymain_add_warning_dev_mode(&pymain->core_config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; } @@ -791,6 +813,10 @@ pymain_add_warnings_options(_PyMain *pymain) pymain->err = _Py_INIT_NO_MEMORY(); return -1; } + if (pymain_add_warning_bytes_flag(pymain->cmdline.bytes_warning) < 0) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; + } return 0; } diff --git a/Python/_warnings.c b/Python/_warnings.c index 9042b328f61e1e..6ab97432f5e506 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -11,9 +11,9 @@ MODULE_NAME " provides basic warning filtering support.\n" _Py_IDENTIFIER(argv); _Py_IDENTIFIER(stderr); +#ifndef Py_DEBUG _Py_IDENTIFIER(ignore); -_Py_IDENTIFIER(error); -_Py_static_string(PyId_default, "default"); +#endif static int check_matched(PyObject *obj, PyObject *arg) @@ -1172,7 +1172,12 @@ create_filter(PyObject *category, _Py_Identifier *id) static PyObject * init_filters(const _PyCoreConfig *config) { - Py_ssize_t count = 5; +#ifdef Py_DEBUG + /* Py_DEBUG builds show all warnings by default */ + return PyList_New(0); +#else + /* Other builds ignore a number of warning categories by default */ + Py_ssize_t count = 4; PyObject *filters = PyList_New(count); if (filters == NULL) return NULL; @@ -1184,35 +1189,8 @@ init_filters(const _PyCoreConfig *config) create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore)); PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ImportWarning, &PyId_ignore)); - - _Py_Identifier *bytes_action; - if (Py_BytesWarningFlag > 1) { - bytes_action = &PyId_error; - } else if (Py_BytesWarningFlag) { - bytes_action = &PyId_default; - } else { - /* We set the 'ignore' action by default, but we don't actually rely - * on the warnings machinery to ignore BytesWarning, as it's too slow. - * - * Instead, the relevant code in bytesobject.c and bytearrayobject.c - * checks Py_BytesWarningFlag directly, and skips the PyErr_WarnEx call - * entirely if it isn't set. - */ - bytes_action = &PyId_ignore; - } - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_BytesWarning, bytes_action)); - - _Py_Identifier *resource_action; - - /* resource usage warnings are enabled by default in pydebug mode */ -#ifdef Py_DEBUG - resource_action = &PyId_default; -#else - resource_action = &PyId_ignore; -#endif PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ResourceWarning, resource_action)); + create_filter(PyExc_ResourceWarning, &PyId_ignore)); for (size_t x = 0; x < pos; x++) { if (PyList_GET_ITEM(filters, x) == NULL) { @@ -1222,6 +1200,7 @@ init_filters(const _PyCoreConfig *config) } return filters; +#endif } static struct PyModuleDef warningsmodule = { From 6e44c431c85acdbc8b698f22271c7b8f78af7f25 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 7 Dec 2017 17:23:12 +1000 Subject: [PATCH 08/11] Second NEWS entry, and update What's New --- Doc/whatsnew/3.7.rst | 29 +++++++++++++++++++ .../2017-12-07-17-22-30.bpo-20361.zQUmbi.rst | 4 +++ 2 files changed, 33 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-12-07-17-22-30.bpo-20361.zQUmbi.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9363730bf1fd64..f90d279e6fbe93 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -454,6 +454,29 @@ Function :func:`~uu.encode` now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) +warnings +-------- + +The initialization of the default warnings filters has changed as follows: + +* warnings enabled via command line options (including those for :option:`-b` + and the new CPython-specific ``-X dev`` option) are always passed to the + warnings machinery via the ``sys.warnoptions`` attribute. +* warnings filters enabled via the command line or the environment now have the + following precedence order: + + * the ``BytesWarning`` filter for :option:`-b` (or ``-bb``) + * any filters specified with :option:`-W` + * any filters specified with :envvar:`PYTHONWARNINGS` + * any other CPython specific filters (e.g. the ``default`` filter added + for the new ``-X dev`` mode) + * any implicit filters defined directly by the warnings machinery +* in CPython debug builds, all warnings are now displayed by default (the + implicit filter list is empty) + +(Contributed by Nick Coghlan and Victor Stinner in :issue:`20361`, +:issue:`32043`, and :issue:`32230`) + xml.etree --------- @@ -827,6 +850,12 @@ Other CPython implementation changes either in embedding applications, or in CPython itself. (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) +* Due to changes in the way the default warnings filters are configured, + setting ``Py_BytesWarningFlag`` to a value greater than one is no longer + sufficient to both emit ``BytesWarning`` messages and have them converted + to exceptions. Instead, the flag must be set (to cause the warnings to be + emitted in the first place), and an explicit ``error::BytesWarning`` warnings + filter added to convert them to exceptions. Documentation ============= diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-07-17-22-30.bpo-20361.zQUmbi.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-07-17-22-30.bpo-20361.zQUmbi.rst new file mode 100644 index 00000000000000..df7aed7f07a2b4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-07-17-22-30.bpo-20361.zQUmbi.rst @@ -0,0 +1,4 @@ +``-b`` and ``-bb`` now inject ``'default::BytesWarning'`` and +``error::BytesWarning`` entries into ``sys.warnoptions``, ensuring that they +take precedence over any other warning filters configured via the ``-W`` +option or the ``PYTHONWARNINGS`` environment variable. From ed37a23abcef7362411fbca5c008d447a99be8e0 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 7 Dec 2017 18:51:32 +1000 Subject: [PATCH 09/11] Add warnings filter test case --- Lib/test/test_cmd_line.py | 40 +++++++++++++++++++++++++++++++++++++++ Lib/warnings.py | 2 -- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 7bbb88952eec0a..9b15aae646699f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -587,6 +587,46 @@ def test_xdev(self): out = self.run_xdev("-c", code) self.assertEqual(out, "True") + def check_warnings_filters(self, cmdline_option, envvar, use_pywarning=False): + if use_pywarning: + code = ("import sys; from test.support import import_fresh_module; " + "warnings = import_fresh_module('warnings', blocked=['_warnings']); ") + else: + code = "import sys, warnings; " + code += ("print(' '.join('%s::%s' % (f[0], f[2].__name__) " + "for f in warnings.filters))") + args = (sys.executable, '-W', cmdline_option, '-bb', '-c', code) + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + env["PYTHONWARNINGS"] = envvar + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() + + def test_warnings_filter_precedence(self): + expected_filters = ("error::BytesWarning " + "once::UserWarning " + "always::UserWarning") + if not hasattr(sys, 'gettotalrefcount'): + expected_filters += (" " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " + "ignore::ResourceWarning") + + out = self.check_warnings_filters("once::UserWarning", + "always::UserWarning") + self.assertEqual(out, expected_filters) + + out = self.check_warnings_filters("once::UserWarning", + "always::UserWarning", + use_pywarning=True) + self.assertEqual(out, expected_filters) + def check_pythonmalloc(self, env_var, name): code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())' env = dict(os.environ) diff --git a/Lib/warnings.py b/Lib/warnings.py index 1334fcb6dc3472..307f5c55de2068 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -519,8 +519,6 @@ def _filters_mutated(): # Module initialization _processoptions(sys.warnoptions) if not _warnings_defaults: - # TODO: Define a test case that ensures this fallback code always remains - # consistent with init_filters() in _warnings.c if not hasattr(sys, 'gettotalrefcount'): # Several warning categories are ignored by default in Py_DEBUG builds simplefilter("ignore", category=DeprecationWarning, append=1) From 77b1f02aa39d5e7f0f9d1f6aeba13ec296b57df1 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 7 Dec 2017 19:19:24 +1000 Subject: [PATCH 10/11] Use the code role to avoid a warning --- Doc/whatsnew/3.7.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index f90d279e6fbe93..cfe7a1bdf6dd50 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -854,8 +854,8 @@ Other CPython implementation changes setting ``Py_BytesWarningFlag`` to a value greater than one is no longer sufficient to both emit ``BytesWarning`` messages and have them converted to exceptions. Instead, the flag must be set (to cause the warnings to be - emitted in the first place), and an explicit ``error::BytesWarning`` warnings - filter added to convert them to exceptions. + emitted in the first place), and an explicit :code:`error::BytesWarning` + warnings filter added to convert them to exceptions. Documentation ============= From a85790be3a816f44ca6f01e2dd6ad596850af733 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 7 Dec 2017 20:22:25 +1000 Subject: [PATCH 11/11] Suppress false alarms from 'make suspicious' --- Doc/tools/susp-ignored.csv | 3 +++ Doc/whatsnew/3.7.rst | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index d52f81b76b52f0..48dd53f85d2d0e 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -327,3 +327,6 @@ whatsnew/changelog,,:end,str[start:end] library/binascii,,`,'`' library/uu,,`,'`' whatsnew/3.7,,`,'`' +whatsnew/3.7,,::,error::BytesWarning +whatsnew/changelog,,::,error::BytesWarning +whatsnew/changelog,,::,default::BytesWarning diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index cfe7a1bdf6dd50..812aa31e1a8d75 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -854,7 +854,7 @@ Other CPython implementation changes setting ``Py_BytesWarningFlag`` to a value greater than one is no longer sufficient to both emit ``BytesWarning`` messages and have them converted to exceptions. Instead, the flag must be set (to cause the warnings to be - emitted in the first place), and an explicit :code:`error::BytesWarning` + emitted in the first place), and an explicit ``error::BytesWarning`` warnings filter added to convert them to exceptions. Documentation