diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index d52f81b76b52f0..48dd53f85d2d0e 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -327,3 +327,6 @@ whatsnew/changelog,,:end,str[start:end] library/binascii,,`,'`' library/uu,,`,'`' whatsnew/3.7,,`,'`' +whatsnew/3.7,,::,error::BytesWarning +whatsnew/changelog,,::,error::BytesWarning +whatsnew/changelog,,::,default::BytesWarning diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9363730bf1fd64..812aa31e1a8d75 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -454,6 +454,29 @@ Function :func:`~uu.encode` now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) +warnings +-------- + +The initialization of the default warnings filters has changed as follows: + +* warnings enabled via command line options (including those for :option:`-b` + and the new CPython-specific ``-X dev`` option) are always passed to the + warnings machinery via the ``sys.warnoptions`` attribute. +* warnings filters enabled via the command line or the environment now have the + following precedence order: + + * the ``BytesWarning`` filter for :option:`-b` (or ``-bb``) + * any filters specified with :option:`-W` + * any filters specified with :envvar:`PYTHONWARNINGS` + * any other CPython specific filters (e.g. the ``default`` filter added + for the new ``-X dev`` mode) + * any implicit filters defined directly by the warnings machinery +* in CPython debug builds, all warnings are now displayed by default (the + implicit filter list is empty) + +(Contributed by Nick Coghlan and Victor Stinner in :issue:`20361`, +:issue:`32043`, and :issue:`32230`) + xml.etree --------- @@ -827,6 +850,12 @@ Other CPython implementation changes either in embedding applications, or in CPython itself. (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) +* Due to changes in the way the default warnings filters are configured, + setting ``Py_BytesWarningFlag`` to a value greater than one is no longer + sufficient to both emit ``BytesWarning`` messages and have them converted + to exceptions. Instead, the flag must be set (to cause the warnings to be + emitted in the first place), and an explicit ``error::BytesWarning`` + warnings filter added to convert them to exceptions. Documentation ============= diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 35bfddde4e92f1..e0e85b76c2a901 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -241,7 +241,7 @@ def _optim_args_from_interpreter_flags(): def _args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current - settings in sys.flags and sys.warnoptions.""" + settings in sys.flags, sys.warnoptions and sys._xoptions.""" flag_opt_map = { 'debug': 'd', # 'inspect': 'i', @@ -262,12 +262,19 @@ def _args_from_interpreter_flags(): args.append('-' + opt * v) # -W options - for opt in sys.warnoptions: + warnopts = sys.warnoptions[:] + bytes_warning = sys.flags.bytes_warning + if bytes_warning > 1: + warnopts.remove("error::BytesWarning") + elif bytes_warning: + warnopts.remove("default::BytesWarning") + for opt in warnopts: args.append('-W' + opt) # -X options xoptions = getattr(sys, '_xoptions', {}) if 'dev' in xoptions: + args.remove('-Wdefault') args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', 'showalloccount', 'showrefcount'): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 383302bad8b549..9b15aae646699f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -541,31 +541,26 @@ def test_xdev(self): code = ("import sys, warnings; " "print(' '.join('%s::%s' % (f[0], f[2].__name__) " "for f in warnings.filters))") + if hasattr(sys, 'gettotalrefcount'): + expected_filters = "default::Warning" + else: + expected_filters = ("default::Warning " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " + "ignore::ResourceWarning") out = self.run_xdev("-c", code) - self.assertEqual(out, - "ignore::BytesWarning " - "default::ResourceWarning " - "default::Warning") + self.assertEqual(out, expected_filters) out = self.run_xdev("-b", "-c", code) - self.assertEqual(out, - "default::BytesWarning " - "default::ResourceWarning " - "default::Warning") + self.assertEqual(out, f"default::BytesWarning {expected_filters}") out = self.run_xdev("-bb", "-c", code) - self.assertEqual(out, - "error::BytesWarning " - "default::ResourceWarning " - "default::Warning") + self.assertEqual(out, f"error::BytesWarning {expected_filters}") out = self.run_xdev("-Werror", "-c", code) - self.assertEqual(out, - "error::Warning " - "ignore::BytesWarning " - "default::ResourceWarning " - "default::Warning") + self.assertEqual(out, f"error::Warning {expected_filters}") # Memory allocator debug hooks try: @@ -592,6 +587,46 @@ def test_xdev(self): out = self.run_xdev("-c", code) self.assertEqual(out, "True") + def check_warnings_filters(self, cmdline_option, envvar, use_pywarning=False): + if use_pywarning: + code = ("import sys; from test.support import import_fresh_module; " + "warnings = import_fresh_module('warnings', blocked=['_warnings']); ") + else: + code = "import sys, warnings; " + code += ("print(' '.join('%s::%s' % (f[0], f[2].__name__) " + "for f in warnings.filters))") + args = (sys.executable, '-W', cmdline_option, '-bb', '-c', code) + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + env["PYTHONWARNINGS"] = envvar + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() + + def test_warnings_filter_precedence(self): + expected_filters = ("error::BytesWarning " + "once::UserWarning " + "always::UserWarning") + if not hasattr(sys, 'gettotalrefcount'): + expected_filters += (" " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " + "ignore::ResourceWarning") + + out = self.check_warnings_filters("once::UserWarning", + "always::UserWarning") + self.assertEqual(out, expected_filters) + + out = self.check_warnings_filters("once::UserWarning", + "always::UserWarning", + use_pywarning=True) + self.assertEqual(out, expected_filters) + def check_pythonmalloc(self, env_var, name): code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())' env = dict(os.environ) diff --git a/Lib/warnings.py b/Lib/warnings.py index 4e7241fe6ca450..307f5c55de2068 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -519,34 +519,11 @@ def _filters_mutated(): # Module initialization _processoptions(sys.warnoptions) if not _warnings_defaults: - dev_mode = ('dev' in getattr(sys, '_xoptions', {})) - py_debug = hasattr(sys, 'gettotalrefcount') - - if not(dev_mode or py_debug): - silence = [ImportWarning, PendingDeprecationWarning] - silence.append(DeprecationWarning) - for cls in silence: - simplefilter("ignore", category=cls) - - bytes_warning = sys.flags.bytes_warning - if bytes_warning > 1: - bytes_action = "error" - elif bytes_warning: - bytes_action = "default" - else: - bytes_action = "ignore" - simplefilter(bytes_action, category=BytesWarning, append=1) - - # resource usage warnings are enabled by default in pydebug mode - if dev_mode or py_debug: - resource_action = "default" - else: - resource_action = "ignore" - simplefilter(resource_action, category=ResourceWarning, append=1) - - if dev_mode: - simplefilter("default", category=Warning, append=1) - - del py_debug, dev_mode + if not hasattr(sys, 'gettotalrefcount'): + # Several warning categories are ignored by default in Py_DEBUG builds + simplefilter("ignore", category=DeprecationWarning, append=1) + simplefilter("ignore", category=PendingDeprecationWarning, append=1) + simplefilter("ignore", category=ImportWarning, append=1) + simplefilter("ignore", category=ResourceWarning, append=1) del _warnings_defaults diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst new file mode 100644 index 00000000000000..b8656e4f25a314 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst @@ -0,0 +1,3 @@ +`-X dev` now injects a ``'default'`` entry into sys.warnoptions, ensuring +that it behaves identically to actually passing ``-Wdefault`` at the command +line. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-07-17-22-30.bpo-20361.zQUmbi.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-07-17-22-30.bpo-20361.zQUmbi.rst new file mode 100644 index 00000000000000..df7aed7f07a2b4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-07-17-22-30.bpo-20361.zQUmbi.rst @@ -0,0 +1,4 @@ +``-b`` and ``-bb`` now inject ``'default::BytesWarning'`` and +``error::BytesWarning`` entries into ``sys.warnoptions``, ensuring that they +take precedence over any other warning filters configured via the ``-W`` +option or the ``PYTHONWARNINGS`` environment variable. diff --git a/Modules/main.c b/Modules/main.c index e2ca72c7dc1a46..3df246cb106db1 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -740,11 +740,71 @@ pymain_add_warnings_optlist(_Py_OptList *warnings) return 0; } +static int +pymain_add_warning_dev_mode(_PyCoreConfig *core_config) +{ + if (core_config->dev_mode) { + PyObject *option = PyUnicode_FromString("default"); + if (option == NULL) { + return -1; + } + if (_PySys_AddWarnOptionWithError(option)) { + Py_DECREF(option); + return -1; + } + Py_DECREF(option); + } + return 0; +} + +static int +pymain_add_warning_bytes_flag(int bytes_warning_flag) +{ + /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c + * don't even try to emit a warning, so we skip setting the filter in that + * case. + */ + if (bytes_warning_flag) { + const char *filter = (bytes_warning_flag > 1) ? "error::BytesWarning": + "default::BytesWarning"; + PyObject *option = PyUnicode_FromString(filter); + if (option == NULL) { + return -1; + } + if (_PySys_AddWarnOptionWithError(option)) { + Py_DECREF(option); + return -1; + } + Py_DECREF(option); + } + return 0; +} + + static int pymain_add_warnings_options(_PyMain *pymain) { PySys_ResetWarnOptions(); + /* The priority order for warnings configuration is (highest precedence + * first): + * + * - the BytesWarning filter, if needed ('-b', '-bb') + * - any '-W' command line options; then + * - the 'PYTHONWARNINGS' environment variable; then + * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then + * - any implicit filters added by _warnings.c/warnings.py + * + * All settings except the last are passed to the warnings module via + * the `sys.warnoptions` list. Since the warnings module works on the basis + * of "the most recently added filter will be checked first", we add + * the lowest precedence entries first so that later entries override them. + */ + + if (pymain_add_warning_dev_mode(&pymain->core_config) < 0) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; + } if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; @@ -753,6 +813,10 @@ pymain_add_warnings_options(_PyMain *pymain) pymain->err = _Py_INIT_NO_MEMORY(); return -1; } + if (pymain_add_warning_bytes_flag(pymain->cmdline.bytes_warning) < 0) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; + } return 0; } diff --git a/Python/_warnings.c b/Python/_warnings.c index 29370369e25d87..6ab97432f5e506 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -11,9 +11,9 @@ MODULE_NAME " provides basic warning filtering support.\n" _Py_IDENTIFIER(argv); _Py_IDENTIFIER(stderr); +#ifndef Py_DEBUG _Py_IDENTIFIER(ignore); -_Py_IDENTIFIER(error); -_Py_static_string(PyId_default, "default"); +#endif static int check_matched(PyObject *obj, PyObject *arg) @@ -1172,57 +1172,25 @@ create_filter(PyObject *category, _Py_Identifier *id) static PyObject * init_filters(const _PyCoreConfig *config) { - int dev_mode = config->dev_mode; - - Py_ssize_t count = 2; - if (dev_mode) { - count++; - } -#ifndef Py_DEBUG - if (!dev_mode) { - count += 3; - } -#endif +#ifdef Py_DEBUG + /* Py_DEBUG builds show all warnings by default */ + return PyList_New(0); +#else + /* Other builds ignore a number of warning categories by default */ + Py_ssize_t count = 4; PyObject *filters = PyList_New(count); if (filters == NULL) return NULL; size_t pos = 0; /* Post-incremented in each use. */ -#ifndef Py_DEBUG - if (!dev_mode) { - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, &PyId_ignore)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ImportWarning, &PyId_ignore)); - } -#endif - - _Py_Identifier *bytes_action; - if (Py_BytesWarningFlag > 1) - bytes_action = &PyId_error; - else if (Py_BytesWarningFlag) - bytes_action = &PyId_default; - else - bytes_action = &PyId_ignore; - PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, - bytes_action)); - - _Py_Identifier *resource_action; - /* resource usage warnings are enabled by default in pydebug mode */ -#ifdef Py_DEBUG - resource_action = &PyId_default; -#else - resource_action = (dev_mode ? &PyId_default: &PyId_ignore); -#endif - PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning, - resource_action)); - - if (dev_mode) { - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_Warning, &PyId_default)); - } + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, &PyId_ignore)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ImportWarning, &PyId_ignore)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ResourceWarning, &PyId_ignore)); for (size_t x = 0; x < pos; x++) { if (PyList_GET_ITEM(filters, x) == NULL) { @@ -1232,6 +1200,7 @@ init_filters(const _PyCoreConfig *config) } return filters; +#endif } static struct PyModuleDef warningsmodule = {