bpo-32230, bpo-20361: Set sys.warnoptions for all command line options

Doc/tools/susp-ignored.csv

@@ -327,3 +327,6 @@ whatsnew/changelog,,:end,str[start:end]
 library/binascii,,`,'`'
 library/uu,,`,'`'
 whatsnew/3.7,,`,'`'
+whatsnew/3.7,,::,error::BytesWarning
+whatsnew/changelog,,::,error::BytesWarning
+whatsnew/changelog,,::,default::BytesWarning

Doc/whatsnew/3.7.rst

@@ -454,6 +454,29 @@ Function :func:`~uu.encode` now accepts an optional *backtick*
 keyword argument.  When it's true, zeros are represented by ``'`'``
 instead of spaces.  (Contributed by Xiang Zhang in :issue:`30103`.)
 
+warnings
+--------
+
+The initialization of the default warnings filters has changed as follows:
+
+* warnings enabled via command line options (including those for :option:`-b`
+  and the new CPython-specific ``-X dev`` option) are always passed to the
+  warnings machinery via the ``sys.warnoptions`` attribute.
+* warnings filters enabled via the command line or the environment now have the
+  following precedence order:
+
+     * the ``BytesWarning`` filter for :option:`-b` (or ``-bb``)
+     * any filters specified with :option:`-W`
+     * any filters specified with :envvar:`PYTHONWARNINGS`
+     * any other CPython specific filters (e.g. the ``default`` filter added
+       for the new ``-X dev`` mode)
+     * any implicit filters defined directly by the warnings machinery
+* in CPython debug builds, all warnings are now displayed by default (the
+  implicit filter list is empty)
+
+(Contributed by Nick Coghlan and Victor Stinner in :issue:`20361`,
+:issue:`32043`, and :issue:`32230`)
+
 xml.etree
 ---------
 
@@ -827,6 +850,12 @@ Other CPython implementation changes
   either in embedding applications, or in CPython itself.
   (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.)
 
+* Due to changes in the way the default warnings filters are configured,
+  setting ``Py_BytesWarningFlag`` to a value greater than one is no longer
+  sufficient to both emit ``BytesWarning`` messages and have them converted
+  to exceptions. Instead, the flag must be set (to cause the warnings to be
+  emitted in the first place), and an explicit ``error::BytesWarning``
+  warnings filter added to convert them to exceptions.
 
 Documentation
 =============

Lib/subprocess.py

@@ -241,7 +241,7 @@ def _optim_args_from_interpreter_flags():
 
 def _args_from_interpreter_flags():
     """Return a list of command-line arguments reproducing the current
-    settings in sys.flags and sys.warnoptions."""
+    settings in sys.flags, sys.warnoptions and sys._xoptions."""
     flag_opt_map = {
         'debug': 'd',
         # 'inspect': 'i',
@@ -262,12 +262,19 @@ def _args_from_interpreter_flags():
             args.append('-' + opt * v)
 
     # -W options
-    for opt in sys.warnoptions:
+    warnopts = sys.warnoptions[:]
+    bytes_warning = sys.flags.bytes_warning
+    if bytes_warning > 1:
+        warnopts.remove("error::BytesWarning")
+    elif bytes_warning:
+        warnopts.remove("default::BytesWarning")
+    for opt in warnopts:
         args.append('-W' + opt)
 
     # -X options
     xoptions = getattr(sys, '_xoptions', {})
     if 'dev' in xoptions:
+        args.remove('-Wdefault')
         args.extend(('-X', 'dev'))
     for opt in ('faulthandler', 'tracemalloc', 'importtime',
                 'showalloccount', 'showrefcount'):

Lib/test/test_cmd_line.py

@@ -541,31 +541,26 @@ def test_xdev(self):
         code = ("import sys, warnings; "
                 "print(' '.join('%s::%s' % (f[0], f[2].__name__) "
                                 "for f in warnings.filters))")
+        if hasattr(sys, 'gettotalrefcount'):
+            expected_filters = "default::Warning"
+        else:
+            expected_filters = ("default::Warning "
+                                "ignore::DeprecationWarning "
+                                "ignore::PendingDeprecationWarning "
+                                "ignore::ImportWarning "
+                                "ignore::ResourceWarning")
 
         out = self.run_xdev("-c", code)
-        self.assertEqual(out,
-                         "ignore::BytesWarning "
-                         "default::ResourceWarning "
-                         "default::Warning")
+        self.assertEqual(out, expected_filters)
 
         out = self.run_xdev("-b", "-c", code)
-        self.assertEqual(out,
-                         "default::BytesWarning "
-                         "default::ResourceWarning "
-                         "default::Warning")
+        self.assertEqual(out, f"default::BytesWarning {expected_filters}")
 
         out = self.run_xdev("-bb", "-c", code)
-        self.assertEqual(out,
-                         "error::BytesWarning "
-                         "default::ResourceWarning "
-                         "default::Warning")
+        self.assertEqual(out, f"error::BytesWarning {expected_filters}")
 
         out = self.run_xdev("-Werror", "-c", code)
-        self.assertEqual(out,
-                         "error::Warning "
-                         "ignore::BytesWarning "
-                         "default::ResourceWarning "
-                         "default::Warning")
+        self.assertEqual(out, f"error::Warning {expected_filters}")
 
         # Memory allocator debug hooks
         try:
@@ -592,6 +587,46 @@ def test_xdev(self):
             out = self.run_xdev("-c", code)
             self.assertEqual(out, "True")
 
+    def check_warnings_filters(self, cmdline_option, envvar, use_pywarning=False):
+        if use_pywarning:
+            code = ("import sys; from test.support import import_fresh_module; "
+                    "warnings = import_fresh_module('warnings', blocked=['_warnings']); ")
+        else:
+            code = "import sys, warnings; "
+        code += ("print(' '.join('%s::%s' % (f[0], f[2].__name__) "
+                                "for f in warnings.filters))")
+        args = (sys.executable, '-W', cmdline_option, '-bb', '-c', code)
+        env = dict(os.environ)
+        env.pop('PYTHONDEVMODE', None)
+        env["PYTHONWARNINGS"] = envvar
+        proc = subprocess.run(args,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.STDOUT,
+                              universal_newlines=True,
+                              env=env)
+        self.assertEqual(proc.returncode, 0, proc)
+        return proc.stdout.rstrip()
+
+    def test_warnings_filter_precedence(self):
+        expected_filters = ("error::BytesWarning "
+                            "once::UserWarning "
+                            "always::UserWarning")
+        if not hasattr(sys, 'gettotalrefcount'):
+            expected_filters += (" "
+                                 "ignore::DeprecationWarning "
+                                 "ignore::PendingDeprecationWarning "
+                                 "ignore::ImportWarning "
+                                 "ignore::ResourceWarning")
+
+        out = self.check_warnings_filters("once::UserWarning",
+                                          "always::UserWarning")
+        self.assertEqual(out, expected_filters)
+
+        out = self.check_warnings_filters("once::UserWarning",
+                                          "always::UserWarning",
+                                          use_pywarning=True)
+        self.assertEqual(out, expected_filters)
+
     def check_pythonmalloc(self, env_var, name):
         code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())'
         env = dict(os.environ)

Lib/warnings.py

@@ -519,34 +519,11 @@ def _filters_mutated():
 # Module initialization
 _processoptions(sys.warnoptions)
 if not _warnings_defaults:
-    dev_mode = ('dev' in getattr(sys, '_xoptions', {}))
-    py_debug = hasattr(sys, 'gettotalrefcount')
-
-    if not(dev_mode or py_debug):
-        silence = [ImportWarning, PendingDeprecationWarning]
-        silence.append(DeprecationWarning)
-        for cls in silence:
-            simplefilter("ignore", category=cls)
-
-    bytes_warning = sys.flags.bytes_warning
-    if bytes_warning > 1:
-        bytes_action = "error"
-    elif bytes_warning:
-        bytes_action = "default"
-    else:
-        bytes_action = "ignore"
-    simplefilter(bytes_action, category=BytesWarning, append=1)
-
-    # resource usage warnings are enabled by default in pydebug mode
-    if dev_mode or py_debug:
-        resource_action = "default"
-    else:
-        resource_action = "ignore"
-    simplefilter(resource_action, category=ResourceWarning, append=1)
-
-    if dev_mode:
-        simplefilter("default", category=Warning, append=1)
-
-    del py_debug, dev_mode
+    if not hasattr(sys, 'gettotalrefcount'):
+        # Several warning categories are ignored by default in Py_DEBUG builds
+        simplefilter("ignore", category=DeprecationWarning, append=1)
+        simplefilter("ignore", category=PendingDeprecationWarning, append=1)
+        simplefilter("ignore", category=ImportWarning, append=1)
+        simplefilter("ignore", category=ResourceWarning, append=1)
 
 del _warnings_defaults

Misc/NEWS.d/next/Core and Builtins/2017-12-06-20-18-34.bpo-32230.PgGQaB.rst

@@ -0,0 +1,3 @@
+`-X dev` now injects a ``'default'`` entry into sys.warnoptions, ensuring
+that it behaves identically to actually passing ``-Wdefault`` at the command
+line.

Misc/NEWS.d/next/Core and Builtins/2017-12-07-17-22-30.bpo-20361.zQUmbi.rst

@@ -0,0 +1,4 @@
+``-b`` and ``-bb`` now inject ``'default::BytesWarning'`` and
+``error::BytesWarning`` entries into ``sys.warnoptions``, ensuring that they
+take precedence over any other warning filters configured via the ``-W``
+option or the ``PYTHONWARNINGS`` environment variable.

Modules/main.c

@@ -740,11 +740,71 @@ pymain_add_warnings_optlist(_Py_OptList *warnings)
     return 0;
 }
 
+static int
+pymain_add_warning_dev_mode(_PyCoreConfig *core_config)
+{
+    if (core_config->dev_mode) {
+        PyObject *option = PyUnicode_FromString("default");
+        if (option == NULL) {
+            return -1;
+        }
+        if (_PySys_AddWarnOptionWithError(option)) {
+            Py_DECREF(option);
+            return -1;
+        }
+        Py_DECREF(option);
+    }
+    return 0;
+}
+
+static int
+pymain_add_warning_bytes_flag(int bytes_warning_flag)
+{
+    /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c
+     * don't even try to emit a warning, so we skip setting the filter in that
+     * case.
+     */
+    if (bytes_warning_flag) {
+        const char *filter = (bytes_warning_flag > 1) ? "error::BytesWarning":
+                                                        "default::BytesWarning";
+        PyObject *option = PyUnicode_FromString(filter);
+        if (option == NULL) {
+            return -1;
+        }
+        if (_PySys_AddWarnOptionWithError(option)) {
+            Py_DECREF(option);
+            return -1;
+        }
+        Py_DECREF(option);
+    }
+    return 0;
+}
+
+
 static int
 pymain_add_warnings_options(_PyMain *pymain)
 {
     PySys_ResetWarnOptions();
 
+    /* The priority order for warnings configuration is (highest precedence
+     * first):
+     *
+     * - the BytesWarning filter, if needed ('-b', '-bb')
+     * - any '-W' command line options; then
+     * - the 'PYTHONWARNINGS' environment variable; then
+     * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then
+     * - any implicit filters added by _warnings.c/warnings.py
+     *
+     * All settings except the last are passed to the warnings module via
+     * the `sys.warnoptions` list. Since the warnings module works on the basis
+     * of "the most recently added filter will be checked first", we add
+     * the lowest precedence entries first so that later entries override them.
+     */
+
+    if (pymain_add_warning_dev_mode(&pymain->core_config) < 0) {
+        pymain->err = _Py_INIT_NO_MEMORY();
+        return -1;
+    }
     if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) {
         pymain->err = _Py_INIT_NO_MEMORY();
         return -1;
@@ -753,6 +813,10 @@ pymain_add_warnings_options(_PyMain *pymain)
         pymain->err = _Py_INIT_NO_MEMORY();
         return -1;
     }
+    if (pymain_add_warning_bytes_flag(pymain->cmdline.bytes_warning) < 0) {
+        pymain->err = _Py_INIT_NO_MEMORY();
+        return -1;
+    }
     return 0;
 }
 

Python/_warnings.c

@@ -11,9 +11,9 @@ MODULE_NAME " provides basic warning filtering support.\n"
 
 _Py_IDENTIFIER(argv);
 _Py_IDENTIFIER(stderr);
+#ifndef Py_DEBUG
 _Py_IDENTIFIER(ignore);
-_Py_IDENTIFIER(error);
-_Py_static_string(PyId_default, "default");
+#endif
 
 static int
 check_matched(PyObject *obj, PyObject *arg)
@@ -1172,57 +1172,25 @@ create_filter(PyObject *category, _Py_Identifier *id)
 static PyObject *
 init_filters(const _PyCoreConfig *config)
 {
-    int dev_mode = config->dev_mode;
-
-    Py_ssize_t count = 2;
-    if (dev_mode) {
-        count++;
-    }
-#ifndef Py_DEBUG
-    if (!dev_mode) {
-        count += 3;
-    }
-#endif
+#ifdef Py_DEBUG
+    /* Py_DEBUG builds show all warnings by default */
+    return PyList_New(0);
+#else
+    /* Other builds ignore a number of warning categories by default */
+    Py_ssize_t count = 4;
     PyObject *filters = PyList_New(count);
     if (filters == NULL)
         return NULL;
 
     size_t pos = 0;  /* Post-incremented in each use. */
-#ifndef Py_DEBUG
-    if (!dev_mode) {
-        PyList_SET_ITEM(filters, pos++,
-                        create_filter(PyExc_DeprecationWarning, &PyId_ignore));
-        PyList_SET_ITEM(filters, pos++,
-                        create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore));
-        PyList_SET_ITEM(filters, pos++,
-                        create_filter(PyExc_ImportWarning, &PyId_ignore));
-    }
-#endif
-
-    _Py_Identifier *bytes_action;
-    if (Py_BytesWarningFlag > 1)
-        bytes_action = &PyId_error;
-    else if (Py_BytesWarningFlag)
-        bytes_action = &PyId_default;
-    else
-        bytes_action = &PyId_ignore;
-    PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning,
-                    bytes_action));
-
-    _Py_Identifier *resource_action;
-    /* resource usage warnings are enabled by default in pydebug mode */
-#ifdef Py_DEBUG
-    resource_action = &PyId_default;
-#else
-    resource_action = (dev_mode ? &PyId_default: &PyId_ignore);
-#endif
-    PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning,
-                    resource_action));
-
-    if (dev_mode) {
-        PyList_SET_ITEM(filters, pos++,
-                        create_filter(PyExc_Warning, &PyId_default));
-    }
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_DeprecationWarning, &PyId_ignore));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_ImportWarning, &PyId_ignore));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_ResourceWarning, &PyId_ignore));
 
     for (size_t x = 0; x < pos; x++) {
         if (PyList_GET_ITEM(filters, x) == NULL) {
@@ -1232,6 +1200,7 @@ init_filters(const _PyCoreConfig *config)
     }
 
     return filters;
+#endif
 }
 
 static struct PyModuleDef warningsmodule = {