diff --git a/Include/moduleobject.h b/Include/moduleobject.h index cf9ad40c0a17a05..90a536f04497965 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -29,7 +29,7 @@ Py_DEPRECATED(3.2) PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *); PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(void) _PyModule_Clear(PyObject *); -PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *); +PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *, PyObject *); PyAPI_FUNC(int) _PyModuleSpec_IsInitializing(PyObject *); #endif PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*); diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 1d44563579fd2f3..d19514ca8b22de9 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -266,14 +266,15 @@ def test_module_repr_source(self): '{!r} does not end with {!r}'.format(r, ends_with)) def test_module_finalization_at_shutdown(self): - # Module globals and builtins should still be available during shutdown + # Most module globals and builtins should still be available during + # shutdown. rc, out, err = assert_python_ok("-c", "from test import final_a") self.assertFalse(err) lines = out.splitlines() self.assertEqual(set(lines), { b"x = a", b"x = b", - b"final_a.x = a", + b"final_a.x = None", b"final_b.x = b", b"len = len", b"shutil.rmtree = rmtree"}) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3860656c181c2bf..20980c0bc87d848 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -901,8 +901,8 @@ def __del__(self, sys=sys): self.assertIn(b'sys.flags', out[0]) self.assertIn(b'sys.float_info', out[1]) - def test_sys_ignores_cleaning_up_user_data(self): - code = """if 1: + def test_sys_cleaning_up_user_data(self): + code = textwrap.dedent(""" import struct, sys class C: @@ -912,11 +912,11 @@ def __del__(self): self.pack('I', -42) sys.x = C() - """ + """) rc, stdout, stderr = assert_python_ok('-c', code) self.assertEqual(rc, 0) self.assertEqual(stdout.rstrip(), b"") - self.assertEqual(stderr.rstrip(), b"") + self.assertIn(b'Exception ignored in: md_dict; - if (d != NULL) - _PyModule_ClearDict(d); + PyModuleObject *module = (PyModuleObject *)m; + _PyModule_ClearDict(module->md_name, module->md_dict); } void -_PyModule_ClearDict(PyObject *d) +_PyModule_ClearDict(PyObject *module_name, PyObject *dict) { + int verbose = _Py_GetConfig()->verbose; + if (verbose) { + PySys_FormatStderr("# cleanup[3] wiping %S module\n", module_name); + } + + // If the module has no dict: there is nothing to do. + if (dict == NULL) { + return; + } + /* To make the execution order of destructors for global objects a bit more predictable, we first zap all objects whose name starts with a single underscore, before we clear @@ -574,50 +583,64 @@ _PyModule_ClearDict(PyObject *d) None, rather than deleting them from the dictionary, to avoid rehashing the dictionary (to some extent). */ - Py_ssize_t pos; - PyObject *key, *value; - - int verbose = _Py_GetConfig()->verbose; + for (int step=1; step <= 2; step++) { + PyObject *reversed = PyObject_CallOneArg((PyObject*)&PyReversed_Type, dict); + if (reversed == NULL) { + PyErr_WriteUnraisable(NULL); + return; + } + PyObject *keys = PyObject_CallOneArg((PyObject*)&PyList_Type, reversed); + Py_DECREF(reversed); + if (keys == NULL) { + PyErr_WriteUnraisable(NULL); + return; + } + PyObject *iter = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iter == NULL) { + PyErr_WriteUnraisable(NULL); + return; + } - /* First, clear only names starting with a single underscore */ - pos = 0; - while (PyDict_Next(d, &pos, &key, &value)) { - if (value != Py_None && PyUnicode_Check(key)) { - if (PyUnicode_READ_CHAR(key, 0) == '_' && - PyUnicode_READ_CHAR(key, 1) != '_') { - if (verbose > 1) { - const char *s = PyUnicode_AsUTF8(key); - if (s != NULL) - PySys_WriteStderr("# clear[1] %s\n", s); - else - PyErr_Clear(); - } - if (PyDict_SetItem(d, key, Py_None) != 0) { - PyErr_WriteUnraisable(NULL); - } + /* First, clear only names starting with a single underscore */ + PyObject *key; + while ((key = PyIter_Next(iter))) { + assert(!PyErr_Occurred()); + PyObject *value = PyObject_GetItem(dict, key); + Py_XDECREF(value); // only the value pointer is useful + if (value == Py_None) { + continue; + } + if (value == NULL) { + // ignore error + PyErr_Clear(); } - } - } - /* Next, clear all names except for __builtins__ */ - pos = 0; - while (PyDict_Next(d, &pos, &key, &value)) { - if (value != Py_None && PyUnicode_Check(key)) { - if (PyUnicode_READ_CHAR(key, 0) != '_' || - !_PyUnicode_EqualToASCIIString(key, "__builtins__")) - { - if (verbose > 1) { - const char *s = PyUnicode_AsUTF8(key); - if (s != NULL) - PySys_WriteStderr("# clear[2] %s\n", s); - else - PyErr_Clear(); + if (PyUnicode_Check(key)) { + if (step == 1) { + if (PyUnicode_READ_CHAR(key, 0) != '_' || + PyUnicode_READ_CHAR(key, 1) == '_') { + continue; + } } - if (PyDict_SetItem(d, key, Py_None) != 0) { - PyErr_WriteUnraisable(NULL); + else { + /* Step 2: clear all names except for __builtins__ */ + if (_PyUnicode_EqualToASCIIString(key, "__builtins__")) { + continue; + } } } + if (verbose > 1) { + PySys_FormatStderr("# clear[%i] %S.%S\n", + step, module_name, key); + } + assert(!PyErr_Occurred()); + if (PyDict_SetItem(dict, key, Py_None) != 0) { + PyErr_WriteUnraisable(NULL); + } + Py_DECREF(key); } + Py_DECREF(iter); } /* Note: we leave __builtins__ in place, so that destructors diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8d744c7bfd4a9d0..c81b199308ea861 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1216,8 +1216,10 @@ _Py_InitializeMain(void) static void -finalize_modules_delete_special(PyThreadState *tstate, int verbose) +finalize_modules_delete_special(PyThreadState *tstate) { + int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; + // List of names to clear in sys static const char * const sys_deletes[] = { "path", "argv", "ps1", "ps2", @@ -1272,75 +1274,99 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) } -static PyObject* -finalize_remove_modules(PyObject *modules, int verbose) +static void +finalize_wipe_module(PyInterpreterState *interp, PyObject *mod, PyObject *name, + int verbose) { - PyObject *weaklist = PyList_New(0); - if (weaklist == NULL) { - PyErr_WriteUnraisable(NULL); + assert(PyModule_Check(mod)); + PyObject *dict = PyModule_GetDict(mod); + if (dict == interp->builtins || dict == interp->sysdict) { + return; } -#define STORE_MODULE_WEAKREF(name, mod) \ - if (weaklist != NULL) { \ - PyObject *wr = PyWeakref_NewRef(mod, NULL); \ - if (wr) { \ - PyObject *tup = PyTuple_Pack(2, name, wr); \ - if (!tup || PyList_Append(weaklist, tup) < 0) { \ - PyErr_WriteUnraisable(NULL); \ - } \ - Py_XDECREF(tup); \ - Py_DECREF(wr); \ - } \ - else { \ - PyErr_WriteUnraisable(NULL); \ - } \ - } + _PyModule_Clear(mod); +} + + +static void +finalize_remove_modules(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; #define CLEAR_MODULE(name, mod) \ + do { \ if (PyModule_Check(mod)) { \ if (verbose && PyUnicode_Check(name)) { \ PySys_FormatStderr("# cleanup[2] removing %U\n", name); \ } \ - STORE_MODULE_WEAKREF(name, mod); \ - if (PyObject_SetItem(modules, name, Py_None) < 0) { \ - PyErr_WriteUnraisable(NULL); \ - } \ - } + finalize_wipe_module(interp, mod, name, verbose); \ + } \ + if (PyObject_SetItem(modules, name, Py_None) < 0) { \ + PyErr_WriteUnraisable(NULL); \ + } \ + } while (0) - if (PyDict_CheckExact(modules)) { - Py_ssize_t pos = 0; - PyObject *key, *value; - while (PyDict_Next(modules, &pos, &key, &value)) { - CLEAR_MODULE(key, value); + PyObject *modules = interp->modules; + + // Wipe the __main__ module + PyObject *main_name = PyUnicode_FromString("__main__"); + if (main_name != NULL) { + PyObject *main_mod = PyObject_GetItem(modules, main_name); + if (main_mod != NULL) { + CLEAR_MODULE(main_name, main_mod); + Py_DECREF(main_mod); + + // Break reference cycles (if nay) + _PyGC_CollectNoFail(tstate); } + else { + PyErr_Clear(); + } + Py_DECREF(main_name); } else { - PyObject *iterator = PyObject_GetIter(modules); - if (iterator == NULL) { + PyErr_Clear(); + } + + PyObject *reversed = PyObject_CallOneArg((PyObject*)&PyReversed_Type, modules); + if (reversed == NULL) { + PyErr_WriteUnraisable(NULL); + return; + } + PyObject *keys = PyObject_CallOneArg((PyObject*)&PyList_Type, reversed); + Py_DECREF(reversed); + if (keys == NULL) { + PyErr_WriteUnraisable(NULL); + return; + } + PyObject *iter = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iter == NULL) { + PyErr_WriteUnraisable(NULL); + return; + } + + PyObject *name; + while ((name = PyIter_Next(iter))) { + PyObject *mod = PyObject_GetItem(modules, name); + if (mod == NULL) { PyErr_WriteUnraisable(NULL); + continue; } - else { - PyObject *key; - while ((key = PyIter_Next(iterator))) { - PyObject *value = PyObject_GetItem(modules, key); - if (value == NULL) { - PyErr_WriteUnraisable(NULL); - continue; - } - CLEAR_MODULE(key, value); - Py_DECREF(value); - Py_DECREF(key); - } - if (PyErr_Occurred()) { - PyErr_WriteUnraisable(NULL); - } - Py_DECREF(iterator); + PyObject *dict = PyModule_Check(mod) ? PyModule_GetDict(mod) : NULL; + // builtins and sys modules are cleared later + if (dict != interp->builtins && dict != interp->sysdict) { + CLEAR_MODULE(name, mod); } + Py_DECREF(mod); + Py_DECREF(name); } + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(NULL); + } + Py_DECREF(iter); #undef CLEAR_MODULE -#undef STORE_MODULE_WEAKREF - - return weaklist; } @@ -1376,46 +1402,27 @@ finalize_restore_builtins(PyThreadState *tstate) static void -finalize_modules_clear_weaklist(PyInterpreterState *interp, - PyObject *weaklist, int verbose) -{ - // First clear modules imported later - for (Py_ssize_t i = PyList_GET_SIZE(weaklist) - 1; i >= 0; i--) { - PyObject *tup = PyList_GET_ITEM(weaklist, i); - PyObject *name = PyTuple_GET_ITEM(tup, 0); - PyObject *mod = PyWeakref_GET_OBJECT(PyTuple_GET_ITEM(tup, 1)); - if (mod == Py_None) { - continue; - } - assert(PyModule_Check(mod)); - PyObject *dict = PyModule_GetDict(mod); - if (dict == interp->builtins || dict == interp->sysdict) { - continue; - } - Py_INCREF(mod); - if (verbose && PyUnicode_Check(name)) { - PySys_FormatStderr("# cleanup[3] wiping %U\n", name); - } - _PyModule_Clear(mod); - Py_DECREF(mod); - } -} - - -static void -finalize_clear_sys_builtins_dict(PyInterpreterState *interp, int verbose) +finalize_clear_sys_builtins_dict(PyInterpreterState *interp) { // Clear sys dict - if (verbose) { - PySys_FormatStderr("# cleanup[3] wiping sys\n"); + PyObject *name = PyUnicode_FromString("sys"); + if (name != NULL) { + _PyModule_ClearDict(name, interp->sysdict); + Py_DECREF(name); + } + else { + PyErr_WriteUnraisable(NULL); } - _PyModule_ClearDict(interp->sysdict); // Clear builtins dict - if (verbose) { - PySys_FormatStderr("# cleanup[3] wiping builtins\n"); + name = PyUnicode_FromString("builtins"); + if (name != NULL) { + _PyModule_ClearDict(name, interp->builtins); + Py_DECREF(name); + } + else { + PyErr_WriteUnraisable(NULL); } - _PyModule_ClearDict(interp->builtins); } @@ -1424,12 +1431,10 @@ static void finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; - PyObject *modules = interp->modules; - if (modules == NULL) { + if (interp->modules == NULL) { // Already done return; } - int verbose = _PyInterpreterState_GetConfig(interp)->verbose; // Delete some special builtins._ and sys attributes first. These are // common places where user values hide and people complain when their @@ -1438,19 +1443,29 @@ finalize_modules(PyThreadState *tstate) // destruction order. Sigh. // // XXX Perhaps these precautions are obsolete. Who knows? - finalize_modules_delete_special(tstate, verbose); + finalize_modules_delete_special(tstate); // Remove all modules from sys.modules, hoping that garbage collection // can reclaim most of them: set all sys.modules values to None. // - // We prepare a list which will receive (name, weakref) tuples of - // modules when they are removed from sys.modules. The name is used - // for diagnosis messages (in verbose mode), while the weakref helps - // detect those modules which have been held alive. - PyObject *weaklist = finalize_remove_modules(modules, verbose); + // * First, clear the __main__ module. + // * Then, clear modules from the most recently imported to the least + // recently imported: reversed(sys.modules). + // * Exception: leave builtins and sys modules unchanged. + // * Module attributes are set to None from the most recently defined to the + // least recently defined: reversed(module.__dict__). + // + // The special treatment of "builtins" here is because even + // when it's not referenced as a module, its dictionary is + // referenced by almost every module's __builtins__. Since + // deleting a module clears its dictionary (even if there are + // references left to it), we need to delete the "builtins" + // module last. Likewise, we don't delete sys until the very + // end because it is implicitly referenced (e.g. by print). + finalize_remove_modules(tstate); // Clear the modules dict - finalize_clear_modules_dict(modules); + finalize_clear_modules_dict(interp->modules); // Restore the original builtins dict, to ensure that any // user data gets cleared. @@ -1463,27 +1478,8 @@ finalize_modules(PyThreadState *tstate) // machinery. _PyGC_DumpShutdownStats(tstate); - if (weaklist != NULL) { - // Now, if there are any modules left alive, clear their globals to - // minimize potential leaks. All C extension modules actually end - // up here, since they are kept alive in the interpreter state. - // - // The special treatment of "builtins" here is because even - // when it's not referenced as a module, its dictionary is - // referenced by almost every module's __builtins__. Since - // deleting a module clears its dictionary (even if there are - // references left to it), we need to delete the "builtins" - // module last. Likewise, we don't delete sys until the very - // end because it is implicitly referenced (e.g. by print). - // - // Since dict is ordered in CPython 3.6+, modules are saved in - // importing order. First clear modules imported later. - finalize_modules_clear_weaklist(interp, weaklist, verbose); - Py_DECREF(weaklist); - } - // Clear sys and builtins modules dict - finalize_clear_sys_builtins_dict(interp, verbose); + finalize_clear_sys_builtins_dict(interp); // Clear module dict copies stored in the interpreter state: // clear PyInterpreterState.modules_by_index and