diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index b86028cf1f6..2fe11d84be0 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -200,6 +200,7 @@ extern void _PyGC_InitState(struct _gc_runtime_state *); extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); extern void _PyGC_ResetHeap(void); +extern void _PyGC_DeferredToImmortal(void); static inline int _PyGC_ShouldCollect(struct _gc_runtime_state *gcstate) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index d23cbb69073..76797376056 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -134,7 +134,9 @@ static inline void _PyObject_GC_TRACK( _PyObject_ASSERT_FROM(op, !_PyObject_GC_IS_TRACKED(op), "object already tracked by the garbage collector", filename, lineno, __func__); - op->ob_gc_bits |= _PyGC_MASK_TRACKED; + if (!_PyObject_IS_IMMORTAL(op)) { + op->ob_gc_bits |= _PyGC_MASK_TRACKED; + } } /* Tell the GC to stop tracking this object. @@ -359,8 +361,16 @@ _PyObject_SetDeferredRefcount(PyObject *op) assert(_Py_ThreadLocal(op) && "non thread-safe"); assert(!_PyObject_HasDeferredRefcount(op) && "already uses deferred refcounting"); assert(PyType_IS_GC(Py_TYPE(op))); - op->ob_ref_local += _Py_REF_DEFERRED_MASK + 1; - op->ob_ref_shared = (op->ob_ref_shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_QUEUED; + if (_PyRuntime.immortalize_deferred) { + _PyObject_SetImmortal(op); + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } + } + else { + op->ob_ref_local += _Py_REF_DEFERRED_MASK + 1; + op->ob_ref_shared = (op->ob_ref_shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_QUEUED; + } } #define _PyObject_SET_DEFERRED_REFCOUNT(op) _PyObject_SetDeferredRefcount(_PyObject_CAST(op)) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 9b51c735511..9d325467407 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -87,6 +87,11 @@ typedef struct pyruntimestate { is created ONLY IF the GIL is disabled. */ int multithreaded; + /* Temporary: if 1, immortalize objects that would use deferred reference + counting in multithreaded programs. (Simluates deferred reference counting + scalability in multi-threaded programs). */ + int immortalize_deferred; + /* Has Python started the process of stopping all threads? Protected by HEAD_LOCK() */ int stop_the_world_requested; diff --git a/Include/object.h b/Include/object.h index 14308c4a0c6..5553df3042f 100644 --- a/Include/object.h +++ b/Include/object.h @@ -665,6 +665,7 @@ _PyObject_IS_IMMORTAL(PyObject *op) { return _Py_REF_IS_IMMORTAL(_Py_atomic_load_uint32_relaxed(&op->ob_ref_local)); } +#define _PyObject_IS_IMMORTAL(op) _PyObject_IS_IMMORTAL(_PyObject_CAST(op)) static inline Py_ssize_t _Py_REF_SHARED_COUNT(Py_ssize_t shared) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index fa472a660ac..dbf135e0cbb 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2192,6 +2192,15 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) +@contextlib.contextmanager +def dont_immortalize(): + """Temporarily change immortalization/deferred behavior.""" + try: + sys._setimmortalize_deferred(False) + yield + finally: + sys._setimmortalize_deferred(True) + def infinite_recursion(max_depth=75): """Set a lower limit for tests that interact with infinite recursions (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index af92430e50f..92718c20ce4 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -1,7 +1,8 @@ import unittest from contextlib import contextmanager, ExitStack -from test.support import catch_unraisable_exception, import_helper, gc_collect +from test.support import (catch_unraisable_exception, import_helper, + gc_collect, dont_immortalize) # Skip this test if the _testcapi module isn't available. @@ -357,6 +358,7 @@ def assert_event_counts(self, exp_created_0, exp_destroyed_0, self.assertEqual( exp_destroyed_1, _testcapi.get_code_watcher_num_destroyed_events(1)) + @dont_immortalize() def test_code_object_events_dispatched(self): # verify that all counts are zero before any watchers are registered self.assert_event_counts(0, 0, 0, 0) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 4ffd57d9d52..296904e2d5c 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -140,7 +140,7 @@ ctypes = None from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, - gc_collect) + gc_collect, dont_immortalize) from test.support.script_helper import assert_python_ok from test.support import threading_helper from opcode import opmap, opname @@ -545,6 +545,7 @@ def test_interned_string_with_null(self): class CodeWeakRefTest(unittest.TestCase): + @dont_immortalize() def test_basic(self): # Create a code object in a clean environment so that we know we have # the only reference to it left. diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 730ab1f595f..d9bfd1eb9bc 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1829,6 +1829,7 @@ def f(): return 1 self.assertEqual(f.cache_parameters(), {'maxsize': 1000, "typed": True}) + @support.dont_immortalize() def test_lru_cache_weakrefable(self): @self.module.lru_cache def test_function(x): diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 70e4efea693..31cd98fbb63 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -1,7 +1,7 @@ # Test the module type import unittest import weakref -from test.support import gc_collect +from test.support import gc_collect, dont_immortalize from test.support import import_helper from test.support.script_helper import assert_python_ok @@ -102,6 +102,7 @@ def f(): gc_collect() self.assertEqual(f().__dict__["bar"], 4) + @dont_immortalize() def test_clear_dict_in_ref_cycle(self): destroyed = [] m = ModuleType("foo") @@ -117,6 +118,7 @@ def __del__(self): gc_collect() self.assertEqual(destroyed, [1]) + @dont_immortalize() def test_weakref(self): m = ModuleType("foo") wr = weakref.ref(m) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 118fcdc73a6..6f11a0cb721 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -941,6 +941,7 @@ def test_debugmallocstats(self): @unittest.skipUnless(hasattr(sys, "getallocatedblocks"), "sys.getallocatedblocks unavailable on this build") + @support.dont_immortalize() def test_getallocatedblocks(self): try: import _testcapi @@ -965,14 +966,6 @@ def test_getallocatedblocks(self): # about the underlying implementation: the function might # return 0 or something greater. self.assertGreaterEqual(a, 0) - try: - # While we could imagine a Python session where the number of - # multiple buffer objects would exceed the sharing of references, - # it is unlikely to happen in a normal test run. - self.assertLess(a, sys.gettotalrefcount()) - except AttributeError: - # gettotalrefcount() not available - pass gc.collect() b = sys.getallocatedblocks() self.assertLessEqual(b, a) diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 8ee4b4fbae2..ce1e4bb295f 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -689,6 +689,7 @@ class D: del c1, c2, C, D gc.collect() + @support.dont_immortalize() def test_callback_in_cycle_resurrection(self): import gc @@ -824,6 +825,7 @@ def test_init(self): # No exception should be raised here gc.collect() + @support.dont_immortalize() def test_classes(self): # Check that classes are weakrefable. class A(object): diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 60ac9c7199c..da1efd71064 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -296,7 +296,7 @@ static bool validate_refcount_visitor(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* args) { VISITOR_BEGIN(block, args); - if (_PyObject_GC_IS_TRACKED(op)) { + if (_PyObject_GC_IS_TRACKED(op) && !_PyObject_IS_IMMORTAL(op)) { assert(_Py_GC_REFCNT(op) >= 0); // assert((op->ob_gc_bits & _PyGC_MASK_TID_REFCOUNT) == 0); } @@ -336,6 +336,31 @@ _PyGC_ResetHeap(void) visit_heaps(reset_heap_visitor, &args); } +static bool +immortalize_heap_visitor(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* args) +{ + VISITOR_BEGIN(block, args); + + Py_ssize_t refcount; + int immortal, deferred; + _PyRef_UnpackLocal(op->ob_ref_local, &refcount, &immortal, &deferred); + + if (deferred) { + _PyObject_SetImmortal(op); + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } + } + return true; +} + +void +_PyGC_DeferredToImmortal(void) +{ + struct visitor_args args; + visit_heaps(immortalize_heap_visitor, &args); +} + /* Subtracts incoming references. */ static int visit_decref(PyObject *op, void *arg) @@ -433,6 +458,15 @@ update_refs(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size return true; }; + if (_PyObject_IS_IMMORTAL(op)) { + _PyObject_GC_UNTRACK(op); + if (gc_is_unreachable(op)) { + gc_clear_unreachable(op); + _PyObject_SetImmortal(op); + } + return true; + } + if (PyTuple_CheckExact(op)) { _PyTuple_MaybeUntrack(op); if (!_PyObject_GC_IS_TRACKED(op)) { diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 07fcd50e3e3..7760c70acb3 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -177,7 +177,9 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname if ((code_obj->co_flags & CO_NESTED) == 0) { _PyObject_SET_DEFERRED_REFCOUNT(op); } - _PyObject_GC_TRACK(op); + if (!_PyObject_IS_IMMORTAL(op)) { + _PyObject_GC_TRACK(op); + } handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return (PyObject *)op; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 7bfe227ca61..cc36f79d493 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -120,7 +120,7 @@ static PyObject * new_module(PyTypeObject *mt, PyObject *args, PyObject *kws) { PyObject *m = (PyObject *)new_module_notrack(mt); - if (m != NULL) { + if (m != NULL && !_PyObject_IS_IMMORTAL(m)) { PyObject_GC_Track(m); } return m; @@ -135,7 +135,9 @@ PyModule_NewObject(PyObject *name) m->md_initialized = 1; if (module_init_dict(m, m->md_dict, name, NULL) != 0) goto fail; - PyObject_GC_Track(m); + if (!_PyObject_IS_IMMORTAL(m)) { + PyObject_GC_Track(m); + } return (PyObject *)m; fail: diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4f152eebcd6..9c211ac6d10 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6862,7 +6862,6 @@ type_ready_managed_dict(PyTypeObject *type) PyHeapTypeObject* et = (PyHeapTypeObject*)type; if (et->ht_cached_keys == NULL) { et->ht_cached_keys = _PyDict_NewKeysForClass(); - assert(_PyObject_GC_IS_TRACKED(et)); if (et->ht_cached_keys == NULL) { PyErr_NoMemory(); return -1; diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 158db40fb1c..c48b1ee9b02 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -456,6 +456,33 @@ sys_setrecursionlimit(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(sys__setimmortalize_deferred__doc__, +"_setimmortalize_deferred($module, immortalize, /)\n" +"--\n" +"\n"); + +#define SYS__SETIMMORTALIZE_DEFERRED_METHODDEF \ + {"_setimmortalize_deferred", (PyCFunction)sys__setimmortalize_deferred, METH_O, sys__setimmortalize_deferred__doc__}, + +static PyObject * +sys__setimmortalize_deferred_impl(PyObject *module, int immortalize); + +static PyObject * +sys__setimmortalize_deferred(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int immortalize; + + immortalize = PyObject_IsTrue(arg); + if (immortalize < 0) { + goto exit; + } + return_value = sys__setimmortalize_deferred_impl(module, immortalize); + +exit: + return return_value; +} + PyDoc_STRVAR(sys_set_coroutine_origin_tracking_depth__doc__, "set_coroutine_origin_tracking_depth($module, /, depth)\n" "--\n" @@ -1337,4 +1364,4 @@ sys_is_stack_trampoline_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=1ef6c758bfe857f7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=42dcbfb142888cb7 input=a9049054013a1b77]*/ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 264b2c0f420..f65cbf4eed3 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -15,6 +15,7 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_list.h" // _PyList_Fini() #include "pycore_long.h" // _PyLong_InitTypes() +#include "pycore_moduleobject.h" // PyModuleObject #include "pycore_mrocache.h" // _Py_mro_cache_init() #include "pycore_object.h" // _PyDebug_PrintTotalRefs() #include "pycore_pathconfig.h" // _PyConfig_WritePathConfig() @@ -1446,6 +1447,15 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) } } +static void +module_swap_dict(PyObject *module) +{ + PyModuleObject *m = (PyModuleObject *)module; + if (_PyObject_IS_IMMORTAL(m->md_dict)) { + PyObject *copy = PyDict_Copy(m->md_dict); + Py_SETREF(m->md_dict, copy); + } +} static PyObject* finalize_remove_modules(PyObject *modules, int verbose) @@ -1476,6 +1486,7 @@ finalize_remove_modules(PyObject *modules, int verbose) if (verbose && PyUnicode_Check(name)) { \ PySys_FormatStderr("# cleanup[2] removing %U\n", name); \ } \ + module_swap_dict(mod); \ STORE_MODULE_WEAKREF(name, mod); \ if (PyObject_SetItem(modules, name, Py_None) < 0) { \ PyErr_WriteUnraisable(NULL); \ diff --git a/Python/pystate.c b/Python/pystate.c index 29e5f3c2ac4..0e44f408098 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1096,6 +1096,8 @@ set_multithreaded(void) if (_PyThreadState_GET() != NULL) { /* creating a new thread from the main thread. */ _Py_atomic_store_int(&_PyRuntime.multithreaded, 1); + _Py_atomic_store_int(&_PyRuntime.immortalize_deferred, 1); + _PyGC_DeferredToImmortal(); } else { /* If we don't have an active thread state, we might be creating a new @@ -1105,6 +1107,8 @@ set_multithreaded(void) */ _PyRuntimeState_StopTheWorld(&_PyRuntime); _Py_atomic_store_int(&_PyRuntime.multithreaded, 1); + _Py_atomic_store_int(&_PyRuntime.immortalize_deferred, 1); + _PyGC_DeferredToImmortal(); _PyRuntimeState_StartTheWorld(&_PyRuntime); } } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 82c68cc3296..1b0fd0558ff 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1224,6 +1224,21 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit) Py_RETURN_NONE; } +/*[clinic input] +sys._setimmortalize_deferred + + immortalize: bool + / +[clinic start generated code]*/ + +static PyObject * +sys__setimmortalize_deferred_impl(PyObject *module, int immortalize) +/*[clinic end generated code: output=42893d14cade9246 input=a4e06e800eb305ba]*/ +{ + _PyRuntime.immortalize_deferred = immortalize; + Py_RETURN_NONE; +} + /*[clinic input] sys.set_coroutine_origin_tracking_depth @@ -2282,6 +2297,7 @@ static PyMethodDef sys_methods[] = { SYS__SETPROFILEALLTHREADS_METHODDEF SYS_GETPROFILE_METHODDEF SYS_SETRECURSIONLIMIT_METHODDEF + SYS__SETIMMORTALIZE_DEFERRED_METHODDEF {"settrace", sys_settrace, METH_O, settrace_doc}, SYS__SETTRACEALLTHREADS_METHODDEF SYS_GETTRACE_METHODDEF