objects: immortalize some objects

This implementation doesn't currently avoid all the reference counts
on deferred RC objects that would be necessary for efficient
multi-threaded scaling. To support this, in multi-threaded programs,
convert objects that use deferred reference counting to immortal
objects. I don't expect this to be a permanent change.

Include/internal/pycore_gc.h

@@ -200,6 +200,7 @@ extern void _PyGC_InitState(struct _gc_runtime_state *);
 
 extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate);
 extern void _PyGC_ResetHeap(void);
+extern void _PyGC_DeferredToImmortal(void);
 
 static inline int
 _PyGC_ShouldCollect(struct _gc_runtime_state *gcstate)

Include/internal/pycore_object.h

@@ -134,7 +134,9 @@ static inline void _PyObject_GC_TRACK(
     _PyObject_ASSERT_FROM(op, !_PyObject_GC_IS_TRACKED(op),
                           "object already tracked by the garbage collector",
                           filename, lineno, __func__);
-    op->ob_gc_bits |= _PyGC_MASK_TRACKED;
+    if (!_PyObject_IS_IMMORTAL(op)) {
+        op->ob_gc_bits |= _PyGC_MASK_TRACKED;
+    }
 }
 
 /* Tell the GC to stop tracking this object.
@@ -359,8 +361,16 @@ _PyObject_SetDeferredRefcount(PyObject *op)
     assert(_Py_ThreadLocal(op) && "non thread-safe");
     assert(!_PyObject_HasDeferredRefcount(op) && "already uses deferred refcounting");
     assert(PyType_IS_GC(Py_TYPE(op)));
-    op->ob_ref_local += _Py_REF_DEFERRED_MASK + 1;
-    op->ob_ref_shared = (op->ob_ref_shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_QUEUED;
+    if (_PyRuntime.immortalize_deferred) {
+        _PyObject_SetImmortal(op);
+        if (_PyObject_GC_IS_TRACKED(op)) {
+            _PyObject_GC_UNTRACK(op);
+        }
+    }
+    else {
+        op->ob_ref_local += _Py_REF_DEFERRED_MASK + 1;
+        op->ob_ref_shared = (op->ob_ref_shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_QUEUED;
+    }
 }
 
 #define _PyObject_SET_DEFERRED_REFCOUNT(op) _PyObject_SetDeferredRefcount(_PyObject_CAST(op))

Include/internal/pycore_runtime.h

@@ -87,6 +87,11 @@ typedef struct pyruntimestate {
        is created ONLY IF the GIL is disabled. */
     int multithreaded;
 
+    /* Temporary: if 1, immortalize objects that would use deferred reference
+       counting in multithreaded programs. (Simluates deferred reference counting
+       scalability in multi-threaded programs). */
+    int immortalize_deferred;
+
     /* Has Python started the process of stopping all threads? Protected by HEAD_LOCK() */
     int stop_the_world_requested;
 

Include/object.h

@@ -665,6 +665,7 @@ _PyObject_IS_IMMORTAL(PyObject *op)
 {
     return _Py_REF_IS_IMMORTAL(_Py_atomic_load_uint32_relaxed(&op->ob_ref_local));
 }
+#define _PyObject_IS_IMMORTAL(op) _PyObject_IS_IMMORTAL(_PyObject_CAST(op))
 
 static inline Py_ssize_t
 _Py_REF_SHARED_COUNT(Py_ssize_t shared)

Lib/test/support/__init__.py

@@ -2192,6 +2192,15 @@ def set_recursion_limit(limit):
     finally:
         sys.setrecursionlimit(original_limit)
 
+@contextlib.contextmanager
+def dont_immortalize():
+    """Temporarily change immortalization/deferred behavior."""
+    try:
+        sys._setimmortalize_deferred(False)
+        yield
+    finally:
+        sys._setimmortalize_deferred(True)
+
 def infinite_recursion(max_depth=75):
     """Set a lower limit for tests that interact with infinite recursions
     (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some

Lib/test/test_capi/test_watchers.py

@@ -1,7 +1,8 @@
 import unittest
 
 from contextlib import contextmanager, ExitStack
-from test.support import catch_unraisable_exception, import_helper, gc_collect
+from test.support import (catch_unraisable_exception, import_helper,
+                          gc_collect, dont_immortalize)
 
 
 # Skip this test if the _testcapi module isn't available.
@@ -357,6 +358,7 @@ def assert_event_counts(self, exp_created_0, exp_destroyed_0,
         self.assertEqual(
             exp_destroyed_1, _testcapi.get_code_watcher_num_destroyed_events(1))
 
+    @dont_immortalize()
     def test_code_object_events_dispatched(self):
         # verify that all counts are zero before any watchers are registered
         self.assert_event_counts(0, 0, 0, 0)

Lib/test/test_code.py

@@ -140,7 +140,7 @@
     ctypes = None
 from test.support import (cpython_only,
                           check_impl_detail, requires_debug_ranges,
-                          gc_collect)
+                          gc_collect, dont_immortalize)
 from test.support.script_helper import assert_python_ok
 from test.support import threading_helper
 from opcode import opmap, opname
@@ -545,6 +545,7 @@ def test_interned_string_with_null(self):
 
 class CodeWeakRefTest(unittest.TestCase):
 
+    @dont_immortalize()
     def test_basic(self):
         # Create a code object in a clean environment so that we know we have
         # the only reference to it left.

Lib/test/test_functools.py

@@ -1829,6 +1829,7 @@ def f():
             return 1
         self.assertEqual(f.cache_parameters(), {'maxsize': 1000, "typed": True})
 
+    @support.dont_immortalize()
     def test_lru_cache_weakrefable(self):
         @self.module.lru_cache
         def test_function(x):

Lib/test/test_module.py

@@ -1,7 +1,7 @@
 # Test the module type
 import unittest
 import weakref
-from test.support import gc_collect
+from test.support import gc_collect, dont_immortalize
 from test.support import import_helper
 from test.support.script_helper import assert_python_ok
 
@@ -102,6 +102,7 @@ def f():
         gc_collect()
         self.assertEqual(f().__dict__["bar"], 4)
 
+    @dont_immortalize()
     def test_clear_dict_in_ref_cycle(self):
         destroyed = []
         m = ModuleType("foo")
@@ -117,6 +118,7 @@ def __del__(self):
         gc_collect()
         self.assertEqual(destroyed, [1])
 
+    @dont_immortalize()
     def test_weakref(self):
         m = ModuleType("foo")
         wr = weakref.ref(m)

Lib/test/test_sys.py

@@ -941,6 +941,7 @@ def test_debugmallocstats(self):
 
     @unittest.skipUnless(hasattr(sys, "getallocatedblocks"),
                          "sys.getallocatedblocks unavailable on this build")
+    @support.dont_immortalize()
     def test_getallocatedblocks(self):
         try:
             import _testcapi
@@ -965,14 +966,6 @@ def test_getallocatedblocks(self):
             # about the underlying implementation: the function might
             # return 0 or something greater.
             self.assertGreaterEqual(a, 0)
-        try:
-            # While we could imagine a Python session where the number of
-            # multiple buffer objects would exceed the sharing of references,
-            # it is unlikely to happen in a normal test run.
-            self.assertLess(a, sys.gettotalrefcount())
-        except AttributeError:
-            # gettotalrefcount() not available
-            pass
         gc.collect()
         b = sys.getallocatedblocks()
         self.assertLessEqual(b, a)

Lib/test/test_weakref.py

@@ -689,6 +689,7 @@ class D:
         del c1, c2, C, D
         gc.collect()
 
+    @support.dont_immortalize()
     def test_callback_in_cycle_resurrection(self):
         import gc
 
@@ -824,6 +825,7 @@ def test_init(self):
         # No exception should be raised here
         gc.collect()
 
+    @support.dont_immortalize()
     def test_classes(self):
         # Check that classes are weakrefable.
         class A(object):

Modules/gcmodule.c

@@ -296,7 +296,7 @@ static bool
 validate_refcount_visitor(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* args)
 {
     VISITOR_BEGIN(block, args);
-    if (_PyObject_GC_IS_TRACKED(op)) {
+    if (_PyObject_GC_IS_TRACKED(op) && !_PyObject_IS_IMMORTAL(op)) {
         assert(_Py_GC_REFCNT(op) >= 0);
         // assert((op->ob_gc_bits & _PyGC_MASK_TID_REFCOUNT) == 0);
     }
@@ -336,6 +336,31 @@ _PyGC_ResetHeap(void)
     visit_heaps(reset_heap_visitor, &args);
 }
 
+static bool
+immortalize_heap_visitor(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* args)
+{
+    VISITOR_BEGIN(block, args);
+
+    Py_ssize_t refcount;
+    int immortal, deferred;
+    _PyRef_UnpackLocal(op->ob_ref_local, &refcount, &immortal, &deferred);
+
+    if (deferred) {
+        _PyObject_SetImmortal(op);
+        if (_PyObject_GC_IS_TRACKED(op)) {
+            _PyObject_GC_UNTRACK(op);
+        }
+    }
+    return true;
+}
+
+void
+_PyGC_DeferredToImmortal(void)
+{
+    struct visitor_args args;
+    visit_heaps(immortalize_heap_visitor, &args);
+}
+
 /* Subtracts incoming references. */
 static int
 visit_decref(PyObject *op, void *arg)
@@ -433,6 +458,15 @@ update_refs(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size
         return true;
     };
 
+    if (_PyObject_IS_IMMORTAL(op)) {
+        _PyObject_GC_UNTRACK(op);
+        if (gc_is_unreachable(op)) {
+            gc_clear_unreachable(op);
+            _PyObject_SetImmortal(op);
+        }
+        return true;
+    }
+
     if (PyTuple_CheckExact(op)) {
         _PyTuple_MaybeUntrack(op);
         if (!_PyObject_GC_IS_TRACKED(op)) {

Objects/funcobject.c

@@ -177,7 +177,9 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname
     if ((code_obj->co_flags & CO_NESTED) == 0) {
         _PyObject_SET_DEFERRED_REFCOUNT(op);
     }
-    _PyObject_GC_TRACK(op);
+    if (!_PyObject_IS_IMMORTAL(op)) {
+        _PyObject_GC_TRACK(op);
+    }
     handle_func_event(PyFunction_EVENT_CREATE, op, NULL);
     return (PyObject *)op;
 

Objects/moduleobject.c

@@ -120,7 +120,7 @@ static PyObject *
 new_module(PyTypeObject *mt, PyObject *args, PyObject *kws)
 {
     PyObject *m = (PyObject *)new_module_notrack(mt);
-    if (m != NULL) {
+    if (m != NULL && !_PyObject_IS_IMMORTAL(m)) {
         PyObject_GC_Track(m);
     }
     return m;
@@ -135,7 +135,9 @@ PyModule_NewObject(PyObject *name)
     m->md_initialized = 1;
     if (module_init_dict(m, m->md_dict, name, NULL) != 0)
         goto fail;
-    PyObject_GC_Track(m);
+    if (!_PyObject_IS_IMMORTAL(m)) {
+        PyObject_GC_Track(m);
+    }
     return (PyObject *)m;
 
  fail:

Objects/typeobject.c

@@ -6862,7 +6862,6 @@ type_ready_managed_dict(PyTypeObject *type)
     PyHeapTypeObject* et = (PyHeapTypeObject*)type;
     if (et->ht_cached_keys == NULL) {
         et->ht_cached_keys = _PyDict_NewKeysForClass();
-        assert(_PyObject_GC_IS_TRACKED(et));
         if (et->ht_cached_keys == NULL) {
             PyErr_NoMemory();
             return -1;

Python/pylifecycle.c

@@ -15,6 +15,7 @@
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_list.h"          // _PyList_Fini()
 #include "pycore_long.h"          // _PyLong_InitTypes()
+#include "pycore_moduleobject.h"  //  PyModuleObject
 #include "pycore_mrocache.h"      // _Py_mro_cache_init()
 #include "pycore_object.h"        // _PyDebug_PrintTotalRefs()
 #include "pycore_pathconfig.h"    // _PyConfig_WritePathConfig()
@@ -1446,6 +1447,15 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose)
     }
 }
 
+static void
+module_swap_dict(PyObject *module)
+{
+    PyModuleObject *m = (PyModuleObject *)module;
+    if (_PyObject_IS_IMMORTAL(m->md_dict)) {
+        PyObject *copy = PyDict_Copy(m->md_dict);
+        Py_SETREF(m->md_dict, copy);
+    }
+}
 
 static PyObject*
 finalize_remove_modules(PyObject *modules, int verbose)
@@ -1476,6 +1486,7 @@ finalize_remove_modules(PyObject *modules, int verbose)
             if (verbose && PyUnicode_Check(name)) { \
                 PySys_FormatStderr("# cleanup[2] removing %U\n", name); \
             } \
+            module_swap_dict(mod); \
             STORE_MODULE_WEAKREF(name, mod); \
             if (PyObject_SetItem(modules, name, Py_None) < 0) { \
                 PyErr_WriteUnraisable(NULL); \

Python/pystate.c

@@ -1096,6 +1096,8 @@ set_multithreaded(void)
     if (_PyThreadState_GET() != NULL) {
         /* creating a new thread from the main thread. */
         _Py_atomic_store_int(&_PyRuntime.multithreaded, 1);
+        _Py_atomic_store_int(&_PyRuntime.immortalize_deferred, 1);
+        _PyGC_DeferredToImmortal();
     }
     else {
         /* If we don't have an active thread state, we might be creating a new
@@ -1105,6 +1107,8 @@ set_multithreaded(void)
          */
         _PyRuntimeState_StopTheWorld(&_PyRuntime);
         _Py_atomic_store_int(&_PyRuntime.multithreaded, 1);
+        _Py_atomic_store_int(&_PyRuntime.immortalize_deferred, 1);
+        _PyGC_DeferredToImmortal();
         _PyRuntimeState_StartTheWorld(&_PyRuntime);
     }
 }

Python/sysmodule.c

@@ -1224,6 +1224,21 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit)
     Py_RETURN_NONE;
 }
 
+/*[clinic input]
+sys._setimmortalize_deferred
+
+    immortalize: bool
+    /
+[clinic start generated code]*/
+
+static PyObject *
+sys__setimmortalize_deferred_impl(PyObject *module, int immortalize)
+/*[clinic end generated code: output=42893d14cade9246 input=a4e06e800eb305ba]*/
+{
+    _PyRuntime.immortalize_deferred = immortalize;
+    Py_RETURN_NONE;
+}
+
 /*[clinic input]
 sys.set_coroutine_origin_tracking_depth
 
@@ -2282,6 +2297,7 @@ static PyMethodDef sys_methods[] = {
     SYS__SETPROFILEALLTHREADS_METHODDEF
     SYS_GETPROFILE_METHODDEF
     SYS_SETRECURSIONLIMIT_METHODDEF
+    SYS__SETIMMORTALIZE_DEFERRED_METHODDEF
     {"settrace", sys_settrace, METH_O, settrace_doc},
     SYS__SETTRACEALLTHREADS_METHODDEF
     SYS_GETTRACE_METHODDEF