bpo-36389: Add gc.enable_object_debugger()

Doc/library/gc.rst

@@ -263,6 +263,39 @@ values but should not rebind them):
    .. versionadded:: 3.3
 
 
+The :mod:`gc` module provides an "object debugger" which checks frequently if
+all Python objects tracked by the garbage collector look valid:
+
+* check that the reference counter is greater than or equal to 1;
+* check that the pointer to the type is not NULL;
+* if debug hooks on memory allocators (:c:func:`PyMem_SetupDebugHooks`) are
+  enabled (:envvar:`PYTHONMALLOC` environment variable set to ``"debug"`` or
+  :option:`-X` ``dev`` command line option), detect freed memory.
+
+This debugger aims to debug bugs in C extensions.
+
+.. function:: enable_object_debugger(threshold)
+
+   Enable the object debugger.
+
+   Check that all Python objects tracked by the garbage collector look valid
+   every *threshold* memory allocation or deallocation made by the garbage
+   collector.
+
+   Low *threshold* can have a significant negative impact on Python
+   performance, but should detect earlier Python objects which look invalid.
+
+   *threshold* must be greater than or equal to 1.
+
+   .. versionadded:: 3.8
+
+.. function:: disable_object_debugger()
+
+   Disable the object debugger.
+
+   .. versionadded:: 3.8
+
+
 The following constants are provided for use with :func:`set_debug`:
 
 

Doc/whatsnew/3.8.rst

@@ -206,6 +206,11 @@ gc
 indicating a generation to get objects from. Contributed in
 :issue:`36016` by Pablo Galindo.
 
+New "object debugger" which checks frequently if all Python objects tracked by
+the garbage collector look valid: :func:`gc.enable_object_debugger` and
+:func:`gc.disable_object_debugger`. This debugger aims to debug bugs in C
+extensions. Contributed in :issue:`36389` by Victor Stinner.
+
 
 gzip
 ----

Include/internal/pycore_object.h

@@ -76,6 +76,8 @@ static inline void _PyObject_GC_UNTRACK_impl(const char *filename, int lineno,
 #define _PyObject_GC_UNTRACK(op) \
     _PyObject_GC_UNTRACK_impl(__FILE__, __LINE__, _PyObject_CAST(op))
 
+PyAPI_FUNC(void) _PyGC_DisableObjectDebugger(void);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_pymem.h

@@ -111,6 +111,15 @@ struct gc_generation_stats {
     Py_ssize_t uncollectable;
 };
 
+struct _gc_object_debugger {
+    int enabled;
+    struct gc_obj_dbg_gen {
+        int threshold; /* collection threshold */
+        int count; /* count of allocations or collections of younger
+                      generations */
+    } generations[NUM_GENERATIONS];
+};
+
 struct _gc_runtime_state {
     /* List of objects that still need to be cleaned up, singly linked
      * via their gc headers' gc_prev pointers.  */
@@ -143,6 +152,8 @@ struct _gc_runtime_state {
        collections, and are awaiting to undergo a full collection for
        the first time. */
     Py_ssize_t long_lived_pending;
+
+    struct _gc_object_debugger object_debugger;
 };
 
 PyAPI_FUNC(void) _PyGC_Initialize(struct _gc_runtime_state *);

Lib/test/test_gc.py

@@ -823,6 +823,35 @@ def test_get_objects_arguments(self):
         self.assertRaises(TypeError, gc.get_objects, 1.234)
 
 
+@unittest.skipUnless(hasattr(gc, 'enable_object_debugger'),
+                     'need gc.enable_object_debugger()')
+class ObjectDebuggerTests(unittest.TestCase):
+    def test_basic(self):
+        try:
+            # Call the object debugger around 2 times
+            # (the test only needs that it's called at least once)
+            gc.enable_object_debugger(10)
+            objs = [{} for _ in range(20)]
+        finally:
+            gc.disable_object_debugger()
+
+    def test_thresholds(self):
+        try:
+            big = 10 ** 7
+            gc.enable_object_debugger(big)
+            gc.enable_object_debugger(big, big)
+            gc.enable_object_debugger(big, big, big)
+        finally:
+            gc.disable_object_debugger()
+
+    def test_invalid_thresholds(self):
+        self.assertRaises(ValueError, gc.enable_object_debugger, -1)
+        self.assertRaises(ValueError, gc.enable_object_debugger, 0)
+        self.assertRaises(OverflowError, gc.enable_object_debugger, 2 ** 100)
+        self.assertRaises(OverflowError, gc.enable_object_debugger, 1, 2 ** 100)
+        self.assertRaises(OverflowError, gc.enable_object_debugger, 1, 1, 2 ** 100)
+
+
 class GCCallbackTests(unittest.TestCase):
     def setUp(self):
         # Save gc state and disable it.

Misc/NEWS.d/next/Core and Builtins/2019-03-21-12-09-05.bpo-36389.EkpMZP.rst

@@ -0,0 +1,4 @@
+New "object debugger" which checks frequently if all Python objects tracked
+by the garbage collector are consistent: :func:`gc.enable_object_debugger`
+and :func:`gc.disable_object_debugger`. Contributed in :issue:`36389` by Victor
+Stinner.