gh-117139: Convert the evaluation stack to stack refs

Include/internal/pycore_call.h

@@ -199,6 +199,24 @@ extern void _PyStack_UnpackDict_FreeNoDecRef(
     PyObject *const *stack,
     PyObject *kwnames);
 
+PyAPI_FUNC(PyObject *)
+PyObject_Vectorcall_Tagged(PyObject *callable,
+                           const _PyStackRef *tagged, size_t nargs, PyObject *kwnames);
+
+PyAPI_FUNC(PyObject *)
+PyObject_TypeVectorcall_Tagged(PyTypeObject *callable,
+                               const _PyStackRef *tagged, size_t nargs, PyObject *kwnames);
+
+PyAPI_FUNC(PyObject *)
+PyObject_PyCFunctionFastCall_Tagged(PyCFunctionFast cfunc,
+                                    PyObject *self,
+                                    const _PyStackRef *tagged, Py_ssize_t nargs);
+PyAPI_FUNC(PyObject *)
+PyObject_PyCFunctionFastWithKeywordsCall_Tagged(PyCFunctionFastWithKeywords cfunc,
+                                                PyObject *self,
+                                                const _PyStackRef *tagged, Py_ssize_t nargs,
+                                                PyObject *kwds);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_ceval.h

@@ -208,7 +208,7 @@ PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *c
 PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs);
 PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs);
 PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys);
-PyAPI_FUNC(int) _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp);
+PyAPI_FUNC(int) _PyEval_UnpackTaggedIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp);
 PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);
 
 

Include/internal/pycore_code.h

@@ -8,6 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_stackref.h" // _PyStackRef
 
 // We hide some of the newer PyCodeObject fields behind macros.
 // This helps with backporting certain changes to 3.12.
@@ -278,7 +279,7 @@ extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub,
 extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr,
                                 int nargs);
 extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
-                                    int oparg, PyObject **locals);
+                                    int oparg, _PyStackRef *locals);
 extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
                                      _Py_CODEUNIT *instr, int oparg);
 extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,

Include/internal/pycore_dict.h

@@ -255,6 +255,14 @@ PyAPI_FUNC(PyObject *)_PyDict_FromItems(
         PyObject *const *keys, Py_ssize_t keys_offset,
         PyObject *const *values, Py_ssize_t values_offset,
         Py_ssize_t length);
+PyAPI_FUNC(PyObject *)_PyDict_FromStackItems(
+    _PyStackRef const *keys, Py_ssize_t keys_offset,
+    _PyStackRef const *values, Py_ssize_t values_offset,
+    Py_ssize_t length);
+PyAPI_FUNC(PyObject *)_PyDict_FromStackItemsUntaggedKeys(
+    PyObject *const *keys, Py_ssize_t keys_offset,
+    _PyStackRef const *values, Py_ssize_t values_offset,
+    Py_ssize_t length);
 
 static inline uint8_t *
 get_insertion_order_array(PyDictValues *values)

Include/internal/pycore_frame.h

@@ -11,6 +11,7 @@ extern "C" {
 #include <stdbool.h>
 #include <stddef.h>               // offsetof()
 #include "pycore_code.h"          // STATS
+#include "pycore_stackref.h"        // _PyStackRef
 
 /* See Objects/frame_layout.md for an explanation of the frame stack
  * including explanation of the PyFrameObject and _PyInterpreterFrame
@@ -67,7 +68,7 @@ typedef struct _PyInterpreterFrame {
     uint16_t return_offset;  /* Only relevant during a function call */
     char owner;
     /* Locals and stack */
-    PyObject *localsplus[1];
+    _PyStackRef localsplus[1];
 } _PyInterpreterFrame;
 
 #define _PyInterpreterFrame_LASTI(IF) \
@@ -78,23 +79,23 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
     return (PyCodeObject *)f->f_executable;
 }
 
-static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) {
-    return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus;
+static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) {
+    return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus);
 }
 
-static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) {
+static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) {
     assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
-    assert(f->localsplus[f->stacktop-1] != NULL);
+    assert(PyStackRef_Get(f->localsplus[f->stacktop-1]) != NULL);
     return f->localsplus[f->stacktop-1];
 }
 
-static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) {
+static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) {
     assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
     f->stacktop--;
     return f->localsplus[f->stacktop];
 }
 
-static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) {
+static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) {
     f->localsplus[f->stacktop] = value;
     f->stacktop++;
 }
@@ -120,6 +121,12 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *
     // Don't leave a dangling pointer to the old frame when creating generators
     // and coroutines:
     dest->previous = NULL;
+#ifdef Py_GIL_DISABLED
+    PyCodeObject *co = (PyCodeObject *)dest->f_executable;
+    for (int i = src->stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) {
+        dest->localsplus[i] = PyStackRef_StealRef(NULL);
+    }
+#endif
 }
 
 /* Consumes reference to func and locals.
@@ -143,14 +150,24 @@ _PyFrame_Initialize(
     frame->owner = FRAME_OWNED_BY_THREAD;
 
     for (int i = null_locals_from; i < code->co_nlocalsplus; i++) {
-        frame->localsplus[i] = NULL;
+        frame->localsplus[i] = PyStackRef_StealRef(NULL);
+    }
+
+#ifdef Py_GIL_DISABLED
+    // On GIL disabled, we walk the entire stack in GC. Since stacktop
+    // is not always in sync with the real stack pointer, we have
+    // no choice but to traverse the entire stack.
+    // This just makes sure we don't pass the GC invalid stack values.
+    for (int i = code->co_nlocalsplus; i < code->co_nlocalsplus + code->co_stacksize; i++) {
+        frame->localsplus[i] = PyStackRef_StealRef(NULL);
     }
+#endif
 }
 
 /* Gets the pointer to the locals array
  * that precedes this frame.
  */
-static inline PyObject**
+static inline _PyStackRef*
 _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
 {
     return frame->localsplus;
@@ -160,16 +177,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
    Having stacktop <= 0 ensures that invalid
    values are not visible to the cycle GC.
    We choose -1 rather than 0 to assist debugging. */
-static inline PyObject**
+static inline _PyStackRef*
 _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 {
-    PyObject **sp = frame->localsplus + frame->stacktop;
+    _PyStackRef *sp = frame->localsplus + frame->stacktop;
     frame->stacktop = -1;
     return sp;
 }
 
 static inline void
-_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer)
+_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
 {
     frame->stacktop = (int)(stack_pointer - frame->localsplus);
 }
@@ -307,6 +324,12 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     frame->instr_ptr = _PyCode_CODE(code);
     frame->owner = FRAME_OWNED_BY_THREAD;
     frame->return_offset = 0;
+#ifdef Py_GIL_DISABLED
+    assert(code->co_nlocalsplus == 0);
+    for (int i = 0; i < code->co_stacksize; i++) {
+        frame->localsplus[i] = PyStackRef_StealRef(NULL);
+    }
+#endif
     return frame;
 }
 

Include/internal/pycore_gc.h

@@ -355,6 +355,9 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
 extern void _Py_ScheduleGC(PyThreadState *tstate);
 extern void _Py_RunGC(PyThreadState *tstate);
 
+extern int _Py_visit_decref(PyObject *op, void *arg);
+extern int _Py_visit_decref_unreachable(PyObject *op, void *data);
+
 #ifdef Py_GIL_DISABLED
 // gh-117783: Immortalize objects that use deferred reference counting
 extern void _PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp);

Include/internal/pycore_jit.h

@@ -11,7 +11,7 @@ extern "C" {
 
 #ifdef _Py_JIT
 
-typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate);
+typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate);
 
 int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
 void _PyJIT_Free(_PyExecutorObject *executor);

Include/internal/pycore_list.h

@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_freelist.h"  // _PyFreeListState
+#include "pycore_stackref.h" // _PyStackRef
 
 PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
 extern void _PyList_DebugMallocStats(FILE *out);
@@ -54,7 +55,7 @@ typedef struct {
     PyListObject *it_seq; /* Set to NULL when iterator is exhausted */
 } _PyListIterObject;
 
-PyAPI_FUNC(PyObject *)_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n);
+PyAPI_FUNC(PyObject *)_PyList_FromStackSteal(_PyStackRef const *src, Py_ssize_t n);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_object.h

@@ -159,20 +159,6 @@ static inline void _Py_ClearImmortal(PyObject *op)
         op = NULL; \
     } while (0)
 
-// Mark an object as supporting deferred reference counting. This is a no-op
-// in the default (with GIL) build. Objects that use deferred reference
-// counting should be tracked by the GC so that they are eventually collected.
-extern void _PyObject_SetDeferredRefcount(PyObject *op);
-
-static inline int
-_PyObject_HasDeferredRefcount(PyObject *op)
-{
-#ifdef Py_GIL_DISABLED
-    return (op->ob_gc_bits & _PyGC_BITS_DEFERRED) != 0;
-#else
-    return 0;
-#endif
-}
 
 #if !defined(Py_GIL_DISABLED)
 static inline void
@@ -709,6 +695,7 @@ PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *);
 extern int _PyObject_IsAbstract(PyObject *);
 
 PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);
+PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyObject *obj, PyObject *name, _PyStackRef *method);
 extern PyObject* _PyObject_NextNotImplemented(PyObject *);
 
 // Pickle support.

Include/internal/pycore_object_deferred.h

@@ -0,0 +1,32 @@
+#ifndef Py_INTERNAL_OBJECT_DEFERRED_H
+#define Py_INTERNAL_OBJECT_DEFERRED_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "pycore_gc.h"
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+// Mark an object as supporting deferred reference counting. This is a no-op
+// in the default (with GIL) build. Objects that use deferred reference
+// counting should be tracked by the GC so that they are eventually collected.
+extern void _PyObject_SetDeferredRefcount(PyObject *op);
+
+static inline int
+_PyObject_HasDeferredRefcount(PyObject *op)
+{
+#ifdef Py_GIL_DISABLED
+    return (op->ob_gc_bits & _PyGC_BITS_DEFERRED) != 0;
+#else
+    return 0;
+#endif
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // !Py_INTERNAL_OBJECT_DEFERRED_H

Include/internal/pycore_optimizer.h

@@ -114,7 +114,7 @@ extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx);
 
 PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
 
-PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr);
+PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_sliceobject.h

@@ -12,7 +12,7 @@ extern "C" {
 /* runtime lifecycle */
 
 PyAPI_FUNC(PyObject *)
-_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop);
+_PyBuildSlice_ConsumeStackRefs(_PyStackRef start, _PyStackRef stop);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_stackref.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_object_deferred.h"
+
 #include <stddef.h>
 
 typedef union {
@@ -52,7 +54,7 @@ _PyStackRef_NewRefDeferred(PyObject *obj)
     // Make sure we don't take an already tagged value.
     assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0);
     assert(obj != NULL);
-    if (_PyObject_HasDeferredRefcount(obj)) {
+    if (_PyObject_HasDeferredRefcount(obj) || _Py_IsImmortal(obj)) {
         return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
     }
     else {
@@ -86,7 +88,8 @@ static inline PyObject *
 PyStackRef_StealObject(_PyStackRef tagged)
 {
     if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
-        assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged)));
+        assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged)) ||
+            _Py_IsImmortal(PyStackRef_Get(tagged)));
         return Py_NewRef(PyStackRef_Get(tagged));
     }
     return PyStackRef_Get(tagged);
@@ -114,7 +117,7 @@ _Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length)
 
 #define PyStackRef_XSETREF(dst, src) \
     do { \
-        _PyStackRef *_tmp_dst_ptr = &(dst) \
+        _PyStackRef *_tmp_dst_ptr = &(dst); \
         _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \
         *_tmp_dst_ptr = (src); \
         PyStackRef_XDECREF(_tmp_old_dst); \

Include/internal/pycore_tuple.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_stackref.h" // _PyStackRef
+
 extern void _PyTuple_MaybeUntrack(PyObject *);
 extern void _PyTuple_DebugMallocStats(FILE *out);
 
@@ -22,6 +24,7 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *);
 
 extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t);
 PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t);
+PyAPI_FUNC(PyObject *)_PyTuple_FromStackSteal(_PyStackRef const *, Py_ssize_t);
 
 typedef struct {
     PyObject_HEAD

Include/internal/pycore_unicodeobject.h

@@ -12,6 +12,7 @@ extern "C" {
 #include "pycore_fileutils.h"     // _Py_error_handler
 #include "pycore_identifier.h"    // _Py_Identifier
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
+#include "pycore_stackref.h" // _PyStackRef
 
 /* --- Characters Type APIs ----------------------------------------------- */
 
@@ -208,6 +209,12 @@ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray(
     Py_ssize_t seqlen
     );
 
+PyAPI_FUNC(PyObject*) _PyUnicode_JoinStack(
+    PyObject *separator,
+    _PyStackRef const *items,
+    Py_ssize_t seqlen
+);
+
 /* Test whether a unicode is equal to ASCII identifier.  Return 1 if true,
    0 otherwise.  The right argument must be ASCII identifier.
    Any error occurs inside will be cleared before return. */