GH-100987: Allow non python frames in frame stack.

Include/cpython/ceval.h

@@ -2,6 +2,7 @@
 #  error "this header file must not be included directly"
 #endif
 
+
 PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *);
 PyAPI_FUNC(void) PyEval_SetProfileAllThreads(Py_tracefunc, PyObject *);
 PyAPI_DATA(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg);

Include/cpython/pystate.h

@@ -59,6 +59,8 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *);
 #define PyTrace_OPCODE 7
 
 
+struct _PyInterpreterFrame;
+
 typedef struct {
     PyCodeObject *code; // The code object for the bounds. May be NULL.
     PyCodeAddressRange bounds; // Only valid if code != NULL.
@@ -79,7 +81,7 @@ typedef struct _PyCFrame {
      */
     uint8_t use_tracing;  // 0 or 255 (or'ed into opcode, hence 8-bit type)
     /* Pointer to the currently executing frame (it can be NULL) */
-    struct _PyInterpreterFrame *current_frame;
+    struct _PyFrame *current_frame;
     struct _PyCFrame *previous;
 } _PyCFrame;
 

Include/internal/pycore_ceval.h

@@ -80,6 +80,8 @@ extern PyStatus _PyPerfTrampoline_AfterFork_Child(void);
 extern _PyPerf_Callbacks _Py_perfmap_callbacks;
 #endif
 
+struct _PyInterpreterFrame;
+
 static inline PyObject*
 _PyEval_EvalFrame(PyThreadState *tstate, struct _PyInterpreterFrame *frame, int throwflag)
 {

Include/internal/pycore_frame.h

@@ -46,9 +46,13 @@ enum _frameowner {
     FRAME_OWNED_BY_CSTACK = 3,
 };
 
+typedef struct _PyFrame {
+    PyObject *f_executable; /* Strong reference */
+    struct _PyFrame *previous;
+} _PyFrame;
+
 typedef struct _PyInterpreterFrame {
-    PyCodeObject *f_code; /* Strong reference */
-    struct _PyInterpreterFrame *previous;
+    _PyFrame base;
     PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */
     PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */
     PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */
@@ -67,20 +71,25 @@ typedef struct _PyInterpreterFrame {
 } _PyInterpreterFrame;
 
 #define _PyInterpreterFrame_LASTI(IF) \
-    ((int)((IF)->prev_instr - _PyCode_CODE((IF)->f_code)))
+    ((int)((IF)->prev_instr - _PyCode_CODE(_PyFrame_GetCode(IF))))
+
+static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
+    assert(PyCode_Check(f->base.f_executable));
+    return (PyCodeObject *)f->base.f_executable;
+}
 
 static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) {
-    return f->localsplus + f->f_code->co_nlocalsplus;
+    return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus;
 }
 
 static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) {
-    assert(f->stacktop > f->f_code->co_nlocalsplus);
+    assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
     assert(f->localsplus[f->stacktop-1] != NULL);
     return f->localsplus[f->stacktop-1];
 }
 
 static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) {
-    assert(f->stacktop > f->f_code->co_nlocalsplus);
+    assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
     f->stacktop--;
     return f->localsplus[f->stacktop];
 }
@@ -113,7 +122,7 @@ _PyFrame_Initialize(
     PyObject *locals, PyCodeObject *code, int null_locals_from)
 {
     frame->f_funcobj = (PyObject *)func;
-    frame->f_code = (PyCodeObject *)Py_NewRef(code);
+    frame->base.f_executable = Py_NewRef(code);
     frame->f_builtins = func->func_builtins;
     frame->f_globals = func->func_globals;
     frame->f_locals = locals;
@@ -158,19 +167,26 @@ _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer)
  * Frames owned by a generator are always complete.
  */
 static inline bool
-_PyFrame_IsIncomplete(_PyInterpreterFrame *frame)
+_PyFrame_IsIncomplete(_PyFrame *frameptr)
 {
+    if (!PyCode_Check(frameptr->f_executable)) {
+        return true;
+    }
+    _PyInterpreterFrame *frame = (_PyInterpreterFrame *)frameptr;
+    if (frame->owner == FRAME_OWNED_BY_CSTACK) {
+        return true;
+    }
     return frame->owner != FRAME_OWNED_BY_GENERATOR &&
-    frame->prev_instr < _PyCode_CODE(frame->f_code) + frame->f_code->_co_firsttraceable;
+    frame->prev_instr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable;
 }
 
 static inline _PyInterpreterFrame *
-_PyFrame_GetFirstComplete(_PyInterpreterFrame *frame)
+_PyFrame_GetFirstComplete(_PyFrame *frame)
 {
-    while (frame && _PyFrame_IsIncomplete(frame)) {
+    while (frame != NULL && _PyFrame_IsIncomplete(frame)) {
         frame = frame->previous;
     }
-    return frame;
+    return (_PyInterpreterFrame *)frame;
 }
 
 static inline _PyInterpreterFrame *
@@ -191,7 +207,7 @@ static inline PyFrameObject *
 _PyFrame_GetFrameObject(_PyInterpreterFrame *frame)
 {
 
-    assert(!_PyFrame_IsIncomplete(frame));
+    assert(!_PyFrame_IsIncomplete(&frame->base));
     PyFrameObject *res =  frame->frame_obj;
     if (res != NULL) {
         return res;

Lib/test/test_capi/test_misc.py

@@ -206,6 +206,7 @@ def test_return_null_without_error(self):
                     r'returned NULL without setting an exception\n'
                 r'\n'
                 r'Current thread.*:\n'
+                    r'  Builtin function return_null_without_error\n'
                 r'  File .*", line 6 in <module>\n')
         else:
             with self.assertRaises(SystemError) as cm:
@@ -240,6 +241,7 @@ def test_return_result_with_error(self):
                         r'returned a result with an exception set\n'
                     r'\n'
                     r'Current thread.*:\n'
+                    r'  Builtin function return_result_with_error\n'
                     r'  File .*, line 6 in <module>\n')
         else:
             with self.assertRaises(SystemError) as cm:
@@ -270,6 +272,7 @@ def test_getitem_with_error(self):
                     r'ValueError: bug\n'
                     r'\n'
                     r'Current thread .* \(most recent call first\):\n'
+                    r'  Builtin function getitem_with_error\n'
                     r'  File .*, line 6 in <module>\n'
                     r'\n'
                     r'Extension modules: _testcapi \(total: 1\)\n')

Lib/test/test_faulthandler.py

@@ -118,7 +118,9 @@ def check_error(self, code, lineno, fatal_error, *,
         # Enable MULTILINE flag
         regex = f'(?m){regex}'
         output, exitcode = self.get_output(code, filename=filename, fd=fd)
-        output = '\n'.join(output)
+        output = '\n'.join([line for line in output if
+                            "Builtin function" not in line and
+                            "Method descriptor" not in line])
         self.assertRegex(output, regex)
         self.assertNotEqual(exitcode, 0)
 
@@ -484,11 +486,19 @@ def funcA():
             lineno = 14
         expected = [
             'Stack (most recent call first):',
+            '  Builtin function dump_traceback',
             '  File "<string>", line %s in funcB' % lineno,
             '  File "<string>", line 17 in funcA',
             '  File "<string>", line 19 in <module>'
         ]
         trace, exitcode = self.get_output(code, filename, fd)
+        if trace != expected:
+            print("Trace")
+            print("-----------------")
+            print(trace)
+            print("Expected")
+            print("-----------------")
+            print(expected)
         self.assertEqual(trace, expected)
         self.assertEqual(exitcode, 0)
 
@@ -522,6 +532,7 @@ def {func_name}():
         )
         expected = [
             'Stack (most recent call first):',
+            '  Builtin function dump_traceback',
             '  File "<string>", line 4 in %s' % truncated,
             '  File "<string>", line 6 in <module>'
         ]
@@ -568,7 +579,9 @@ def run(self):
             """
         code = code.format(filename=repr(filename))
         output, exitcode = self.get_output(code, filename)
-        output = '\n'.join(output)
+        output = '\n'.join([line for line in output if
+                            "Builtin function" not in line and
+                            "Method descriptor" not in line])
         if filename:
             lineno = 8
         else:
@@ -644,8 +657,9 @@ def func(timeout, repeat, cancel, file, loops):
             fd=fd,
         )
         trace, exitcode = self.get_output(code, filename)
-        trace = '\n'.join(trace)
-
+        trace = '\n'.join([line for line in trace if
+                            "Builtin function" not in line and
+                            "Method descriptor" not in line])
         if not cancel:
             count = loops
             if repeat:
@@ -747,7 +761,7 @@ def handler(signum, frame):
             fd=fd,
         )
         trace, exitcode = self.get_output(code, filename)
-        trace = '\n'.join(trace)
+        trace = '\n'.join([line for line in trace if "Builtin function" not in line ])
         if not unregister:
             if all_threads:
                 regex = r'Current thread 0x[0-9a-f]+ \(most recent call first\):\n'

Modules/_tracemalloc.c

@@ -263,7 +263,7 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame)
     }
     frame->lineno = (unsigned int)lineno;
 
-    PyObject *filename = pyframe->f_code->co_filename;
+    PyObject *filename = _PyFrame_GetCode(pyframe)->co_filename;
 
     if (filename == NULL) {
 #ifdef TRACE_DEBUG
@@ -358,7 +358,7 @@ traceback_get_frames(traceback_t *traceback)
         if (traceback->total_nframe < UINT16_MAX) {
             traceback->total_nframe++;
         }
-        pyframe = _PyFrame_GetFirstComplete(pyframe->previous);
+        pyframe = _PyFrame_GetFirstComplete(pyframe->base.previous);
     }
 }
 

Modules/_xxsubinterpretersmodule.c

@@ -385,8 +385,7 @@ _is_running(PyInterpreterState *interp)
     }
 
     assert(!PyErr_Occurred());
-    _PyInterpreterFrame *frame = tstate->cframe->current_frame;
-    if (frame == NULL) {
+    if (tstate->cframe->current_frame == NULL) {
         return 0;
     }
     return 1;