gh-142598: Add new type of executable to support external JITs

Include/cpython/pyframe.h

@@ -40,6 +40,7 @@ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *
 #define PyUnstable_EXECUTABLE_KIND_PY_FUNCTION 1
 #define PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION 3
 #define PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4
-#define PyUnstable_EXECUTABLE_KINDS 5
+#define PyUnstable_EXECUTABLE_KIND_JIT 5
+#define PyUnstable_EXECUTABLE_KINDS 6
 
-PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1];
+PyAPI_DATA(PyTypeObject *) PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1];

Include/internal/pycore_debug_offsets.h

@@ -131,6 +131,10 @@ typedef struct _Py_DebugOffsets {
         uint64_t tlbc_index;
     } interpreter_frame;
 
+    struct _interpreter_frame_metadata {
+        uintptr_t executable_kinds;
+    } interpreter_frame_metadata;
+
     // Code object offset;
     struct _code_object {
         uint64_t size;
@@ -146,6 +150,10 @@ typedef struct _Py_DebugOffsets {
         uint64_t co_tlbc;
     } code_object;
 
+    struct _jit_executable {
+        uint64_t code;
+    } jit_executable;
+
     // PyObject offset;
     struct _pyobject {
         uint64_t size;
@@ -305,6 +313,9 @@ typedef struct _Py_DebugOffsets {
         .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \
         .tlbc_index = _Py_Debug_interpreter_frame_tlbc_index, \
     }, \
+    .interpreter_frame_metadata = { \
+        .executable_kinds = (uintptr_t)PyUnstable_ExecutableKinds, \
+    }, \
     .code_object = { \
         .size = sizeof(PyCodeObject), \
         .filename = offsetof(PyCodeObject, co_filename), \
@@ -318,6 +329,9 @@ typedef struct _Py_DebugOffsets {
         .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \
         .co_tlbc = _Py_Debug_code_object_co_tlbc, \
     }, \
+    .jit_executable = { \
+        .code = offsetof(PyUnstable_PyJitExecutable, je_code), \
+    }, \
     .pyobject = { \
         .size = sizeof(PyObject), \
         .ob_type = offsetof(PyObject, ob_type), \

Include/internal/pycore_interpframe.h

@@ -17,9 +17,29 @@ extern "C" {
 #define _PyInterpreterFrame_LASTI(IF) \
     ((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF))))
 
+PyAPI_DATA(PyTypeObject) PyUnstable_JITExecutable_Type;
+
+#define PyUnstable_JITExecutable_Check(op) Py_IS_TYPE((op), &PyUnstable_JITExecutable_Type)
+
+PyAPI_FUNC(PyObject *) PyUnstable_MakeJITExecutable(_PyFrame_Reifier reifier, PyCodeObject *code, PyObject *state);
+
+PyAPI_FUNC(int) _PyFrame_InitializeExternalFrame(_PyInterpreterFrame *frame);
+
+static inline int
+_PyFrame_EnsureFrameFullyInitialized(_PyInterpreterFrame *frame)
+{
+    if (PyUnstable_JITExecutable_Check(PyStackRef_AsPyObjectBorrow(frame->f_executable))) {
+        return _PyFrame_InitializeExternalFrame(frame);
+    }
+    return 0;
+}
+
 static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
     assert(!PyStackRef_IsNull(f->f_executable));
     PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable);
+    if (PyUnstable_JITExecutable_Check(executable)) {
+        return ((PyUnstable_PyJitExecutable *)executable)->je_code;
+    }
     assert(PyCode_Check(executable));
     return (PyCodeObject *)executable;
 }
@@ -48,6 +68,12 @@ _PyFrame_SafeGetCode(_PyInterpreterFrame *f)
     if (_PyObject_IsFreed(executable)) {
         return NULL;
     }
+    if (PyUnstable_JITExecutable_Check(executable)) {
+        executable = (PyObject *)((PyUnstable_PyJitExecutable *)executable)->je_code;
+        if (_PyObject_IsFreed(executable)) {
+            return NULL;
+        }
+    }
     if (!PyCode_Check(executable)) {
         return NULL;
     }
@@ -58,6 +84,10 @@ static inline _Py_CODEUNIT *
 _PyFrame_GetBytecode(_PyInterpreterFrame *f)
 {
 #ifdef Py_GIL_DISABLED
+    if (_PyFrame_EnsureFrameFullyInitialized(f) < 0) {
+        return NULL;
+    }
+
     PyCodeObject *co = _PyFrame_GetCode(f);
     _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co);
     assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size);
@@ -80,6 +110,10 @@ _PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f)
         return -1;
     }
 
+    if (_PyFrame_EnsureFrameFullyInitialized(f) < 0) {
+        return -1;
+    }
+
     _Py_CODEUNIT *bytecode;
 #ifdef Py_GIL_DISABLED
     _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co);
@@ -277,6 +311,22 @@ _PyThreadState_GetFrame(PyThreadState *tstate)
     return _PyFrame_GetFirstComplete(tstate->current_frame);
 }
 
+static inline PyObject *
+_PyFrame_GetGlobals(_PyInterpreterFrame *frame) {
+    if (_PyFrame_EnsureFrameFullyInitialized(frame) < 0) {
+        return NULL;
+    }
+    return frame->f_globals;
+}
+
+static inline PyObject *
+_PyFrame_GetBuiltins(_PyInterpreterFrame *frame) {
+    if (_PyFrame_EnsureFrameFullyInitialized(frame) < 0) {
+        return NULL;
+    }
+    return frame->f_builtins;
+}
+
 /* For use by _PyFrame_GetFrameObject
   Do not call directly. */
 PyFrameObject *
@@ -288,6 +338,9 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame);
 static inline PyFrameObject *
 _PyFrame_GetFrameObject(_PyInterpreterFrame *frame)
 {
+    if (PyUnstable_JITExecutable_Check(PyStackRef_AsPyObjectBorrow(frame->f_executable))) {
+        return _PyFrame_MakeAndSetFrameObject(frame);
+    }
 
     assert(!_PyFrame_IsIncomplete(frame));
     PyFrameObject *res =  frame->frame_obj;
@@ -309,7 +362,7 @@ _PyFrame_ClearLocals(_PyInterpreterFrame *frame);
  * take should  be set to 1 for heap allocated
  * frames like the ones in generators and coroutines.
  */
-void
+PyAPI_FUNC(void)
 _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame);
 
 int

Include/internal/pycore_interpframe_structs.h

@@ -85,6 +85,15 @@ struct _PyAsyncGenObject {
     _PyGenObject_HEAD(ag)
 };
 
+typedef int (*_PyFrame_Reifier)(struct _PyInterpreterFrame *, PyObject *reifier);
+
+typedef struct {
+    PyObject_HEAD
+    PyCodeObject *je_code;
+    PyObject *je_state;
+    _PyFrame_Reifier je_reifier;
+} PyUnstable_PyJitExecutable;
+
 #undef _PyGenObject_HEAD
 
 

Lib/test/test_capi/test_misc.py

@@ -4,6 +4,7 @@
 import _thread
 from collections import deque
 import contextlib
+import dis
 import importlib.machinery
 import importlib.util
 import json
@@ -2857,6 +2858,85 @@ def func():
         names = ["func", "outer", "outer", "inner", "inner", "outer", "inner"]
         self.do_test(func, names)
 
+    def test_jit_frame(self):
+        def fakefunc():
+            pass
+
+        def f():
+            return sys._getframe(1)
+
+        res = _testinternalcapi.call_with_jit_frame(fakefunc, f, ())
+
+    def test_jit_frame_globals(self):
+        """jit executable can fill in globals when accessed"""
+        def fakefunc():
+            pass
+
+        fake_globals = {"abc":42}
+        def callback():
+            return {"globals": fake_globals}
+
+        res = _testinternalcapi.call_with_jit_frame(fakefunc, globals, (), callback)
+        self.assertEqual(res, fake_globals)
+
+    def test_jit_frame_builtins(self):
+        """jit executable can fill in builtins when accessed"""
+        def fakefunc():
+            pass
+
+        fake_builtins = {"abc":42}
+        def callback():
+            return {"builtins": fake_builtins}
+
+        res = _testinternalcapi.call_with_jit_frame(fakefunc, _testlimitedcapi.eval_getbuiltins, (), callback)
+        self.assertEqual(res, fake_builtins)
+
+    def test_jit_frame_instr_ptr(self):
+        """jit executable can fill in the instr ptr each time the frame is queried"""
+        def fakefunc():
+            pass
+            pass
+            pass
+            pass
+
+        offset = 0
+        linenos = []
+        def test():
+            for op in dis.get_instructions(fakefunc):
+                if op.opname in ("RESUME", "NOP", "RETURN_VALUE"):
+                    nonlocal offset
+                    offset = op.offset//2
+                    linenos.append(sys._getframe(1).f_lineno)
+
+        def callback():
+            return {"instr_ptr": offset}
+
+        _testinternalcapi.call_with_jit_frame(fakefunc, test, (), callback)
+        base = fakefunc.__code__.co_firstlineno
+        self.assertEqual(linenos, [base, base + 1, base + 2, base + 3, base + 4])
+
+    def test_jit_frame_code(self):
+        """internal C api checks the for a code executor"""
+        def fakefunc():
+            pass
+
+        def callback():
+            return _testinternalcapi.iframe_getcode(sys._getframe(1))
+
+        res = _testinternalcapi.call_with_jit_frame(fakefunc, callback, ())
+        self.assertEqual(res, fakefunc.__code__)
+
+    def test_jit_frame_line(self):
+        """internal C api checks the for a code executor"""
+        def fakefunc():
+            pass
+
+        def callback():
+            return _testinternalcapi.iframe_getline(sys._getframe(1))
+
+        res = _testinternalcapi.call_with_jit_frame(fakefunc, callback, ())
+        self.assertEqual(res, fakefunc.__code__.co_firstlineno)
+
 
 @unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED')
 class TestPyThreadId(unittest.TestCase):

Lib/test/test_external_inspection.py

@@ -3534,5 +3534,89 @@ def test_get_stats_disabled_raises(self):
             client_socket.sendall(b"done")
 
 
+class TestNonCodeExecutable(RemoteInspectionTestBase):
+    @skip_if_not_supported
+    @unittest.skipIf(
+        sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
+        "Test only runs on Linux with process_vm_readv support",
+    )
+    def test_remote_stack_trace(self):
+        port = find_unused_port()
+        script = textwrap.dedent(
+            f"""\
+            import time, sys, socket, threading
+            import _testinternalcapi
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect(('localhost', {port}))
+
+            def bar():
+                for x in range(100):
+                    if x == 50:
+                        _testinternalcapi.call_with_jit_frame(baz, foo, ())
+
+            def baz():
+                pass
+
+            def foo():
+                sock.sendall(b"ready:thread\\n"); time.sleep(10_000)
+
+            t = threading.Thread(target=bar)
+            t.start()
+            sock.sendall(b"ready:main\\n"); t.join()
+            """
+        )
+
+        with os_helper.temp_dir() as work_dir:
+            script_dir = os.path.join(work_dir, "script_pkg")
+            os.mkdir(script_dir)
+
+            server_socket = _create_server_socket(port)
+            script_name = _make_test_script(script_dir, "script", script)
+            client_socket = None
+
+            try:
+                with _managed_subprocess([sys.executable, script_name]) as p:
+                    client_socket, _ = server_socket.accept()
+                    server_socket.close()
+                    server_socket = None
+
+                    _wait_for_signal(
+                        client_socket, [b"ready:main", b"ready:thread"]
+                    )
+
+                    try:
+                        stack_trace = get_stack_trace(p.pid)
+                    except PermissionError:
+                        self.skipTest(
+                            "Insufficient permissions to read the stack trace"
+                        )
+
+                    # Find expected thread stack by funcname
+                    found_thread = self._find_thread_with_frame(
+                        stack_trace,
+                        lambda f: f.funcname == "foo" and f.location.lineno == 15,
+                    )
+                    self.assertIsNotNone(
+                        found_thread, "Expected thread stack trace not found"
+                    )
+                    # Check the funcnames in order
+                    funcnames = [f.funcname for f in found_thread.frame_info]
+                    self.assertEqual(
+                        funcnames[:6],
+                        ["foo", "baz", "bar", "Thread.run", "Thread._bootstrap_inner", "Thread._bootstrap"]
+                    )
+
+                    # Check main thread
+                    found_main = self._find_frame_in_trace(
+                        stack_trace,
+                        lambda f: f.funcname == "<module>" and f.location.lineno == 19,
+                    )
+                    self.assertIsNotNone(
+                        found_main, "Main thread stack trace not found"
+                    )
+            finally:
+                _cleanup_sockets(client_socket, server_socket)
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core_and_Builtins/2025-12-11-20-23-56.gh-issue-142598.ftYjy7.rst

@@ -0,0 +1 @@
+Adds a new executable type for _PyInterpreterFrame.f_executable PEP 523 JITs to plug in and have their frames visible to external introspection tools.

Modules/_remote_debugging/_remote_debugging.h

@@ -238,6 +238,7 @@ typedef struct {
     RemoteDebuggingState *cached_state;
     FrameCacheEntry *frame_cache;  // preallocated array of FRAME_CACHE_MAX_THREADS entries
     UnwinderStats stats;  // statistics for performance analysis
+    uintptr_t frame_executable_types[PyUnstable_EXECUTABLE_KINDS];
 #ifdef Py_GIL_DISABLED
     uint32_t tlbc_generation;
     _Py_hashtable_t *tlbc_cache;
@@ -332,7 +333,7 @@ extern long read_py_long(RemoteUnwinderObject *unwinder, uintptr_t address);
  * CODE OBJECT FUNCTION DECLARATIONS
  * ============================================================================ */
 
-extern int parse_code_object(
+extern int parse_executable_object(
     RemoteUnwinderObject *unwinder,
     PyObject **result,
     uintptr_t address,
@@ -473,6 +474,8 @@ extern int populate_initial_state_data(
     uintptr_t *tstate
 );
 
+extern int populate_frame_executable_types(RemoteUnwinderObject *unwinder);
+
 extern int find_running_frame(
     RemoteUnwinderObject *unwinder,
     uintptr_t address_of_thread,