JIT: invalid memory read in _PyEval_EvalFrameDefault

[devdanzin]

Crash report

What happened?

ASan will detect a heap buffer overflow from running the code below in a JIT build. In a patched build, the error comes at iteration 175, while on an unpatched build it happens at around 12.000 iterations.

The necessary patch to speed up the overflow:

diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h
index 7f60eb49508..fd80dedb27e 100644
--- a/Include/internal/pycore_backoff.h
+++ b/Include/internal/pycore_backoff.h
@@ -124,7 +124,7 @@ trigger_backoff_counter(void)
 // For example, 4095 does not work for the nqueens benchmark on pyperformance
 // as we always end up tracing the loop iteration's
 // exhaustion iteration. Which aborts our current tracer.
-#define JUMP_BACKWARD_INITIAL_VALUE 4000
+#define JUMP_BACKWARD_INITIAL_VALUE 63
 #define JUMP_BACKWARD_INITIAL_BACKOFF 6
 static inline _Py_BackoffCounter
 initial_jump_backoff_counter(void)
@@ -137,7 +137,7 @@ initial_jump_backoff_counter(void)
  * Must be larger than ADAPTIVE_COOLDOWN_VALUE,
  * otherwise when a side exit warms up we may construct
  * a new trace before the Tier 1 code has properly re-specialized. */
-#define SIDE_EXIT_INITIAL_VALUE 4000
+#define SIDE_EXIT_INITIAL_VALUE 63
 #define SIDE_EXIT_INITIAL_BACKOFF 6

 static inline _Py_BackoffCounter

The code that triggers the overflow:

str_v1 = ''
tuple_v2 = (None, None, None, None, None)
small_int_v3 = 4


def f1():
    class StatefulAbs:
        def __abs__(self):
            return 123
 
    evil_abs_obj = StatefulAbs()
    for _ in range(10):
        try:
            abs(evil_abs_obj)
        except Exception:
            pass

    tuple_v2[small_int_v3]
    tuple_v2[small_int_v3]
    tuple_v2[small_int_v3]

    def recursive_wrapper_4569():
        str_v1 > str_v1
        str_v1 <= str_v1
        str_v1 > str_v1
        recursive_wrapper_4569()

    try:
        recursive_wrapper_4569()
    except RecursionError:
        pass

# The number of iterations is this high because an unpatched build may need over 12.000 iterations to crash
for i_f1 in range(19000):
    print(i_f1)
    f1()

ASan output:

1
2
[...]
175
=================================================================
==92971==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x6e88fd4a8738 at pc 0x5edcfcfffe9f bp 0x7fffe8f80370 sp 0x7fffe8f80368
READ of size 8 at 0x6e88fd4a8738 thread T0
    #0 0x5edcfcfffe9e in _PyEval_EvalFrameDefault /home/danzin/projects/jit_cpython/Python/generated_cases.c.h:5479:43
    #1 0x5edcfcf9a74a in _PyEval_EvalFrame /home/danzin/projects/jit_cpython/./Include/internal/pycore_ceval.h:121:16
    #2 0x5edcfcf9a74a in _PyEval_Vector /home/danzin/projects/jit_cpython/Python/ceval.c:2159:12
    #3 0x5edcfcf9a164 in PyEval_EvalCode /home/danzin/projects/jit_cpython/Python/ceval.c:995:21
    #4 0x5edcfd30beae in run_eval_code_obj /home/danzin/projects/jit_cpython/Python/pythonrun.c:1372:12
    #5 0x5edcfd30b07b in run_mod /home/danzin/projects/jit_cpython/Python/pythonrun.c:1475:19
    #6 0x5edcfd30567c in pyrun_file /home/danzin/projects/jit_cpython/Python/pythonrun.c:1300:15
    #7 0x5edcfd303212 in _PyRun_SimpleFileObject /home/danzin/projects/jit_cpython/Python/pythonrun.c:521:13
    #8 0x5edcfd30258d in _PyRun_AnyFileObject /home/danzin/projects/jit_cpython/Python/pythonrun.c:81:15
    #9 0x5edcfd37d5ba in pymain_run_file_obj /home/danzin/projects/jit_cpython/Modules/main.c:410:15
    #10 0x5edcfd37d5ba in pymain_run_file /home/danzin/projects/jit_cpython/Modules/main.c:429:15
    #11 0x5edcfd37b683 in pymain_run_python /home/danzin/projects/jit_cpython/Modules/main.c:691:21
    #12 0x5edcfd37b683 in Py_RunMain /home/danzin/projects/jit_cpython/Modules/main.c:772:5
    #13 0x5edcfd37c586 in pymain_main /home/danzin/projects/jit_cpython/Modules/main.c:802:12
    #14 0x5edcfd37c6f7 in Py_BytesMain /home/danzin/projects/jit_cpython/Modules/main.c:826:12
    #15 0x7228fe02a574 in __libc_start_call_main csu/../sysdeps/nptl/libc_start_call_main.h:58:16
    #16 0x7228fe02a627 in __libc_start_main csu/../csu/libc-start.c:360:3
    #17 0x5edcfc95f4f4 in _start (/home/danzin/projects/jit_cpython/python+0x2bb4f4) (BuildId: 80666b118b34989342403a590b39bd333cc40fa4)

0x6e88fd4a8738 is located 1016 bytes after 64-byte region [0x6e88fd4a8300,0x6e88fd4a8340)
allocated by thread T0 here:
    #0 0x5edcfca04a98 in malloc (/home/danzin/projects/jit_cpython/python+0x360a98) (BuildId: 80666b118b34989342403a590b39bd333cc40fa4)
    #1 0x5edcfcd72f22 in _PyMem_DebugRawAlloc /home/danzin/projects/jit_cpython/Objects/obmalloc.c:2887:24
    #2 0x5edcfcd72f22 in _PyMem_DebugRawRealloc /home/danzin/projects/jit_cpython/Objects/obmalloc.c:2963:16
    #3 0x5edcfd275126 in get_index_for_executor /home/danzin/projects/jit_cpython/Python/optimizer.c:77:33
    #4 0x5edcfd275126 in _PyOptimizer_Optimize /home/danzin/projects/jit_cpython/Python/optimizer.c:171:21
    #5 0x5edcfd011410 in stop_tracing_and_jit /home/danzin/projects/jit_cpython/Python/ceval.c:1108:15
    #6 0x5edcfcfbe5c2 in _PyEval_EvalFrameDefault /home/danzin/projects/jit_cpython/Python/generated_cases.c.h:11712:27
    #7 0x5edcfcf9a74a in _PyEval_EvalFrame /home/danzin/projects/jit_cpython/./Include/internal/pycore_ceval.h:121:16
    #8 0x5edcfcf9a74a in _PyEval_Vector /home/danzin/projects/jit_cpython/Python/ceval.c:2159:12
    #9 0x5edcfcf9a164 in PyEval_EvalCode /home/danzin/projects/jit_cpython/Python/ceval.c:995:21
    #10 0x5edcfd30beae in run_eval_code_obj /home/danzin/projects/jit_cpython/Python/pythonrun.c:1372:12
    #11 0x5edcfd30b07b in run_mod /home/danzin/projects/jit_cpython/Python/pythonrun.c:1475:19
    #12 0x5edcfd30567c in pyrun_file /home/danzin/projects/jit_cpython/Python/pythonrun.c:1300:15
    #13 0x5edcfd303212 in _PyRun_SimpleFileObject /home/danzin/projects/jit_cpython/Python/pythonrun.c:521:13
    #14 0x5edcfd30258d in _PyRun_AnyFileObject /home/danzin/projects/jit_cpython/Python/pythonrun.c:81:15
    #15 0x5edcfd37d5ba in pymain_run_file_obj /home/danzin/projects/jit_cpython/Modules/main.c:410:15
    #16 0x5edcfd37d5ba in pymain_run_file /home/danzin/projects/jit_cpython/Modules/main.c:429:15
    #17 0x5edcfd37b683 in pymain_run_python /home/danzin/projects/jit_cpython/Modules/main.c:691:21
    #18 0x5edcfd37b683 in Py_RunMain /home/danzin/projects/jit_cpython/Modules/main.c:772:5
    #19 0x5edcfd37c586 in pymain_main /home/danzin/projects/jit_cpython/Modules/main.c:802:12
    #20 0x5edcfd37c6f7 in Py_BytesMain /home/danzin/projects/jit_cpython/Modules/main.c:826:12
    #21 0x7228fe02a574 in __libc_start_call_main csu/../sysdeps/nptl/libc_start_call_main.h:58:16
    #22 0x7228fe02a627 in __libc_start_main csu/../csu/libc-start.c:360:3
    #23 0x5edcfc95f4f4 in _start (/home/danzin/projects/jit_cpython/python+0x2bb4f4) (BuildId: 80666b118b34989342403a590b39bd333cc40fa4)

SUMMARY: AddressSanitizer: heap-buffer-overflow /home/danzin/projects/jit_cpython/Python/generated_cases.c.h:5479:43 in _PyEval_EvalFrameDefault
Shadow bytes around the buggy address:
  0x6e88fd4a8480: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x6e88fd4a8500: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x6e88fd4a8580: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x6e88fd4a8600: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x6e88fd4a8680: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
=>0x6e88fd4a8700: fa fa fa fa fa fa fa[fa]fa fa fa fa fa fa fa fa
  0x6e88fd4a8780: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x6e88fd4a8800: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x6e88fd4a8880: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x6e88fd4a8900: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x6e88fd4a8980: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07
  Heap left redzone:       fa
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
==92971==ABORTING

Output from running with PYTHON_LLTRACE=4:
139_overflow_lltrace.txt

Output from running with PYTHON_DEBUG=4:
139_overflow_opt_debug.txt

Interestingly, the original fuzzing code would trigger the overflow at iteration 127 instead of 175, something I cut out when reducing increased the necessary number of iterations.

Found using lafleur.

CPython versions tested on:

CPython main branch

Operating systems tested on:

Linux

Output from running 'python -VV' on the command line:

Python 3.15.0a2+ (heads/main-dirty:dc9d2eea587, Nov 22 2025, 19:28:52) [Clang 21.1.2 (2ubuntu6)]

Linked PRs

• GH-141861: Fix invalid memory read in the ENTER_EXECUTOR #141921
• GH-141861: Fix TRACE_RECORD if full #141959

[Fidget-Spinner]

I can reproduce this, but I'm completely stumped by exactly what is causing it. It seems to be a mixture of executor insertion and freeing, but I don't know. We seem to somehow grab a freed executor???

In any case, it's not a buffer overflow. Rather, it's a invalid memory read.

[Fidget-Spinner]

Anyone else is free to try debug this as well and provide a fix. I will try but I can't promise anything.

[sergey-miryanov]

I would like to look into it.

[Fidget-Spinner]

This bug is causing the sphinx benchmark to crash.

[Fidget-Spinner]

@sergey-miryanov I found a fix, could you please apply this patch and add the test case in a PR? Thank you.

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 12ee506e4f2..6129ea2e723 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -3018,7 +3018,7 @@ dummy_func(
                 goto stop_tracing;
             }
             PyCodeObject *code = _PyFrame_GetCode(frame);
-            _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
+            _PyExecutorObject *executor = code->co_executors->executors[this_instr->op.arg];
             assert(executor->vm_data.index == INSTR_OFFSET() - 1);
             assert(executor->vm_data.code == code);
             assert(executor->vm_data.valid);
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index b83b7c528e9..47805c270f9 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -5476,7 +5476,7 @@
                 JUMP_TO_LABEL(stop_tracing);
             }
             PyCodeObject *code = _PyFrame_GetCode(frame);
-            _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
+            _PyExecutorObject *executor = code->co_executors->executors[this_instr->op.arg];
             assert(executor->vm_data.index == INSTR_OFFSET() - 1);
             assert(executor->vm_data.code == code);
             assert(executor->vm_data.valid);

The problem is that it's possible for oparg to change under our feet while tracing right before an opcode switches to ENTER_EXECUTOR, thus causing the oparg and the index to be different.

[Fidget-Spinner]

This was a very tricky bug to find.

[sergey-miryanov]

A slightly reduced repro:

import sys
sys.setrecursionlimit(30) # reduce time of the run

str_v1 = ''
tuple_v2 = (None, None, None, None, None)
small_int_v3 = 4


def f1():

    for _ in range(10):
        abs(0)

    tuple_v2[small_int_v3]
    tuple_v2[small_int_v3]
    tuple_v2[small_int_v3]

    def recursive_wrapper_4569():
        str_v1 > str_v1
        str_v1 > str_v1
        str_v1 > str_v1
        recursive_wrapper_4569()

    recursive_wrapper_4569()


for i_f1 in range(19000):
    try:
        f1()
    except RecursionError:
        pass

[Fidget-Spinner]

Thank you for the bug report @devdanzin and thank you for the PR @sergey-miryanov !