From a4293bc398317be3d8c724c9cf73f5b75dd3ab02 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 16 Oct 2025 21:43:10 +0100 Subject: [PATCH 1/8] Fix wrong instruction pointer for exceptions --- Lib/test/test_capi/test_opt.py | 11 +++++++++++ Python/bytecodes.c | 5 ++++- Python/executor_cases.c.h | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f121f27174875e..f863d766b14c25 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2637,6 +2637,17 @@ def f2(): f2() + def test_next_instr_for_exception_handler_set(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + for i in range(TIER2_THRESHOLD + 3): + try: + g(i) + except Exception: + pass + + f() + def global_identity(x): return x diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f9f14322df0a5e..ef45a9e9d92d83 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5384,7 +5384,10 @@ dummy_func( tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; + // gh-140104: The exception handler expects frame->instr_ptr to be pointing to next_instr, not this_instr! + frame->instr_ptr = next_instr; SYNC_SP(); GOTO_TIER_ONE(NULL); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0e4d86463761a0..441f73b457debb 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7438,7 +7438,9 @@ oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; + frame->instr_ptr = next_instr; GOTO_TIER_ONE(NULL); break; } From 55967c983fa21d44e6db2000f910de9bb2131408 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 16 Oct 2025 21:47:09 +0100 Subject: [PATCH 2/8] Create 2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst Co-Authored-By: devdanzin <74280297+devdanzin@users.noreply.github.com> --- .../2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst b/Misc/NEWS.d/next/Core and Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst new file mode 100644 index 00000000000000..1c18cbc9ad0588 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst @@ -0,0 +1,2 @@ +Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported +by Daniel Diniz. From 008979a4c1e455ec3c12efd8dd0d1a01949dcbb5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 16 Oct 2025 22:53:04 +0100 Subject: [PATCH 3/8] Move files --- .../2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/{Core and Builtins => Core_and_Builtins}/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst (100%) diff --git a/Misc/NEWS.d/next/Core and Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst rename to Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst From d9eb7a92b000ced493124c79d41a68894fbf531b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 16:29:26 +0100 Subject: [PATCH 4/8] Fix externs Co-Authored-By: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> --- Include/internal/pycore_opcode_metadata.h | 4 ++-- Tools/cases_generator/opcode_metadata_generator.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bd6b84ec7fd908..ffb59971599e82 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1744,7 +1744,7 @@ const char *_PyOpcode_OpName[267] = { }; #endif -extern const uint8_t _PyOpcode_Caches[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { [TO_BOOL] = 3, @@ -1769,7 +1769,7 @@ const uint8_t _PyOpcode_Caches[256] = { }; #endif -extern const uint8_t _PyOpcode_Deopt[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { [121] = 121, diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index b649b38123388d..e3ac335ed48dd9 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -147,7 +147,7 @@ def generate_instruction_formats(analysis: Analysis, out: CWriter) -> None: def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint8_t _PyOpcode_Deopt[256];\n") + out.emit("PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256];\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint8_t _PyOpcode_Deopt[256] = {\n") deopts: list[tuple[str, str]] = [] @@ -170,7 +170,7 @@ def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: def generate_cache_table(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint8_t _PyOpcode_Caches[256];\n") + out.emit("PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256];\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint8_t _PyOpcode_Caches[256] = {\n") for inst in analysis.instructions.values(): From 4f4577628a8c7d2438b6cb53b7a3ed6176285da6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 15:01:21 +0100 Subject: [PATCH 5/8] Apply suggestions --- Include/internal/pycore_opcode_metadata.h | 4 ++-- Python/bytecodes.c | 4 ---- Python/ceval_macros.h | 4 +++- Python/executor_cases.c.h | 3 --- Tools/cases_generator/opcode_metadata_generator.py | 4 ++-- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index ffb59971599e82..bd6b84ec7fd908 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1744,7 +1744,7 @@ const char *_PyOpcode_OpName[267] = { }; #endif -PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; +extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { [TO_BOOL] = 3, @@ -1769,7 +1769,7 @@ const uint8_t _PyOpcode_Caches[256] = { }; #endif -PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; +extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { [121] = 121, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ef45a9e9d92d83..b8c8503606f8e5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5384,10 +5384,6 @@ dummy_func( tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); - _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; - _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; - // gh-140104: The exception handler expects frame->instr_ptr to be pointing to next_instr, not this_instr! - frame->instr_ptr = next_instr; SYNC_SP(); GOTO_TIER_ONE(NULL); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 4ed03b7fb01bdf..43fb8806625d7b 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -358,7 +358,9 @@ do { \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ - next_instr = frame->instr_ptr; \ + /* gh-140104: The exception handler expects frame->instr_ptr + to be pointing to next_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[frame->instr_ptr->op.code]]; \ JUMP_TO_LABEL(error); \ } \ DISPATCH(); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 441f73b457debb..03555f779fd45e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7438,9 +7438,6 @@ oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); assert(oparg == 0); - _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; - _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; - frame->instr_ptr = next_instr; GOTO_TIER_ONE(NULL); break; } diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index e3ac335ed48dd9..b649b38123388d 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -147,7 +147,7 @@ def generate_instruction_formats(analysis: Analysis, out: CWriter) -> None: def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: - out.emit("PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256];\n") + out.emit("extern const uint8_t _PyOpcode_Deopt[256];\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint8_t _PyOpcode_Deopt[256] = {\n") deopts: list[tuple[str, str]] = [] @@ -170,7 +170,7 @@ def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: def generate_cache_table(analysis: Analysis, out: CWriter) -> None: - out.emit("PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256];\n") + out.emit("extern const uint8_t _PyOpcode_Caches[256];\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint8_t _PyOpcode_Caches[256] = {\n") for inst in analysis.instructions.values(): From ccc3d6446911ef16cf5e37542869a21b08e8d01b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 20:57:49 +0100 Subject: [PATCH 6/8] Apply suggestions --- Python/ceval_macros.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 43fb8806625d7b..8083913b1a1cfc 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -359,8 +359,8 @@ do { \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ /* gh-140104: The exception handler expects frame->instr_ptr - to be pointing to next_instr, not this_instr! */ \ - next_instr = frame->instr_ptr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[frame->instr_ptr->op.code]]; \ + to after this_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ DISPATCH(); \ From 1e6b2e4c39448e899319bcf62dc67397d493c526 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 22:21:48 +0100 Subject: [PATCH 7/8] Address review partially --- Lib/test/test_capi/test_opt.py | 2 +- Python/bytecodes.c | 1 + Python/executor_cases.c.h | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f863d766b14c25..2afd890424fdd4 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2642,7 +2642,7 @@ def test_next_instr_for_exception_handler_set(self): def f(): for i in range(TIER2_THRESHOLD + 3): try: - g(i) + undefined_variable(i) except Exception: pass diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b8c8503606f8e5..f9f14322df0a5e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5384,6 +5384,7 @@ dummy_func( tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SYNC_SP(); GOTO_TIER_ONE(NULL); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 03555f779fd45e..0e4d86463761a0 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7438,6 +7438,7 @@ oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); assert(oparg == 0); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; GOTO_TIER_ONE(NULL); break; } From ab4a8c922b3c457696a5995109182260b7d763bd Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:28:11 +0000 Subject: [PATCH 8/8] Address review; add test --- Lib/test/test_capi/test_opt.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 2afd890424fdd4..4e94f62d35eba2 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2648,6 +2648,19 @@ def f(): f() + def test_next_instr_for_exception_handler_set_lasts_instr(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + a_list = [] + for _ in range(TIER2_THRESHOLD + 3): + try: + a_list[""] = 0 + except Exception: + pass + + f() + + def global_identity(x): return x