From 8807c08ffac949a162e755d1f532f872cba6b6ca Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 12 Nov 2025 12:16:59 +0000 Subject: [PATCH 1/2] Support switch/case dispatch with the tracing interpreter. --- .github/workflows/jit.yml | 26 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 9 +- Include/internal/pycore_optimizer.h | 2 +- Include/internal/pycore_uop_ids.h | 1 + Include/opcode_ids.h | 47 +-- Lib/_opcode_metadata.py | 47 +-- Python/bytecodes.c | 8 +- Python/ceval.c | 4 + Python/ceval_macros.h | 10 +- Python/executor_cases.c.h | 2 + Python/generated_cases.c.h | 112 +++--- Python/instrumentation.c | 4 +- Python/opcode_targets.h | 460 +++++++++++----------- Python/optimizer_cases.c.h | 2 + Tools/cases_generator/analyzer.py | 7 +- Tools/cases_generator/target_generator.py | 2 +- Tools/cases_generator/tier1_generator.py | 2 +- 18 files changed, 389 insertions(+), 359 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 3349eb042425dd..62325250bd368e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,10 +57,9 @@ jobs: fail-fast: false matrix: target: -# To re-enable later when we support these. -# - i686-pc-windows-msvc/msvc -# - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -71,16 +70,15 @@ jobs: llvm: - 21 include: -# To re-enable later when we support these. -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-2022 -# - target: x86_64-pc-windows-msvc/msvc -# architecture: x64 -# runner: windows-2022 -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-11-arm + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-2022 + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-2022 + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 7ec7bd1c695516..2fb46a6df50bb3 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,6 +286,7 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) Python 3.16 will start with 3700 @@ -299,7 +300,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3655 +#define PYC_MAGIC_NUMBER 3656 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 548627dc7982ec..cca88818c575df 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -488,6 +488,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -971,6 +973,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -1287,6 +1291,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1738,6 +1743,7 @@ const char *_PyOpcode_OpName[267] = { [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [TRACE_RECORD] = "TRACE_RECORD", [UNARY_INVERT] = "UNARY_INVERT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", @@ -1809,7 +1815,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, - [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -2025,6 +2030,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [TO_BOOL_LIST] = TO_BOOL, [TO_BOOL_NONE] = TO_BOOL, [TO_BOOL_STR] = TO_BOOL, + [TRACE_RECORD] = TRACE_RECORD, [UNARY_INVERT] = UNARY_INVERT, [UNARY_NEGATIVE] = UNARY_NEGATIVE, [UNARY_NOT] = UNARY_NOT, @@ -2070,7 +2076,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 230: \ case 231: \ case 232: \ - case 233: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 0307a174e77346..e7177552cf666e 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -364,7 +364,7 @@ extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); -int +PyAPI_FUNC(int) _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 7a33a5b84fd21a..c38f28f9db161b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -352,6 +352,7 @@ extern "C" { #define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE #define _TO_BOOL_STR 551 +#define _TRACE_RECORD TRACE_RECORD #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefcd35..0d066c169019a7 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -213,28 +213,29 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 207 #define UNPACK_SEQUENCE_TUPLE 208 #define UNPACK_SEQUENCE_TWO_TUPLE 209 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 +#define INSTRUMENTED_END_FOR 233 +#define INSTRUMENTED_POP_ITER 234 +#define INSTRUMENTED_END_SEND 235 +#define INSTRUMENTED_FOR_ITER 236 +#define INSTRUMENTED_INSTRUCTION 237 +#define INSTRUMENTED_JUMP_FORWARD 238 +#define INSTRUMENTED_NOT_TAKEN 239 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 +#define INSTRUMENTED_POP_JUMP_IF_NONE 242 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 +#define INSTRUMENTED_RESUME 244 +#define INSTRUMENTED_RETURN_VALUE 245 +#define INSTRUMENTED_YIELD_VALUE 246 +#define INSTRUMENTED_END_ASYNC_FOR 247 +#define INSTRUMENTED_LOAD_SUPER_ATTR 248 +#define INSTRUMENTED_CALL 249 +#define INSTRUMENTED_CALL_KW 250 +#define INSTRUMENTED_CALL_FUNCTION_EX 251 +#define INSTRUMENTED_JUMP_BACKWARD 252 +#define INSTRUMENTED_LINE 253 +#define ENTER_EXECUTOR 254 +#define TRACE_RECORD 255 #define ANNOTATIONS_PLACEHOLDER 256 #define JUMP 257 #define JUMP_IF_FALSE 258 @@ -249,7 +250,7 @@ extern "C" { #define HAVE_ARGUMENT 43 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 +#define MIN_INSTRUMENTED_OPCODE 233 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a32948..e681cb17e43e04 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -208,8 +208,9 @@ 'CACHE': 0, 'RESERVED': 17, 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, + 'INSTRUMENTED_LINE': 253, + 'ENTER_EXECUTOR': 254, + 'TRACE_RECORD': 255, 'BINARY_SLICE': 1, 'BUILD_TEMPLATE': 2, 'CALL_FUNCTION_EX': 4, @@ -328,26 +329,26 @@ 'UNPACK_EX': 118, 'UNPACK_SEQUENCE': 119, 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, + 'INSTRUMENTED_END_FOR': 233, + 'INSTRUMENTED_POP_ITER': 234, + 'INSTRUMENTED_END_SEND': 235, + 'INSTRUMENTED_FOR_ITER': 236, + 'INSTRUMENTED_INSTRUCTION': 237, + 'INSTRUMENTED_JUMP_FORWARD': 238, + 'INSTRUMENTED_NOT_TAKEN': 239, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 240, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 241, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 242, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 243, + 'INSTRUMENTED_RESUME': 244, + 'INSTRUMENTED_RETURN_VALUE': 245, + 'INSTRUMENTED_YIELD_VALUE': 246, + 'INSTRUMENTED_END_ASYNC_FOR': 247, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 248, + 'INSTRUMENTED_CALL': 249, + 'INSTRUMENTED_CALL_KW': 250, + 'INSTRUMENTED_CALL_FUNCTION_EX': 251, + 'INSTRUMENTED_JUMP_BACKWARD': 252, 'ANNOTATIONS_PLACEHOLDER': 256, 'JUMP': 257, 'JUMP_IF_FALSE': 258, @@ -362,4 +363,4 @@ } HAVE_ARGUMENT = 43 -MIN_INSTRUMENTED_OPCODE = 234 +MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 565eaa7a599175..c2690ba5ea68ee 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5636,10 +5636,12 @@ dummy_func( DISPATCH(); } - label(record_previous_inst) { + inst(TRACE_RECORD, (--)) { #if _Py_TIER2 assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; bool stop_tracing = (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); @@ -5675,7 +5677,7 @@ dummy_func( } DISPATCH_GOTO_NON_TRACING(); #else - Py_FatalError("JIT label executed in non-jit build."); + Py_FatalError("JIT instruction executed in non-jit build."); #endif } diff --git a/Python/ceval.c b/Python/ceval.c index 25294ebd993f6c..14fef42ea967d6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1179,6 +1179,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); +#if !USE_COMPUTED_GOTOS + uint8_t tracing_mode = 0; + uint8_t dispatch_code; +#endif #endif _PyEntryFrame entry; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 05a2760671e847..c30638c221a03f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,8 +134,8 @@ # define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: -# define DISPATCH_GOTO() goto dispatch_opcode -# define DISPATCH_GOTO_NON_TRACING() goto dispatch_opcode +# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -148,9 +148,9 @@ # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; #else -# define IS_JIT_TRACING() (0) -# define ENTER_TRACING() -# define LEAVE_TRACING() +# define IS_JIT_TRACING() (tracing_mode != 0) +# define ENTER_TRACING() tracing_mode = 255 +# define LEAVE_TRACING() tracing_mode = 0 #endif /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6796abf84ac5f4..e1edd20b778d27 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7579,5 +7579,7 @@ break; } + /* _TRACE_RECORD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0d4678df68ce2d..f3bbe3568ef8f3 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11,7 +11,7 @@ #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif { #endif /* _Py_TAIL_CALL_INTERP */ @@ -11683,6 +11683,67 @@ DISPATCH(); } + TARGET(TRACE_RECORD) { + #if _Py_TAIL_CALL_INTERP + int opcode = TRACE_RECORD; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TRACE_RECORD); + opcode = TRACE_RECORD; + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT instruction executed in non-jit build."); + #endif + } + TARGET(UNARY_INVERT) { #if _Py_TAIL_CALL_INTERP int opcode = UNARY_INVERT; @@ -12254,55 +12315,6 @@ JUMP_TO_LABEL(error); DISPATCH(); } - LABEL(record_previous_inst) - { - #if _Py_TIER2 - assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; - bool stop_tracing = (opcode == WITH_EXCEPT_START || - opcode == RERAISE || opcode == CLEANUP_THROW || - opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); - _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (full) { - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - DISPATCH_GOTO_NON_TRACING(); - } - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && - opcode == POP_TOP) || - (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && - opcode == STORE_FAST)) { - _tstate->jit_tracer_state.prev_state.instr_is_super = true; - } - else { - _tstate->jit_tracer_state.prev_state.instr = next_instr; - } - PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _tstate->jit_tracer_state.prev_state.instr_frame = frame; - _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; - _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - (&next_instr[1])->counter = trigger_backoff_counter(); - } - DISPATCH_GOTO_NON_TRACING(); - #else - Py_FatalError("JIT label executed in non-jit build."); - #endif - } - LABEL(stop_tracing) { #if _Py_TIER2 diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 81e46a331e0b9e..72b7433022fdea 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -191,7 +191,7 @@ is_instrumented(int opcode) { assert(opcode != 0); assert(opcode != RESERVED); - return opcode != ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; + return opcode < ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; } #ifndef NDEBUG @@ -526,7 +526,7 @@ valid_opcode(int opcode) if (IS_VALID_OPCODE(opcode) && opcode != CACHE && opcode != RESERVED && - opcode < 255) + opcode < 254) { return true; } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 1b9196503b570b..867d7a5a605eb9 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -233,7 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -256,220 +255,220 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, + &&TARGET_TRACE_RECORD, }; #if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -493,28 +492,29 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, }; #endif #else /* _Py_TAIL_CALL_INTERP */ @@ -528,7 +528,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); @@ -746,6 +745,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); @@ -983,6 +983,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, @@ -1023,7 +1024,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [BINARY_OP] = _TAIL_CALL_record_previous_inst, @@ -1241,6 +1241,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, + [TRACE_RECORD] = _TAIL_CALL_record_previous_inst, [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, [UNARY_NOT] = _TAIL_CALL_record_previous_inst, @@ -1281,6 +1282,5 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 01263fe8c7a78f..9ebd113df2dabf 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3483,3 +3483,5 @@ break; } + /* _TRACE_RECORD is not a viable micro-op for tier 2 */ + diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index d39013db4f7fd6..93aa4899fe6ec8 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1195,8 +1195,9 @@ def assign_opcodes( # This is an historical oddity. instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 - instmap["INSTRUMENTED_LINE"] = 254 - instmap["ENTER_EXECUTOR"] = 255 + instmap["INSTRUMENTED_LINE"] = 253 + instmap["ENTER_EXECUTOR"] = 254 + instmap["TRACE_RECORD"] = 255 instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] @@ -1221,7 +1222,7 @@ def assign_opcodes( # Specialized ops appear in their own section # Instrumented opcodes are at the end of the valid range min_internal = instmap["RESUME"] + 1 - min_instrumented = 254 - (len(instrumented) - 1) + min_instrumented = 254 - len(instrumented) assert min_internal + len(specialized) < min_instrumented next_opcode = 1 diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 36fa1d7fa4908b..c4554f8efb1219 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: targets = ["&&_unknown_opcode,\n"] * 256 for name, op in analysis.opmap.items(): if op < 256: - targets[op] = f"&&record_previous_inst,\n" + targets[op] = f"&&TARGET_TRACE_RECORD,\n" out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 94ffb0118f0786..c7ff5de681e6fa 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -160,7 +160,7 @@ def generate_tier1( #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif {{ #endif /* _Py_TAIL_CALL_INTERP */ From de9f2f897d4b3ed23201941b82a8aa586190d2ef Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 18 Nov 2025 10:41:11 +0000 Subject: [PATCH 2/2] Fix tail calling interpreter --- Python/bytecodes.c | 1 + Python/generated_cases.c.h | 1 + Python/opcode_targets.h | 452 +++++++++++----------- Tools/cases_generator/target_generator.py | 2 +- 4 files changed, 229 insertions(+), 227 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c2690ba5ea68ee..12ee506e4f2bc4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5677,6 +5677,7 @@ dummy_func( } DISPATCH_GOTO_NON_TRACING(); #else + (void)prev_instr; Py_FatalError("JIT instruction executed in non-jit build."); #endif } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f3bbe3568ef8f3..b83b7c528e9150 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11740,6 +11740,7 @@ } DISPATCH_GOTO_NON_TRACING(); #else + (void)prev_instr; Py_FatalError("JIT instruction executed in non-jit build."); #endif } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 867d7a5a605eb9..b2fa7d01e8f6c2 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -1026,232 +1026,232 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { - [BINARY_OP] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_EXTEND] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_SLICE] = _TAIL_CALL_record_previous_inst, - [BUILD_INTERPOLATION] = _TAIL_CALL_record_previous_inst, - [BUILD_LIST] = _TAIL_CALL_record_previous_inst, - [BUILD_MAP] = _TAIL_CALL_record_previous_inst, - [BUILD_SET] = _TAIL_CALL_record_previous_inst, - [BUILD_SLICE] = _TAIL_CALL_record_previous_inst, - [BUILD_STRING] = _TAIL_CALL_record_previous_inst, - [BUILD_TEMPLATE] = _TAIL_CALL_record_previous_inst, - [BUILD_TUPLE] = _TAIL_CALL_record_previous_inst, - [CACHE] = _TAIL_CALL_record_previous_inst, - [CALL] = _TAIL_CALL_record_previous_inst, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_record_previous_inst, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_FAST] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_O] = _TAIL_CALL_record_previous_inst, - [CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, - [CALL_INTRINSIC_1] = _TAIL_CALL_record_previous_inst, - [CALL_INTRINSIC_2] = _TAIL_CALL_record_previous_inst, - [CALL_ISINSTANCE] = _TAIL_CALL_record_previous_inst, - [CALL_KW] = _TAIL_CALL_record_previous_inst, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_record_previous_inst, - [CALL_KW_NON_PY] = _TAIL_CALL_record_previous_inst, - [CALL_KW_PY] = _TAIL_CALL_record_previous_inst, - [CALL_LEN] = _TAIL_CALL_record_previous_inst, - [CALL_LIST_APPEND] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_record_previous_inst, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, - [CALL_PY_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_STR_1] = _TAIL_CALL_record_previous_inst, - [CALL_TUPLE_1] = _TAIL_CALL_record_previous_inst, - [CALL_TYPE_1] = _TAIL_CALL_record_previous_inst, - [CHECK_EG_MATCH] = _TAIL_CALL_record_previous_inst, - [CHECK_EXC_MATCH] = _TAIL_CALL_record_previous_inst, - [CLEANUP_THROW] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_FLOAT] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_INT] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_STR] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP_DICT] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP_SET] = _TAIL_CALL_record_previous_inst, - [CONVERT_VALUE] = _TAIL_CALL_record_previous_inst, - [COPY] = _TAIL_CALL_record_previous_inst, - [COPY_FREE_VARS] = _TAIL_CALL_record_previous_inst, - [DELETE_ATTR] = _TAIL_CALL_record_previous_inst, - [DELETE_DEREF] = _TAIL_CALL_record_previous_inst, - [DELETE_FAST] = _TAIL_CALL_record_previous_inst, - [DELETE_GLOBAL] = _TAIL_CALL_record_previous_inst, - [DELETE_NAME] = _TAIL_CALL_record_previous_inst, - [DELETE_SUBSCR] = _TAIL_CALL_record_previous_inst, - [DICT_MERGE] = _TAIL_CALL_record_previous_inst, - [DICT_UPDATE] = _TAIL_CALL_record_previous_inst, - [END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, - [END_FOR] = _TAIL_CALL_record_previous_inst, - [END_SEND] = _TAIL_CALL_record_previous_inst, - [ENTER_EXECUTOR] = _TAIL_CALL_record_previous_inst, - [EXIT_INIT_CHECK] = _TAIL_CALL_record_previous_inst, - [EXTENDED_ARG] = _TAIL_CALL_record_previous_inst, - [FORMAT_SIMPLE] = _TAIL_CALL_record_previous_inst, - [FORMAT_WITH_SPEC] = _TAIL_CALL_record_previous_inst, - [FOR_ITER] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_GEN] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_LIST] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_RANGE] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_TUPLE] = _TAIL_CALL_record_previous_inst, - [GET_AITER] = _TAIL_CALL_record_previous_inst, - [GET_ANEXT] = _TAIL_CALL_record_previous_inst, - [GET_AWAITABLE] = _TAIL_CALL_record_previous_inst, - [GET_ITER] = _TAIL_CALL_record_previous_inst, - [GET_LEN] = _TAIL_CALL_record_previous_inst, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_record_previous_inst, - [IMPORT_FROM] = _TAIL_CALL_record_previous_inst, - [IMPORT_NAME] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_LINE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_RESUME] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_record_previous_inst, - [INTERPRETER_EXIT] = _TAIL_CALL_record_previous_inst, - [IS_OP] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_record_previous_inst, - [JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, - [LIST_APPEND] = _TAIL_CALL_record_previous_inst, - [LIST_EXTEND] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_CLASS] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_MODULE] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, - [LOAD_BUILD_CLASS] = _TAIL_CALL_record_previous_inst, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_record_previous_inst, - [LOAD_CONST] = _TAIL_CALL_record_previous_inst, - [LOAD_DEREF] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_CHECK] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_record_previous_inst, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_record_previous_inst, - [LOAD_LOCALS] = _TAIL_CALL_record_previous_inst, - [LOAD_NAME] = _TAIL_CALL_record_previous_inst, - [LOAD_SMALL_INT] = _TAIL_CALL_record_previous_inst, - [LOAD_SPECIAL] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_record_previous_inst, - [MAKE_CELL] = _TAIL_CALL_record_previous_inst, - [MAKE_FUNCTION] = _TAIL_CALL_record_previous_inst, - [MAP_ADD] = _TAIL_CALL_record_previous_inst, - [MATCH_CLASS] = _TAIL_CALL_record_previous_inst, - [MATCH_KEYS] = _TAIL_CALL_record_previous_inst, - [MATCH_MAPPING] = _TAIL_CALL_record_previous_inst, - [MATCH_SEQUENCE] = _TAIL_CALL_record_previous_inst, - [NOP] = _TAIL_CALL_record_previous_inst, - [NOT_TAKEN] = _TAIL_CALL_record_previous_inst, - [POP_EXCEPT] = _TAIL_CALL_record_previous_inst, - [POP_ITER] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, - [POP_TOP] = _TAIL_CALL_record_previous_inst, - [PUSH_EXC_INFO] = _TAIL_CALL_record_previous_inst, - [PUSH_NULL] = _TAIL_CALL_record_previous_inst, - [RAISE_VARARGS] = _TAIL_CALL_record_previous_inst, - [RERAISE] = _TAIL_CALL_record_previous_inst, - [RESERVED] = _TAIL_CALL_record_previous_inst, - [RESUME] = _TAIL_CALL_record_previous_inst, - [RESUME_CHECK] = _TAIL_CALL_record_previous_inst, - [RETURN_GENERATOR] = _TAIL_CALL_record_previous_inst, - [RETURN_VALUE] = _TAIL_CALL_record_previous_inst, - [SEND] = _TAIL_CALL_record_previous_inst, - [SEND_GEN] = _TAIL_CALL_record_previous_inst, - [SETUP_ANNOTATIONS] = _TAIL_CALL_record_previous_inst, - [SET_ADD] = _TAIL_CALL_record_previous_inst, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_record_previous_inst, - [SET_UPDATE] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, - [STORE_DEREF] = _TAIL_CALL_record_previous_inst, - [STORE_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_GLOBAL] = _TAIL_CALL_record_previous_inst, - [STORE_NAME] = _TAIL_CALL_record_previous_inst, - [STORE_SLICE] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, - [SWAP] = _TAIL_CALL_record_previous_inst, - [TO_BOOL] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_BOOL] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_INT] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, - [TRACE_RECORD] = _TAIL_CALL_record_previous_inst, - [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, - [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, - [UNARY_NOT] = _TAIL_CALL_record_previous_inst, - [UNPACK_EX] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_record_previous_inst, - [WITH_EXCEPT_START] = _TAIL_CALL_record_previous_inst, - [YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACE_RECORD, + [BUILD_LIST] = _TAIL_CALL_TRACE_RECORD, + [BUILD_MAP] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SET] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_STRING] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [CACHE] = _TAIL_CALL_TRACE_RECORD, + [CALL] = _TAIL_CALL_TRACE_RECORD, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CLEANUP_THROW] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_INT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_STR] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACE_RECORD, + [CONVERT_VALUE] = _TAIL_CALL_TRACE_RECORD, + [COPY] = _TAIL_CALL_TRACE_RECORD, + [COPY_FREE_VARS] = _TAIL_CALL_TRACE_RECORD, + [DELETE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [DELETE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [DELETE_FAST] = _TAIL_CALL_TRACE_RECORD, + [DELETE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [DELETE_NAME] = _TAIL_CALL_TRACE_RECORD, + [DELETE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, + [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_SEND] = _TAIL_CALL_TRACE_RECORD, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_GEN] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [GET_AITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ANEXT] = _TAIL_CALL_TRACE_RECORD, + [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER] = _TAIL_CALL_TRACE_RECORD, + [GET_LEN] = _TAIL_CALL_TRACE_RECORD, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACE_RECORD, + [IS_OP] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_CONST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_LOCALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_NAME] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SPECIAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACE_RECORD, + [MAKE_CELL] = _TAIL_CALL_TRACE_RECORD, + [MAKE_FUNCTION] = _TAIL_CALL_TRACE_RECORD, + [MAP_ADD] = _TAIL_CALL_TRACE_RECORD, + [MATCH_CLASS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_KEYS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_MAPPING] = _TAIL_CALL_TRACE_RECORD, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [NOP] = _TAIL_CALL_TRACE_RECORD, + [NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [POP_EXCEPT] = _TAIL_CALL_TRACE_RECORD, + [POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [POP_TOP] = _TAIL_CALL_TRACE_RECORD, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, + [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, + [RERAISE] = _TAIL_CALL_TRACE_RECORD, + [RESERVED] = _TAIL_CALL_TRACE_RECORD, + [RESUME] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD, + [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, + [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, + [SET_ADD] = _TAIL_CALL_TRACE_RECORD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, + [SET_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [STORE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [STORE_NAME] = _TAIL_CALL_TRACE_RECORD, + [STORE_SLICE] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [SWAP] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_INT] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_LIST] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_NONE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_STR] = _TAIL_CALL_TRACE_RECORD, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NOT] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_EX] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, + [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index c4554f8efb1219..f633f704485819 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -84,7 +84,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit the tracing dispatch table. out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_record_previous_inst,\n") + out.emit(f"[{name}] = _TAIL_CALL_TRACE_RECORD,\n") named_values = analysis.opmap.values() for rest in range(256): if rest not in named_values: