From 82d12596fb4ecae5b4bac1dc8049b48a3c3ea59a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 21 Aug 2025 22:20:54 +0800 Subject: [PATCH 01/16] Brandt's changes Co-Authored-By: Brandt Bucher --- Python/ceval_macros.h | 11 +- Python/opcode_targets.h | 464 +++++++++++----------- Tools/cases_generator/target_generator.py | 2 +- 3 files changed, 240 insertions(+), 237 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 64ca7716fdbdee..8311fa42227535 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -79,11 +79,14 @@ #endif #if Py_TAIL_CALL_INTERP - // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. -# define Py_MUSTTAIL [[clang::musttail]] -# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) +# ifdef _MSC_VER +# define Py_MUSTTAIL [[msvc::musttail]] +# define Py_PRESERVE_NONE_CC __preserve_none +# else +# define Py_MUSTTAIL [[clang::musttail]] +# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); - +# endif # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 1d6dcddab4b12d..a3d0e4c6736be6 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -260,240 +260,240 @@ static void *opcode_targets[256] = { #else /* Py_TAIL_CALL_INTERP */ static py_tail_call_funcptr INSTRUCTION_TABLE[256]; -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index ca151ff640a30c..7282c4ba9ba483 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: out.emit("#else /* Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: - return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" + return f"static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: From 32486589ae668d3453921f01c1b82f12524c8a75 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 21 Aug 2025 22:57:28 +0800 Subject: [PATCH 02/16] Hulon's and mine changes Co-Authored-By: Hulon Jenkins <109993038+hulonjenkins@users.noreply.github.com> --- Include/pyport.h | 18 ++++++++++++++++++ PCbuild/pythoncore.vcxproj | 1 + Python/bytecodes.c | 14 ++++++++++---- Python/ceval_macros.h | 4 ++-- Python/executor_cases.c.h | 11 ++++++++--- Python/generated_cases.c.h | 25 ++++++++++++++++++------- Tools/cases_generator/analyzer.py | 2 ++ 7 files changed, 59 insertions(+), 16 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 62db8d07701d1d..2f680ca1d57139 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -385,6 +385,24 @@ extern "C" { # define Py_NO_INLINE #endif +// Any function annotated with this MUST not modify global state. +// It can only modify state referenced by its parameters. +// This is useful for optimizations on certain compilers. +// Please see https://learn.microsoft.com/en-us/cpp/cpp/noalias +#if defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) +# define Py_NOALIAS +#elif defined(_MSC_VER) +# define Py_NOALIAS __declspec(noalias) +#else +# define Py_NOALIAS +#endif + +// A no-op at compile time. Hints to the programmer +// That any local variable defined within this block MUST +// not escape from the current definition. +# define Py_BEGIN_LOCALS_MUST_NOT_ESCAPE() { +# define Py_END_LOCALS_MUST_NOT_ESCAPE() } + #include "exports.h" #ifdef Py_LIMITED_API diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 517103acea8d8e..e7d41c41225b9d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -108,6 +108,7 @@ _Py_JIT;%(PreprocessorDefinitions) _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) Py_TAIL_CALL_INTERP=1;%(PreprocessorDefinitions) + /std:clatest %(AdditionalOptions) HAVE_COMPUTED_GOTOS;%(PreprocessorDefinitions) Py_REMOTE_DEBUG;%(PreprocessorDefinitions) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7f89c312b9a815..220d592398f365 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2226,8 +2226,11 @@ dummy_func( } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + PyObject *super; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + PyObject *restrict stack[] = {class, self}; + super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + Py_END_LOCALS_MUST_NOT_ESCAPE(); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { @@ -3509,10 +3512,13 @@ dummy_func( } assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject* res_o; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + PyObject *restrict stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + Py_END_LOCALS_MUST_NOT_ESCAPE(); Py_XDECREF(original_tb); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 8311fa42227535..4f8712e09b6306 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -85,9 +85,9 @@ # else # define Py_MUSTTAIL [[clang::musttail]] # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) - Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); # endif -# define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) + typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS); +# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ Py_MUSTTAIL return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3dcb2decc43737..c0a753a2f127ee 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4678,11 +4678,16 @@ } assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject* res_o; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + PyObject *restrict stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_END_LOCALS_MUST_NOT_ESCAPE(); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_XDECREF(original_tb); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7547eaad125370..0aa74832d5c9f4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7093,10 +7093,13 @@ JUMP_TO_LABEL(error); } } - PyObject *stack[] = {class, self}; + PyObject *super; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + PyObject *restrict stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + Py_END_LOCALS_MUST_NOT_ESCAPE(); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { @@ -9489,10 +9492,13 @@ JUMP_TO_LABEL(error); } } - PyObject *stack[] = {class, self}; + PyObject *super; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + PyObject *restrict stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + Py_END_LOCALS_MUST_NOT_ESCAPE(); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { @@ -12035,11 +12041,16 @@ } assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject* res_o; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + PyObject *restrict stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_END_LOCALS_MUST_NOT_ESCAPE(); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_XDECREF(original_tb); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 9dd7e5dbfbae7b..6de88135340789 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -692,6 +692,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Wrap", "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", + "Py_BEGIN_LOCALS_MUST_NOT_ESCAPE", + "Py_END_LOCALS_MUST_NOT_ESCAPE", ) From 9ac430d8e988fbddfde6fead59109bf1d510d333 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 22 Aug 2025 01:44:33 +0800 Subject: [PATCH 03/16] push stackrefs into tstate --- Include/internal/pycore_tstate.h | 4 ++++ Python/ceval_macros.h | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index bad968428c73a1..de58f859528d9c 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -21,6 +21,9 @@ struct _gc_thread_state { }; #endif +/* How much scratch space to give stackref to PyObject* conversion. */ +#define MAX_STACKREF_SCRATCH 10 + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -47,6 +50,7 @@ typedef struct _PyThreadStateImpl { struct _qsbr_thread_state *qsbr; // only used by free-threaded build struct llist_node mem_free_queue; // delayed free queue + PyObject *stackref_scratch[MAX_STACKREF_SCRATCH + 1]; #ifdef Py_GIL_DISABLED // Stack references for the current thread that exist on the C stack struct _PyCStackRef *c_stack_refs; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 4f8712e09b6306..542acf3cf43fb5 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -393,13 +393,19 @@ do { \ /* Stackref macros */ -/* How much scratch space to give stackref to PyObject* conversion. */ -#define MAX_STACKREF_SCRATCH 10 +#if Py_TAIL_CALL_INTERP && defined(_MSC_VER) +#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ + /* +1 because vectorcall might use -1 to write self */ \ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; \ + PyObject **NAME##_temp = _tstate->stackref_scratch; \ + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); +#else #define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ /* +1 because vectorcall might use -1 to write self */ \ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); +#endif #define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ /* +1 because we +1 previously */ \ From 085c1d7df618e3c404d1bb89fdd6b9266ce599b5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 22 Aug 2025 02:04:37 +0800 Subject: [PATCH 04/16] Fix last few remaining problems --- Include/internal/pycore_ceval.h | 5 +++- Include/internal/pycore_dict.h | 4 +-- Include/internal/pycore_unicodeobject.h | 4 +-- Include/pyport.h | 14 +++------ Objects/abstract.c | 6 ++++ Objects/dictobject.c | 6 ++-- Objects/unicodeobject.c | 2 +- Python/bytecodes.c | 33 +++++++++------------ Python/executor_cases.c.h | 30 +++++++------------ Python/generated_cases.c.h | 38 ++++++++++--------------- 10 files changed, 62 insertions(+), 80 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a03fe42668f15a..10f88a44742512 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -303,7 +303,7 @@ PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, Py PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); -PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); +PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *restrict input, Py_ssize_t nargs, PyObject **restrict scratch); PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch); @@ -391,6 +391,9 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 +/* Special counterparts of ceval functions for performance reasons */ +PyAPI_FUNC(int) _PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6ab569393e5ce1..cadd0ce3186f6a 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -290,8 +290,8 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj); PyAPI_FUNC(PyObject *)_PyDict_FromItems( - PyObject *const *keys, Py_ssize_t keys_offset, - PyObject *const *values, Py_ssize_t values_offset, + PyObject *const *restrict keys, Py_ssize_t keys_offset, + PyObject *const *restrict values, Py_ssize_t values_offset, Py_ssize_t length); static inline uint8_t * diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 3791b913c17546..cc724b4eb6b3f3 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -206,8 +206,8 @@ PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( /* --- Methods & Slots ---------------------------------------------------- */ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( - PyObject *separator, - PyObject *const *items, + PyObject *restrict separator, + PyObject *const *restrict items, Py_ssize_t seqlen ); diff --git a/Include/pyport.h b/Include/pyport.h index 2f680ca1d57139..703d59480bb72e 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -385,19 +385,13 @@ extern "C" { # define Py_NO_INLINE #endif -// Any function annotated with this MUST not modify global state. -// It can only modify state referenced by its parameters. -// This is useful for optimizations on certain compilers. -// Please see https://learn.microsoft.com/en-us/cpp/cpp/noalias -#if defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) -# define Py_NOALIAS -#elif defined(_MSC_VER) -# define Py_NOALIAS __declspec(noalias) +#if defined(_MSC_VER) && Py_TAIL_CALL_INTERP +# define Py_NO_INLINE_MSVC_TAILCALL Py_NO_INLINE #else -# define Py_NOALIAS +# define Py_NO_INLINE_MSVC_TAILCALL #endif -// A no-op at compile time. Hints to the programmer +// Just a scope. Hints to the programmer // That any local variable defined within this block MUST // not escape from the current definition. # define Py_BEGIN_LOCALS_MUST_NOT_ESCAPE() { diff --git a/Objects/abstract.c b/Objects/abstract.c index 8adad8407d04d4..aa68efee0e0f5f 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -224,6 +224,12 @@ PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) return 0; } +Py_NO_INLINE_MSVC_TAILCALL int +_PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **restrict result) +{ + return PyMapping_GetOptionalItem(obj, key, result); +} + int PyObject_SetItem(PyObject *o, PyObject *key, PyObject *value) { diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 06e0c1b61cbcec..fc014f5d1591d2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2225,9 +2225,9 @@ _PyDict_NewPresized(Py_ssize_t minused) return dict_new_presized(minused, false); } -PyObject * -_PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, - PyObject *const *values, Py_ssize_t values_offset, +Py_NO_INLINE_MSVC_TAILCALL PyObject * +_PyDict_FromItems(PyObject *const *restrict keys, Py_ssize_t keys_offset, + PyObject *const *restrict values, Py_ssize_t values_offset, Py_ssize_t length) { bool unicode = true; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 9300a99a72144d..0bb2a375249f0f 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -10234,7 +10234,7 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) } PyObject * -_PyUnicode_JoinArray(PyObject *separator, PyObject *const *items, Py_ssize_t seqlen) +_PyUnicode_JoinArray(PyObject *restrict separator, PyObject *const *restrict items, Py_ssize_t seqlen) { PyObject *res = NULL; /* the result */ PyObject *sep = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 220d592398f365..74709efc278d58 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1534,8 +1534,8 @@ dummy_func( } inst(LOAD_BUILD_CLASS, ( -- bc)) { - PyObject *bc_o; - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + PyObject *restrict bc_o; + int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); ERROR_IF(err < 0); if (bc_o == NULL) { _PyErr_SetString(tstate, PyExc_NameError, @@ -1738,8 +1738,8 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + PyObject *restrict v_o; + int err = _PyEval_Mapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); PyStackRef_CLOSE(mod_or_class_dict); ERROR_IF(err < 0); if (v_o == NULL) { @@ -1762,11 +1762,11 @@ dummy_func( else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + int err = _PyEval_Mapping_GetOptionalItem(GLOBALS(), name, &v_o); ERROR_IF(err < 0); if (v_o == NULL) { /* namespace 2: builtins */ - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), name, &v_o); ERROR_IF(err < 0); if (v_o == NULL) { _PyEval_FormatExcCheckArg( @@ -1925,14 +1925,14 @@ dummy_func( } inst(LOAD_FROM_DICT_OR_DEREF, (class_dict_st -- value)) { - PyObject *value_o; + PyObject *restrict value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + int err = _PyEval_Mapping_GetOptionalItem(class_dict, name, &value_o); if (err < 0) { ERROR_NO_POP(); } @@ -2115,14 +2115,14 @@ dummy_func( } inst(SETUP_ANNOTATIONS, (--)) { - PyObject *ann_dict; + PyObject *restrict ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); ERROR_IF(true); } /* check if __annotations__ in locals()... */ - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + int err = _PyEval_Mapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); ERROR_IF(err < 0); if (ann_dict == NULL) { ann_dict = PyDict_New(); @@ -2289,18 +2289,13 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, NULL); if (attr_o == NULL) { ERROR_NO_POP(); } - if (method_found) { - self_or_null = self_st; // transfer ownership - DEAD(self_st); - } else { - PyStackRef_CLOSE(self_st); - self_or_null = PyStackRef_NULL; - } + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; + DECREF_INPUTS(); attr = PyStackRef_FromPyObjectSteal(attr_o); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c0a753a2f127ee..62c4167ebde747 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2132,9 +2132,9 @@ case _LOAD_BUILD_CLASS: { _PyStackRef bc; - PyObject *bc_o; + PyObject *restrict bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_ERROR(); @@ -2640,14 +2640,14 @@ _PyStackRef value; oparg = CURRENT_OPARG(); class_dict_st = stack_pointer[-1]; - PyObject *value_o; + PyObject *restrict value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + int err = _PyEval_Mapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_ERROR(); @@ -3042,7 +3042,7 @@ } case _SETUP_ANNOTATIONS: { - PyObject *ann_dict; + PyObject *restrict ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, @@ -3051,7 +3051,7 @@ JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + int err = _PyEval_Mapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_ERROR(); @@ -3254,26 +3254,18 @@ PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { JUMP_TO_ERROR(); } - if (method_found) { - self_or_null = self_st; - } else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0aa74832d5c9f4..70168c5600a4cc 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8787,9 +8787,9 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; - PyObject *bc_o; + PyObject *restrict bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_LABEL(error); @@ -9000,14 +9000,14 @@ _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; - PyObject *value_o; + PyObject *restrict value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + int err = _PyEval_Mapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_LABEL(error); @@ -9046,9 +9046,9 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; + PyObject *restrict v_o; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + int err = _PyEval_Mapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -9079,14 +9079,14 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + int err = _PyEval_Mapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_LABEL(error); @@ -9664,26 +9664,18 @@ PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } - if (method_found) { - self_or_null = self_st; - } else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; @@ -10598,7 +10590,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyObject *ann_dict; + PyObject *restrict ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, @@ -10607,7 +10599,7 @@ JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + int err = _PyEval_Mapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_LABEL(error); From 0b12f2efb486d62b5286d258f16a37dc9e877fcc Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 24 Aug 2025 22:10:53 +0800 Subject: [PATCH 05/16] fix handling of recursion --- Include/internal/pycore_ceval.h | 4 ++-- Include/internal/pycore_tstate.h | 5 +++-- Python/bytecodes.c | 22 +++++++++++----------- Python/ceval.c | 21 +++++++++++++++------ Python/ceval_macros.h | 17 ++++------------- Python/executor_cases.c.h | 18 +++++++++--------- Python/generated_cases.c.h | 26 +++++++++++++------------- Python/pystate.c | 1 + 8 files changed, 58 insertions(+), 56 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 10f88a44742512..7edd29472d0a58 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -303,9 +303,9 @@ PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, Py PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); -PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *restrict input, Py_ssize_t nargs, PyObject **restrict scratch); +PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyThreadStateImpl *_tstate, _PyStackRef *restrict input, Py_ssize_t nargs); -PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch); +PyAPI_FUNC(void) _PyObjectArray_Free(_PyThreadStateImpl *_tstate, PyObject **array, Py_ssize_t nargs, PyObject **temp_arr); PyAPI_FUNC(PyObject *) _PyEval_GetANext(PyObject *aiter); PyAPI_FUNC(void) _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name, _PyStackRef *writeto); diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index de58f859528d9c..8edaf5c6a5ae67 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -22,7 +22,7 @@ struct _gc_thread_state { #endif /* How much scratch space to give stackref to PyObject* conversion. */ -#define MAX_STACKREF_SCRATCH 10 +#define MAX_STACKREF_SCRATCH 1024 // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields @@ -50,7 +50,8 @@ typedef struct _PyThreadStateImpl { struct _qsbr_thread_state *qsbr; // only used by free-threaded build struct llist_node mem_free_queue; // delayed free queue - PyObject *stackref_scratch[MAX_STACKREF_SCRATCH + 1]; + PyObject *stackref_scratch[MAX_STACKREF_SCRATCH]; + int n_stackref_scratch_used; #ifdef Py_GIL_DISABLED // Stack references for the current thread that exist on the C stack struct _PyCStackRef *c_stack_refs; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 74709efc278d58..4225ec443450b6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1983,7 +1983,7 @@ dummy_func( ERROR_IF(true); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o, oparg); DECREF_INPUTS(); ERROR_IF(str_o == NULL); str = PyStackRef_FromPyObjectSteal(str_o); @@ -2108,7 +2108,7 @@ dummy_func( values_o, 2, values_o+1, 2, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o, oparg*2); DECREF_INPUTS(); ERROR_IF(map_o == NULL); map = PyStackRef_FromPyObjectStealMortal(map_o); @@ -3755,7 +3755,7 @@ dummy_func( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -3910,7 +3910,7 @@ dummy_func( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); DECREF_INPUTS(); ERROR_IF(res_o == NULL); @@ -4193,7 +4193,7 @@ dummy_func( ERROR_IF(true); } PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); DECREF_INPUTS(); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); @@ -4264,7 +4264,7 @@ dummy_func( PyCFunction_GET_SELF(callable_o), args_o, total_args); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); DECREF_INPUTS(); ERROR_IF(res_o == NULL); @@ -4300,7 +4300,7 @@ dummy_func( ERROR_IF(true); } PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); DECREF_INPUTS(); ERROR_IF(res_o == NULL); @@ -4480,7 +4480,7 @@ dummy_func( PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); DECREF_INPUTS(); ERROR_IF(res_o == NULL); @@ -4559,7 +4559,7 @@ dummy_func( } PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); PyObject *res_o = cfunc(self, (args_o + 1), nargs); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); DECREF_INPUTS(); ERROR_IF(res_o == NULL); @@ -4658,7 +4658,7 @@ dummy_func( callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -4815,7 +4815,7 @@ dummy_func( positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); PyStackRef_CLOSE(kwnames); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); DECREF_INPUTS(); ERROR_IF(res_o == NULL); diff --git a/Python/ceval.c b/Python/ceval.c index 578c5d2a8b1420..92b7718d33affc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -945,10 +945,12 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); PyObject ** -_PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch) +_PyObjectArray_FromStackRefArray(_PyThreadStateImpl *_tstate, _PyStackRef *input, Py_ssize_t nargs) { PyObject **result; - if (nargs > MAX_STACKREF_SCRATCH) { + /* +1 because vectorcall might use -1 to write self */ + PyObject **target = _tstate->stackref_scratch + _tstate->n_stackref_scratch_used + nargs + 1; + if (target > _tstate->stackref_scratch + MAX_STACKREF_SCRATCH) { // +1 in case PY_VECTORCALL_ARGUMENTS_OFFSET is set. result = PyMem_Malloc((nargs + 1) * sizeof(PyObject *)); if (result == NULL) { @@ -957,7 +959,9 @@ _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject result++; } else { - result = scratch; + result = _tstate->stackref_scratch + _tstate->n_stackref_scratch_used; + _tstate->n_stackref_scratch_used += (int)nargs + 1; + assert(_tstate->n_stackref_scratch_used < MAX_STACKREF_SCRATCH); } for (int i = 0; i < nargs; i++) { result[i] = PyStackRef_AsPyObjectBorrow(input[i]); @@ -966,10 +970,15 @@ _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject } void -_PyObjectArray_Free(PyObject **array, PyObject **scratch) +_PyObjectArray_Free(_PyThreadStateImpl *_tstate, PyObject **array, Py_ssize_t nargs, PyObject **temp_arr) { - if (array != scratch) { - PyMem_Free(array); + /* -1 because we +1 previously */ + if (array == temp_arr) { + _tstate->n_stackref_scratch_used -= ((int)nargs + 1); + assert(_tstate->n_stackref_scratch_used >= 0); + } + else { + PyMem_Free(array-1); } } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 542acf3cf43fb5..9cdc481ae76edc 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -394,22 +394,13 @@ do { \ /* Stackref macros */ -#if Py_TAIL_CALL_INTERP && defined(_MSC_VER) #define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ - /* +1 because vectorcall might use -1 to write self */ \ _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; \ - PyObject **NAME##_temp = _tstate->stackref_scratch; \ - PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); -#else -#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ - /* +1 because vectorcall might use -1 to write self */ \ - PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ - PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); -#endif + PyObject **NAME##_temp = _tstate->stackref_scratch + _tstate->n_stackref_scratch_used; \ + PyObject **NAME = _PyObjectArray_FromStackRefArray(_tstate, ARGS, ARG_COUNT); -#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ - /* +1 because we +1 previously */ \ - _PyObjectArray_Free(NAME - 1, NAME##_temp); +#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME, ARG_COUNT) \ + _PyObjectArray_Free(_tstate, NAME, ARG_COUNT, NAME##_temp); #define CONVERSION_FAILED(NAME) ((NAME) == NULL) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 62c4167ebde747..b8211cd5d20e4d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2744,7 +2744,7 @@ JUMP_TO_ERROR(); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { @@ -3020,7 +3020,7 @@ values_o+1, 2, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o, oparg*2); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { @@ -5064,7 +5064,7 @@ total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -5650,7 +5650,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { @@ -5790,7 +5790,7 @@ args_o, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -5874,7 +5874,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -6208,7 +6208,7 @@ _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -6367,7 +6367,7 @@ PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); PyObject *res_o = cfunc(self, (args_o + 1), nargs); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -6613,7 +6613,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 70168c5600a4cc..b3ace9a906327f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1274,7 +1274,7 @@ values_o+1, 2, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o, oparg*2); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { @@ -1413,7 +1413,7 @@ JUMP_TO_LABEL(error); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { @@ -1622,7 +1622,7 @@ total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -2143,7 +2143,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { @@ -2251,7 +2251,7 @@ args_o, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -2360,7 +2360,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -2922,7 +2922,7 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -3195,7 +3195,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -3586,7 +3586,7 @@ PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); PyObject *res_o = cfunc(self, (args_o + 1), nargs); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -3709,7 +3709,7 @@ _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -4023,7 +4023,7 @@ total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp; @@ -6365,7 +6365,7 @@ total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -6725,7 +6725,7 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o, total_args); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); diff --git a/Python/pystate.c b/Python/pystate.c index 9091057f6f62cf..3ac3291f5386b4 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1495,6 +1495,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, } tstate->_status.initialized = 1; + _tstate->n_stackref_scratch_used = 0; } static void From acf48f5d94aeea9f947307ca1ce67ffb9a24ca4c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Oct 2025 17:19:21 +0100 Subject: [PATCH 06/16] Changes by Chris to support clang-cl Co-Authored-By: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> --- PCbuild/pythoncore.vcxproj | 5 +++-- Python/ceval_macros.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index e7d41c41225b9d..a56d4324580849 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -108,7 +108,6 @@ _Py_JIT;%(PreprocessorDefinitions) _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) Py_TAIL_CALL_INTERP=1;%(PreprocessorDefinitions) - /std:clatest %(AdditionalOptions) HAVE_COMPUTED_GOTOS;%(PreprocessorDefinitions) Py_REMOTE_DEBUG;%(PreprocessorDefinitions) @@ -591,7 +590,9 @@ - + + /std:clatest %(AdditionalOptions) + diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 9cdc481ae76edc..d4fba68a3ab2ef 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -79,7 +79,7 @@ #endif #if Py_TAIL_CALL_INTERP -# ifdef _MSC_VER +# if defined(_MSC_VER) && !defined(__clang__) # define Py_MUSTTAIL [[msvc::musttail]] # define Py_PRESERVE_NONE_CC __preserve_none # else From 35e96c17459519255648423fc162d36ae4c37578 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Oct 2025 17:45:26 +0100 Subject: [PATCH 07/16] Make the restrict MSVC only for now --- Include/internal/pycore_ceval.h | 2 +- Include/pyport.h | 8 +++++++- Objects/abstract.c | 2 +- Python/bytecodes.c | 14 +++++++------- Python/ceval.c | 2 +- Python/executor_cases.c.h | 8 ++++---- Python/generated_cases.c.h | 14 +++++++------- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 7edd29472d0a58..498d9030165c09 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -303,7 +303,7 @@ PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, Py PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); -PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyThreadStateImpl *_tstate, _PyStackRef *restrict input, Py_ssize_t nargs); +PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyThreadStateImpl *_tstate, _PyStackRef *input, Py_ssize_t nargs); PyAPI_FUNC(void) _PyObjectArray_Free(_PyThreadStateImpl *_tstate, PyObject **array, Py_ssize_t nargs, PyObject **temp_arr); diff --git a/Include/pyport.h b/Include/pyport.h index 703d59480bb72e..415a68da9e1363 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -385,12 +385,18 @@ extern "C" { # define Py_NO_INLINE #endif -#if defined(_MSC_VER) && Py_TAIL_CALL_INTERP +#if defined(_MSC_VER) && !defined(__clang__) && Py_TAIL_CALL_INTERP # define Py_NO_INLINE_MSVC_TAILCALL Py_NO_INLINE #else # define Py_NO_INLINE_MSVC_TAILCALL #endif +#if defined(_MSC_VER) && !defined(__clang__) +# define Py_MSVC_RESTRICT restrict +#else +# define Py_MSVC_RESTRICT +#endif + // Just a scope. Hints to the programmer // That any local variable defined within this block MUST // not escape from the current definition. diff --git a/Objects/abstract.c b/Objects/abstract.c index aa68efee0e0f5f..0e0b4dbf42c160 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -225,7 +225,7 @@ PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) } Py_NO_INLINE_MSVC_TAILCALL int -_PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **restrict result) +_PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **Py_MSVC_RESTRICT result) { return PyMapping_GetOptionalItem(obj, key, result); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4225ec443450b6..795cbe359af271 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1534,7 +1534,7 @@ dummy_func( } inst(LOAD_BUILD_CLASS, ( -- bc)) { - PyObject *restrict bc_o; + PyObject *Py_MSVC_RESTRICT bc_o; int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); ERROR_IF(err < 0); if (bc_o == NULL) { @@ -1738,7 +1738,7 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *restrict v_o; + PyObject *Py_MSVC_RESTRICT v_o; int err = _PyEval_Mapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); PyStackRef_CLOSE(mod_or_class_dict); ERROR_IF(err < 0); @@ -1925,7 +1925,7 @@ dummy_func( } inst(LOAD_FROM_DICT_OR_DEREF, (class_dict_st -- value)) { - PyObject *restrict value_o; + PyObject *Py_MSVC_RESTRICT value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); @@ -2115,7 +2115,7 @@ dummy_func( } inst(SETUP_ANNOTATIONS, (--)) { - PyObject *restrict ann_dict; + PyObject *Py_MSVC_RESTRICT ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); @@ -2228,7 +2228,7 @@ dummy_func( // handle any case whose performance we care about PyObject *super; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); - PyObject *restrict stack[] = {class, self}; + PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); Py_END_LOCALS_MUST_NOT_ESCAPE(); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { @@ -2295,7 +2295,7 @@ dummy_func( } PyStackRef_CLOSE(self_st); self_or_null = PyStackRef_NULL; - + DECREF_INPUTS(); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -3509,7 +3509,7 @@ dummy_func( (void)lasti; // Shut up compiler warning if asserts are off PyObject* res_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); - PyObject *restrict stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); diff --git a/Python/ceval.c b/Python/ceval.c index 92b7718d33affc..04342031198591 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -945,7 +945,7 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); PyObject ** -_PyObjectArray_FromStackRefArray(_PyThreadStateImpl *_tstate, _PyStackRef *input, Py_ssize_t nargs) +_PyObjectArray_FromStackRefArray(_PyThreadStateImpl *_tstate, _PyStackRef *Py_MSVC_RESTRICT input, Py_ssize_t nargs) { PyObject **result; /* +1 because vectorcall might use -1 to write self */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b8211cd5d20e4d..5f1bbfc108dc08 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2132,7 +2132,7 @@ case _LOAD_BUILD_CLASS: { _PyStackRef bc; - PyObject *restrict bc_o; + PyObject *Py_MSVC_RESTRICT bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2640,7 +2640,7 @@ _PyStackRef value; oparg = CURRENT_OPARG(); class_dict_st = stack_pointer[-1]; - PyObject *restrict value_o; + PyObject *Py_MSVC_RESTRICT value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); @@ -3042,7 +3042,7 @@ } case _SETUP_ANNOTATIONS: { - PyObject *restrict ann_dict; + PyObject *Py_MSVC_RESTRICT ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, @@ -4672,7 +4672,7 @@ (void)lasti; PyObject* res_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); - PyObject *restrict stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b3ace9a906327f..6c1655ca6f2625 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7095,7 +7095,7 @@ } PyObject *super; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); - PyObject *restrict stack[] = {class, self}; + PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8787,7 +8787,7 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; - PyObject *restrict bc_o; + PyObject *Py_MSVC_RESTRICT bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9000,7 +9000,7 @@ _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; - PyObject *restrict value_o; + PyObject *Py_MSVC_RESTRICT value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); @@ -9046,7 +9046,7 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *restrict v_o; + PyObject *Py_MSVC_RESTRICT v_o; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_Mapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9494,7 +9494,7 @@ } PyObject *super; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); - PyObject *restrict stack[] = {class, self}; + PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10590,7 +10590,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyObject *restrict ann_dict; + PyObject *Py_MSVC_RESTRICT ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, @@ -12035,7 +12035,7 @@ (void)lasti; PyObject* res_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); - PyObject *restrict stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, From 86f19cf7880f62de288d28bf0f7440f07a1aea2a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Oct 2025 17:58:02 +0100 Subject: [PATCH 08/16] Change restricts to MSVC only --- Include/internal/pycore_dict.h | 4 ++-- Include/internal/pycore_unicodeobject.h | 4 ++-- Include/pyport.h | 2 +- Objects/dictobject.c | 4 ++-- Objects/unicodeobject.c | 2 +- Python/ceval_macros.h | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index cc299dc4e009e3..92b6b819ef2625 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -290,8 +290,8 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj); PyAPI_FUNC(PyObject *)_PyDict_FromItems( - PyObject *const *restrict keys, Py_ssize_t keys_offset, - PyObject *const *restrict values, Py_ssize_t values_offset, + PyObject *const *keys, Py_ssize_t keys_offset, + PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length); static inline uint8_t * diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index aab67242497022..da8775937bb725 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -274,8 +274,8 @@ PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( /* --- Methods & Slots ---------------------------------------------------- */ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( - PyObject *restrict separator, - PyObject *const *restrict items, + PyObject * separator, + PyObject *const * items, Py_ssize_t seqlen ); diff --git a/Include/pyport.h b/Include/pyport.h index 415a68da9e1363..ee18f3580950c6 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -385,7 +385,7 @@ extern "C" { # define Py_NO_INLINE #endif -#if defined(_MSC_VER) && !defined(__clang__) && Py_TAIL_CALL_INTERP +#if defined(_MSC_VER) && !defined(__clang__) && _Py_TAIL_CALL_INTERP # define Py_NO_INLINE_MSVC_TAILCALL Py_NO_INLINE #else # define Py_NO_INLINE_MSVC_TAILCALL diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 9b43e5ab2b2568..035f656afd6a89 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2226,8 +2226,8 @@ _PyDict_NewPresized(Py_ssize_t minused) } Py_NO_INLINE_MSVC_TAILCALL PyObject * -_PyDict_FromItems(PyObject *const *restrict keys, Py_ssize_t keys_offset, - PyObject *const *restrict values, Py_ssize_t values_offset, +_PyDict_FromItems(PyObject *const *Py_MSVC_RESTRICT keys, Py_ssize_t keys_offset, + PyObject *const *Py_MSVC_RESTRICT values, Py_ssize_t values_offset, Py_ssize_t length) { bool unicode = true; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d6859d1b4467a9..7a08cc224efed6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -10068,7 +10068,7 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) } PyObject * -_PyUnicode_JoinArray(PyObject *restrict separator, PyObject *const *restrict items, Py_ssize_t seqlen) +_PyUnicode_JoinArray(PyObject *Py_MSVC_RESTRICT separator, PyObject *const *Py_MSVC_RESTRICT items, Py_ssize_t seqlen) { PyObject *res = NULL; /* the result */ PyObject *sep = NULL; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a8616424b97463..5f2bd449a15572 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -78,7 +78,7 @@ # define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg #endif -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP # if defined(_MSC_VER) && !defined(__clang__) # define Py_MUSTTAIL [[msvc::musttail]] # define Py_PRESERVE_NONE_CC __preserve_none From 40013cc5c8410a54999c967841fd23482f8f33ce Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 11 Oct 2025 17:01:28 +0000 Subject: [PATCH 09/16] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-10-11-17-01-21.gh-issue-139922.RUkXyd.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-11-17-01-21.gh-issue-139922.RUkXyd.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-11-17-01-21.gh-issue-139922.RUkXyd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-11-17-01-21.gh-issue-139922.RUkXyd.rst new file mode 100644 index 00000000000000..d498db07f91454 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-11-17-01-21.gh-issue-139922.RUkXyd.rst @@ -0,0 +1 @@ +Allow building CPython with the tail calling interpreter on Visual Studio 2026 MSVC. This provides a performance gain over the prior interpreter for MSVC. Patch by Ken Jin, Brandt Bucher, and Chris Eibl. With help from the MSVC team including Hulon Jenkins. From 48db59e6e2c768089395ec7d19134f666434ef8c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Oct 2025 18:02:46 +0100 Subject: [PATCH 10/16] Reduce diff --- Include/internal/pycore_unicodeobject.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index da8775937bb725..b83039c1869f23 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -274,8 +274,8 @@ PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( /* --- Methods & Slots ---------------------------------------------------- */ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( - PyObject * separator, - PyObject *const * items, + PyObject *separator, + PyObject *const *items, Py_ssize_t seqlen ); From 19e02c28e09769b5c5173bf9777fccf712189caa Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Oct 2025 18:34:17 +0100 Subject: [PATCH 11/16] Work around specialization --- Python/bytecodes.c | 16 +++++++++++----- Python/executor_cases.c.h | 20 ++++++++++++++------ Python/generated_cases.c.h | 20 ++++++++++++++------ 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4dbfb5554b4cf6..927e2d173bd281 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2288,14 +2288,20 @@ dummy_func( STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - PyObject *attr_o = _PySuper_Lookup(cls, self, name, NULL); + // Note: not actually a pointer, just so that we can use restrict on it. + int *Py_MSVC_RESTRICT method_found = NULL; + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? (int *)&method_found : NULL); if (attr_o == NULL) { ERROR_NO_POP(); } - PyStackRef_CLOSE(self_st); - self_or_null = PyStackRef_NULL; - + if (method_found) { + self_or_null = self_st; // transfer ownership + DEAD(self_st); + } else { + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; + } DECREF_INPUTS(); attr = PyStackRef_FromPyObjectSteal(attr_o); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bee46826067fd1..4cf5b0d32443bc 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3252,20 +3252,28 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; + int *Py_MSVC_RESTRICT method_found = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, NULL); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? (int *)&method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { JUMP_TO_ERROR(); } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a620b619631b01..e60fe52b2d4b37 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9663,20 +9663,28 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; + int *Py_MSVC_RESTRICT method_found = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, NULL); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? (int *)&method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; From 66ec774074592353e73276df8e34a7865fc07cd1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Oct 2025 01:46:42 +0800 Subject: [PATCH 12/16] Fix VS 2026 --- Python/bytecodes.c | 11 +++++++---- Python/executor_cases.c.h | 10 +++++++--- Python/generated_cases.c.h | 10 +++++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 927e2d173bd281..705a9f1864ab78 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2288,10 +2288,13 @@ dummy_func( STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; - // Note: not actually a pointer, just so that we can use restrict on it. - int *Py_MSVC_RESTRICT method_found = NULL; - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? (int *)&method_found : NULL); + int method_found = 0; + PyObject *attr_o; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE();. + int *Py_MSVC_RESTRICT method_found_ptr = &method_found; + attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); + Py_END_LOCALS_MUST_NOT_ESCAPE(); if (attr_o == NULL) { ERROR_NO_POP(); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4cf5b0d32443bc..e91bc57782f03d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3252,11 +3252,15 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; - int *Py_MSVC_RESTRICT method_found = NULL; + int method_found = 0; + PyObject *attr_o; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + int *Py_MSVC_RESTRICT method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? (int *)&method_found : NULL); + attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + Py_END_LOCALS_MUST_NOT_ESCAPE(); if (attr_o == NULL) { JUMP_TO_ERROR(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e60fe52b2d4b37..a06fabf74e3007 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9663,11 +9663,15 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; - int *Py_MSVC_RESTRICT method_found = NULL; + int method_found = 0; + PyObject *attr_o; + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + int *Py_MSVC_RESTRICT method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? (int *)&method_found : NULL); + attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + Py_END_LOCALS_MUST_NOT_ESCAPE(); if (attr_o == NULL) { JUMP_TO_LABEL(error); } From 50f8ff7a145bcaa5607e0a420a2f1648e68b3799 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Oct 2025 02:27:46 +0800 Subject: [PATCH 13/16] fix a typo --- Python/bytecodes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 705a9f1864ab78..886ce44a600eb8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2290,7 +2290,7 @@ dummy_func( PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; PyObject *attr_o; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE();. + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); int *Py_MSVC_RESTRICT method_found_ptr = &method_found; attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); From 5d908b4f7911a6ece9caa9786cdc75c2304c6caa Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Oct 2025 22:38:11 +0100 Subject: [PATCH 14/16] Move to macros to internal header --- Include/internal/pycore_ceval.h | 18 ++++++++++++++++++ Include/pyport.h | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 9f4bd9989faf5a..2fa382215e9342 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -394,6 +394,24 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame /* Special counterparts of ceval functions for performance reasons */ PyAPI_FUNC(int) _PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result); +#if defined(_MSC_VER) && !defined(__clang__) && _Py_TAIL_CALL_INTERP +# define Py_NO_INLINE_MSVC_TAILCALL Py_NO_INLINE +#else +# define Py_NO_INLINE_MSVC_TAILCALL +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +# define Py_MSVC_RESTRICT restrict +#else +# define Py_MSVC_RESTRICT +#endif + +// Just a scope. Hints to the programmer +// That any local variable defined within this block MUST +// not escape from the current definition. +# define Py_BEGIN_LOCALS_MUST_NOT_ESCAPE() { +# define Py_END_LOCALS_MUST_NOT_ESCAPE() } + #ifdef __cplusplus } #endif diff --git a/Include/pyport.h b/Include/pyport.h index ee18f3580950c6..62db8d07701d1d 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -385,24 +385,6 @@ extern "C" { # define Py_NO_INLINE #endif -#if defined(_MSC_VER) && !defined(__clang__) && _Py_TAIL_CALL_INTERP -# define Py_NO_INLINE_MSVC_TAILCALL Py_NO_INLINE -#else -# define Py_NO_INLINE_MSVC_TAILCALL -#endif - -#if defined(_MSC_VER) && !defined(__clang__) -# define Py_MSVC_RESTRICT restrict -#else -# define Py_MSVC_RESTRICT -#endif - -// Just a scope. Hints to the programmer -// That any local variable defined within this block MUST -// not escape from the current definition. -# define Py_BEGIN_LOCALS_MUST_NOT_ESCAPE() { -# define Py_END_LOCALS_MUST_NOT_ESCAPE() } - #include "exports.h" #ifdef Py_LIMITED_API From e699d40c065b1b257ce40c470fb5f418aa4b4f11 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Oct 2025 04:49:00 +0800 Subject: [PATCH 15/16] Reduce number of restricts Co-Authored-By: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> --- Include/pyport.h | 4 ++-- Objects/dictobject.c | 4 ++-- Objects/unicodeobject.c | 2 +- Python/bytecodes.c | 26 +++++++++++++------------- Python/ceval.c | 2 +- Python/executor_cases.c.h | 14 +++++++------- Python/generated_cases.c.h | 24 ++++++++++++------------ Tools/cases_generator/analyzer.py | 2 -- 8 files changed, 38 insertions(+), 40 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index ee18f3580950c6..bcbc613f865962 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -400,8 +400,8 @@ extern "C" { // Just a scope. Hints to the programmer // That any local variable defined within this block MUST // not escape from the current definition. -# define Py_BEGIN_LOCALS_MUST_NOT_ESCAPE() { -# define Py_END_LOCALS_MUST_NOT_ESCAPE() } +# define Py_BEGIN_LOCALS_MUST_NOT_ESCAPE { +# define Py_END_LOCALS_MUST_NOT_ESCAPE } #include "exports.h" diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 035f656afd6a89..56b1da3e07c13f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2226,8 +2226,8 @@ _PyDict_NewPresized(Py_ssize_t minused) } Py_NO_INLINE_MSVC_TAILCALL PyObject * -_PyDict_FromItems(PyObject *const *Py_MSVC_RESTRICT keys, Py_ssize_t keys_offset, - PyObject *const *Py_MSVC_RESTRICT values, Py_ssize_t values_offset, +_PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, + PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length) { bool unicode = true; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7a08cc224efed6..a67bf9b1c5337b 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -10068,7 +10068,7 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) } PyObject * -_PyUnicode_JoinArray(PyObject *Py_MSVC_RESTRICT separator, PyObject *const *Py_MSVC_RESTRICT items, Py_ssize_t seqlen) +_PyUnicode_JoinArray(PyObject *separator, PyObject *const *items, Py_ssize_t seqlen) { PyObject *res = NULL; /* the result */ PyObject *sep = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 886ce44a600eb8..4da24c3848f3b0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1534,7 +1534,7 @@ dummy_func( } inst(LOAD_BUILD_CLASS, ( -- bc)) { - PyObject *Py_MSVC_RESTRICT bc_o; + PyObject *bc_o; int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); ERROR_IF(err < 0); if (bc_o == NULL) { @@ -1738,7 +1738,7 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *Py_MSVC_RESTRICT v_o; + PyObject *v_o; int err = _PyEval_Mapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); PyStackRef_CLOSE(mod_or_class_dict); ERROR_IF(err < 0); @@ -1925,7 +1925,7 @@ dummy_func( } inst(LOAD_FROM_DICT_OR_DEREF, (class_dict_st -- value)) { - PyObject *Py_MSVC_RESTRICT value_o; + PyObject *value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); @@ -2115,7 +2115,7 @@ dummy_func( } inst(SETUP_ANNOTATIONS, (--)) { - PyObject *Py_MSVC_RESTRICT ann_dict; + PyObject *ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); @@ -2227,10 +2227,10 @@ dummy_func( // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *super; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); - PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; - super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; + PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; + super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + Py_END_LOCALS_MUST_NOT_ESCAPE; if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { @@ -2290,11 +2290,11 @@ dummy_func( PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; PyObject *attr_o; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; int *Py_MSVC_RESTRICT method_found_ptr = &method_found; attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); + Py_END_LOCALS_MUST_NOT_ESCAPE; if (attr_o == NULL) { ERROR_NO_POP(); } @@ -3518,12 +3518,12 @@ dummy_func( assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject* res_o; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_END_LOCALS_MUST_NOT_ESCAPE; Py_XDECREF(original_tb); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); diff --git a/Python/ceval.c b/Python/ceval.c index 1256b773035262..cf890d2c22681b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -956,7 +956,7 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); PyObject ** -_PyObjectArray_FromStackRefArray(_PyThreadStateImpl *_tstate, _PyStackRef *Py_MSVC_RESTRICT input, Py_ssize_t nargs) +_PyObjectArray_FromStackRefArray(_PyThreadStateImpl *_tstate, _PyStackRef *restrict input, Py_ssize_t nargs) { PyObject **result; /* +1 because vectorcall might use -1 to write self */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e91bc57782f03d..6328cd044dd774 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2132,7 +2132,7 @@ case _LOAD_BUILD_CLASS: { _PyStackRef bc; - PyObject *Py_MSVC_RESTRICT bc_o; + PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2640,7 +2640,7 @@ _PyStackRef value; oparg = CURRENT_OPARG(); class_dict_st = stack_pointer[-1]; - PyObject *Py_MSVC_RESTRICT value_o; + PyObject *value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); @@ -3042,7 +3042,7 @@ } case _SETUP_ANNOTATIONS: { - PyObject *Py_MSVC_RESTRICT ann_dict; + PyObject *ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, @@ -3254,13 +3254,13 @@ PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; PyObject *attr_o; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; int *Py_MSVC_RESTRICT method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_END_LOCALS_MUST_NOT_ESCAPE; if (attr_o == NULL) { JUMP_TO_ERROR(); } @@ -4683,14 +4683,14 @@ assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; PyObject* res_o; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_END_LOCALS_MUST_NOT_ESCAPE; _PyFrame_SetStackPointer(frame, stack_pointer); Py_XDECREF(original_tb); stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a06fabf74e3007..16adee25589cf4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7094,12 +7094,12 @@ } } PyObject *super; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_END_LOCALS_MUST_NOT_ESCAPE; if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { @@ -8788,7 +8788,7 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; - PyObject *Py_MSVC_RESTRICT bc_o; + PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_Mapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9001,7 +9001,7 @@ _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; - PyObject *Py_MSVC_RESTRICT value_o; + PyObject *value_o; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); @@ -9047,7 +9047,7 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *Py_MSVC_RESTRICT v_o; + PyObject *v_o; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_Mapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9494,12 +9494,12 @@ } } PyObject *super; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_END_LOCALS_MUST_NOT_ESCAPE; if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { @@ -9665,13 +9665,13 @@ PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; PyObject *attr_o; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; int *Py_MSVC_RESTRICT method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_END_LOCALS_MUST_NOT_ESCAPE; if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -10603,7 +10603,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyObject *Py_MSVC_RESTRICT ann_dict; + PyObject *ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, @@ -12047,14 +12047,14 @@ assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; PyObject* res_o; - Py_BEGIN_LOCALS_MUST_NOT_ESCAPE(); + Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_END_LOCALS_MUST_NOT_ESCAPE(); + Py_END_LOCALS_MUST_NOT_ESCAPE; _PyFrame_SetStackPointer(frame, stack_pointer); Py_XDECREF(original_tb); stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 6de88135340789..9dd7e5dbfbae7b 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -692,8 +692,6 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Wrap", "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", - "Py_BEGIN_LOCALS_MUST_NOT_ESCAPE", - "Py_END_LOCALS_MUST_NOT_ESCAPE", ) From 5584fec64e358f55ceeb58136f1d2ba16b35e8dd Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Oct 2025 05:08:14 +0800 Subject: [PATCH 16/16] Reduce restrict use even more, reduce usage --- Include/internal/pycore_ceval.h | 13 +++++++------ Objects/abstract.c | 2 +- Python/bytecodes.c | 6 +++--- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 8 ++++---- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 2fa382215e9342..8b2c573181f1a7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -391,7 +391,7 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 -/* Special counterparts of ceval functions for performance reasons */ +// Special counterparts of ceval functions for performance reasons PyAPI_FUNC(int) _PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result); #if defined(_MSC_VER) && !defined(__clang__) && _Py_TAIL_CALL_INTERP @@ -400,17 +400,18 @@ PyAPI_FUNC(int) _PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, Py # define Py_NO_INLINE_MSVC_TAILCALL #endif +// Tells the compiler that this variable cannot be alised. #if defined(_MSC_VER) && !defined(__clang__) -# define Py_MSVC_RESTRICT restrict +# define Py_UNALIASED(var) restrict var #else -# define Py_MSVC_RESTRICT +# define Py_UNALIASED(var) var #endif -// Just a scope. Hints to the programmer +// Just a scope. Hints to the programmer and compiler // That any local variable defined within this block MUST // not escape from the current definition. -# define Py_BEGIN_LOCALS_MUST_NOT_ESCAPE() { -# define Py_END_LOCALS_MUST_NOT_ESCAPE() } +# define Py_BEGIN_LOCALS_MUST_NOT_ESCAPE { +# define Py_END_LOCALS_MUST_NOT_ESCAPE } #ifdef __cplusplus } diff --git a/Objects/abstract.c b/Objects/abstract.c index 0e0b4dbf42c160..cf58ea5d793f5a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -225,7 +225,7 @@ PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) } Py_NO_INLINE_MSVC_TAILCALL int -_PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **Py_MSVC_RESTRICT result) +_PyEval_Mapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **Py_UNALIASED(result)) { return PyMapping_GetOptionalItem(obj, key, result); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4da24c3848f3b0..fce9d2311d3a73 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2228,7 +2228,7 @@ dummy_func( // handle any case whose performance we care about PyObject *super; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; + PyObject *stack[] = {class, self}; super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); Py_END_LOCALS_MUST_NOT_ESCAPE; if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { @@ -2291,7 +2291,7 @@ dummy_func( int method_found = 0; PyObject *attr_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - int *Py_MSVC_RESTRICT method_found_ptr = &method_found; + int *method_found_ptr = &method_found; attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); Py_END_LOCALS_MUST_NOT_ESCAPE; @@ -3519,7 +3519,7 @@ dummy_func( (void)lasti; // Shut up compiler warning if asserts are off PyObject* res_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6328cd044dd774..5fe7e2fd949245 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3255,7 +3255,7 @@ int method_found = 0; PyObject *attr_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - int *Py_MSVC_RESTRICT method_found_ptr = &method_found; + int *method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); @@ -4684,7 +4684,7 @@ (void)lasti; PyObject* res_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 16adee25589cf4..9f9e149cdead83 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7095,7 +7095,7 @@ } PyObject *super; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; + PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9495,7 +9495,7 @@ } PyObject *super; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - PyObject *Py_MSVC_RESTRICT stack[] = {class, self}; + PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9666,7 +9666,7 @@ int method_found = 0; PyObject *attr_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - int *Py_MSVC_RESTRICT method_found_ptr = &method_found; + int *method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); @@ -12048,7 +12048,7 @@ (void)lasti; PyObject* res_o; Py_BEGIN_LOCALS_MUST_NOT_ESCAPE; - PyObject *Py_MSVC_RESTRICT stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self,