From d6d9de8a25af7b6d6ec10fb3ba58502342b16681 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sun, 23 Mar 2025 00:03:36 +0000 Subject: [PATCH 1/9] gh-131586: Avoid refcount contention in context managers This avoid reference count contention in the free threading build when calling special methods like `__enter__` and `__exit__`. --- Include/internal/pycore_object.h | 2 +- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 175 +++++++++++----------- Include/internal/pycore_uop_metadata.h | 6 +- Objects/typeobject.c | 43 +++--- Python/bytecodes.c | 24 +-- Python/executor_cases.c.h | 36 +++-- Python/generated_cases.c.h | 54 +++---- Python/optimizer_cases.c.h | 11 ++ Tools/cases_generator/stack.py | 2 +- Tools/ftscalingbench/ftscalingbench.py | 13 ++ 11 files changed, 202 insertions(+), 166 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index e5034ff4dcc42b..b7e162c8abcabf 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -950,7 +950,7 @@ extern int _PyObject_IsInstanceDictEmpty(PyObject *); // Export for 'math' shared extension PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); -PyAPI_FUNC(PyObject*) _PyObject_LookupSpecialMethod(PyObject *self, PyObject *attr, PyObject **self_or_null); +PyAPI_FUNC(int) _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self); // Calls the method named `attr` on `self`, but does not set an exception if // the attribute does not exist. diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 9c101302591872..3f623ae349c9be 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1415,7 +1415,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, OPARG_SIMPLE, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, OPARG_SIMPLE, 0 } } }, [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, OPARG_SIMPLE, 0 } } }, - [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 2, .uops = { { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, OPARG_SIMPLE, 1 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index fb5486171ef4ca..7133f830d79198 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -146,6 +146,7 @@ extern "C" { #define _INIT_CALL_PY_EXACT_ARGS_2 390 #define _INIT_CALL_PY_EXACT_ARGS_3 391 #define _INIT_CALL_PY_EXACT_ARGS_4 392 +#define _INSERT_NULL 393 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -155,137 +156,137 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 393 +#define _IS_NONE 394 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 394 -#define _ITER_CHECK_RANGE 395 -#define _ITER_CHECK_TUPLE 396 -#define _ITER_JUMP_LIST 397 -#define _ITER_JUMP_RANGE 398 -#define _ITER_JUMP_TUPLE 399 -#define _ITER_NEXT_LIST 400 -#define _ITER_NEXT_LIST_TIER_TWO 401 -#define _ITER_NEXT_RANGE 402 -#define _ITER_NEXT_TUPLE 403 -#define _JUMP_TO_TOP 404 +#define _ITER_CHECK_LIST 395 +#define _ITER_CHECK_RANGE 396 +#define _ITER_CHECK_TUPLE 397 +#define _ITER_JUMP_LIST 398 +#define _ITER_JUMP_RANGE 399 +#define _ITER_JUMP_TUPLE 400 +#define _ITER_NEXT_LIST 401 +#define _ITER_NEXT_LIST_TIER_TWO 402 +#define _ITER_NEXT_RANGE 403 +#define _ITER_NEXT_TUPLE 404 +#define _JUMP_TO_TOP 405 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 405 -#define _LOAD_ATTR_CLASS 406 +#define _LOAD_ATTR 406 +#define _LOAD_ATTR_CLASS 407 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 407 -#define _LOAD_ATTR_METHOD_LAZY_DICT 408 -#define _LOAD_ATTR_METHOD_NO_DICT 409 -#define _LOAD_ATTR_METHOD_WITH_VALUES 410 -#define _LOAD_ATTR_MODULE 411 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 412 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 413 -#define _LOAD_ATTR_PROPERTY_FRAME 414 -#define _LOAD_ATTR_SLOT 415 -#define _LOAD_ATTR_WITH_HINT 416 +#define _LOAD_ATTR_INSTANCE_VALUE 408 +#define _LOAD_ATTR_METHOD_LAZY_DICT 409 +#define _LOAD_ATTR_METHOD_NO_DICT 410 +#define _LOAD_ATTR_METHOD_WITH_VALUES 411 +#define _LOAD_ATTR_MODULE 412 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 413 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 414 +#define _LOAD_ATTR_PROPERTY_FRAME 415 +#define _LOAD_ATTR_SLOT 416 +#define _LOAD_ATTR_WITH_HINT 417 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 417 +#define _LOAD_BYTECODE 418 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 418 -#define _LOAD_CONST_INLINE_BORROW 419 +#define _LOAD_CONST_INLINE 419 +#define _LOAD_CONST_INLINE_BORROW 420 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 420 -#define _LOAD_FAST_0 421 -#define _LOAD_FAST_1 422 -#define _LOAD_FAST_2 423 -#define _LOAD_FAST_3 424 -#define _LOAD_FAST_4 425 -#define _LOAD_FAST_5 426 -#define _LOAD_FAST_6 427 -#define _LOAD_FAST_7 428 +#define _LOAD_FAST 421 +#define _LOAD_FAST_0 422 +#define _LOAD_FAST_1 423 +#define _LOAD_FAST_2 424 +#define _LOAD_FAST_3 425 +#define _LOAD_FAST_4 426 +#define _LOAD_FAST_5 427 +#define _LOAD_FAST_6 428 +#define _LOAD_FAST_7 429 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 429 -#define _LOAD_GLOBAL_BUILTINS 430 -#define _LOAD_GLOBAL_MODULE 431 +#define _LOAD_GLOBAL 430 +#define _LOAD_GLOBAL_BUILTINS 431 +#define _LOAD_GLOBAL_MODULE 432 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 432 -#define _LOAD_SMALL_INT_0 433 -#define _LOAD_SMALL_INT_1 434 -#define _LOAD_SMALL_INT_2 435 -#define _LOAD_SMALL_INT_3 436 -#define _LOAD_SPECIAL LOAD_SPECIAL +#define _LOAD_SMALL_INT 433 +#define _LOAD_SMALL_INT_0 434 +#define _LOAD_SMALL_INT_1 435 +#define _LOAD_SMALL_INT_2 436 +#define _LOAD_SMALL_INT_3 437 +#define _LOAD_SPECIAL 438 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 437 +#define _MAKE_CALLARGS_A_TUPLE 439 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 438 +#define _MAKE_WARM 440 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 439 -#define _MAYBE_EXPAND_METHOD_KW 440 -#define _MONITOR_CALL 441 -#define _MONITOR_CALL_KW 442 -#define _MONITOR_JUMP_BACKWARD 443 -#define _MONITOR_RESUME 444 +#define _MAYBE_EXPAND_METHOD 441 +#define _MAYBE_EXPAND_METHOD_KW 442 +#define _MONITOR_CALL 443 +#define _MONITOR_CALL_KW 444 +#define _MONITOR_JUMP_BACKWARD 445 +#define _MONITOR_RESUME 446 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 445 -#define _POP_JUMP_IF_TRUE 446 +#define _POP_JUMP_IF_FALSE 447 +#define _POP_JUMP_IF_TRUE 448 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 447 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 448 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 449 +#define _POP_TOP_LOAD_CONST_INLINE 449 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 450 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 451 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 450 +#define _PUSH_FRAME 452 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 451 -#define _PY_FRAME_GENERAL 452 -#define _PY_FRAME_KW 453 -#define _QUICKEN_RESUME 454 -#define _REPLACE_WITH_TRUE 455 +#define _PUSH_NULL_CONDITIONAL 453 +#define _PY_FRAME_GENERAL 454 +#define _PY_FRAME_KW 455 +#define _QUICKEN_RESUME 456 +#define _REPLACE_WITH_TRUE 457 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 456 -#define _SEND 457 -#define _SEND_GEN_FRAME 458 +#define _SAVE_RETURN_OFFSET 458 +#define _SEND 459 +#define _SEND_GEN_FRAME 460 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 459 -#define _STORE_ATTR 460 -#define _STORE_ATTR_INSTANCE_VALUE 461 -#define _STORE_ATTR_SLOT 462 -#define _STORE_ATTR_WITH_HINT 463 +#define _START_EXECUTOR 461 +#define _STORE_ATTR 462 +#define _STORE_ATTR_INSTANCE_VALUE 463 +#define _STORE_ATTR_SLOT 464 +#define _STORE_ATTR_WITH_HINT 465 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 464 -#define _STORE_FAST_0 465 -#define _STORE_FAST_1 466 -#define _STORE_FAST_2 467 -#define _STORE_FAST_3 468 -#define _STORE_FAST_4 469 -#define _STORE_FAST_5 470 -#define _STORE_FAST_6 471 -#define _STORE_FAST_7 472 +#define _STORE_FAST 466 +#define _STORE_FAST_0 467 +#define _STORE_FAST_1 468 +#define _STORE_FAST_2 469 +#define _STORE_FAST_3 470 +#define _STORE_FAST_4 471 +#define _STORE_FAST_5 472 +#define _STORE_FAST_6 473 +#define _STORE_FAST_7 474 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 473 -#define _STORE_SUBSCR 474 +#define _STORE_SLICE 475 +#define _STORE_SUBSCR 476 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 475 -#define _TO_BOOL 476 +#define _TIER2_RESUME_CHECK 477 +#define _TO_BOOL 478 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -295,13 +296,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 477 +#define _UNPACK_SEQUENCE 479 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 477 +#define MAX_UOP_ID 479 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 59572fca3753d6..c1999af61891cb 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -193,6 +193,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_INSERT_NULL] = 0, [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, @@ -424,6 +425,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", + [_INSERT_NULL] = "_INSERT_NULL", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", @@ -907,8 +909,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _FOR_ITER_GEN_FRAME: return 0; - case _LOAD_SPECIAL: + case _INSERT_NULL: return 1; + case _LOAD_SPECIAL: + return 0; case _WITH_EXCEPT_START: return 0; case _PUSH_EXC_INFO: diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b92eaefc90d0af..03436e8b3df567 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2774,32 +2774,37 @@ _PyObject_LookupSpecial(PyObject *self, PyObject *attr) return res; } -/* Steals a reference to self */ -PyObject * -_PyObject_LookupSpecialMethod(PyObject *self, PyObject *attr, PyObject **self_or_null) +// Lookup the method name `attr` on `self`. On entry, `method_and_self[0]` +// is null and `method_and_self[1]` is `self`. On exit, `method_and_self[0]` +// is the method object and `method_and_self[1]` is `self` if the method is +// not bound. +// Return 0 on success, -1 on error or if the method is missing. +int +_PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self) { - PyObject *res; - - res = _PyType_LookupRef(Py_TYPE(self), attr); - if (res == NULL) { - Py_DECREF(self); - *self_or_null = NULL; - return NULL; + PyObject *self = PyStackRef_AsPyObjectBorrow(method_and_self[1]); + _PyType_LookupStackRefAndVersion(Py_TYPE(self), attr, &method_and_self[0]); + PyObject *method_o = PyStackRef_AsPyObjectBorrow(method_and_self[0]); + if (method_o == NULL) { + return -1; } - if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { + if (_PyType_HasFeature(Py_TYPE(method_o), Py_TPFLAGS_METHOD_DESCRIPTOR)) { /* Avoid temporary PyMethodObject */ - *self_or_null = self; + return 0; } - else { - descrgetfunc f = Py_TYPE(res)->tp_descr_get; - if (f != NULL) { - Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + + descrgetfunc f = Py_TYPE(method_o)->tp_descr_get; + if (f != NULL) { + PyObject *func = f(method_o, self, (PyObject *)(Py_TYPE(self))); + if (func == NULL) { + return -1; } - *self_or_null = NULL; - Py_DECREF(self); + PyStackRef_CLEAR(method_and_self[0]); // clear method + method_and_self[0] = PyStackRef_FromPyObjectSteal(func); } - return res; + PyStackRef_CLEAR(method_and_self[1]); // clear self + return 0; } static int diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b3b7441c31569a..3edbb15e116d42 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3345,25 +3345,29 @@ dummy_func( _FOR_ITER_GEN_FRAME + _PUSH_FRAME; - inst(LOAD_SPECIAL, (owner -- attr, self_or_null)) { - assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + op(_INSERT_NULL, (arg -- args[2])) { + args[0] = PyStackRef_NULL; + args[1] = arg; + DEAD(arg); + } + + op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { PyObject *name = _Py_SpecialMethods[oparg].name; - PyObject *self_or_null_o; - PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); - if (attr_o == NULL) { + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + if (err < 0) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, - Py_TYPE(owner_o)->tp_name); + PyStackRef_TYPE(method_and_self[1])->tp_name); } ERROR_IF(true, error); } - attr = PyStackRef_FromPyObjectSteal(attr_o); - self_or_null = self_or_null_o == NULL ? - PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); } + macro(LOAD_SPECIAL) = + _INSERT_NULL + + _LOAD_SPECIAL; + inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) { /* At the top of the stack are 4 values: - val: TOP = exc_info() diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9306a6aea35435..ecc0b59582c229 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4399,38 +4399,36 @@ break; } + case _INSERT_NULL: { + _PyStackRef arg; + _PyStackRef *args; + arg = stack_pointer[-1]; + args = &stack_pointer[-1]; + args[0] = PyStackRef_NULL; + args[1] = arg; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _LOAD_SPECIAL: { - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; + _PyStackRef *method_and_self; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + method_and_self = &stack_pointer[-2]; PyObject *name = _Py_SpecialMethods[oparg].name; - PyObject *self_or_null_o; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { + if (err < 0) { if (!_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, - Py_TYPE(owner_o)->tp_name); + PyStackRef_TYPE(method_and_self[1])->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } JUMP_TO_ERROR(); } - attr = PyStackRef_FromPyObjectSteal(attr_o); - self_or_null = self_or_null_o == NULL ? - PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f1e22f6d3dd700..cea28453b3f75d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9315,36 +9315,36 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - owner = stack_pointer[-1]; - assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); - PyObject *name = _Py_SpecialMethods[oparg].name; - PyObject *self_or_null_o; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + _PyStackRef arg; + _PyStackRef *args; + _PyStackRef *method_and_self; + // _INSERT_NULL + { + arg = stack_pointer[-1]; + args = &stack_pointer[-1]; + args[0] = PyStackRef_NULL; + args[1] = arg; + } + // _LOAD_SPECIAL + { + method_and_self = &stack_pointer[-1]; + PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, - Py_TYPE(owner_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_TYPE(method_and_self[1])->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); } - JUMP_TO_LABEL(error); } - attr = PyStackRef_FromPyObjectSteal(attr_o); - self_or_null = self_or_null_o == NULL ? - PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 85fa8f4a2a7910..062d41a015f95d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1520,6 +1520,17 @@ break; } + case _INSERT_NULL: { + JitOptSymbol **args; + args = &stack_pointer[-1]; + for (int _i = 2; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _LOAD_SPECIAL: { JitOptSymbol *attr; JitOptSymbol *self_or_null; diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 94a5d395064191..bf941d37752648 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -259,7 +259,7 @@ def pop(self, var: StackItem) -> tuple[str, Local]: else: rename = "" if not popped.in_local: - assert popped.memory_offset is not None + # assert popped.memory_offset is not None if var.is_array(): defn = f"{var.name} = &stack_pointer[{self.top_offset.to_c()}];\n" else: diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 364c465bc91b0b..926bc66b944c6f 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -65,6 +65,19 @@ def object_lookup_special(): for i in range(N): round(i / N) +class MyContextManager: + def __enter__(self): + pass + def __exit__(self, exc_type, exc_value, traceback): + pass + +@register_benchmark +def context_manager(): + N = 1000 * WORK_SCALE + for i in range(N): + with MyContextManager(): + pass + @register_benchmark def mult_constant(): x = 1.0 From c8d811d0f5b2655e47aaf47ef5866e9002436962 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 28 Mar 2025 16:03:11 +0000 Subject: [PATCH 2/9] Struggle with the cases generator --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 4 ++-- Python/bytecodes.c | 14 +++++++++----- Python/executor_cases.c.h | 17 +++++++++++++---- Python/generated_cases.c.h | 18 ++++++++++++++---- Python/optimizer_cases.c.h | 11 ++++++----- Tools/cases_generator/stack.py | 2 +- 7 files changed, 46 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3f623ae349c9be..dbb3a43f8e62da 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1208,7 +1208,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index c1999af61891cb..89a3662c5e03a1 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -194,7 +194,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_INSERT_NULL] = 0, - [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, @@ -912,7 +912,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _INSERT_NULL: return 1; case _LOAD_SPECIAL: - return 0; + return 2; case _WITH_EXCEPT_START: return 0; case _PUSH_EXC_INFO: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3edbb15e116d42..b570ba0a4740d4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3345,13 +3345,15 @@ dummy_func( _FOR_ITER_GEN_FRAME + _PUSH_FRAME; - op(_INSERT_NULL, (arg -- args[2])) { - args[0] = PyStackRef_NULL; - args[1] = arg; + op(_INSERT_NULL, (arg -- arg1, arg2)) { + arg1 = PyStackRef_NULL; + arg2 = arg; DEAD(arg); } - op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { + op(_LOAD_SPECIAL, (null, self -- method_and_self[2])) { + method_and_self[0] = null; + method_and_self[1] = self; PyObject *name = _Py_SpecialMethods[oparg].name; int err = _PyObject_LookupSpecialMethod(name, method_and_self); if (err < 0) { @@ -3359,9 +3361,11 @@ dummy_func( _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, PyStackRef_TYPE(method_and_self[1])->tp_name); + ERROR_NO_POP(); } - ERROR_IF(true, error); + ERROR_NO_POP(); } + INPUTS_DEAD(); } macro(LOAD_SPECIAL) = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ecc0b59582c229..b1f32e76ee243c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4401,20 +4401,28 @@ case _INSERT_NULL: { _PyStackRef arg; - _PyStackRef *args; + _PyStackRef arg1; + _PyStackRef arg2; arg = stack_pointer[-1]; - args = &stack_pointer[-1]; - args[0] = PyStackRef_NULL; - args[1] = arg; + arg1 = PyStackRef_NULL; + arg2 = arg; + stack_pointer[-1] = arg1; + stack_pointer[0] = arg2; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_SPECIAL: { + _PyStackRef self; + _PyStackRef null; _PyStackRef *method_and_self; oparg = CURRENT_OPARG(); + self = stack_pointer[-1]; + null = stack_pointer[-2]; method_and_self = &stack_pointer[-2]; + method_and_self[0] = null; + method_and_self[1] = self; PyObject *name = _Py_SpecialMethods[oparg].name; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); @@ -4426,6 +4434,7 @@ _Py_SpecialMethods[oparg].error, PyStackRef_TYPE(method_and_self[1])->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); } JUMP_TO_ERROR(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cea28453b3f75d..b24c2368571191 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9316,19 +9316,27 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef arg; - _PyStackRef *args; + _PyStackRef arg1; + _PyStackRef arg2; + _PyStackRef null; + _PyStackRef self; _PyStackRef *method_and_self; // _INSERT_NULL { arg = stack_pointer[-1]; - args = &stack_pointer[-1]; - args[0] = PyStackRef_NULL; - args[1] = arg; + arg1 = PyStackRef_NULL; + arg2 = arg; } // _LOAD_SPECIAL { + self = arg2; + null = arg1; method_and_self = &stack_pointer[-1]; + method_and_self[0] = null; + method_and_self[1] = self; PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer[-1] = null; + stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9336,11 +9344,13 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { if (!_PyErr_Occurred(tstate)) { + stack_pointer[-1] = self; _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, PyStackRef_TYPE(method_and_self[1])->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } JUMP_TO_LABEL(error); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 062d41a015f95d..16850f03e4409c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1521,11 +1521,12 @@ } case _INSERT_NULL: { - JitOptSymbol **args; - args = &stack_pointer[-1]; - for (int _i = 2; --_i >= 0;) { - args[_i] = sym_new_not_null(ctx); - } + JitOptSymbol *arg1; + JitOptSymbol *arg2; + arg1 = sym_new_not_null(ctx); + arg2 = sym_new_not_null(ctx); + stack_pointer[-1] = arg1; + stack_pointer[0] = arg2; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index bf941d37752648..94a5d395064191 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -259,7 +259,7 @@ def pop(self, var: StackItem) -> tuple[str, Local]: else: rename = "" if not popped.in_local: - # assert popped.memory_offset is not None + assert popped.memory_offset is not None if var.is_array(): defn = f"{var.name} = &stack_pointer[{self.top_offset.to_c()}];\n" else: From 2af6677e32792553e1d5983cdeb988134b18e14a Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 4 Apr 2025 21:54:05 +0000 Subject: [PATCH 3/9] Adjust return values --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 197 +++++++++++----------- Include/internal/pycore_uop_metadata.h | 6 +- Objects/typeobject.c | 6 +- Python/bytecodes.c | 28 +-- Python/executor_cases.c.h | 44 ++--- Python/generated_cases.c.h | 54 +++--- Python/optimizer_cases.c.h | 12 -- 8 files changed, 146 insertions(+), 203 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 446a797fa812b4..4134eda4963386 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1427,7 +1427,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, OPARG_SIMPLE, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, OPARG_SIMPLE, 0 } } }, [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, OPARG_SIMPLE, 0 } } }, - [LOAD_SPECIAL] = { .nuops = 2, .uops = { { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, OPARG_SIMPLE, 1 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 994f540fdb9945..2b79fffc14bc33 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -144,7 +144,6 @@ extern "C" { #define _INIT_CALL_PY_EXACT_ARGS_2 391 #define _INIT_CALL_PY_EXACT_ARGS_3 392 #define _INIT_CALL_PY_EXACT_ARGS_4 393 -#define _INSERT_NULL 394 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -154,163 +153,163 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 395 +#define _IS_NONE 394 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 396 -#define _ITER_CHECK_RANGE 397 -#define _ITER_CHECK_TUPLE 398 -#define _ITER_JUMP_LIST 399 -#define _ITER_JUMP_RANGE 400 -#define _ITER_JUMP_TUPLE 401 -#define _ITER_NEXT_LIST 402 -#define _ITER_NEXT_LIST_TIER_TWO 403 -#define _ITER_NEXT_RANGE 404 -#define _ITER_NEXT_TUPLE 405 -#define _JUMP_TO_TOP 406 +#define _ITER_CHECK_LIST 395 +#define _ITER_CHECK_RANGE 396 +#define _ITER_CHECK_TUPLE 397 +#define _ITER_JUMP_LIST 398 +#define _ITER_JUMP_RANGE 399 +#define _ITER_JUMP_TUPLE 400 +#define _ITER_NEXT_LIST 401 +#define _ITER_NEXT_LIST_TIER_TWO 402 +#define _ITER_NEXT_RANGE 403 +#define _ITER_NEXT_TUPLE 404 +#define _JUMP_TO_TOP 405 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 407 -#define _LOAD_ATTR_CLASS 408 +#define _LOAD_ATTR 406 +#define _LOAD_ATTR_CLASS 407 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 409 -#define _LOAD_ATTR_METHOD_LAZY_DICT 410 -#define _LOAD_ATTR_METHOD_NO_DICT 411 -#define _LOAD_ATTR_METHOD_WITH_VALUES 412 -#define _LOAD_ATTR_MODULE 413 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 414 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 415 -#define _LOAD_ATTR_PROPERTY_FRAME 416 -#define _LOAD_ATTR_SLOT 417 -#define _LOAD_ATTR_WITH_HINT 418 +#define _LOAD_ATTR_INSTANCE_VALUE 408 +#define _LOAD_ATTR_METHOD_LAZY_DICT 409 +#define _LOAD_ATTR_METHOD_NO_DICT 410 +#define _LOAD_ATTR_METHOD_WITH_VALUES 411 +#define _LOAD_ATTR_MODULE 412 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 413 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 414 +#define _LOAD_ATTR_PROPERTY_FRAME 415 +#define _LOAD_ATTR_SLOT 416 +#define _LOAD_ATTR_WITH_HINT 417 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 419 +#define _LOAD_BYTECODE 418 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 420 -#define _LOAD_CONST_INLINE_BORROW 421 +#define _LOAD_CONST_INLINE 419 +#define _LOAD_CONST_INLINE_BORROW 420 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 422 -#define _LOAD_FAST_0 423 -#define _LOAD_FAST_1 424 -#define _LOAD_FAST_2 425 -#define _LOAD_FAST_3 426 -#define _LOAD_FAST_4 427 -#define _LOAD_FAST_5 428 -#define _LOAD_FAST_6 429 -#define _LOAD_FAST_7 430 +#define _LOAD_FAST 421 +#define _LOAD_FAST_0 422 +#define _LOAD_FAST_1 423 +#define _LOAD_FAST_2 424 +#define _LOAD_FAST_3 425 +#define _LOAD_FAST_4 426 +#define _LOAD_FAST_5 427 +#define _LOAD_FAST_6 428 +#define _LOAD_FAST_7 429 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 431 -#define _LOAD_FAST_BORROW_0 432 -#define _LOAD_FAST_BORROW_1 433 -#define _LOAD_FAST_BORROW_2 434 -#define _LOAD_FAST_BORROW_3 435 -#define _LOAD_FAST_BORROW_4 436 -#define _LOAD_FAST_BORROW_5 437 -#define _LOAD_FAST_BORROW_6 438 -#define _LOAD_FAST_BORROW_7 439 +#define _LOAD_FAST_BORROW 430 +#define _LOAD_FAST_BORROW_0 431 +#define _LOAD_FAST_BORROW_1 432 +#define _LOAD_FAST_BORROW_2 433 +#define _LOAD_FAST_BORROW_3 434 +#define _LOAD_FAST_BORROW_4 435 +#define _LOAD_FAST_BORROW_5 436 +#define _LOAD_FAST_BORROW_6 437 +#define _LOAD_FAST_BORROW_7 438 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 440 -#define _LOAD_GLOBAL_BUILTINS 441 -#define _LOAD_GLOBAL_MODULE 442 +#define _LOAD_GLOBAL 439 +#define _LOAD_GLOBAL_BUILTINS 440 +#define _LOAD_GLOBAL_MODULE 441 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 443 -#define _LOAD_SMALL_INT_0 444 -#define _LOAD_SMALL_INT_1 445 -#define _LOAD_SMALL_INT_2 446 -#define _LOAD_SMALL_INT_3 447 -#define _LOAD_SPECIAL 448 +#define _LOAD_SMALL_INT 442 +#define _LOAD_SMALL_INT_0 443 +#define _LOAD_SMALL_INT_1 444 +#define _LOAD_SMALL_INT_2 445 +#define _LOAD_SMALL_INT_3 446 +#define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 449 +#define _MAKE_CALLARGS_A_TUPLE 447 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 450 +#define _MAKE_WARM 448 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 451 -#define _MAYBE_EXPAND_METHOD_KW 452 -#define _MONITOR_CALL 453 -#define _MONITOR_CALL_KW 454 -#define _MONITOR_JUMP_BACKWARD 455 -#define _MONITOR_RESUME 456 +#define _MAYBE_EXPAND_METHOD 449 +#define _MAYBE_EXPAND_METHOD_KW 450 +#define _MONITOR_CALL 451 +#define _MONITOR_CALL_KW 452 +#define _MONITOR_JUMP_BACKWARD 453 +#define _MONITOR_RESUME 454 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 457 -#define _POP_JUMP_IF_TRUE 458 +#define _POP_JUMP_IF_FALSE 455 +#define _POP_JUMP_IF_TRUE 456 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 459 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 460 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 461 +#define _POP_TOP_LOAD_CONST_INLINE 457 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 458 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 459 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 462 +#define _PUSH_FRAME 460 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 463 -#define _PY_FRAME_GENERAL 464 -#define _PY_FRAME_KW 465 -#define _QUICKEN_RESUME 466 -#define _REPLACE_WITH_TRUE 467 +#define _PUSH_NULL_CONDITIONAL 461 +#define _PY_FRAME_GENERAL 462 +#define _PY_FRAME_KW 463 +#define _QUICKEN_RESUME 464 +#define _REPLACE_WITH_TRUE 465 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 468 -#define _SEND 469 -#define _SEND_GEN_FRAME 470 +#define _SAVE_RETURN_OFFSET 466 +#define _SEND 467 +#define _SEND_GEN_FRAME 468 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 471 -#define _STORE_ATTR 472 -#define _STORE_ATTR_INSTANCE_VALUE 473 -#define _STORE_ATTR_SLOT 474 -#define _STORE_ATTR_WITH_HINT 475 +#define _START_EXECUTOR 469 +#define _STORE_ATTR 470 +#define _STORE_ATTR_INSTANCE_VALUE 471 +#define _STORE_ATTR_SLOT 472 +#define _STORE_ATTR_WITH_HINT 473 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 476 -#define _STORE_FAST_0 477 -#define _STORE_FAST_1 478 -#define _STORE_FAST_2 479 -#define _STORE_FAST_3 480 -#define _STORE_FAST_4 481 -#define _STORE_FAST_5 482 -#define _STORE_FAST_6 483 -#define _STORE_FAST_7 484 +#define _STORE_FAST 474 +#define _STORE_FAST_0 475 +#define _STORE_FAST_1 476 +#define _STORE_FAST_2 477 +#define _STORE_FAST_3 478 +#define _STORE_FAST_4 479 +#define _STORE_FAST_5 480 +#define _STORE_FAST_6 481 +#define _STORE_FAST_7 482 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 485 -#define _STORE_SUBSCR 486 +#define _STORE_SLICE 483 +#define _STORE_SUBSCR 484 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT -#define _STORE_SUBSCR_LIST_INT 487 +#define _STORE_SUBSCR_LIST_INT 485 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 488 -#define _TO_BOOL 489 +#define _TIER2_RESUME_CHECK 486 +#define _TO_BOOL 487 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 490 +#define _TO_BOOL_STR 488 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 491 +#define _UNPACK_SEQUENCE 489 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 491 +#define MAX_UOP_ID 489 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index c3c386b4eedbc1..250cecfba9e6f3 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -202,7 +202,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_INSERT_NULL] = 0, [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, @@ -432,7 +431,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", - [_INSERT_NULL] = "_INSERT_NULL", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", @@ -944,10 +942,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _FOR_ITER_GEN_FRAME: return 0; - case _INSERT_NULL: - return 1; case _LOAD_SPECIAL: - return 2; + return 1; case _WITH_EXCEPT_START: return 0; case _PUSH_EXC_INFO: diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 03436e8b3df567..a4e8b55da2817b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2786,12 +2786,12 @@ _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self) _PyType_LookupStackRefAndVersion(Py_TYPE(self), attr, &method_and_self[0]); PyObject *method_o = PyStackRef_AsPyObjectBorrow(method_and_self[0]); if (method_o == NULL) { - return -1; + return 0; } if (_PyType_HasFeature(Py_TYPE(method_o), Py_TPFLAGS_METHOD_DESCRIPTOR)) { /* Avoid temporary PyMethodObject */ - return 0; + return 1; } descrgetfunc f = Py_TYPE(method_o)->tp_descr_get; @@ -2804,7 +2804,7 @@ _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self) method_and_self[0] = PyStackRef_FromPyObjectSteal(func); } PyStackRef_CLEAR(method_and_self[1]); // clear self - return 0; + return 1; } static int diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fc37a1305094a6..b3cde05dc2d901 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3365,33 +3365,23 @@ dummy_func( _FOR_ITER_GEN_FRAME + _PUSH_FRAME; - op(_INSERT_NULL, (arg -- arg1, arg2)) { - arg1 = PyStackRef_NULL; - arg2 = arg; - DEAD(arg); - } - - op(_LOAD_SPECIAL, (null, self -- method_and_self[2])) { - method_and_self[0] = null; - method_and_self[1] = self; + inst(LOAD_SPECIAL, (owner -- method_and_self[2])) { + method_and_self[0] = PyStackRef_NULL; + method_and_self[1] = owner; PyObject *name = _Py_SpecialMethods[oparg].name; int err = _PyObject_LookupSpecialMethod(name, method_and_self); if (err < 0) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - PyStackRef_TYPE(method_and_self[1])->tp_name); - ERROR_NO_POP(); - } + ERROR_NO_POP(); + } + else if (err == 0) { + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + PyStackRef_TYPE(method_and_self[1])->tp_name); ERROR_NO_POP(); } INPUTS_DEAD(); } - macro(LOAD_SPECIAL) = - _INSERT_NULL + - _LOAD_SPECIAL; - inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) { /* At the top of the stack are 4 values: - val: TOP = exc_info() diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b8d6218754abd9..afd721287d5848 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4373,45 +4373,31 @@ break; } - case _INSERT_NULL: { - _PyStackRef arg; - _PyStackRef arg1; - _PyStackRef arg2; - arg = stack_pointer[-1]; - arg1 = PyStackRef_NULL; - arg2 = arg; - stack_pointer[-1] = arg1; - stack_pointer[0] = arg2; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - case _LOAD_SPECIAL: { - _PyStackRef self; - _PyStackRef null; + _PyStackRef owner; _PyStackRef *method_and_self; oparg = CURRENT_OPARG(); - self = stack_pointer[-1]; - null = stack_pointer[-2]; - method_and_self = &stack_pointer[-2]; - method_and_self[0] = null; - method_and_self[1] = self; + owner = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[0] = PyStackRef_NULL; + method_and_self[1] = owner; PyObject *name = _Py_SpecialMethods[oparg].name; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - PyStackRef_TYPE(method_and_self[1])->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); - } JUMP_TO_ERROR(); } + else if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + PyStackRef_TYPE(method_and_self[1])->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); + } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c5c27cf660e5c4..2ceae44160949e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9268,45 +9268,29 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); - _PyStackRef arg; - _PyStackRef arg1; - _PyStackRef arg2; - _PyStackRef null; - _PyStackRef self; + _PyStackRef owner; _PyStackRef *method_and_self; - // _INSERT_NULL - { - arg = stack_pointer[-1]; - arg1 = PyStackRef_NULL; - arg2 = arg; - } - // _LOAD_SPECIAL - { - self = arg2; - null = arg1; - method_and_self = &stack_pointer[-1]; - method_and_self[0] = null; - method_and_self[1] = self; - PyObject *name = _Py_SpecialMethods[oparg].name; - stack_pointer[-1] = null; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + owner = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[0] = PyStackRef_NULL; + method_and_self[1] = owner; + PyObject *name = _Py_SpecialMethods[oparg].name; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + else if (err == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyObject_LookupSpecialMethod(name, method_and_self); + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + PyStackRef_TYPE(method_and_self[1])->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - PyStackRef_TYPE(method_and_self[1])->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - JUMP_TO_LABEL(error); - } + JUMP_TO_LABEL(error); } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index fe0ffad0342183..f5cd1697fbacda 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1482,18 +1482,6 @@ break; } - case _INSERT_NULL: { - JitOptSymbol *arg1; - JitOptSymbol *arg2; - arg1 = sym_new_not_null(ctx); - arg2 = sym_new_not_null(ctx); - stack_pointer[-1] = arg1; - stack_pointer[0] = arg2; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - case _LOAD_SPECIAL: { JitOptSymbol *attr; JitOptSymbol *self_or_null; From 1f25dd9fbcf0d218654c15c7ce36806682e050ac Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 4 Apr 2025 21:56:59 +0000 Subject: [PATCH 4/9] Use simpler formulation --- Python/bytecodes.c | 6 +++--- Python/executor_cases.c.h | 10 ++++++---- Python/generated_cases.c.h | 10 ++++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b3cde05dc2d901..a9fe4ef0df1d6a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3365,9 +3365,10 @@ dummy_func( _FOR_ITER_GEN_FRAME + _PUSH_FRAME; - inst(LOAD_SPECIAL, (owner -- method_and_self[2])) { + inst(LOAD_SPECIAL, (self -- method_and_self[2])) { + method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; - method_and_self[1] = owner; + DEAD(self); PyObject *name = _Py_SpecialMethods[oparg].name; int err = _PyObject_LookupSpecialMethod(name, method_and_self); if (err < 0) { @@ -3379,7 +3380,6 @@ dummy_func( PyStackRef_TYPE(method_and_self[1])->tp_name); ERROR_NO_POP(); } - INPUTS_DEAD(); } inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index afd721287d5848..92dc4ec34c808c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4374,14 +4374,16 @@ } case _LOAD_SPECIAL: { - _PyStackRef owner; + _PyStackRef self; _PyStackRef *method_and_self; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; - method_and_self[1] = owner; PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4396,7 +4398,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - stack_pointer += 1; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2ceae44160949e..6d3311a0205f94 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9268,13 +9268,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); - _PyStackRef owner; + _PyStackRef self; _PyStackRef *method_and_self; - owner = stack_pointer[-1]; + self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; - method_and_self[1] = owner; PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9289,7 +9291,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } - stack_pointer += 1; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } From 8224da80d1721b0cee9c400d811421b844d5eeb7 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 10 Apr 2025 19:08:23 +0000 Subject: [PATCH 5/9] Split LOAD_SPECIAL into two ops again. Make sure that `method_and_self` stays alive across the call to _PyObject_LookupSpecialMethod. When written as a single op, the input is marked as dead, but the outputs aren't considered alive until the end of the op. --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 381 +++++++++++----------- Include/internal/pycore_uop_metadata.h | 6 +- Python/bytecodes.c | 9 +- Python/executor_cases.c.h | 16 +- Python/generated_cases.c.h | 41 ++- Python/optimizer_cases.c.h | 11 + 7 files changed, 249 insertions(+), 217 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4134eda4963386..108d8b813ab280 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1427,7 +1427,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, OPARG_SIMPLE, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, OPARG_SIMPLE, 0 } } }, [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, OPARG_SIMPLE, 0 } } }, - [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 2, .uops = { { INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, OPARG_SIMPLE, 1 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 2b79fffc14bc33..61ab8ed4ca6504 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,139 +11,140 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _BINARY_OP 302 -#define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_EXTEND 306 -#define _BINARY_OP_INPLACE_ADD_UNICODE 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_MULTIPLY_INT 309 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 310 +#define INSERT_NULL 302 +#define _BINARY_OP 303 +#define _BINARY_OP_ADD_FLOAT 304 +#define _BINARY_OP_ADD_INT 305 +#define _BINARY_OP_ADD_UNICODE 306 +#define _BINARY_OP_EXTEND 307 +#define _BINARY_OP_INPLACE_ADD_UNICODE 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_MULTIPLY_INT 310 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 311 #define _BINARY_OP_SUBSCR_DICT BINARY_OP_SUBSCR_DICT -#define _BINARY_OP_SUBSCR_INIT_CALL 311 -#define _BINARY_OP_SUBSCR_LIST_INT 312 -#define _BINARY_OP_SUBSCR_STR_INT 313 -#define _BINARY_OP_SUBSCR_TUPLE_INT 314 -#define _BINARY_OP_SUBTRACT_FLOAT 315 -#define _BINARY_OP_SUBTRACT_INT 316 -#define _BINARY_SLICE 317 +#define _BINARY_OP_SUBSCR_INIT_CALL 312 +#define _BINARY_OP_SUBSCR_LIST_INT 313 +#define _BINARY_OP_SUBSCR_STR_INT 314 +#define _BINARY_OP_SUBSCR_TUPLE_INT 315 +#define _BINARY_OP_SUBTRACT_FLOAT 316 +#define _BINARY_OP_SUBTRACT_INT 317 +#define _BINARY_SLICE 318 #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP #define _BUILD_SET BUILD_SET #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 318 -#define _CALL_BUILTIN_FAST 319 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 320 -#define _CALL_BUILTIN_O 321 +#define _CALL_BUILTIN_CLASS 319 +#define _CALL_BUILTIN_FAST 320 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 321 +#define _CALL_BUILTIN_O 322 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_KW_NON_PY 322 +#define _CALL_KW_NON_PY 323 #define _CALL_LEN CALL_LEN #define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 323 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 324 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 325 -#define _CALL_METHOD_DESCRIPTOR_O 326 -#define _CALL_NON_PY_GENERAL 327 -#define _CALL_STR_1 328 -#define _CALL_TUPLE_1 329 +#define _CALL_METHOD_DESCRIPTOR_FAST 324 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 325 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 326 +#define _CALL_METHOD_DESCRIPTOR_O 327 +#define _CALL_NON_PY_GENERAL 328 +#define _CALL_STR_1 329 +#define _CALL_TUPLE_1 330 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_AND_ALLOCATE_OBJECT 330 -#define _CHECK_ATTR_CLASS 331 -#define _CHECK_ATTR_METHOD_LAZY_DICT 332 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 333 +#define _CHECK_AND_ALLOCATE_OBJECT 331 +#define _CHECK_ATTR_CLASS 332 +#define _CHECK_ATTR_METHOD_LAZY_DICT 333 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 334 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 334 -#define _CHECK_FUNCTION_EXACT_ARGS 335 -#define _CHECK_FUNCTION_VERSION 336 -#define _CHECK_FUNCTION_VERSION_INLINE 337 -#define _CHECK_FUNCTION_VERSION_KW 338 -#define _CHECK_IS_NOT_PY_CALLABLE 339 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 340 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 341 -#define _CHECK_METHOD_VERSION 342 -#define _CHECK_METHOD_VERSION_KW 343 -#define _CHECK_PEP_523 344 -#define _CHECK_PERIODIC 345 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 346 -#define _CHECK_STACK_SPACE 347 -#define _CHECK_STACK_SPACE_OPERAND 348 -#define _CHECK_VALIDITY 349 -#define _COMPARE_OP 350 -#define _COMPARE_OP_FLOAT 351 -#define _COMPARE_OP_INT 352 -#define _COMPARE_OP_STR 353 -#define _CONTAINS_OP 354 +#define _CHECK_FUNCTION 335 +#define _CHECK_FUNCTION_EXACT_ARGS 336 +#define _CHECK_FUNCTION_VERSION 337 +#define _CHECK_FUNCTION_VERSION_INLINE 338 +#define _CHECK_FUNCTION_VERSION_KW 339 +#define _CHECK_IS_NOT_PY_CALLABLE 340 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 341 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 342 +#define _CHECK_METHOD_VERSION 343 +#define _CHECK_METHOD_VERSION_KW 344 +#define _CHECK_PEP_523 345 +#define _CHECK_PERIODIC 346 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 347 +#define _CHECK_STACK_SPACE 348 +#define _CHECK_STACK_SPACE_OPERAND 349 +#define _CHECK_VALIDITY 350 +#define _COMPARE_OP 351 +#define _COMPARE_OP_FLOAT 352 +#define _COMPARE_OP_INT 353 +#define _COMPARE_OP_STR 354 +#define _CONTAINS_OP 355 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 355 +#define _CREATE_INIT_FRAME 356 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 356 +#define _DEOPT 357 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 357 -#define _DO_CALL_FUNCTION_EX 358 -#define _DO_CALL_KW 359 +#define _DO_CALL 358 +#define _DO_CALL_FUNCTION_EX 359 +#define _DO_CALL_KW 360 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 360 +#define _ERROR_POP_N 361 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 361 -#define _EXPAND_METHOD_KW 362 -#define _FATAL_ERROR 363 +#define _EXPAND_METHOD 362 +#define _EXPAND_METHOD_KW 363 +#define _FATAL_ERROR 364 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 364 -#define _FOR_ITER_GEN_FRAME 365 -#define _FOR_ITER_TIER_TWO 366 +#define _FOR_ITER 365 +#define _FOR_ITER_GEN_FRAME 366 +#define _FOR_ITER_TIER_TWO 367 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 367 -#define _GUARD_DORV_NO_DICT 368 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 369 -#define _GUARD_GLOBALS_VERSION 370 -#define _GUARD_IS_FALSE_POP 371 -#define _GUARD_IS_NONE_POP 372 -#define _GUARD_IS_NOT_NONE_POP 373 -#define _GUARD_IS_TRUE_POP 374 -#define _GUARD_KEYS_VERSION 375 -#define _GUARD_NOS_FLOAT 376 -#define _GUARD_NOS_INT 377 -#define _GUARD_NOS_UNICODE 378 -#define _GUARD_NOT_EXHAUSTED_LIST 379 -#define _GUARD_NOT_EXHAUSTED_RANGE 380 -#define _GUARD_NOT_EXHAUSTED_TUPLE 381 -#define _GUARD_TOS_FLOAT 382 -#define _GUARD_TOS_INT 383 -#define _GUARD_TOS_UNICODE 384 -#define _GUARD_TYPE_VERSION 385 -#define _GUARD_TYPE_VERSION_AND_LOCK 386 +#define _GUARD_BINARY_OP_EXTEND 368 +#define _GUARD_DORV_NO_DICT 369 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 370 +#define _GUARD_GLOBALS_VERSION 371 +#define _GUARD_IS_FALSE_POP 372 +#define _GUARD_IS_NONE_POP 373 +#define _GUARD_IS_NOT_NONE_POP 374 +#define _GUARD_IS_TRUE_POP 375 +#define _GUARD_KEYS_VERSION 376 +#define _GUARD_NOS_FLOAT 377 +#define _GUARD_NOS_INT 378 +#define _GUARD_NOS_UNICODE 379 +#define _GUARD_NOT_EXHAUSTED_LIST 380 +#define _GUARD_NOT_EXHAUSTED_RANGE 381 +#define _GUARD_NOT_EXHAUSTED_TUPLE 382 +#define _GUARD_TOS_FLOAT 383 +#define _GUARD_TOS_INT 384 +#define _GUARD_TOS_UNICODE 385 +#define _GUARD_TYPE_VERSION 386 +#define _GUARD_TYPE_VERSION_AND_LOCK 387 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 387 -#define _INIT_CALL_PY_EXACT_ARGS 388 -#define _INIT_CALL_PY_EXACT_ARGS_0 389 -#define _INIT_CALL_PY_EXACT_ARGS_1 390 -#define _INIT_CALL_PY_EXACT_ARGS_2 391 -#define _INIT_CALL_PY_EXACT_ARGS_3 392 -#define _INIT_CALL_PY_EXACT_ARGS_4 393 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 388 +#define _INIT_CALL_PY_EXACT_ARGS 389 +#define _INIT_CALL_PY_EXACT_ARGS_0 390 +#define _INIT_CALL_PY_EXACT_ARGS_1 391 +#define _INIT_CALL_PY_EXACT_ARGS_2 392 +#define _INIT_CALL_PY_EXACT_ARGS_3 393 +#define _INIT_CALL_PY_EXACT_ARGS_4 394 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -153,163 +154,163 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 394 +#define _IS_NONE 395 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 395 -#define _ITER_CHECK_RANGE 396 -#define _ITER_CHECK_TUPLE 397 -#define _ITER_JUMP_LIST 398 -#define _ITER_JUMP_RANGE 399 -#define _ITER_JUMP_TUPLE 400 -#define _ITER_NEXT_LIST 401 -#define _ITER_NEXT_LIST_TIER_TWO 402 -#define _ITER_NEXT_RANGE 403 -#define _ITER_NEXT_TUPLE 404 -#define _JUMP_TO_TOP 405 +#define _ITER_CHECK_LIST 396 +#define _ITER_CHECK_RANGE 397 +#define _ITER_CHECK_TUPLE 398 +#define _ITER_JUMP_LIST 399 +#define _ITER_JUMP_RANGE 400 +#define _ITER_JUMP_TUPLE 401 +#define _ITER_NEXT_LIST 402 +#define _ITER_NEXT_LIST_TIER_TWO 403 +#define _ITER_NEXT_RANGE 404 +#define _ITER_NEXT_TUPLE 405 +#define _JUMP_TO_TOP 406 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 406 -#define _LOAD_ATTR_CLASS 407 +#define _LOAD_ATTR 407 +#define _LOAD_ATTR_CLASS 408 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 408 -#define _LOAD_ATTR_METHOD_LAZY_DICT 409 -#define _LOAD_ATTR_METHOD_NO_DICT 410 -#define _LOAD_ATTR_METHOD_WITH_VALUES 411 -#define _LOAD_ATTR_MODULE 412 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 413 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 414 -#define _LOAD_ATTR_PROPERTY_FRAME 415 -#define _LOAD_ATTR_SLOT 416 -#define _LOAD_ATTR_WITH_HINT 417 +#define _LOAD_ATTR_INSTANCE_VALUE 409 +#define _LOAD_ATTR_METHOD_LAZY_DICT 410 +#define _LOAD_ATTR_METHOD_NO_DICT 411 +#define _LOAD_ATTR_METHOD_WITH_VALUES 412 +#define _LOAD_ATTR_MODULE 413 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 414 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 415 +#define _LOAD_ATTR_PROPERTY_FRAME 416 +#define _LOAD_ATTR_SLOT 417 +#define _LOAD_ATTR_WITH_HINT 418 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 418 +#define _LOAD_BYTECODE 419 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 419 -#define _LOAD_CONST_INLINE_BORROW 420 +#define _LOAD_CONST_INLINE 420 +#define _LOAD_CONST_INLINE_BORROW 421 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 421 -#define _LOAD_FAST_0 422 -#define _LOAD_FAST_1 423 -#define _LOAD_FAST_2 424 -#define _LOAD_FAST_3 425 -#define _LOAD_FAST_4 426 -#define _LOAD_FAST_5 427 -#define _LOAD_FAST_6 428 -#define _LOAD_FAST_7 429 +#define _LOAD_FAST 422 +#define _LOAD_FAST_0 423 +#define _LOAD_FAST_1 424 +#define _LOAD_FAST_2 425 +#define _LOAD_FAST_3 426 +#define _LOAD_FAST_4 427 +#define _LOAD_FAST_5 428 +#define _LOAD_FAST_6 429 +#define _LOAD_FAST_7 430 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 430 -#define _LOAD_FAST_BORROW_0 431 -#define _LOAD_FAST_BORROW_1 432 -#define _LOAD_FAST_BORROW_2 433 -#define _LOAD_FAST_BORROW_3 434 -#define _LOAD_FAST_BORROW_4 435 -#define _LOAD_FAST_BORROW_5 436 -#define _LOAD_FAST_BORROW_6 437 -#define _LOAD_FAST_BORROW_7 438 +#define _LOAD_FAST_BORROW 431 +#define _LOAD_FAST_BORROW_0 432 +#define _LOAD_FAST_BORROW_1 433 +#define _LOAD_FAST_BORROW_2 434 +#define _LOAD_FAST_BORROW_3 435 +#define _LOAD_FAST_BORROW_4 436 +#define _LOAD_FAST_BORROW_5 437 +#define _LOAD_FAST_BORROW_6 438 +#define _LOAD_FAST_BORROW_7 439 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 439 -#define _LOAD_GLOBAL_BUILTINS 440 -#define _LOAD_GLOBAL_MODULE 441 +#define _LOAD_GLOBAL 440 +#define _LOAD_GLOBAL_BUILTINS 441 +#define _LOAD_GLOBAL_MODULE 442 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 442 -#define _LOAD_SMALL_INT_0 443 -#define _LOAD_SMALL_INT_1 444 -#define _LOAD_SMALL_INT_2 445 -#define _LOAD_SMALL_INT_3 446 -#define _LOAD_SPECIAL LOAD_SPECIAL +#define _LOAD_SMALL_INT 443 +#define _LOAD_SMALL_INT_0 444 +#define _LOAD_SMALL_INT_1 445 +#define _LOAD_SMALL_INT_2 446 +#define _LOAD_SMALL_INT_3 447 +#define _LOAD_SPECIAL 448 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 447 +#define _MAKE_CALLARGS_A_TUPLE 449 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 448 +#define _MAKE_WARM 450 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 449 -#define _MAYBE_EXPAND_METHOD_KW 450 -#define _MONITOR_CALL 451 -#define _MONITOR_CALL_KW 452 -#define _MONITOR_JUMP_BACKWARD 453 -#define _MONITOR_RESUME 454 +#define _MAYBE_EXPAND_METHOD 451 +#define _MAYBE_EXPAND_METHOD_KW 452 +#define _MONITOR_CALL 453 +#define _MONITOR_CALL_KW 454 +#define _MONITOR_JUMP_BACKWARD 455 +#define _MONITOR_RESUME 456 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 455 -#define _POP_JUMP_IF_TRUE 456 +#define _POP_JUMP_IF_FALSE 457 +#define _POP_JUMP_IF_TRUE 458 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 457 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 458 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 459 +#define _POP_TOP_LOAD_CONST_INLINE 459 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 460 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 461 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 460 +#define _PUSH_FRAME 462 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 461 -#define _PY_FRAME_GENERAL 462 -#define _PY_FRAME_KW 463 -#define _QUICKEN_RESUME 464 -#define _REPLACE_WITH_TRUE 465 +#define _PUSH_NULL_CONDITIONAL 463 +#define _PY_FRAME_GENERAL 464 +#define _PY_FRAME_KW 465 +#define _QUICKEN_RESUME 466 +#define _REPLACE_WITH_TRUE 467 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 466 -#define _SEND 467 -#define _SEND_GEN_FRAME 468 +#define _SAVE_RETURN_OFFSET 468 +#define _SEND 469 +#define _SEND_GEN_FRAME 470 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 469 -#define _STORE_ATTR 470 -#define _STORE_ATTR_INSTANCE_VALUE 471 -#define _STORE_ATTR_SLOT 472 -#define _STORE_ATTR_WITH_HINT 473 +#define _START_EXECUTOR 471 +#define _STORE_ATTR 472 +#define _STORE_ATTR_INSTANCE_VALUE 473 +#define _STORE_ATTR_SLOT 474 +#define _STORE_ATTR_WITH_HINT 475 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 474 -#define _STORE_FAST_0 475 -#define _STORE_FAST_1 476 -#define _STORE_FAST_2 477 -#define _STORE_FAST_3 478 -#define _STORE_FAST_4 479 -#define _STORE_FAST_5 480 -#define _STORE_FAST_6 481 -#define _STORE_FAST_7 482 +#define _STORE_FAST 476 +#define _STORE_FAST_0 477 +#define _STORE_FAST_1 478 +#define _STORE_FAST_2 479 +#define _STORE_FAST_3 480 +#define _STORE_FAST_4 481 +#define _STORE_FAST_5 482 +#define _STORE_FAST_6 483 +#define _STORE_FAST_7 484 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 483 -#define _STORE_SUBSCR 484 +#define _STORE_SLICE 485 +#define _STORE_SUBSCR 486 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT -#define _STORE_SUBSCR_LIST_INT 485 +#define _STORE_SUBSCR_LIST_INT 487 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 486 -#define _TO_BOOL 487 +#define _TIER2_RESUME_CHECK 488 +#define _TO_BOOL 489 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 488 +#define _TO_BOOL_STR 490 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 489 +#define _UNPACK_SEQUENCE 491 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 489 +#define MAX_UOP_ID 491 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 250cecfba9e6f3..daf55fe20ab336 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -202,6 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [INSERT_NULL] = 0, [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, @@ -301,6 +302,7 @@ const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { + [INSERT_NULL] = "INSERT_NULL", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", @@ -942,8 +944,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _FOR_ITER_GEN_FRAME: return 0; - case _LOAD_SPECIAL: + case INSERT_NULL: return 1; + case _LOAD_SPECIAL: + return 0; case _WITH_EXCEPT_START: return 0; case _PUSH_EXC_INFO: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a9fe4ef0df1d6a..9b099e05a9ff68 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3365,10 +3365,13 @@ dummy_func( _FOR_ITER_GEN_FRAME + _PUSH_FRAME; - inst(LOAD_SPECIAL, (self -- method_and_self[2])) { + op(_INSERT_NULL, (self -- method_and_self[2])) { method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; DEAD(self); + } + + op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { PyObject *name = _Py_SpecialMethods[oparg].name; int err = _PyObject_LookupSpecialMethod(name, method_and_self); if (err < 0) { @@ -3382,6 +3385,10 @@ dummy_func( } } + macro(LOAD_SPECIAL) = + _INSERT_NULL + + _LOAD_SPECIAL; + inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) { /* At the top of the stack are 4 values: - val: TOP = exc_info() diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 92dc4ec34c808c..e3d931c7a7554a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4373,17 +4373,23 @@ break; } - case _LOAD_SPECIAL: { + case INSERT_NULL: { _PyStackRef self; _PyStackRef *method_and_self; - oparg = CURRENT_OPARG(); self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; - PyObject *name = _Py_SpecialMethods[oparg].name; - stack_pointer += -1; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SPECIAL: { + _PyStackRef *method_and_self; + oparg = CURRENT_OPARG(); + method_and_self = &stack_pointer[-2]; + PyObject *name = _Py_SpecialMethods[oparg].name; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4398,8 +4404,6 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6d3311a0205f94..d4fb3da66ff918 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9270,29 +9270,34 @@ INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef self; _PyStackRef *method_and_self; - self = stack_pointer[-1]; - method_and_self = &stack_pointer[-1]; - method_and_self[1] = self; - method_and_self[0] = PyStackRef_NULL; - PyObject *name = _Py_SpecialMethods[oparg].name; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); + // INSERT_NULL + { + self = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; + method_and_self[0] = PyStackRef_NULL; } - else if (err == 0) { + // _LOAD_SPECIAL + { + method_and_self = &stack_pointer[-1]; + PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + else if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, PyStackRef_TYPE(method_and_self[1])->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } } - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f5cd1697fbacda..2cf0a726e7a484 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1482,6 +1482,17 @@ break; } + case INSERT_NULL: { + JitOptSymbol **method_and_self; + method_and_self = &stack_pointer[-1]; + for (int _i = 2; --_i >= 0;) { + method_and_self[_i] = sym_new_not_null(ctx); + } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _LOAD_SPECIAL: { JitOptSymbol *attr; JitOptSymbol *self_or_null; From c0614f3c07cba9c5dae7fc75ee261f6eecb369bc Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 15 Apr 2025 17:08:53 +0000 Subject: [PATCH 6/9] Refactor error code paths --- Python/bytecodes.c | 13 ++++++------- Python/executor_cases.c.h | 17 ++++++++--------- Python/generated_cases.c.h | 17 ++++++++--------- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5cba767d7a7045..689d1b22017aab 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3427,13 +3427,12 @@ dummy_func( op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { PyObject *name = _Py_SpecialMethods[oparg].name; int err = _PyObject_LookupSpecialMethod(name, method_and_self); - if (err < 0) { - ERROR_NO_POP(); - } - else if (err == 0) { - _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - PyStackRef_TYPE(method_and_self[1])->tp_name); + if (err <= 0) { + if (err == 0) { + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + PyStackRef_TYPE(method_and_self[1])->tp_name); + } ERROR_NO_POP(); } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 72266093b4d6b7..622ea72dc9c0e4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4428,15 +4428,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_ERROR(); - } - else if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - PyStackRef_TYPE(method_and_self[1])->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (err <= 0) { + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + PyStackRef_TYPE(method_and_self[1])->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } JUMP_TO_ERROR(); } break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e225eca095ee51..f34560f1f7b9b0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9326,15 +9326,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - else if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - PyStackRef_TYPE(method_and_self[1])->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (err <= 0) { + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + PyStackRef_TYPE(method_and_self[1])->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } JUMP_TO_LABEL(error); } } From 36e5637a0e7ede591a39df0679a10499bd7a6ec7 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 15 Apr 2025 17:14:19 +0000 Subject: [PATCH 7/9] Update optimizer_bytecodes.c --- Python/optimizer_bytecodes.c | 11 ++++++++--- Python/optimizer_cases.c.h | 19 ++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 8e1eacfec83e95..5e266fc3dcbfba 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -897,9 +897,14 @@ dummy_func(void) { } } - op(_LOAD_SPECIAL, (owner -- attr, self_or_null)) { - attr = sym_new_not_null(ctx); - self_or_null = sym_new_unknown(ctx); + op(_INSERT_NULL, (self -- method_and_self[2])) { + method_and_self[1] = self; + method_and_self[0] = sym_new_null(ctx); + } + + op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { + method_and_self[0] = sym_new_not_null(ctx); + method_and_self[1] = sym_new_unknown(ctx); } op(_JUMP_TO_TOP, (--)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 842bac8e93e744..7b9d8a0793d8d4 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1553,25 +1553,22 @@ } case _INSERT_NULL: { + JitOptSymbol *self; JitOptSymbol **method_and_self; + self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; - for (int _i = 2; --_i >= 0;) { - method_and_self[_i] = sym_new_not_null(ctx); - } + method_and_self[1] = self; + method_and_self[0] = sym_new_null(ctx); stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_SPECIAL: { - JitOptSymbol *attr; - JitOptSymbol *self_or_null; - attr = sym_new_not_null(ctx); - self_or_null = sym_new_unknown(ctx); - stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + JitOptSymbol **method_and_self; + method_and_self = &stack_pointer[-2]; + method_and_self[0] = sym_new_not_null(ctx); + method_and_self[1] = sym_new_unknown(ctx); break; } From 6937b5320a2557cedf9b5163a87a08c8721b1048 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 15 Apr 2025 17:18:11 +0000 Subject: [PATCH 8/9] Adjust formatting and ordering --- Python/bytecodes.c | 4 ++-- Python/optimizer_bytecodes.c | 2 +- Python/optimizer_cases.c.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 689d1b22017aab..92c5390425d8b1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3430,8 +3430,8 @@ dummy_func( if (err <= 0) { if (err == 0) { _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - PyStackRef_TYPE(method_and_self[1])->tp_name); + _Py_SpecialMethods[oparg].error, + PyStackRef_TYPE(method_and_self[1])->tp_name); } ERROR_NO_POP(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 5e266fc3dcbfba..f7696a94d3f8dc 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -898,8 +898,8 @@ dummy_func(void) { } op(_INSERT_NULL, (self -- method_and_self[2])) { - method_and_self[1] = self; method_and_self[0] = sym_new_null(ctx); + method_and_self[1] = self; } op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 7b9d8a0793d8d4..8ff2481583eee2 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1557,8 +1557,8 @@ JitOptSymbol **method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; - method_and_self[1] = self; method_and_self[0] = sym_new_null(ctx); + method_and_self[1] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; From 500afcb6893272c32e1ed6c003d69128f5c85046 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 16 Apr 2025 14:15:14 +0000 Subject: [PATCH 9/9] Update code comment --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3a8bae45118b09..cef9d4ab0c6a33 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2777,7 +2777,7 @@ _PyObject_LookupSpecial(PyObject *self, PyObject *attr) // is null and `method_and_self[1]` is `self`. On exit, `method_and_self[0]` // is the method object and `method_and_self[1]` is `self` if the method is // not bound. -// Return 0 on success, -1 on error or if the method is missing. +// Return 1 on success, -1 on error, and 0 if the method is missing. int _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self) {