diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9eac58af3e2622..f64a7778121c71 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -810,7 +810,7 @@ dummy_func( assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = (int)CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0_16(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index d4ea1162eba444..26312d0ea5aa5c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -450,8 +450,12 @@ do { \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) -#define CURRENT_OPERAND0() (next_uop[-1].operand0) -#define CURRENT_OPERAND1() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_64() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_64() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_32() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_32() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_16() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_16() (next_uop[-1].operand1) #define CURRENT_TARGET() (next_uop[-1].target) #define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a64fd5b28f1d57..24f343c994038b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4645,7 +4645,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = (int)CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0_16(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); @@ -4692,7 +4692,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; left = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; @@ -4730,7 +4730,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; left = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); @@ -6736,7 +6736,7 @@ case _GUARD_GLOBALS_VERSION_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6759,7 +6759,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6784,7 +6784,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6811,7 +6811,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6837,8 +6837,8 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef res; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6883,8 +6883,8 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef res; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); PyDictObject *dict = (PyDictObject *)BUILTINS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -7936,7 +7936,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -7958,7 +7958,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -7979,7 +7979,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -8002,7 +8002,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -8023,7 +8023,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { @@ -8054,7 +8054,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { @@ -8084,7 +8084,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { @@ -8116,7 +8116,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { @@ -8235,7 +8235,7 @@ _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); @@ -8280,8 +8280,8 @@ _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); + uint32_t dict_version = (uint32_t)CURRENT_OPERAND0_32(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UOP_STAT_INC(uopcode, miss); @@ -8342,7 +8342,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + uint16_t hint = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); @@ -8428,7 +8428,7 @@ _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); @@ -8472,7 +8472,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -8499,7 +8499,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -8525,7 +8525,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -8553,7 +8553,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -8581,7 +8581,7 @@ _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -8612,7 +8612,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *fget = (PyObject *)CURRENT_OPERAND0(); + PyObject *fget = (PyObject *)CURRENT_OPERAND0_64(); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; @@ -8763,7 +8763,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; value = _stack_item_0; - uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); @@ -8798,7 +8798,7 @@ oparg = CURRENT_OPARG(); owner = _stack_item_1; value = _stack_item_0; - uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + uint16_t hint = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); @@ -8874,7 +8874,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; value = _stack_item_0; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!LOCK_OBJECT(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -11133,7 +11133,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -11156,7 +11156,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -11178,7 +11178,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -11202,7 +11202,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -11227,7 +11227,7 @@ _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11252,7 +11252,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11276,7 +11276,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11299,7 +11299,7 @@ _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -11325,7 +11325,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -11350,7 +11350,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -11374,7 +11374,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11398,7 +11398,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -11420,7 +11420,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -11442,7 +11442,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -11463,7 +11463,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -11486,7 +11486,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -11510,7 +11510,7 @@ _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11535,7 +11535,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11559,7 +11559,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11656,7 +11656,7 @@ _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -11677,8 +11677,8 @@ case _CHECK_FUNCTION_VERSION_INLINE_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { @@ -11695,8 +11695,8 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { @@ -11715,8 +11715,8 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { @@ -11737,8 +11737,8 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { @@ -11762,7 +11762,7 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); @@ -12941,7 +12941,7 @@ oparg = CURRENT_OPARG(); self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { UOP_STAT_INC(uopcode, miss); @@ -14123,7 +14123,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -14151,7 +14151,7 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); @@ -15403,7 +15403,7 @@ case _SET_IP_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); @@ -15414,7 +15414,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); @@ -15427,7 +15427,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; @@ -15442,7 +15442,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; _tos_cache2 = _stack_item_2; _tos_cache1 = _stack_item_1; @@ -15455,7 +15455,7 @@ case _CHECK_STACK_SPACE_OPERAND_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -15476,7 +15476,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -15499,7 +15499,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -15524,7 +15524,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -15619,7 +15619,7 @@ case _EXIT_TRACE_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) @@ -15646,7 +15646,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) @@ -15681,7 +15681,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) @@ -15719,7 +15719,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) @@ -15756,7 +15756,7 @@ case _DYNAMIC_EXIT_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -15780,7 +15780,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -15812,7 +15812,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -15847,7 +15847,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -15946,7 +15946,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectNew(ptr); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); @@ -15959,7 +15959,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectNew(ptr); _tos_cache1 = value; _tos_cache0 = _stack_item_0; @@ -15974,7 +15974,7 @@ _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectNew(ptr); _tos_cache2 = value; _tos_cache1 = _stack_item_1; @@ -15991,7 +15991,7 @@ _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; pop = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -16008,7 +16008,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); @@ -16021,7 +16021,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache1 = value; _tos_cache0 = _stack_item_0; @@ -16036,7 +16036,7 @@ _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache2 = value; _tos_cache1 = _stack_item_1; @@ -16147,7 +16147,7 @@ _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; pop = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -16170,7 +16170,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; pop2 = _stack_item_1; pop1 = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); stack_pointer[0] = pop1; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -16201,7 +16201,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; null = _stack_item_1; callable = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); (void)null; _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); @@ -16228,7 +16228,7 @@ pop = _stack_item_2; null = _stack_item_1; callable = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); stack_pointer[0] = callable; stack_pointer[1] = null; stack_pointer += 2; @@ -16266,7 +16266,7 @@ pop1 = _stack_item_1; null = _stack_item_0; callable = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); stack_pointer[0] = null; stack_pointer[1] = pop1; stack_pointer += 2; @@ -16301,7 +16301,7 @@ _PyStackRef value; _PyStackRef new; old = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectNew(ptr); _tos_cache1 = new; @@ -16321,7 +16321,7 @@ _PyStackRef new; _PyStackRef _stack_item_0 = _tos_cache0; old = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectNew(ptr); _tos_cache1 = new; @@ -16340,7 +16340,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; old = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectNew(ptr); _tos_cache2 = new; @@ -16358,7 +16358,7 @@ _PyStackRef value; _PyStackRef new; old = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache1 = new; @@ -16378,7 +16378,7 @@ _PyStackRef new; _PyStackRef _stack_item_0 = _tos_cache0; old = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache1 = new; @@ -16397,7 +16397,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; old = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache2 = new; @@ -16411,7 +16411,7 @@ case _START_EXECUTOR_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - PyObject *executor = (PyObject *)CURRENT_OPERAND0(); + PyObject *executor = (PyObject *)CURRENT_OPERAND0_64(); #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); #endif @@ -16646,7 +16646,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); oparg = CURRENT_OPARG(); - uint32_t target = (uint32_t)CURRENT_OPERAND0(); + uint32_t target = (uint32_t)CURRENT_OPERAND0_32(); assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SET_CURRENT_CACHED_VALUES(0); @@ -16972,7 +16972,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); #define OFFSET_OF__PUSH_FRAME ((0)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF__PUSH_FRAME; @@ -16993,7 +16993,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; #define OFFSET_OF__PUSH_FRAME ((0)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF__PUSH_FRAME; @@ -17016,7 +17016,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #define OFFSET_OF__PUSH_FRAME ((0)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF__PUSH_FRAME; @@ -17041,7 +17041,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #define OFFSET_OF__PUSH_FRAME ((0)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF__PUSH_FRAME; @@ -17064,7 +17064,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_YIELD_VALUE; @@ -17085,7 +17085,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_YIELD_VALUE; @@ -17108,7 +17108,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_YIELD_VALUE; @@ -17133,7 +17133,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_YIELD_VALUE; @@ -17156,7 +17156,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_VALUE; @@ -17177,7 +17177,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_VALUE; @@ -17200,7 +17200,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_VALUE; @@ -17225,7 +17225,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_VALUE; @@ -17248,7 +17248,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; @@ -17269,7 +17269,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; @@ -17292,7 +17292,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; @@ -17317,7 +17317,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 420cdcaed03339..ba80886f6dc0c7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -371,7 +371,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = (int)CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0_16(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 7e50a6ebafbfd3..9c86382cdc5610 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -222,12 +222,13 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, t idx = 0 for cache in uop.caches: if cache.name != "unused": + bits = cache.size*16 if cache.size == 4: type = cast = "PyObject *" else: - type = f"uint{cache.size*16}_t " - cast = f"uint{cache.size*16}_t" - emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n") + type = f"uint{bits}_t " + cast = f"uint{bits}_t" + emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}_{bits}();\n") idx += 1 reachable, storage = emitter.emit_tokens(uop, storage, None, False) if reachable: diff --git a/Tools/jit/_optimizers.py b/Tools/jit/_optimizers.py index 297b9517f6a27a..08d4e700652deb 100644 --- a/Tools/jit/_optimizers.py +++ b/Tools/jit/_optimizers.py @@ -96,6 +96,8 @@ class InstructionKind(enum.Enum): LONG_BRANCH = enum.auto() SHORT_BRANCH = enum.auto() RETURN = enum.auto() + SMALL_CONST_1 = enum.auto() + SMALL_CONST_2 = enum.auto() OTHER = enum.auto() @@ -172,6 +174,7 @@ class Optimizer: ) # Override everything that follows in subclasses: _supports_external_relocations = True + supports_small_constants = False _branches: typing.ClassVar[dict[str, tuple[str | None, str | None]]] = {} # Short branches are instructions that can branch within a micro-op, # but might not have the reach to branch anywhere within a trace. @@ -184,6 +187,9 @@ class Optimizer: _re_return: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH text: str = "" globals: set[str] = dataclasses.field(default_factory=set) + _re_small_const_1 = _RE_NEVER_MATCH + _re_small_const_2 = _RE_NEVER_MATCH + const_reloc = "" def __post_init__(self) -> None: # Split the code into a linked list of basic blocks. A basic block is an @@ -253,6 +259,14 @@ def _parse_instruction(self, line: str) -> Instruction: elif match := self._re_return.match(line): name = line kind = InstructionKind.RETURN + elif match := self._re_small_const_1.match(line): + target = match["value"] + name = match["instruction"] + kind = InstructionKind.SMALL_CONST_1 + elif match := self._re_small_const_2.match(line): + target = match["value"] + name = match["instruction"] + kind = InstructionKind.SMALL_CONST_2 else: name, *_ = line.split(" ") kind = InstructionKind.OTHER @@ -385,7 +399,7 @@ def _remove_redundant_jumps(self) -> None: block.fallthrough = True block.instructions.pop() # Before: - # br ? FOO: + # branch FOO: # ... # FOO: # jump BAR @@ -461,6 +475,70 @@ def _fixup_external_labels(self) -> None: ) block.instructions.append(branch.update_target("0")) + def _make_temp_label(self, index: int) -> Instruction: + marker = f"jit_temp_{index}:" + return Instruction(InstructionKind.OTHER, "", marker, None) + + def _fixup_constants(self) -> None: + if not self.supports_small_constants: + return + index = 0 + for block in self._blocks(): + fixed: list[Instruction] = [] + small_const_index = -1 + for inst in block.instructions: + if inst.kind == InstructionKind.SMALL_CONST_1: + marker = f"jit_pending_{inst.target}{index}:" + fixed.append(self._make_temp_label(index)) + index += 1 + small_const_index = len(fixed) + fixed.append(inst) + elif inst.kind == InstructionKind.SMALL_CONST_2: + if small_const_index < 0: + fixed.append(inst) + continue + small_const_1 = fixed[small_const_index] + if not self._small_consts_match(small_const_1, inst): + small_const_index = -1 + fixed.append(inst) + continue + assert small_const_1.target is not None + if small_const_1.target.endswith("16"): + fixed[small_const_index] = self._make_temp_label(index) + index += 1 + else: + assert small_const_1.target.endswith("32") + patch_kind, replacement = self._small_const_1(small_const_1) + if replacement is not None: + label = f"{self.const_reloc}{patch_kind}_JIT_RELOCATION_CONST{small_const_1.target[:-3]}_JIT_RELOCATION_{index}:" + index += 1 + fixed[small_const_index - 1] = Instruction( + InstructionKind.OTHER, "", label, None + ) + fixed[small_const_index] = replacement + patch_kind, replacement = self._small_const_2(inst) + if replacement is not None: + assert inst.target is not None + label = f"{self.const_reloc}{patch_kind}_JIT_RELOCATION_CONST{inst.target[:-3]}_JIT_RELOCATION_{index}:" + index += 1 + fixed.append( + Instruction(InstructionKind.OTHER, "", label, None) + ) + fixed.append(replacement) + small_const_index = -1 + else: + fixed.append(inst) + block.instructions = fixed + + def _small_const_1(self, inst: Instruction) -> tuple[str, Instruction | None]: + raise NotImplementedError() + + def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]: + raise NotImplementedError() + + def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool: + raise NotImplementedError() + def run(self) -> None: """Run this optimizer.""" self._insert_continue_label() @@ -472,6 +550,7 @@ def run(self) -> None: self._remove_redundant_jumps() self._remove_unreachable() self._fixup_external_labels() + self._fixup_constants() self.path.write_text(self._body()) @@ -492,6 +571,54 @@ class OptimizerAArch64(Optimizer): # pylint: disable = too-few-public-methods # https://developer.arm.com/documentation/ddi0602/2025-09/Base-Instructions/RET--Return-from-subroutine- _re_return = re.compile(r"\s*ret\b") + supports_small_constants = True + _re_small_const_1 = re.compile( + r"\s*(?Padrp)\s+.*(?P_JIT_OP(ARG|ERAND(0|1))_(16|32)).*" + ) + _re_small_const_2 = re.compile( + r"\s*(?Pldr)\s+.*(?P_JIT_OP(ARG|ERAND(0|1))_(16|32)).*" + ) + const_reloc = "CUSTOM_AARCH64_CONST" + + def _get_reg(self, inst: Instruction) -> str: + _, rest = inst.text.split(inst.name) + reg, *_ = rest.split(",") + return reg.strip() + + def _small_const_1(self, inst: Instruction) -> tuple[str, Instruction | None]: + assert inst.kind is InstructionKind.SMALL_CONST_1 + assert inst.target is not None + if "16" in inst.target: + return "", None + pre, _ = inst.text.split(inst.name) + return "16a", Instruction( + InstructionKind.OTHER, "movz", f"{pre}movz {self._get_reg(inst)}, 0", None + ) + + def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]: + assert inst.kind is InstructionKind.SMALL_CONST_2 + assert inst.target is not None + pre, _ = inst.text.split(inst.name) + if "16" in inst.target: + return "16a", Instruction( + InstructionKind.OTHER, + "movz", + f"{pre}movz {self._get_reg(inst)}, 0", + None, + ) + else: + return "16b", Instruction( + InstructionKind.OTHER, + "movk", + f"{pre}movk {self._get_reg(inst)}, 0, lsl #16", + None, + ) + + def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool: + reg1 = self._get_reg(inst1) + reg2 = self._get_reg(inst2) + return reg1 == reg2 + class OptimizerX86(Optimizer): # pylint: disable = too-few-public-methods """i686-pc-windows-msvc/x86_64-apple-darwin/x86_64-unknown-linux-gnu""" diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 4e86abe604972e..964e8bdbdc1010 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -9,6 +9,9 @@ "ARM64_RELOC_PAGE21", "ARM64_RELOC_PAGEOFF12", "ARM64_RELOC_UNSIGNED", + "CUSTOM_AARCH64_BRANCH19", + "CUSTOM_AARCH64_CONST_16", + "CUSTOM_AARCH64_CONST_32", "IMAGE_REL_AMD64_REL32", "IMAGE_REL_ARM64_BRANCH19", "IMAGE_REL_ARM64_BRANCH26", diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index cdffd953ef9838..2b78d8013af5db 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -32,6 +32,12 @@ class HoleValue(enum.Enum): # The current uop's operand0 on 32-bit platforms (exposed as _JIT_OPERAND0_HI/LO): OPERAND0_HI = enum.auto() OPERAND0_LO = enum.auto() + # 16 and 32 bit versions of OPARG, OPERAND0 and OPERAND1 + OPARG_16 = enum.auto() + OPERAND0_16 = enum.auto() + OPERAND1_16 = enum.auto() + OPERAND0_32 = enum.auto() + OPERAND1_32 = enum.auto() # The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND1): OPERAND1 = enum.auto() # The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND1_HI/LO): @@ -59,6 +65,8 @@ class HoleValue(enum.Enum): "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12", "ARM64_RELOC_UNSIGNED": "patch_64", "CUSTOM_AARCH64_BRANCH19": "patch_aarch64_19r", + "CUSTOM_AARCH64_CONST16a": "patch_aarch64_16a", + "CUSTOM_AARCH64_CONST16b": "patch_aarch64_16b", # x86_64-pc-windows-msvc: "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx", # aarch64-pc-windows-msvc: @@ -95,6 +103,7 @@ class HoleValue(enum.Enum): "X86_64_RELOC_SIGNED": "patch_32r", "X86_64_RELOC_UNSIGNED": "patch_64", } + # Translate HoleValues to C expressions: _HOLE_EXPRS = { HoleValue.CODE: "(uintptr_t)code", @@ -103,10 +112,15 @@ class HoleValue(enum.Enum): HoleValue.GOT: "", # These should all have been turned into DATA values by process_relocations: HoleValue.OPARG: "instruction->oparg", + HoleValue.OPARG_16: "instruction->oparg", HoleValue.OPERAND0: "instruction->operand0", + HoleValue.OPERAND0_16: "instruction->operand0", + HoleValue.OPERAND0_32: "instruction->operand0", HoleValue.OPERAND0_HI: "(instruction->operand0 >> 32)", HoleValue.OPERAND0_LO: "(instruction->operand0 & UINT32_MAX)", HoleValue.OPERAND1: "instruction->operand1", + HoleValue.OPERAND1_16: "instruction->operand1", + HoleValue.OPERAND1_32: "instruction->operand1", HoleValue.OPERAND1_HI: "(instruction->operand1 >> 32)", HoleValue.OPERAND1_LO: "(instruction->operand1 & UINT32_MAX)", HoleValue.TARGET: "instruction->target", @@ -201,7 +215,10 @@ def as_c(self, where: str) -> str: if self.symbol: if value: value += " + " - value += f"(uintptr_t)&{self.symbol}" + if self.symbol.startswith("CONST"): + value += f"instruction->{self.symbol[10:].lower()}" + else: + value += f"(uintptr_t)&{self.symbol}" if _signed(self.addend) or not value: if value: value += " + " diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index f92f3eac13bde5..5895e91c3c44ce 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -138,6 +138,7 @@ async def _compile( f"--target={self.triple}", "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", + f"-DSUPPORTS_SMALL_CONSTS={1 if self.optimizer.supports_small_constants else 0}", f"-D_JIT_OPCODE={opname}", "-D_PyJIT_ACTIVE", "-D_Py_JIT", diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 93067977791c42..064b401bc3aca4 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -34,14 +34,38 @@ #include "jit.h" + +#undef CURRENT_OPERAND0_64 +#define CURRENT_OPERAND0_64() (_operand0_64) + +#undef CURRENT_OPERAND1_64 +#define CURRENT_OPERAND1_64() (_operand1_64) + + #undef CURRENT_OPARG +#undef CURRENT_OPERAND0_16 +#undef CURRENT_OPERAND0_32 +#undef CURRENT_OPERAND1_16 +#undef CURRENT_OPERAND1_32 + +#if SUPPORTS_SMALL_CONSTS + +#define CURRENT_OPARG() (_oparg_16) +#define CURRENT_OPERAND0_32() (_operand0_32) +#define CURRENT_OPERAND0_16() (_operand0_16) +#define CURRENT_OPERAND1_32() (_operand1_32) +#define CURRENT_OPERAND1_16() (_operand1_16) + +#else + #define CURRENT_OPARG() (_oparg) +#define CURRENT_OPERAND0_32() (_operand0_64) +#define CURRENT_OPERAND0_16() (_operand0_64) +#define CURRENT_OPERAND1_32() (_operand1_64) +#define CURRENT_OPERAND1_16() (_operand1_64) -#undef CURRENT_OPERAND0 -#define CURRENT_OPERAND0() (_operand0) +#endif -#undef CURRENT_OPERAND1 -#define CURRENT_OPERAND1() (_operand1) #undef CURRENT_TARGET #define CURRENT_TARGET() (_target) @@ -105,18 +129,26 @@ _JIT_ENTRY( int uopcode = _JIT_OPCODE; _Py_CODEUNIT *next_instr; // Other stuff we need handy: - PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) #if SIZEOF_VOID_P == 8 - PATCH_VALUE(uint64_t, _operand0, _JIT_OPERAND0) - PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1) + PATCH_VALUE(uint64_t, _operand0_64, _JIT_OPERAND0) + PATCH_VALUE(uint64_t, _operand1_64, _JIT_OPERAND1) #else assert(SIZEOF_VOID_P == 4); PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI) PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO) - uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; + uint64_t _operand0_64 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI) PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO) - uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; + uint64_t _operand1_64 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; +#endif +#if SUPPORTS_SMALL_CONSTS + PATCH_VALUE(uint32_t, _operand0_32, _JIT_OPERAND0_32) + PATCH_VALUE(uint32_t, _operand1_32, _JIT_OPERAND1_32) + PATCH_VALUE(uint16_t, _operand0_16, _JIT_OPERAND0_16) + PATCH_VALUE(uint16_t, _operand1_16, _JIT_OPERAND1_16) + PATCH_VALUE(uint16_t, _oparg_16, _JIT_OPARG_16) +#else + PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) #endif PATCH_VALUE(uint32_t, _target, _JIT_TARGET) OPT_STAT_INC(uops_executed);