From 7bf8b08be316079384f138daf60233f9961b2cff Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 1 Oct 2024 21:28:23 +0800 Subject: [PATCH 01/10] Strength reduce function guards --- Include/internal/pycore_uop_ids.h | 279 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 4 + Python/bytecodes.c | 7 + Python/executor_cases.c.h | 13 ++ Python/optimizer_bytecodes.c | 20 +- Python/optimizer_cases.c.h | 25 ++- 6 files changed, 207 insertions(+), 141 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 927dae88c1fa73..9f5dfbf215a62f 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -63,88 +63,89 @@ extern "C" { #define _CHECK_FUNCTION 333 #define _CHECK_FUNCTION_EXACT_ARGS 334 #define _CHECK_FUNCTION_VERSION 335 -#define _CHECK_FUNCTION_VERSION_KW 336 -#define _CHECK_IS_NOT_PY_CALLABLE 337 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 338 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 339 -#define _CHECK_METHOD_VERSION 340 -#define _CHECK_METHOD_VERSION_KW 341 -#define _CHECK_PEP_523 342 -#define _CHECK_PERIODIC 343 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 344 -#define _CHECK_STACK_SPACE 345 -#define _CHECK_STACK_SPACE_OPERAND 346 -#define _CHECK_VALIDITY 347 -#define _CHECK_VALIDITY_AND_SET_IP 348 -#define _COMPARE_OP 349 -#define _COMPARE_OP_FLOAT 350 -#define _COMPARE_OP_INT 351 -#define _COMPARE_OP_STR 352 -#define _CONTAINS_OP 353 +#define _CHECK_FUNCTION_VERSION_INLINE 336 +#define _CHECK_FUNCTION_VERSION_KW 337 +#define _CHECK_IS_NOT_PY_CALLABLE 338 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 339 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 340 +#define _CHECK_METHOD_VERSION 341 +#define _CHECK_METHOD_VERSION_KW 342 +#define _CHECK_PEP_523 343 +#define _CHECK_PERIODIC 344 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 345 +#define _CHECK_STACK_SPACE 346 +#define _CHECK_STACK_SPACE_OPERAND 347 +#define _CHECK_VALIDITY 348 +#define _CHECK_VALIDITY_AND_SET_IP 349 +#define _COMPARE_OP 350 +#define _COMPARE_OP_FLOAT 351 +#define _COMPARE_OP_INT 352 +#define _COMPARE_OP_STR 353 +#define _CONTAINS_OP 354 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 354 +#define _CREATE_INIT_FRAME 355 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 355 +#define _DEOPT 356 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 356 -#define _DO_CALL_KW 357 -#define _DYNAMIC_EXIT 358 +#define _DO_CALL 357 +#define _DO_CALL_KW 358 +#define _DYNAMIC_EXIT 359 #define _END_SEND END_SEND -#define _ERROR_POP_N 359 +#define _ERROR_POP_N 360 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 360 -#define _EXPAND_METHOD_KW 361 -#define _FATAL_ERROR 362 +#define _EXPAND_METHOD 361 +#define _EXPAND_METHOD_KW 362 +#define _FATAL_ERROR 363 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 363 -#define _FOR_ITER_GEN_FRAME 364 -#define _FOR_ITER_TIER_TWO 365 +#define _FOR_ITER 364 +#define _FOR_ITER_GEN_FRAME 365 +#define _FOR_ITER_TIER_TWO 366 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 366 -#define _GUARD_BOTH_INT 367 -#define _GUARD_BOTH_UNICODE 368 -#define _GUARD_BUILTINS_VERSION 369 -#define _GUARD_DORV_NO_DICT 370 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 371 -#define _GUARD_GLOBALS_VERSION 372 -#define _GUARD_IS_FALSE_POP 373 -#define _GUARD_IS_NONE_POP 374 -#define _GUARD_IS_NOT_NONE_POP 375 -#define _GUARD_IS_TRUE_POP 376 -#define _GUARD_KEYS_VERSION 377 -#define _GUARD_NOS_FLOAT 378 -#define _GUARD_NOS_INT 379 -#define _GUARD_NOT_EXHAUSTED_LIST 380 -#define _GUARD_NOT_EXHAUSTED_RANGE 381 -#define _GUARD_NOT_EXHAUSTED_TUPLE 382 -#define _GUARD_TOS_FLOAT 383 -#define _GUARD_TOS_INT 384 -#define _GUARD_TYPE_VERSION 385 +#define _GUARD_BOTH_FLOAT 367 +#define _GUARD_BOTH_INT 368 +#define _GUARD_BOTH_UNICODE 369 +#define _GUARD_BUILTINS_VERSION 370 +#define _GUARD_DORV_NO_DICT 371 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 372 +#define _GUARD_GLOBALS_VERSION 373 +#define _GUARD_IS_FALSE_POP 374 +#define _GUARD_IS_NONE_POP 375 +#define _GUARD_IS_NOT_NONE_POP 376 +#define _GUARD_IS_TRUE_POP 377 +#define _GUARD_KEYS_VERSION 378 +#define _GUARD_NOS_FLOAT 379 +#define _GUARD_NOS_INT 380 +#define _GUARD_NOT_EXHAUSTED_LIST 381 +#define _GUARD_NOT_EXHAUSTED_RANGE 382 +#define _GUARD_NOT_EXHAUSTED_TUPLE 383 +#define _GUARD_TOS_FLOAT 384 +#define _GUARD_TOS_INT 385 +#define _GUARD_TYPE_VERSION 386 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 386 -#define _INIT_CALL_PY_EXACT_ARGS 387 -#define _INIT_CALL_PY_EXACT_ARGS_0 388 -#define _INIT_CALL_PY_EXACT_ARGS_1 389 -#define _INIT_CALL_PY_EXACT_ARGS_2 390 -#define _INIT_CALL_PY_EXACT_ARGS_3 391 -#define _INIT_CALL_PY_EXACT_ARGS_4 392 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 387 +#define _INIT_CALL_PY_EXACT_ARGS 388 +#define _INIT_CALL_PY_EXACT_ARGS_0 389 +#define _INIT_CALL_PY_EXACT_ARGS_1 390 +#define _INIT_CALL_PY_EXACT_ARGS_2 391 +#define _INIT_CALL_PY_EXACT_ARGS_3 392 +#define _INIT_CALL_PY_EXACT_ARGS_4 393 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -156,65 +157,65 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 393 -#define _IS_NONE 394 +#define _INTERNAL_INCREMENT_OPT_COUNTER 394 +#define _IS_NONE 395 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 395 -#define _ITER_CHECK_RANGE 396 -#define _ITER_CHECK_TUPLE 397 -#define _ITER_JUMP_LIST 398 -#define _ITER_JUMP_RANGE 399 -#define _ITER_JUMP_TUPLE 400 -#define _ITER_NEXT_LIST 401 -#define _ITER_NEXT_RANGE 402 -#define _ITER_NEXT_TUPLE 403 -#define _JUMP_TO_TOP 404 +#define _ITER_CHECK_LIST 396 +#define _ITER_CHECK_RANGE 397 +#define _ITER_CHECK_TUPLE 398 +#define _ITER_JUMP_LIST 399 +#define _ITER_JUMP_RANGE 400 +#define _ITER_JUMP_TUPLE 401 +#define _ITER_NEXT_LIST 402 +#define _ITER_NEXT_RANGE 403 +#define _ITER_NEXT_TUPLE 404 +#define _JUMP_TO_TOP 405 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 405 -#define _LOAD_ATTR_CLASS 406 -#define _LOAD_ATTR_CLASS_0 407 -#define _LOAD_ATTR_CLASS_1 408 +#define _LOAD_ATTR 406 +#define _LOAD_ATTR_CLASS 407 +#define _LOAD_ATTR_CLASS_0 408 +#define _LOAD_ATTR_CLASS_1 409 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 409 -#define _LOAD_ATTR_INSTANCE_VALUE_0 410 -#define _LOAD_ATTR_INSTANCE_VALUE_1 411 -#define _LOAD_ATTR_METHOD_LAZY_DICT 412 -#define _LOAD_ATTR_METHOD_NO_DICT 413 -#define _LOAD_ATTR_METHOD_WITH_VALUES 414 -#define _LOAD_ATTR_MODULE 415 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 416 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 417 -#define _LOAD_ATTR_PROPERTY_FRAME 418 -#define _LOAD_ATTR_SLOT 419 -#define _LOAD_ATTR_SLOT_0 420 -#define _LOAD_ATTR_SLOT_1 421 -#define _LOAD_ATTR_WITH_HINT 422 +#define _LOAD_ATTR_INSTANCE_VALUE 410 +#define _LOAD_ATTR_INSTANCE_VALUE_0 411 +#define _LOAD_ATTR_INSTANCE_VALUE_1 412 +#define _LOAD_ATTR_METHOD_LAZY_DICT 413 +#define _LOAD_ATTR_METHOD_NO_DICT 414 +#define _LOAD_ATTR_METHOD_WITH_VALUES 415 +#define _LOAD_ATTR_MODULE 416 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 417 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 418 +#define _LOAD_ATTR_PROPERTY_FRAME 419 +#define _LOAD_ATTR_SLOT 420 +#define _LOAD_ATTR_SLOT_0 421 +#define _LOAD_ATTR_SLOT_1 422 +#define _LOAD_ATTR_WITH_HINT 423 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 423 -#define _LOAD_CONST_INLINE_BORROW 424 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 425 -#define _LOAD_CONST_INLINE_WITH_NULL 426 +#define _LOAD_CONST_INLINE 424 +#define _LOAD_CONST_INLINE_BORROW 425 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 426 +#define _LOAD_CONST_INLINE_WITH_NULL 427 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 427 -#define _LOAD_FAST_0 428 -#define _LOAD_FAST_1 429 -#define _LOAD_FAST_2 430 -#define _LOAD_FAST_3 431 -#define _LOAD_FAST_4 432 -#define _LOAD_FAST_5 433 -#define _LOAD_FAST_6 434 -#define _LOAD_FAST_7 435 +#define _LOAD_FAST 428 +#define _LOAD_FAST_0 429 +#define _LOAD_FAST_1 430 +#define _LOAD_FAST_2 431 +#define _LOAD_FAST_3 432 +#define _LOAD_FAST_4 433 +#define _LOAD_FAST_5 434 +#define _LOAD_FAST_6 435 +#define _LOAD_FAST_7 436 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 436 -#define _LOAD_GLOBAL_BUILTINS 437 -#define _LOAD_GLOBAL_MODULE 438 +#define _LOAD_GLOBAL 437 +#define _LOAD_GLOBAL_BUILTINS 438 +#define _LOAD_GLOBAL_MODULE 439 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL @@ -222,65 +223,65 @@ extern "C" { #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 439 +#define _MAKE_WARM 440 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 440 -#define _MONITOR_CALL 441 -#define _MONITOR_JUMP_BACKWARD 442 -#define _MONITOR_RESUME 443 +#define _MAYBE_EXPAND_METHOD 441 +#define _MONITOR_CALL 442 +#define _MONITOR_JUMP_BACKWARD 443 +#define _MONITOR_RESUME 444 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 444 -#define _POP_JUMP_IF_TRUE 445 +#define _POP_JUMP_IF_FALSE 445 +#define _POP_JUMP_IF_TRUE 446 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 446 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 447 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 447 +#define _PUSH_FRAME 448 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 448 -#define _PY_FRAME_KW 449 -#define _QUICKEN_RESUME 450 -#define _REPLACE_WITH_TRUE 451 +#define _PY_FRAME_GENERAL 449 +#define _PY_FRAME_KW 450 +#define _QUICKEN_RESUME 451 +#define _REPLACE_WITH_TRUE 452 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 452 -#define _SEND 453 -#define _SEND_GEN_FRAME 454 +#define _SAVE_RETURN_OFFSET 453 +#define _SEND 454 +#define _SEND_GEN_FRAME 455 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 455 -#define _STORE_ATTR 456 -#define _STORE_ATTR_INSTANCE_VALUE 457 -#define _STORE_ATTR_SLOT 458 -#define _STORE_ATTR_WITH_HINT 459 +#define _START_EXECUTOR 456 +#define _STORE_ATTR 457 +#define _STORE_ATTR_INSTANCE_VALUE 458 +#define _STORE_ATTR_SLOT 459 +#define _STORE_ATTR_WITH_HINT 460 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 460 -#define _STORE_FAST_0 461 -#define _STORE_FAST_1 462 -#define _STORE_FAST_2 463 -#define _STORE_FAST_3 464 -#define _STORE_FAST_4 465 -#define _STORE_FAST_5 466 -#define _STORE_FAST_6 467 -#define _STORE_FAST_7 468 +#define _STORE_FAST 461 +#define _STORE_FAST_0 462 +#define _STORE_FAST_1 463 +#define _STORE_FAST_2 464 +#define _STORE_FAST_3 465 +#define _STORE_FAST_4 466 +#define _STORE_FAST_5 467 +#define _STORE_FAST_6 468 +#define _STORE_FAST_7 469 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 469 -#define _STORE_SUBSCR 470 +#define _STORE_SLICE 470 +#define _STORE_SUBSCR 471 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 471 -#define _TO_BOOL 472 +#define _TIER2_RESUME_CHECK 472 +#define _TO_BOOL 473 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -290,14 +291,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 473 +#define _UNPACK_SEQUENCE 474 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE #define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 473 +#define MAX_UOP_ID 474 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 07606135d7a356..abdaaf5bac578d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -206,6 +206,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_FUNCTION_VERSION_INLINE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD] = HAS_ARG_FLAG, [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -341,6 +342,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", @@ -928,6 +930,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + oparg; case _CHECK_FUNCTION_VERSION: return 2 + oparg; + case _CHECK_FUNCTION_VERSION_INLINE: + return 0; case _CHECK_METHOD_VERSION: return 2 + oparg; case _EXPAND_METHOD: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8535306d9c7a03..2a4107db83b344 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3365,6 +3365,13 @@ dummy_func( EXIT_IF(func->func_version != func_version); } + tier2 op(_CHECK_FUNCTION_VERSION_INLINE, (callable_o/4 --)) { + uint16_t func_version = oparg; + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + EXIT_IF(func->func_version != func_version); + } + macro(CALL_PY_GENERAL) = unused/1 + // Skip over the counter _CHECK_PEP_523 + diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 650bf4533a3a86..0453e73f8d6750 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3636,6 +3636,19 @@ break; } + case _CHECK_FUNCTION_VERSION_INLINE: { + oparg = CURRENT_OPARG(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND(); + uint16_t func_version = oparg; + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _CHECK_METHOD_VERSION: { _PyStackRef *null; _PyStackRef callable; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 9a1b9da52f4bb5..e50bbd42e4641e 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -558,8 +558,26 @@ dummy_func(void) { self = sym_new_not_null(ctx); } - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(callable))); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, sym_get_const(callable)); + } sym_set_type(callable, &PyFunction_Type); + } + + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + assert((this_instr-1)->opcode == _CHECK_FUNCTION_VERSION || (this_instr-1)->opcode == _CHECK_FUNCTION_VERSION_INLINE); + if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); + PyCodeObject *co = func->func_code; + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + // Note: this is only valid if CHECK_FUNCTION_VERSION precedes this instruction. + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + } + } (void)self_or_null; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4d172e3c762704..76353e8116ef9c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1679,6 +1679,19 @@ } case _CHECK_FUNCTION_VERSION: { + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)this_instr->operand; + if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(callable))); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, sym_get_const(callable)); + } + sym_set_type(callable, &PyFunction_Type); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE: { break; } @@ -1748,7 +1761,17 @@ _Py_UopsSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - sym_set_type(callable, &PyFunction_Type); + assert((this_instr-1)->opcode == _CHECK_FUNCTION_VERSION || (this_instr-1)->opcode == _CHECK_FUNCTION_VERSION_INLINE); + if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); + PyCodeObject *co = func->func_code; + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + // Note: this is only valid if CHECK_FUNCTION_VERSION precedes this instruction. + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + } + } (void)self_or_null; break; } From 9d38c213f8171b10a7717ae0ce9ae430b4a69c2b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 1 Oct 2024 22:40:58 +0800 Subject: [PATCH 02/10] fix warnings --- Python/optimizer_bytecodes.c | 5 +++-- Python/optimizer_cases.c.h | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e50bbd42e4641e..ca003288e28941 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -559,9 +559,10 @@ dummy_func(void) { } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, sym_get_const(callable)); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, (uintptr_t)sym_get_const(callable)); } sym_set_type(callable, &PyFunction_Type); } @@ -571,7 +572,7 @@ dummy_func(void) { if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); - PyCodeObject *co = func->func_code; + PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { // Note: this is only valid if CHECK_FUNCTION_VERSION precedes this instruction. REPLACE_OP(this_instr, _NOP, 0 ,0); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 76353e8116ef9c..8b2f096eb3a155 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1681,11 +1681,13 @@ case _CHECK_FUNCTION_VERSION: { _Py_UopsSymbol *self_or_null; _Py_UopsSymbol *callable; + self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand; + (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, sym_get_const(callable)); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, (uintptr_t)sym_get_const(callable)); } sym_set_type(callable, &PyFunction_Type); break; @@ -1765,7 +1767,7 @@ if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); - PyCodeObject *co = func->func_code; + PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { // Note: this is only valid if CHECK_FUNCTION_VERSION precedes this instruction. REPLACE_OP(this_instr, _NOP, 0 ,0); From 53f2396b45b52a5ff6bdba15ac59640d3ede0e86 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 2 Oct 2024 01:37:38 +0800 Subject: [PATCH 03/10] Add test --- Lib/test/test_capi/test_opt.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f1ab72180d714d..166b941381f31b 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1481,6 +1481,30 @@ def fn(a): fn(A()) + def test_func_guards_removed_or_reduced(self): + def testfunc(n): + for i in range(n): + # Only works on functions promoted to constants + global_identity(i) + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + # Strength reduced version + self.assertIn("_CHECK_FUNCTION_VERSION_INLINE", uops) + self.assertNotIn("_CHECK_FUNCTION_VERSION", uops) + # Removed guard + self.assertNotIn("_CHECK_FUNCTION_EXACT_ARGS", uops) + + +def global_identity(x): + return x + if __name__ == "__main__": unittest.main() From cc5bf06cb1710887177a5c46d8fe85c64e689732 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:35:29 +0800 Subject: [PATCH 04/10] check for overflow --- Python/optimizer_bytecodes.c | 4 +++- Python/optimizer_cases.c.h | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index ca003288e28941..834893e0fe1e91 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -562,7 +562,9 @@ dummy_func(void) { (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, (uintptr_t)sym_get_const(callable)); + if (func_version == (uint16_t)func_version) { + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, (uintptr_t)sym_get_const(callable)); + } } sym_set_type(callable, &PyFunction_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8b2f096eb3a155..59af6a93b392de 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1687,7 +1687,9 @@ (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, (uintptr_t)sym_get_const(callable)); + if (func_version == (uint16_t)func_version) { + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, (uintptr_t)sym_get_const(callable)); + } } sym_set_type(callable, &PyFunction_Type); break; From 6d6263cd87c5603f9cde479f6f7d8c8a4279d59d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:01:37 +0800 Subject: [PATCH 05/10] Add 2-operand support --- Include/internal/pycore_optimizer.h | 1 + Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 3 +-- Python/ceval_macros.h | 1 + Python/executor_cases.c.h | 5 ++--- Python/optimizer_bytecodes.c | 5 ++--- Python/optimizer_cases.c.h | 5 ++--- Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/tier2_generator.py | 7 +++++-- Tools/jit/_stencils.py | 8 ++++++++ Tools/jit/template.c | 11 +++++++++++ 11 files changed, 35 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index f92c0a0cddf906..7b4944f206fe20 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -59,6 +59,7 @@ typedef struct { }; }; uint64_t operand; // A cache entry + uint64_t operand1; } _PyUOpInstruction; typedef struct { diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 34648fc44533f2..3de6b0b1f7621a 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -213,7 +213,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_CHECK_FUNCTION_VERSION_INLINE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD] = HAS_ARG_FLAG, [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3b4ae9ede65e51..8fd90bc7ce7e23 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3442,8 +3442,7 @@ dummy_func( EXIT_IF(func->func_version != func_version); } - tier2 op(_CHECK_FUNCTION_VERSION_INLINE, (callable_o/4 --)) { - uint16_t func_version = oparg; + tier2 op(_CHECK_FUNCTION_VERSION_INLINE, (func_version/2, callable_o/4 --)) { assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; EXIT_IF(func->func_version != func_version); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 6674c4ccf9f693..c36c470806c664 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -410,6 +410,7 @@ do { \ #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND() (next_uop[-1].operand) +#define CURRENT_OPERAND1() (next_uop[-1].operand1) #define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target #define JUMP_TO_ERROR() goto jump_to_error_target diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 5db8395c050052..bc7238c5266537 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3972,9 +3972,8 @@ } case _CHECK_FUNCTION_VERSION_INLINE: { - oparg = CURRENT_OPARG(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND(); - uint16_t func_version = oparg; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index daa585a00ffcdc..d8f489cf6766ae 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -593,9 +593,8 @@ dummy_func(void) { (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); - if (func_version == (uint16_t)func_version) { - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, (uintptr_t)sym_get_const(callable)); - } + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)sym_get_const(callable); } sym_set_type(callable, &PyFunction_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 7a0d0b6d3ee79a..a7abe553ac3335 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1755,9 +1755,8 @@ (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); - if (func_version == (uint16_t)func_version) { - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, func_version, (uintptr_t)sym_get_const(callable)); - } + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)sym_get_const(callable); } sym_set_type(callable, &PyFunction_Type); break; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index a725ec10d4e52a..96b32445fb62e2 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -200,7 +200,7 @@ def why_not_viable(self) -> str | None: return "has tier 1 control flow" if self.properties.needs_this: return "uses the 'this_instr' variable" - if len([c for c in self.caches if c.name != "unused"]) > 1: + if len([c for c in self.caches if c.name != "unused"]) > 2: return "has unused cache entries" if self.properties.error_with_pop and self.properties.error_without_pop: return "has both popping and not-popping errors" diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index ce761495cca435..a1debd62466c87 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -181,14 +181,17 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: code_list, storage = Storage.for_uop(stack, uop) for code in code_list: emitter.emit(code) - for cache in uop.caches: + for idx, cache in enumerate(uop.caches): if cache.name != "unused": if cache.size == 4: type = cast = "PyObject *" else: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" - emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") + if idx == 0: + emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") + else: + emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n") storage = emitter.emit_tokens(uop, storage, None) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) from None diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 2cd051b0a77b8d..f2831d3623f46d 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -34,6 +34,11 @@ class HoleValue(enum.Enum): # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): OPERAND_HI = enum.auto() OPERAND_LO = enum.auto() + # The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND): + OPERAND1 = enum.auto() + # The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): + OPERAND1_HI = enum.auto() + OPERAND1_LO = enum.auto() # The current uop's target (exposed as _JIT_TARGET): TARGET = enum.auto() # The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET): @@ -102,6 +107,9 @@ class HoleValue(enum.Enum): HoleValue.OPERAND: "instruction->operand", HoleValue.OPERAND_HI: "(instruction->operand >> 32)", HoleValue.OPERAND_LO: "(instruction->operand & UINT32_MAX)", + HoleValue.OPERAND1: "instruction->operand1", + HoleValue.OPERAND1_HI: "(instruction->operand1 >> 32)", + HoleValue.OPERAND1_LO: "(instruction->operand1 & UINT32_MAX)", HoleValue.TARGET: "instruction->target", HoleValue.JUMP_TARGET: "state->instruction_starts[instruction->jump_target]", HoleValue.ERROR_TARGET: "state->instruction_starts[instruction->error_target]", diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 57c1006ab423e9..1a41d883cbf729 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -29,6 +29,9 @@ #undef CURRENT_OPERAND #define CURRENT_OPERAND() (_operand) +#undef CURRENT_OPERAND1 +#define CURRENT_OPERAND1() (_operand1) + #undef DEOPT_IF #define DEOPT_IF(COND, INSTNAME) \ do { \ @@ -105,6 +108,14 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI) PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO) uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo; +#endif +#if SIZEOF_VOID_P == 8 + PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1) +#else + assert(SIZEOF_VOID_P == 4); + PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI) + PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO) + uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; #endif PATCH_VALUE(uint32_t, _target, _JIT_TARGET) From f0733d9e65829a47f3653c7f2e7e27e6a2cfe2a0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 6 Nov 2024 22:08:05 +0800 Subject: [PATCH 06/10] Rename operand to operand0 --- Include/internal/pycore_optimizer.h | 2 +- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 98 ++++++++++---------- Python/generated_cases.c.h | 2 +- Python/optimizer.c | 22 ++--- Python/optimizer_analysis.c | 28 +++--- Python/optimizer_bytecodes.c | 2 +- Python/optimizer_cases.c.h | 44 ++++----- Tools/cases_generator/optimizer_generator.py | 2 +- Tools/cases_generator/tier2_generator.py | 5 +- Tools/jit/_stencils.py | 12 +-- Tools/jit/template.c | 12 +-- 12 files changed, 114 insertions(+), 117 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 7b4944f206fe20..6d70b42f708854 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -58,7 +58,7 @@ typedef struct { uint16_t error_target; }; }; - uint64_t operand; // A cache entry + uint64_t operand0; // A cache entry uint64_t operand1; } _PyUOpInstruction; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8fd90bc7ce7e23..fd060101957b90 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -637,7 +637,7 @@ dummy_func( assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND(); + next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); DEOPT_IF(!PyStackRef_Is(*target_local, left)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc7238c5266537..6cbdf0b0b942fe 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -822,7 +822,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND(); + next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); if (!PyStackRef_Is(*target_local, left)) { @@ -1855,7 +1855,7 @@ } case _GUARD_GLOBALS_VERSION: { - uint16_t version = (uint16_t)CURRENT_OPERAND(); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -1871,7 +1871,7 @@ case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { PyDictKeysObject *globals_keys; - uint16_t version = (uint16_t)CURRENT_OPERAND(); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -1891,7 +1891,7 @@ case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { PyDictKeysObject *builtins_keys; - uint16_t version = (uint16_t)CURRENT_OPERAND(); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)BUILTINS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -1915,7 +1915,7 @@ _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = entries[index].me_value; stack_pointer += -1; @@ -1941,7 +1941,7 @@ _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = entries[index].me_value; stack_pointer += -1; @@ -2514,7 +2514,7 @@ case _GUARD_TYPE_VERSION: { _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (tp->tp_version_tag != type_version) { @@ -2543,7 +2543,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = *value_ptr; @@ -2566,7 +2566,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = *value_ptr; @@ -2591,7 +2591,7 @@ case _CHECK_ATTR_MODULE: { _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); + uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyModule_CheckExact(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -2612,7 +2612,7 @@ _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); @@ -2655,7 +2655,7 @@ _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)CURRENT_OPERAND(); + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; PyDictObject *dict = _PyObject_GetManagedDict(owner_o); @@ -2696,7 +2696,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; PyObject *attr_o = *(PyObject **)addr; @@ -2718,7 +2718,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; PyObject *attr_o = *(PyObject **)addr; @@ -2742,7 +2742,7 @@ case _CHECK_ATTR_CLASS: { _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -2762,7 +2762,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -2778,7 +2778,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -2798,7 +2798,7 @@ _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *fget = (PyObject *)CURRENT_OPERAND(); + PyObject *fget = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; @@ -2850,7 +2850,7 @@ _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t offset = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); @@ -2877,7 +2877,7 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t hint = (uint16_t)CURRENT_OPERAND(); + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); @@ -2928,7 +2928,7 @@ _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); @@ -3770,7 +3770,7 @@ case _GUARD_KEYS_VERSION: { _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { @@ -3786,7 +3786,7 @@ _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -3807,7 +3807,7 @@ _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3827,7 +3827,7 @@ _PyStackRef attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3842,7 +3842,7 @@ _PyStackRef attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3856,7 +3856,7 @@ case _CHECK_ATTR_METHOD_LAZY_DICT: { _PyStackRef owner; owner = stack_pointer[-1]; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ @@ -3873,7 +3873,7 @@ _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3957,7 +3957,7 @@ _PyStackRef *callable; oparg = CURRENT_OPARG(); callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -3972,7 +3972,7 @@ } case _CHECK_FUNCTION_VERSION_INLINE: { - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -3989,7 +3989,7 @@ oparg = CURRENT_OPARG(); null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); @@ -4445,7 +4445,7 @@ callable = &stack_pointer[-2 - oparg]; init = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyStackRef_IsNull(null[0])) { UOP_STAT_INC(uopcode, miss); @@ -5203,7 +5203,7 @@ _PyStackRef *callable; oparg = CURRENT_OPARG(); callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -5223,7 +5223,7 @@ oparg = CURRENT_OPARG(); null = &stack_pointer[-2 - oparg]; callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); @@ -5661,13 +5661,13 @@ } case _SET_IP: { - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } case _CHECK_STACK_SPACE_OPERAND: { - uint32_t framesize = (uint32_t)CURRENT_OPERAND(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -5692,7 +5692,7 @@ } case _EXIT_TRACE: { - PyObject *exit_p = (PyObject *)CURRENT_OPERAND(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; @@ -5757,7 +5757,7 @@ case _LOAD_CONST_INLINE: { _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -5767,7 +5767,7 @@ case _LOAD_CONST_INLINE_BORROW: { _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectImmortal(ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -5779,7 +5779,7 @@ _PyStackRef pop; _PyStackRef value; pop = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); PyStackRef_CLOSE(pop); value = PyStackRef_FromPyObjectImmortal(ptr); stack_pointer[-1] = value; @@ -5789,7 +5789,7 @@ case _LOAD_CONST_INLINE_WITH_NULL: { _PyStackRef value; _PyStackRef null; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectNew(ptr); null = PyStackRef_NULL; stack_pointer[0] = value; @@ -5802,7 +5802,7 @@ case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { _PyStackRef value; _PyStackRef null; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectImmortal(ptr); null = PyStackRef_NULL; stack_pointer[0] = value; @@ -5813,7 +5813,7 @@ } case _CHECK_FUNCTION: { - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); if (func->func_version != func_version) { @@ -5827,7 +5827,7 @@ _PyStackRef res; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); PyObject *res_o = entries[index].me_value; @@ -5849,7 +5849,7 @@ _PyStackRef res; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); PyObject *res_o = entries[index].me_value; @@ -5878,7 +5878,7 @@ } case _DYNAMIC_EXIT: { - PyObject *exit_p = (PyObject *)CURRENT_OPERAND(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); tstate->previous_executor = (PyObject *)current_executor; _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -5925,7 +5925,7 @@ } case _START_EXECUTOR: { - PyObject *executor = (PyObject *)CURRENT_OPERAND(); + PyObject *executor = (PyObject *)CURRENT_OPERAND0(); Py_DECREF(tstate->previous_executor); tstate->previous_executor = NULL; #ifndef _Py_JIT @@ -5951,7 +5951,7 @@ } case _CHECK_VALIDITY_AND_SET_IP: { - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5967,7 +5967,7 @@ case _ERROR_POP_N: { oparg = CURRENT_OPARG(); - uint32_t target = (uint32_t)CURRENT_OPERAND(); + uint32_t target = (uint32_t)CURRENT_OPERAND0(); frame->instr_ptr = ((_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive) + target; stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 632cbc7790a4d8..7d0a038b69a702 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -192,7 +192,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND(); + next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); DEOPT_IF(!PyStackRef_Is(*target_local, left), BINARY_OP); diff --git a/Python/optimizer.c b/Python/optimizer.c index b876b6c2bd72fd..bc2ecc098b0e15 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -288,13 +288,13 @@ _PyUOpPrint(const _PyUOpInstruction *uop) printf(" (%d, target=%d, operand=%#" PRIx64, uop->oparg, uop->target, - (uint64_t)uop->operand); + (uint64_t)uop->operand0); break; case UOP_FORMAT_JUMP: printf(" (%d, jump_target=%d, operand=%#" PRIx64, uop->oparg, uop->jump_target, - (uint64_t)uop->operand); + (uint64_t)uop->operand0); break; default: printf(" (%d, Unknown format)", uop->oparg); @@ -340,7 +340,7 @@ uop_item(_PyExecutorObject *self, Py_ssize_t index) Py_DECREF(oname); return NULL; } - PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand); + PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand0); if (operand == NULL) { Py_DECREF(target); Py_DECREF(oparg); @@ -463,7 +463,7 @@ add_to_trace( trace[trace_length].format = UOP_FORMAT_TARGET; trace[trace_length].target = target; trace[trace_length].oparg = oparg; - trace[trace_length].operand = operand; + trace[trace_length].operand0 = operand; return trace_length + 1; } @@ -970,7 +970,7 @@ static void make_exit(_PyUOpInstruction *inst, int opcode, int target) { inst->opcode = opcode; inst->oparg = 0; - inst->operand = 0; + inst->operand0 = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; } @@ -1033,7 +1033,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_error_target = target; make_exit(&buffer[next_spare], _ERROR_POP_N, 0); buffer[next_spare].oparg = popped; - buffer[next_spare].operand = target; + buffer[next_spare].operand0 = target; next_spare++; } buffer[i].error_target = current_error; @@ -1150,7 +1150,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; assert(buffer[0].opcode == _START_EXECUTOR); - buffer[0].operand = (uint64_t)executor; + buffer[0].operand0 = (uint64_t)executor; for (int i = length-1; i >= 0; i--) { int opcode = buffer[i].opcode; dest--; @@ -1159,13 +1159,13 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil if (opcode == _EXIT_TRACE) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; - dest->operand = (uint64_t)exit; + dest->operand0 = (uint64_t)exit; next_exit--; } if (opcode == _DYNAMIC_EXIT) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = 0; - dest->operand = (uint64_t)exit; + dest->operand0 = (uint64_t)exit; next_exit--; } } @@ -1312,7 +1312,7 @@ _PyOptimizer_NewUOpOptimizer(void) static void counter_dealloc(_PyExecutorObject *self) { /* The optimizer is the operand of the second uop. */ - PyObject *opt = (PyObject *)self->trace[1].operand; + PyObject *opt = (PyObject *)self->trace[1].operand0; Py_DECREF(opt); uop_dealloc(self); } @@ -1352,7 +1352,7 @@ counter_optimize( _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; _PyUOpInstruction buffer[4] = { { .opcode = _START_EXECUTOR, .jump_target = 3, .format=UOP_FORMAT_JUMP }, - { .opcode = _LOAD_CONST_INLINE, .operand = (uintptr_t)self }, + { .opcode = _LOAD_CONST_INLINE, .operand0 = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 25166bc2dc5c02..a4a0472b64e57c 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -100,11 +100,11 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) PyDictObject *dict = (PyDictObject *)obj; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - assert(inst->operand <= UINT16_MAX); - if ((int)inst->operand >= dict->ma_keys->dk_nentries) { + assert(inst->operand0 <= UINT16_MAX); + if ((int)inst->operand0 >= dict->ma_keys->dk_nentries) { return NULL; } - PyObject *res = entries[inst->operand].me_value; + PyObject *res = entries[inst->operand0].me_value; if (res == NULL) { return NULL; } @@ -114,7 +114,7 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) else { inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; } - inst->operand = (uint64_t)res; + inst->operand0 = (uint64_t)res; return res; } @@ -125,7 +125,7 @@ incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) return 1; } PyDictObject *dict = (PyDictObject *)obj; - if (dict->ma_keys->dk_version != inst->operand) { + if (dict->ma_keys->dk_version != inst->operand0) { return 1; } return 0; @@ -215,7 +215,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, } else { buffer[pc].opcode = _CHECK_FUNCTION; - buffer[pc].operand = function_version; + buffer[pc].operand0 = function_version; function_checked |= 1; } // We're no longer pushing the builtins keys; rewrite the @@ -248,7 +248,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, } else { buffer[pc].opcode = _CHECK_FUNCTION; - buffer[pc].operand = function_version; + buffer[pc].operand0 = function_version; function_checked |= 1; } if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS) { @@ -273,7 +273,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, builtins_watched <<= 1; globals_watched <<= 1; function_checked <<= 1; - uint64_t operand = buffer[pc].operand; + uint64_t operand = buffer[pc].operand0; if (operand == 0 || (operand & 1)) { // It's either a code object or NULL, so bail return 1; @@ -301,7 +301,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, builtins_watched >>= 1; globals_watched >>= 1; function_checked >>= 1; - uint64_t operand = buffer[pc].operand; + uint64_t operand = buffer[pc].operand0; if (operand == 0 || (operand & 1)) { // It's either a code object or NULL, so bail return 1; @@ -317,7 +317,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, break; } case _CHECK_FUNCTION_EXACT_ARGS: - prechecked_function_version = (uint32_t)buffer[pc].operand; + prechecked_function_version = (uint32_t)buffer[pc].operand0; break; default: if (is_terminator(inst)) { @@ -343,7 +343,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define REPLACE_OP(INST, OP, ARG, OPERAND) \ INST->opcode = OP; \ INST->oparg = ARG; \ - INST->operand = OPERAND; + INST->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null @@ -409,7 +409,7 @@ get_code(_PyUOpInstruction *op) { assert(op->opcode == _PUSH_FRAME || op->opcode == _RETURN_VALUE || op->opcode == _RETURN_GENERATOR); PyCodeObject *co = NULL; - uint64_t operand = op->operand; + uint64_t operand = op->operand0; if (operand == 0) { return NULL; } @@ -429,7 +429,7 @@ static PyCodeObject * get_code_with_logging(_PyUOpInstruction *op) { PyCodeObject *co = NULL; - uint64_t push_operand = op->operand; + uint64_t push_operand = op->operand0; if (push_operand & 1) { co = (PyCodeObject *)(push_operand & ~1); DPRINTF(3, "code=%p ", co); @@ -534,7 +534,7 @@ optimize_uops( assert(max_space <= INT_MAX); assert(max_space <= INT32_MAX); first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; - first_valid_check_stack->operand = max_space; + first_valid_check_stack->operand0 = max_space; } return trace_len; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d8f489cf6766ae..c1435b2dc4b2dc 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -346,7 +346,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyUnicode_Type); } // _STORE_FAST: - GETLOCAL(this_instr->operand) = res; + GETLOCAL(this_instr->operand0) = res; } op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _Py_UOpsAbstractFrame *)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a7abe553ac3335..4922b4cd0dc550 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -523,7 +523,7 @@ res = sym_new_type(ctx, &PyUnicode_Type); } // _STORE_FAST: - GETLOCAL(this_instr->operand) = res; + GETLOCAL(this_instr->operand0) = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -886,7 +886,7 @@ case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { _Py_UopsSymbol *globals_keys; - uint16_t version = (uint16_t)this_instr->operand; + uint16_t version = (uint16_t)this_instr->operand0; globals_keys = sym_new_unknown(ctx); (void)version; stack_pointer[0] = globals_keys; @@ -897,7 +897,7 @@ case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { _Py_UopsSymbol *builtins_keys; - uint16_t version = (uint16_t)this_instr->operand; + uint16_t version = (uint16_t)this_instr->operand0; builtins_keys = sym_new_unknown(ctx); (void)version; stack_pointer[0] = builtins_keys; @@ -1088,7 +1088,7 @@ case _GUARD_TYPE_VERSION: { _Py_UopsSymbol *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)this_instr->operand; + uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); if (sym_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1120,7 +1120,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)this_instr->operand; + uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)offset; @@ -1135,7 +1135,7 @@ case _CHECK_ATTR_MODULE: { _Py_UopsSymbol *owner; owner = stack_pointer[-1]; - uint32_t dict_version = (uint32_t)this_instr->operand; + uint32_t dict_version = (uint32_t)this_instr->operand0; (void)dict_version; if (sym_is_const(owner)) { PyObject *cnst = sym_get_const(owner); @@ -1158,7 +1158,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; + uint16_t index = (uint16_t)this_instr->operand0; (void)index; null = sym_new_null(ctx); attr = NULL; @@ -1200,7 +1200,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)this_instr->operand; + uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)hint; @@ -1217,7 +1217,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; + uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)index; @@ -1238,7 +1238,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)descr; @@ -1254,7 +1254,7 @@ _Py_UopsSymbol *owner; _Py_UOpsAbstractFrame *new_frame; owner = stack_pointer[-1]; - PyObject *fget = (PyObject *)this_instr->operand; + PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; (void)owner; new_frame = NULL; @@ -1637,7 +1637,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; attr = sym_new_not_null(ctx); self = owner; @@ -1653,7 +1653,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; attr = sym_new_not_null(ctx); self = owner; @@ -1687,7 +1687,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; attr = sym_new_not_null(ctx); self = owner; @@ -1751,7 +1751,7 @@ _Py_UopsSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)this_instr->operand; + uint32_t func_version = (uint32_t)this_instr->operand0; (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); @@ -1963,7 +1963,7 @@ null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; args = &stack_pointer[-oparg]; - uint32_t type_version = (uint32_t)this_instr->operand; + uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; (void)callable; (void)null; @@ -2423,7 +2423,7 @@ } case _CHECK_STACK_SPACE_OPERAND: { - uint32_t framesize = (uint32_t)this_instr->operand; + uint32_t framesize = (uint32_t)this_instr->operand0; (void)framesize; /* We should never see _CHECK_STACK_SPACE_OPERANDs. * They are only created at the end of this pass. */ @@ -2436,7 +2436,7 @@ } case _EXIT_TRACE: { - PyObject *exit_p = (PyObject *)this_instr->operand; + PyObject *exit_p = (PyObject *)this_instr->operand0; (void)exit_p; ctx->done = true; break; @@ -2448,7 +2448,7 @@ case _LOAD_CONST_INLINE: { _Py_UopsSymbol *value; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -2458,7 +2458,7 @@ case _LOAD_CONST_INLINE_BORROW: { _Py_UopsSymbol *value; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -2476,7 +2476,7 @@ case _LOAD_CONST_INLINE_WITH_NULL: { _Py_UopsSymbol *value; _Py_UopsSymbol *null; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); stack_pointer[0] = value; @@ -2489,7 +2489,7 @@ case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { _Py_UopsSymbol *value; _Py_UopsSymbol *null; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); stack_pointer[0] = value; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 7a1dfe1b85bf1a..d08b621aed552b 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -143,7 +143,7 @@ def write_uop( else: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" - out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") + out.emit(f"{type}{cache.name} = ({cast})this_instr->operand0;\n") if override: emitter = OptimizerEmitter(out) # No reference management of inputs needed. diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index a1debd62466c87..dd16a1a7eb28b5 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -188,10 +188,7 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: else: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" - if idx == 0: - emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") - else: - emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n") + emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n") storage = emitter.emit_tokens(uop, storage, None) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) from None diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index f2831d3623f46d..594e197ef2dcf6 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -30,10 +30,10 @@ class HoleValue(enum.Enum): # The current uop's oparg (exposed as _JIT_OPARG): OPARG = enum.auto() # The current uop's operand on 64-bit platforms (exposed as _JIT_OPERAND): - OPERAND = enum.auto() + OPERAND0 = enum.auto() # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): - OPERAND_HI = enum.auto() - OPERAND_LO = enum.auto() + OPERAND0_HI = enum.auto() + OPERAND0_LO = enum.auto() # The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND): OPERAND1 = enum.auto() # The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): @@ -104,9 +104,9 @@ class HoleValue(enum.Enum): # These should all have been turned into DATA values by process_relocations: # HoleValue.GOT: "", HoleValue.OPARG: "instruction->oparg", - HoleValue.OPERAND: "instruction->operand", - HoleValue.OPERAND_HI: "(instruction->operand >> 32)", - HoleValue.OPERAND_LO: "(instruction->operand & UINT32_MAX)", + HoleValue.OPERAND0: "instruction->operand0", + HoleValue.OPERAND0_HI: "(instruction->operand0 >> 32)", + HoleValue.OPERAND0_LO: "(instruction->operand0 & UINT32_MAX)", HoleValue.OPERAND1: "instruction->operand1", HoleValue.OPERAND1_HI: "(instruction->operand1 >> 32)", HoleValue.OPERAND1_LO: "(instruction->operand1 & UINT32_MAX)", diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 1a41d883cbf729..1253585014ae8e 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -26,8 +26,8 @@ #undef CURRENT_OPARG #define CURRENT_OPARG() (_oparg) -#undef CURRENT_OPERAND -#define CURRENT_OPERAND() (_operand) +#undef CURRENT_OPERAND0 +#define CURRENT_OPERAND0() (_operand0) #undef CURRENT_OPERAND1 #define CURRENT_OPERAND1() (_operand1) @@ -102,12 +102,12 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) #if SIZEOF_VOID_P == 8 - PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) + PATCH_VALUE(uint64_t, _operand0, _JIT_OPERAND0) #else assert(SIZEOF_VOID_P == 4); - PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI) - PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO) - uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo; + PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI) + PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO) + uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; #endif #if SIZEOF_VOID_P == 8 PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1) From 1201f7c7664e5154a03096bb48f3724732b052fd Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:09:57 +0800 Subject: [PATCH 07/10] Use sym_matches_type PyFunction_Type --- Python/ceval_macros.h | 2 +- Python/optimizer_bytecodes.c | 3 +-- Python/optimizer_cases.c.h | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c36c470806c664..074ae5ddfea1a8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -409,7 +409,7 @@ do { \ #define CURRENT_OPARG() (next_uop[-1].oparg) -#define CURRENT_OPERAND() (next_uop[-1].operand) +#define CURRENT_OPERAND0() (next_uop[-1].operand0) #define CURRENT_OPERAND1() (next_uop[-1].operand1) #define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index c1435b2dc4b2dc..200bec6b7743e3 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -600,13 +600,12 @@ dummy_func(void) { } op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - assert((this_instr-1)->opcode == _CHECK_FUNCTION_VERSION || (this_instr-1)->opcode == _CHECK_FUNCTION_VERSION_INLINE); + assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { - // Note: this is only valid if CHECK_FUNCTION_VERSION precedes this instruction. REPLACE_OP(this_instr, _NOP, 0 ,0); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4922b4cd0dc550..b8fa0cad7db8a2 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1830,13 +1830,12 @@ _Py_UopsSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert((this_instr-1)->opcode == _CHECK_FUNCTION_VERSION || (this_instr-1)->opcode == _CHECK_FUNCTION_VERSION_INLINE); + assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { - // Note: this is only valid if CHECK_FUNCTION_VERSION precedes this instruction. REPLACE_OP(this_instr, _NOP, 0 ,0); } } From 555d18098879eecdfd6c5d114e88b6a441581426 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:14:17 +0800 Subject: [PATCH 08/10] Update pycore_uop_ids.h --- Include/internal/pycore_uop_ids.h | 124 +++++++++++++++--------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d6460b2e8c06e8..fab4ce6a25b347 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -194,107 +194,107 @@ extern "C" { #define _LOAD_ATTR_SLOT_1 424 #define _LOAD_ATTR_WITH_HINT 425 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 425 +#define _LOAD_BYTECODE 426 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 426 -#define _LOAD_CONST_INLINE_BORROW 427 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 428 -#define _LOAD_CONST_INLINE_WITH_NULL 429 +#define _LOAD_CONST_INLINE 427 +#define _LOAD_CONST_INLINE_BORROW 428 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 429 +#define _LOAD_CONST_INLINE_WITH_NULL 430 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 430 -#define _LOAD_FAST_0 431 -#define _LOAD_FAST_1 432 -#define _LOAD_FAST_2 433 -#define _LOAD_FAST_3 434 -#define _LOAD_FAST_4 435 -#define _LOAD_FAST_5 436 -#define _LOAD_FAST_6 437 -#define _LOAD_FAST_7 438 +#define _LOAD_FAST 431 +#define _LOAD_FAST_0 432 +#define _LOAD_FAST_1 433 +#define _LOAD_FAST_2 434 +#define _LOAD_FAST_3 435 +#define _LOAD_FAST_4 436 +#define _LOAD_FAST_5 437 +#define _LOAD_FAST_6 438 +#define _LOAD_FAST_7 439 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 439 -#define _LOAD_GLOBAL_BUILTINS 440 -#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 441 -#define _LOAD_GLOBAL_MODULE 442 -#define _LOAD_GLOBAL_MODULE_FROM_KEYS 443 +#define _LOAD_GLOBAL 440 +#define _LOAD_GLOBAL_BUILTINS 441 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 442 +#define _LOAD_GLOBAL_MODULE 443 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 444 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 444 -#define _LOAD_SMALL_INT_0 445 -#define _LOAD_SMALL_INT_1 446 -#define _LOAD_SMALL_INT_2 447 -#define _LOAD_SMALL_INT_3 448 +#define _LOAD_SMALL_INT 445 +#define _LOAD_SMALL_INT_0 446 +#define _LOAD_SMALL_INT_1 447 +#define _LOAD_SMALL_INT_2 448 +#define _LOAD_SMALL_INT_3 449 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 449 +#define _MAKE_CALLARGS_A_TUPLE 450 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 450 +#define _MAKE_WARM 451 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 451 -#define _MAYBE_EXPAND_METHOD_KW 452 -#define _MONITOR_CALL 453 -#define _MONITOR_JUMP_BACKWARD 454 -#define _MONITOR_RESUME 455 +#define _MAYBE_EXPAND_METHOD 452 +#define _MAYBE_EXPAND_METHOD_KW 453 +#define _MONITOR_CALL 454 +#define _MONITOR_JUMP_BACKWARD 455 +#define _MONITOR_RESUME 456 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 456 -#define _POP_JUMP_IF_TRUE 457 +#define _POP_JUMP_IF_FALSE 457 +#define _POP_JUMP_IF_TRUE 458 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 458 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 459 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 459 +#define _PUSH_FRAME 460 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 460 -#define _PY_FRAME_KW 461 -#define _QUICKEN_RESUME 462 -#define _REPLACE_WITH_TRUE 463 +#define _PY_FRAME_GENERAL 461 +#define _PY_FRAME_KW 462 +#define _QUICKEN_RESUME 463 +#define _REPLACE_WITH_TRUE 464 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 464 -#define _SEND 465 -#define _SEND_GEN_FRAME 466 +#define _SAVE_RETURN_OFFSET 465 +#define _SEND 466 +#define _SEND_GEN_FRAME 467 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 467 -#define _STORE_ATTR 468 -#define _STORE_ATTR_INSTANCE_VALUE 469 -#define _STORE_ATTR_SLOT 470 -#define _STORE_ATTR_WITH_HINT 471 +#define _START_EXECUTOR 468 +#define _STORE_ATTR 469 +#define _STORE_ATTR_INSTANCE_VALUE 470 +#define _STORE_ATTR_SLOT 471 +#define _STORE_ATTR_WITH_HINT 472 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 472 -#define _STORE_FAST_0 473 -#define _STORE_FAST_1 474 -#define _STORE_FAST_2 475 -#define _STORE_FAST_3 476 -#define _STORE_FAST_4 477 -#define _STORE_FAST_5 478 -#define _STORE_FAST_6 479 -#define _STORE_FAST_7 480 +#define _STORE_FAST 473 +#define _STORE_FAST_0 474 +#define _STORE_FAST_1 475 +#define _STORE_FAST_2 476 +#define _STORE_FAST_3 477 +#define _STORE_FAST_4 478 +#define _STORE_FAST_5 479 +#define _STORE_FAST_6 480 +#define _STORE_FAST_7 481 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 481 -#define _STORE_SUBSCR 482 +#define _STORE_SLICE 482 +#define _STORE_SUBSCR 483 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 483 -#define _TO_BOOL 484 +#define _TIER2_RESUME_CHECK 484 +#define _TO_BOOL 485 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -304,13 +304,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 485 +#define _UNPACK_SEQUENCE 486 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 485 +#define MAX_UOP_ID 486 #ifdef __cplusplus } From 075a9a9531c43fc3b2583e1355715f2beaa4b84f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 8 Nov 2024 08:17:21 +0800 Subject: [PATCH 09/10] Address reviews --- Python/optimizer_bytecodes.c | 2 +- Python/optimizer_cases.c.h | 2 +- Tools/jit/template.c | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 200bec6b7743e3..42bdbd9ca8d0cd 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -601,7 +601,7 @@ dummy_func(void) { op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + if (sym_is_const(callable)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); PyCodeObject *co = (PyCodeObject *)func->func_code; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 495a44de92bba1..f77a5aa35bdf82 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1833,7 +1833,7 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + if (sym_is_const(callable)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); PyCodeObject *co = (PyCodeObject *)func->func_code; diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 1253585014ae8e..95c90bda70f352 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -103,16 +103,13 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) #if SIZEOF_VOID_P == 8 PATCH_VALUE(uint64_t, _operand0, _JIT_OPERAND0) + PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1) #else assert(SIZEOF_VOID_P == 4); PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI) PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO) uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; -#endif -#if SIZEOF_VOID_P == 8 - PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1) -#else - assert(SIZEOF_VOID_P == 4); + PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI) PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO) uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; From 68251db8aa50f4c4d5c7d0a57d222e903d5a220d Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sat, 9 Nov 2024 10:34:43 +0800 Subject: [PATCH 10/10] Apply suggestions from code review Co-authored-by: Brandt Bucher --- Tools/jit/_stencils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 594e197ef2dcf6..61be8fd3bbdf55 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -29,14 +29,14 @@ class HoleValue(enum.Enum): GOT = enum.auto() # The current uop's oparg (exposed as _JIT_OPARG): OPARG = enum.auto() - # The current uop's operand on 64-bit platforms (exposed as _JIT_OPERAND): + # The current uop's operand0 on 64-bit platforms (exposed as _JIT_OPERAND0): OPERAND0 = enum.auto() - # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): + # The current uop's operand0 on 32-bit platforms (exposed as _JIT_OPERAND0_HI/LO): OPERAND0_HI = enum.auto() OPERAND0_LO = enum.auto() - # The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND): + # The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND1): OPERAND1 = enum.auto() - # The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): + # The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND1_HI/LO): OPERAND1_HI = enum.auto() OPERAND1_LO = enum.auto() # The current uop's target (exposed as _JIT_TARGET):