From c78b66496ae8d96dc5231998114b3e8cd82809c2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 15 Dec 2025 14:49:19 +0000 Subject: [PATCH] JIT: Fix crash due to incorrect caching on side exits when exiting jitted code. * Make sure that stack is in correct state at side exits with TOS cached values * Simplify choice of cached items for side exits --- Include/internal/pycore_uop_ids.h | 416 +++++------ Include/internal/pycore_uop_metadata.h | 162 ++--- Lib/test/test_capi/test_opt.py | 22 + ...-12-15-15-07-40.gh-issue-142718.zjiGjS.rst | 1 + Python/executor_cases.c.h | 652 +++++++++++++++++- Tools/cases_generator/analyzer.py | 8 - Tools/cases_generator/tier2_generator.py | 48 +- 7 files changed, 979 insertions(+), 330 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-15-15-07-40.gh-issue-142718.zjiGjS.rst diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 64e51bd2b8bb58..f91838814f6d8a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -903,212 +903,216 @@ extern "C" { #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1096 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1097 #define _LOAD_FAST_CHECK_r01 1098 -#define _LOAD_FAST_LOAD_FAST_r02 1099 -#define _LOAD_FAST_LOAD_FAST_r13 1100 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1101 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1102 -#define _LOAD_GLOBAL_r00 1103 -#define _LOAD_GLOBAL_BUILTINS_r01 1104 -#define _LOAD_GLOBAL_MODULE_r01 1105 -#define _LOAD_LOCALS_r01 1106 -#define _LOAD_NAME_r01 1107 -#define _LOAD_SMALL_INT_r01 1108 -#define _LOAD_SMALL_INT_r12 1109 -#define _LOAD_SMALL_INT_r23 1110 -#define _LOAD_SMALL_INT_0_r01 1111 -#define _LOAD_SMALL_INT_0_r12 1112 -#define _LOAD_SMALL_INT_0_r23 1113 -#define _LOAD_SMALL_INT_1_r01 1114 -#define _LOAD_SMALL_INT_1_r12 1115 -#define _LOAD_SMALL_INT_1_r23 1116 -#define _LOAD_SMALL_INT_2_r01 1117 -#define _LOAD_SMALL_INT_2_r12 1118 -#define _LOAD_SMALL_INT_2_r23 1119 -#define _LOAD_SMALL_INT_3_r01 1120 -#define _LOAD_SMALL_INT_3_r12 1121 -#define _LOAD_SMALL_INT_3_r23 1122 -#define _LOAD_SPECIAL_r00 1123 -#define _LOAD_SUPER_ATTR_ATTR_r31 1124 -#define _LOAD_SUPER_ATTR_METHOD_r32 1125 -#define _MAKE_CALLARGS_A_TUPLE_r33 1126 -#define _MAKE_CELL_r00 1127 -#define _MAKE_FUNCTION_r11 1128 -#define _MAKE_WARM_r00 1129 -#define _MAKE_WARM_r11 1130 -#define _MAKE_WARM_r22 1131 -#define _MAKE_WARM_r33 1132 -#define _MAP_ADD_r20 1133 -#define _MATCH_CLASS_r31 1134 -#define _MATCH_KEYS_r23 1135 -#define _MATCH_MAPPING_r02 1136 -#define _MATCH_MAPPING_r12 1137 -#define _MATCH_MAPPING_r23 1138 -#define _MATCH_SEQUENCE_r02 1139 -#define _MATCH_SEQUENCE_r12 1140 -#define _MATCH_SEQUENCE_r23 1141 -#define _MAYBE_EXPAND_METHOD_r00 1142 -#define _MAYBE_EXPAND_METHOD_KW_r11 1143 -#define _MONITOR_CALL_r00 1144 -#define _MONITOR_CALL_KW_r11 1145 -#define _MONITOR_JUMP_BACKWARD_r00 1146 -#define _MONITOR_JUMP_BACKWARD_r11 1147 -#define _MONITOR_JUMP_BACKWARD_r22 1148 -#define _MONITOR_JUMP_BACKWARD_r33 1149 -#define _MONITOR_RESUME_r00 1150 -#define _NOP_r00 1151 -#define _NOP_r11 1152 -#define _NOP_r22 1153 -#define _NOP_r33 1154 -#define _POP_CALL_r20 1155 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1156 -#define _POP_CALL_ONE_r30 1157 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1158 -#define _POP_CALL_TWO_r30 1159 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1160 -#define _POP_EXCEPT_r10 1161 -#define _POP_ITER_r20 1162 -#define _POP_JUMP_IF_FALSE_r00 1163 -#define _POP_JUMP_IF_FALSE_r10 1164 -#define _POP_JUMP_IF_FALSE_r21 1165 -#define _POP_JUMP_IF_FALSE_r32 1166 -#define _POP_JUMP_IF_TRUE_r00 1167 -#define _POP_JUMP_IF_TRUE_r10 1168 -#define _POP_JUMP_IF_TRUE_r21 1169 -#define _POP_JUMP_IF_TRUE_r32 1170 -#define _POP_TOP_r10 1171 -#define _POP_TOP_FLOAT_r00 1172 -#define _POP_TOP_FLOAT_r10 1173 -#define _POP_TOP_FLOAT_r21 1174 -#define _POP_TOP_FLOAT_r32 1175 -#define _POP_TOP_INT_r00 1176 -#define _POP_TOP_INT_r10 1177 -#define _POP_TOP_INT_r21 1178 -#define _POP_TOP_INT_r32 1179 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1180 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1181 -#define _POP_TOP_NOP_r00 1182 -#define _POP_TOP_NOP_r10 1183 -#define _POP_TOP_NOP_r21 1184 -#define _POP_TOP_NOP_r32 1185 -#define _POP_TOP_UNICODE_r00 1186 -#define _POP_TOP_UNICODE_r10 1187 -#define _POP_TOP_UNICODE_r21 1188 -#define _POP_TOP_UNICODE_r32 1189 -#define _POP_TWO_r20 1190 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1191 -#define _PUSH_EXC_INFO_r02 1192 -#define _PUSH_EXC_INFO_r12 1193 -#define _PUSH_EXC_INFO_r23 1194 -#define _PUSH_FRAME_r10 1195 -#define _PUSH_NULL_r01 1196 -#define _PUSH_NULL_r12 1197 -#define _PUSH_NULL_r23 1198 -#define _PUSH_NULL_CONDITIONAL_r00 1199 -#define _PY_FRAME_GENERAL_r01 1200 -#define _PY_FRAME_KW_r11 1201 -#define _QUICKEN_RESUME_r00 1202 -#define _QUICKEN_RESUME_r11 1203 -#define _QUICKEN_RESUME_r22 1204 -#define _QUICKEN_RESUME_r33 1205 -#define _REPLACE_WITH_TRUE_r11 1206 -#define _RESUME_CHECK_r00 1207 -#define _RESUME_CHECK_r11 1208 -#define _RESUME_CHECK_r22 1209 -#define _RESUME_CHECK_r33 1210 -#define _RETURN_GENERATOR_r01 1211 -#define _RETURN_VALUE_r11 1212 -#define _SAVE_RETURN_OFFSET_r00 1213 -#define _SAVE_RETURN_OFFSET_r11 1214 -#define _SAVE_RETURN_OFFSET_r22 1215 -#define _SAVE_RETURN_OFFSET_r33 1216 -#define _SEND_r22 1217 -#define _SEND_GEN_FRAME_r22 1218 -#define _SETUP_ANNOTATIONS_r00 1219 -#define _SET_ADD_r10 1220 -#define _SET_FUNCTION_ATTRIBUTE_r01 1221 -#define _SET_FUNCTION_ATTRIBUTE_r11 1222 -#define _SET_FUNCTION_ATTRIBUTE_r21 1223 -#define _SET_FUNCTION_ATTRIBUTE_r32 1224 -#define _SET_IP_r00 1225 -#define _SET_IP_r11 1226 -#define _SET_IP_r22 1227 -#define _SET_IP_r33 1228 -#define _SET_UPDATE_r10 1229 -#define _SPILL_OR_RELOAD_r01 1230 -#define _SPILL_OR_RELOAD_r02 1231 -#define _SPILL_OR_RELOAD_r03 1232 -#define _SPILL_OR_RELOAD_r10 1233 -#define _SPILL_OR_RELOAD_r12 1234 -#define _SPILL_OR_RELOAD_r13 1235 -#define _SPILL_OR_RELOAD_r20 1236 -#define _SPILL_OR_RELOAD_r21 1237 -#define _SPILL_OR_RELOAD_r23 1238 -#define _SPILL_OR_RELOAD_r30 1239 -#define _SPILL_OR_RELOAD_r31 1240 -#define _SPILL_OR_RELOAD_r32 1241 -#define _START_EXECUTOR_r00 1242 -#define _STORE_ATTR_r20 1243 -#define _STORE_ATTR_INSTANCE_VALUE_r20 1244 -#define _STORE_ATTR_SLOT_r20 1245 -#define _STORE_ATTR_WITH_HINT_r20 1246 -#define _STORE_DEREF_r10 1247 -#define _STORE_FAST_r10 1248 -#define _STORE_FAST_0_r10 1249 -#define _STORE_FAST_1_r10 1250 -#define _STORE_FAST_2_r10 1251 -#define _STORE_FAST_3_r10 1252 -#define _STORE_FAST_4_r10 1253 -#define _STORE_FAST_5_r10 1254 -#define _STORE_FAST_6_r10 1255 -#define _STORE_FAST_7_r10 1256 -#define _STORE_FAST_LOAD_FAST_r11 1257 -#define _STORE_FAST_STORE_FAST_r20 1258 -#define _STORE_GLOBAL_r10 1259 -#define _STORE_NAME_r10 1260 -#define _STORE_SLICE_r30 1261 -#define _STORE_SUBSCR_r30 1262 -#define _STORE_SUBSCR_DICT_r30 1263 -#define _STORE_SUBSCR_LIST_INT_r32 1264 -#define _SWAP_r11 1265 -#define _SWAP_2_r02 1266 -#define _SWAP_2_r12 1267 -#define _SWAP_2_r22 1268 -#define _SWAP_2_r33 1269 -#define _SWAP_3_r03 1270 -#define _SWAP_3_r13 1271 -#define _SWAP_3_r23 1272 -#define _SWAP_3_r33 1273 -#define _TIER2_RESUME_CHECK_r00 1274 -#define _TIER2_RESUME_CHECK_r11 1275 -#define _TIER2_RESUME_CHECK_r22 1276 -#define _TIER2_RESUME_CHECK_r33 1277 -#define _TO_BOOL_r11 1278 -#define _TO_BOOL_BOOL_r01 1279 -#define _TO_BOOL_BOOL_r11 1280 -#define _TO_BOOL_BOOL_r22 1281 -#define _TO_BOOL_BOOL_r33 1282 -#define _TO_BOOL_INT_r11 1283 -#define _TO_BOOL_LIST_r11 1284 -#define _TO_BOOL_NONE_r01 1285 -#define _TO_BOOL_NONE_r11 1286 -#define _TO_BOOL_NONE_r22 1287 -#define _TO_BOOL_NONE_r33 1288 -#define _TO_BOOL_STR_r11 1289 -#define _TRACE_RECORD_r00 1290 -#define _UNARY_INVERT_r11 1291 -#define _UNARY_NEGATIVE_r11 1292 -#define _UNARY_NOT_r01 1293 -#define _UNARY_NOT_r11 1294 -#define _UNARY_NOT_r22 1295 -#define _UNARY_NOT_r33 1296 -#define _UNPACK_EX_r10 1297 -#define _UNPACK_SEQUENCE_r10 1298 -#define _UNPACK_SEQUENCE_LIST_r10 1299 -#define _UNPACK_SEQUENCE_TUPLE_r10 1300 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1301 -#define _WITH_EXCEPT_START_r33 1302 -#define _YIELD_VALUE_r11 1303 -#define MAX_UOP_REGS_ID 1303 +#define _LOAD_FAST_CHECK_r12 1099 +#define _LOAD_FAST_CHECK_r23 1100 +#define _LOAD_FAST_LOAD_FAST_r02 1101 +#define _LOAD_FAST_LOAD_FAST_r13 1102 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1103 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1104 +#define _LOAD_GLOBAL_r00 1105 +#define _LOAD_GLOBAL_BUILTINS_r01 1106 +#define _LOAD_GLOBAL_MODULE_r01 1107 +#define _LOAD_LOCALS_r01 1108 +#define _LOAD_LOCALS_r12 1109 +#define _LOAD_LOCALS_r23 1110 +#define _LOAD_NAME_r01 1111 +#define _LOAD_SMALL_INT_r01 1112 +#define _LOAD_SMALL_INT_r12 1113 +#define _LOAD_SMALL_INT_r23 1114 +#define _LOAD_SMALL_INT_0_r01 1115 +#define _LOAD_SMALL_INT_0_r12 1116 +#define _LOAD_SMALL_INT_0_r23 1117 +#define _LOAD_SMALL_INT_1_r01 1118 +#define _LOAD_SMALL_INT_1_r12 1119 +#define _LOAD_SMALL_INT_1_r23 1120 +#define _LOAD_SMALL_INT_2_r01 1121 +#define _LOAD_SMALL_INT_2_r12 1122 +#define _LOAD_SMALL_INT_2_r23 1123 +#define _LOAD_SMALL_INT_3_r01 1124 +#define _LOAD_SMALL_INT_3_r12 1125 +#define _LOAD_SMALL_INT_3_r23 1126 +#define _LOAD_SPECIAL_r00 1127 +#define _LOAD_SUPER_ATTR_ATTR_r31 1128 +#define _LOAD_SUPER_ATTR_METHOD_r32 1129 +#define _MAKE_CALLARGS_A_TUPLE_r33 1130 +#define _MAKE_CELL_r00 1131 +#define _MAKE_FUNCTION_r11 1132 +#define _MAKE_WARM_r00 1133 +#define _MAKE_WARM_r11 1134 +#define _MAKE_WARM_r22 1135 +#define _MAKE_WARM_r33 1136 +#define _MAP_ADD_r20 1137 +#define _MATCH_CLASS_r31 1138 +#define _MATCH_KEYS_r23 1139 +#define _MATCH_MAPPING_r02 1140 +#define _MATCH_MAPPING_r12 1141 +#define _MATCH_MAPPING_r23 1142 +#define _MATCH_SEQUENCE_r02 1143 +#define _MATCH_SEQUENCE_r12 1144 +#define _MATCH_SEQUENCE_r23 1145 +#define _MAYBE_EXPAND_METHOD_r00 1146 +#define _MAYBE_EXPAND_METHOD_KW_r11 1147 +#define _MONITOR_CALL_r00 1148 +#define _MONITOR_CALL_KW_r11 1149 +#define _MONITOR_JUMP_BACKWARD_r00 1150 +#define _MONITOR_JUMP_BACKWARD_r11 1151 +#define _MONITOR_JUMP_BACKWARD_r22 1152 +#define _MONITOR_JUMP_BACKWARD_r33 1153 +#define _MONITOR_RESUME_r00 1154 +#define _NOP_r00 1155 +#define _NOP_r11 1156 +#define _NOP_r22 1157 +#define _NOP_r33 1158 +#define _POP_CALL_r20 1159 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1160 +#define _POP_CALL_ONE_r30 1161 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1162 +#define _POP_CALL_TWO_r30 1163 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1164 +#define _POP_EXCEPT_r10 1165 +#define _POP_ITER_r20 1166 +#define _POP_JUMP_IF_FALSE_r00 1167 +#define _POP_JUMP_IF_FALSE_r10 1168 +#define _POP_JUMP_IF_FALSE_r21 1169 +#define _POP_JUMP_IF_FALSE_r32 1170 +#define _POP_JUMP_IF_TRUE_r00 1171 +#define _POP_JUMP_IF_TRUE_r10 1172 +#define _POP_JUMP_IF_TRUE_r21 1173 +#define _POP_JUMP_IF_TRUE_r32 1174 +#define _POP_TOP_r10 1175 +#define _POP_TOP_FLOAT_r00 1176 +#define _POP_TOP_FLOAT_r10 1177 +#define _POP_TOP_FLOAT_r21 1178 +#define _POP_TOP_FLOAT_r32 1179 +#define _POP_TOP_INT_r00 1180 +#define _POP_TOP_INT_r10 1181 +#define _POP_TOP_INT_r21 1182 +#define _POP_TOP_INT_r32 1183 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1184 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1185 +#define _POP_TOP_NOP_r00 1186 +#define _POP_TOP_NOP_r10 1187 +#define _POP_TOP_NOP_r21 1188 +#define _POP_TOP_NOP_r32 1189 +#define _POP_TOP_UNICODE_r00 1190 +#define _POP_TOP_UNICODE_r10 1191 +#define _POP_TOP_UNICODE_r21 1192 +#define _POP_TOP_UNICODE_r32 1193 +#define _POP_TWO_r20 1194 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1195 +#define _PUSH_EXC_INFO_r02 1196 +#define _PUSH_EXC_INFO_r12 1197 +#define _PUSH_EXC_INFO_r23 1198 +#define _PUSH_FRAME_r10 1199 +#define _PUSH_NULL_r01 1200 +#define _PUSH_NULL_r12 1201 +#define _PUSH_NULL_r23 1202 +#define _PUSH_NULL_CONDITIONAL_r00 1203 +#define _PY_FRAME_GENERAL_r01 1204 +#define _PY_FRAME_KW_r11 1205 +#define _QUICKEN_RESUME_r00 1206 +#define _QUICKEN_RESUME_r11 1207 +#define _QUICKEN_RESUME_r22 1208 +#define _QUICKEN_RESUME_r33 1209 +#define _REPLACE_WITH_TRUE_r11 1210 +#define _RESUME_CHECK_r00 1211 +#define _RESUME_CHECK_r11 1212 +#define _RESUME_CHECK_r22 1213 +#define _RESUME_CHECK_r33 1214 +#define _RETURN_GENERATOR_r01 1215 +#define _RETURN_VALUE_r11 1216 +#define _SAVE_RETURN_OFFSET_r00 1217 +#define _SAVE_RETURN_OFFSET_r11 1218 +#define _SAVE_RETURN_OFFSET_r22 1219 +#define _SAVE_RETURN_OFFSET_r33 1220 +#define _SEND_r22 1221 +#define _SEND_GEN_FRAME_r22 1222 +#define _SETUP_ANNOTATIONS_r00 1223 +#define _SET_ADD_r10 1224 +#define _SET_FUNCTION_ATTRIBUTE_r01 1225 +#define _SET_FUNCTION_ATTRIBUTE_r11 1226 +#define _SET_FUNCTION_ATTRIBUTE_r21 1227 +#define _SET_FUNCTION_ATTRIBUTE_r32 1228 +#define _SET_IP_r00 1229 +#define _SET_IP_r11 1230 +#define _SET_IP_r22 1231 +#define _SET_IP_r33 1232 +#define _SET_UPDATE_r10 1233 +#define _SPILL_OR_RELOAD_r01 1234 +#define _SPILL_OR_RELOAD_r02 1235 +#define _SPILL_OR_RELOAD_r03 1236 +#define _SPILL_OR_RELOAD_r10 1237 +#define _SPILL_OR_RELOAD_r12 1238 +#define _SPILL_OR_RELOAD_r13 1239 +#define _SPILL_OR_RELOAD_r20 1240 +#define _SPILL_OR_RELOAD_r21 1241 +#define _SPILL_OR_RELOAD_r23 1242 +#define _SPILL_OR_RELOAD_r30 1243 +#define _SPILL_OR_RELOAD_r31 1244 +#define _SPILL_OR_RELOAD_r32 1245 +#define _START_EXECUTOR_r00 1246 +#define _STORE_ATTR_r20 1247 +#define _STORE_ATTR_INSTANCE_VALUE_r20 1248 +#define _STORE_ATTR_SLOT_r20 1249 +#define _STORE_ATTR_WITH_HINT_r20 1250 +#define _STORE_DEREF_r10 1251 +#define _STORE_FAST_r10 1252 +#define _STORE_FAST_0_r10 1253 +#define _STORE_FAST_1_r10 1254 +#define _STORE_FAST_2_r10 1255 +#define _STORE_FAST_3_r10 1256 +#define _STORE_FAST_4_r10 1257 +#define _STORE_FAST_5_r10 1258 +#define _STORE_FAST_6_r10 1259 +#define _STORE_FAST_7_r10 1260 +#define _STORE_FAST_LOAD_FAST_r11 1261 +#define _STORE_FAST_STORE_FAST_r20 1262 +#define _STORE_GLOBAL_r10 1263 +#define _STORE_NAME_r10 1264 +#define _STORE_SLICE_r30 1265 +#define _STORE_SUBSCR_r30 1266 +#define _STORE_SUBSCR_DICT_r30 1267 +#define _STORE_SUBSCR_LIST_INT_r32 1268 +#define _SWAP_r11 1269 +#define _SWAP_2_r02 1270 +#define _SWAP_2_r12 1271 +#define _SWAP_2_r22 1272 +#define _SWAP_2_r33 1273 +#define _SWAP_3_r03 1274 +#define _SWAP_3_r13 1275 +#define _SWAP_3_r23 1276 +#define _SWAP_3_r33 1277 +#define _TIER2_RESUME_CHECK_r00 1278 +#define _TIER2_RESUME_CHECK_r11 1279 +#define _TIER2_RESUME_CHECK_r22 1280 +#define _TIER2_RESUME_CHECK_r33 1281 +#define _TO_BOOL_r11 1282 +#define _TO_BOOL_BOOL_r01 1283 +#define _TO_BOOL_BOOL_r11 1284 +#define _TO_BOOL_BOOL_r22 1285 +#define _TO_BOOL_BOOL_r33 1286 +#define _TO_BOOL_INT_r11 1287 +#define _TO_BOOL_LIST_r11 1288 +#define _TO_BOOL_NONE_r01 1289 +#define _TO_BOOL_NONE_r11 1290 +#define _TO_BOOL_NONE_r22 1291 +#define _TO_BOOL_NONE_r33 1292 +#define _TO_BOOL_STR_r11 1293 +#define _TRACE_RECORD_r00 1294 +#define _UNARY_INVERT_r11 1295 +#define _UNARY_NEGATIVE_r11 1296 +#define _UNARY_NOT_r01 1297 +#define _UNARY_NOT_r11 1298 +#define _UNARY_NOT_r22 1299 +#define _UNARY_NOT_r33 1300 +#define _UNPACK_EX_r10 1301 +#define _UNPACK_SEQUENCE_r10 1302 +#define _UNPACK_SEQUENCE_LIST_r10 1303 +#define _UNPACK_SEQUENCE_TUPLE_r10 1304 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1305 +#define _WITH_EXCEPT_START_r33 1306 +#define _YIELD_VALUE_r11 1307 +#define MAX_UOP_REGS_ID 1307 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 5fa375a8ce6b4a..3ce6fb3b820521 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -405,11 +405,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_LOAD_FAST_CHECK] = { - .best = { 0, 0, 0, 0 }, + .best = { 0, 1, 2, 2 }, .entries = { { 1, 0, _LOAD_FAST_CHECK_r01 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 2, 1, _LOAD_FAST_CHECK_r12 }, + { 3, 2, _LOAD_FAST_CHECK_r23 }, { -1, -1, -1 }, }, }, @@ -813,7 +813,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _UNARY_NEGATIVE_r11 }, + { 1, 1, _UNARY_NEGATIVE_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -831,7 +831,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _TO_BOOL_r11 }, + { 1, 1, _TO_BOOL_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -939,7 +939,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _UNARY_INVERT_r11 }, + { 1, 1, _UNARY_INVERT_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1093,7 +1093,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 0, _BINARY_OP_INPLACE_ADD_UNICODE_r20 }, + { 0, 2, _BINARY_OP_INPLACE_ADD_UNICODE_r20 }, { -1, -1, -1 }, }, }, @@ -1102,7 +1102,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 2, 0, _GUARD_BINARY_OP_EXTEND_r22 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_r22 }, { -1, -1, -1 }, }, }, @@ -1111,7 +1111,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _BINARY_OP_EXTEND_r21 }, + { 1, 2, _BINARY_OP_EXTEND_r21 }, { -1, -1, -1 }, }, }, @@ -1121,7 +1121,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _BINARY_SLICE_r31 }, + { 1, 3, _BINARY_SLICE_r31 }, }, }, [_STORE_SLICE] = { @@ -1130,7 +1130,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 0, _STORE_SLICE_r30 }, + { 0, 3, _STORE_SLICE_r30 }, }, }, [_BINARY_OP_SUBSCR_LIST_INT] = { @@ -1138,7 +1138,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _BINARY_OP_SUBSCR_LIST_INT_r21 }, + { 1, 2, _BINARY_OP_SUBSCR_LIST_INT_r21 }, { -1, -1, -1 }, }, }, @@ -1147,7 +1147,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _BINARY_OP_SUBSCR_LIST_SLICE_r21 }, + { 1, 2, _BINARY_OP_SUBSCR_LIST_SLICE_r21 }, { -1, -1, -1 }, }, }, @@ -1210,7 +1210,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _BINARY_OP_SUBSCR_DICT_r21 }, + { 1, 2, _BINARY_OP_SUBSCR_DICT_r21 }, { -1, -1, -1 }, }, }, @@ -1245,7 +1245,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _SET_ADD_r10 }, + { 0, 1, _SET_ADD_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1256,7 +1256,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 0, _STORE_SUBSCR_r30 }, + { 0, 3, _STORE_SUBSCR_r30 }, }, }, [_STORE_SUBSCR_LIST_INT] = { @@ -1274,7 +1274,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 0, _STORE_SUBSCR_DICT_r30 }, + { 0, 3, _STORE_SUBSCR_DICT_r30 }, }, }, [_DELETE_SUBSCR] = { @@ -1282,7 +1282,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 0, _DELETE_SUBSCR_r20 }, + { 0, 2, _DELETE_SUBSCR_r20 }, { -1, -1, -1 }, }, }, @@ -1290,7 +1290,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _CALL_INTRINSIC_1_r11 }, + { 1, 1, _CALL_INTRINSIC_1_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1300,7 +1300,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _CALL_INTRINSIC_2_r21 }, + { 1, 2, _CALL_INTRINSIC_2_r21 }, { -1, -1, -1 }, }, }, @@ -1308,7 +1308,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _RETURN_VALUE_r11 }, + { 1, 1, _RETURN_VALUE_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1317,7 +1317,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _GET_AITER_r11 }, + { 1, 1, _GET_AITER_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1326,7 +1326,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 2, 0, _GET_ANEXT_r12 }, + { 2, 1, _GET_ANEXT_r12 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1335,7 +1335,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _GET_AWAITABLE_r11 }, + { 1, 1, _GET_AWAITABLE_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1362,7 +1362,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _POP_EXCEPT_r10 }, + { 0, 1, _POP_EXCEPT_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1389,7 +1389,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _STORE_NAME_r10 }, + { 0, 1, _STORE_NAME_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1407,7 +1407,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _UNPACK_SEQUENCE_r10 }, + { 0, 1, _UNPACK_SEQUENCE_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1443,7 +1443,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _UNPACK_EX_r10 }, + { 0, 1, _UNPACK_EX_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1453,7 +1453,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 0, _STORE_ATTR_r20 }, + { 0, 2, _STORE_ATTR_r20 }, { -1, -1, -1 }, }, }, @@ -1461,7 +1461,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _DELETE_ATTR_r10 }, + { 0, 1, _DELETE_ATTR_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1470,7 +1470,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _STORE_GLOBAL_r10 }, + { 0, 1, _STORE_GLOBAL_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1485,11 +1485,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_LOAD_LOCALS] = { - .best = { 0, 0, 0, 0 }, + .best = { 0, 1, 2, 2 }, .entries = { { 1, 0, _LOAD_LOCALS_r01 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 2, 1, _LOAD_LOCALS_r12 }, + { 3, 2, _LOAD_LOCALS_r23 }, { -1, -1, -1 }, }, }, @@ -1578,7 +1578,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _LOAD_FROM_DICT_OR_DEREF_r11 }, + { 1, 1, _LOAD_FROM_DICT_OR_DEREF_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1596,7 +1596,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _STORE_DEREF_r10 }, + { 0, 1, _STORE_DEREF_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1633,7 +1633,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _BUILD_TEMPLATE_r21 }, + { 1, 2, _BUILD_TEMPLATE_r21 }, { -1, -1, -1 }, }, }, @@ -1659,7 +1659,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _LIST_EXTEND_r10 }, + { 0, 1, _LIST_EXTEND_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1668,7 +1668,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _SET_UPDATE_r10 }, + { 0, 1, _SET_UPDATE_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1704,7 +1704,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _DICT_UPDATE_r10 }, + { 0, 1, _DICT_UPDATE_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1713,7 +1713,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _DICT_MERGE_r10 }, + { 0, 1, _DICT_MERGE_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1723,7 +1723,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 0, _MAP_ADD_r20 }, + { 0, 2, _MAP_ADD_r20 }, { -1, -1, -1 }, }, }, @@ -1733,7 +1733,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _LOAD_SUPER_ATTR_ATTR_r31 }, + { 1, 3, _LOAD_SUPER_ATTR_ATTR_r31 }, }, }, [_LOAD_SUPER_ATTR_METHOD] = { @@ -1742,14 +1742,14 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 2, 0, _LOAD_SUPER_ATTR_METHOD_r32 }, + { 2, 3, _LOAD_SUPER_ATTR_METHOD_r32 }, }, }, [_LOAD_ATTR] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _LOAD_ATTR_r10 }, + { 0, 1, _LOAD_ATTR_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1867,7 +1867,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 0, _STORE_ATTR_WITH_HINT_r20 }, + { 0, 2, _STORE_ATTR_WITH_HINT_r20 }, { -1, -1, -1 }, }, }, @@ -1885,7 +1885,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _COMPARE_OP_r21 }, + { 1, 2, _COMPARE_OP_r21 }, { -1, -1, -1 }, }, }, @@ -1930,7 +1930,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _CONTAINS_OP_r21 }, + { 1, 2, _CONTAINS_OP_r21 }, { -1, -1, -1 }, }, }, @@ -1948,7 +1948,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _CONTAINS_OP_SET_r21 }, + { 1, 2, _CONTAINS_OP_SET_r21 }, { -1, -1, -1 }, }, }, @@ -1957,7 +1957,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _CONTAINS_OP_DICT_r21 }, + { 1, 2, _CONTAINS_OP_DICT_r21 }, { -1, -1, -1 }, }, }, @@ -1966,7 +1966,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 2, 0, _CHECK_EG_MATCH_r22 }, + { 2, 2, _CHECK_EG_MATCH_r22 }, { -1, -1, -1 }, }, }, @@ -1975,7 +1975,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 2, 0, _CHECK_EXC_MATCH_r22 }, + { 2, 2, _CHECK_EXC_MATCH_r22 }, { -1, -1, -1 }, }, }, @@ -1984,7 +1984,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _IMPORT_NAME_r21 }, + { 1, 2, _IMPORT_NAME_r21 }, { -1, -1, -1 }, }, }, @@ -1992,7 +1992,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 2, 0, _IMPORT_FROM_r12 }, + { 2, 1, _IMPORT_FROM_r12 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2010,7 +2010,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 2, 0, _GET_LEN_r12 }, + { 2, 1, _GET_LEN_r12 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2021,7 +2021,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _MATCH_CLASS_r31 }, + { 1, 3, _MATCH_CLASS_r31 }, }, }, [_MATCH_MAPPING] = { @@ -2047,7 +2047,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 3, 0, _MATCH_KEYS_r23 }, + { 3, 2, _MATCH_KEYS_r23 }, { -1, -1, -1 }, }, }, @@ -2055,7 +2055,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 2, 0, _GET_ITER_r12 }, + { 2, 1, _GET_ITER_r12 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2064,7 +2064,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _GET_YIELD_FROM_ITER_r11 }, + { 1, 1, _GET_YIELD_FROM_ITER_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2074,7 +2074,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 3, 0, _FOR_ITER_TIER_TWO_r23 }, + { 3, 2, _FOR_ITER_TIER_TWO_r23 }, { -1, -1, -1 }, }, }, @@ -2101,7 +2101,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 3, 0, _ITER_NEXT_LIST_TIER_TWO_r23 }, + { 3, 2, _ITER_NEXT_LIST_TIER_TWO_r23 }, { -1, -1, -1 }, }, }, @@ -2192,7 +2192,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 3, 0, _WITH_EXCEPT_START_r33 }, + { 3, 3, _WITH_EXCEPT_START_r33 }, }, }, [_PUSH_EXC_INFO] = { @@ -2288,7 +2288,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_PY_FRAME_GENERAL] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _PY_FRAME_GENERAL_r01 }, + { 1, 1, _PY_FRAME_GENERAL_r01 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -2525,7 +2525,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 2, 0, _CALL_STR_1_r32 }, + { 2, 3, _CALL_STR_1_r32 }, }, }, [_GUARD_CALLABLE_TUPLE_1] = { @@ -2543,7 +2543,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 2, 0, _CALL_TUPLE_1_r32 }, + { 2, 3, _CALL_TUPLE_1_r32 }, }, }, [_CHECK_AND_ALLOCATE_OBJECT] = { @@ -2558,7 +2558,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CREATE_INIT_FRAME] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CREATE_INIT_FRAME_r01 }, + { 1, 1, _CREATE_INIT_FRAME_r01 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -2568,7 +2568,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 0, _EXIT_INIT_CHECK_r10 }, + { 0, 1, _EXIT_INIT_CHECK_r10 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2624,7 +2624,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 3, 0, _CALL_LEN_r33 }, + { 3, 3, _CALL_LEN_r33 }, }, }, [_GUARD_CALLABLE_ISINSTANCE] = { @@ -2642,7 +2642,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _CALL_ISINSTANCE_r31 }, + { 1, 3, _CALL_ISINSTANCE_r31 }, }, }, [_GUARD_CALLABLE_LIST_APPEND] = { @@ -2712,7 +2712,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _PY_FRAME_KW_r11 }, + { 1, 1, _PY_FRAME_KW_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2757,7 +2757,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _CALL_KW_NON_PY_r11 }, + { 1, 1, _CALL_KW_NON_PY_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2768,14 +2768,14 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 3, 0, _MAKE_CALLARGS_A_TUPLE_r33 }, + { 3, 3, _MAKE_CALLARGS_A_TUPLE_r33 }, }, }, [_MAKE_FUNCTION] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _MAKE_FUNCTION_r11 }, + { 1, 1, _MAKE_FUNCTION_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2811,7 +2811,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _CONVERT_VALUE_r11 }, + { 1, 1, _CONVERT_VALUE_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2820,7 +2820,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 0, _FORMAT_SIMPLE_r11 }, + { 1, 1, _FORMAT_SIMPLE_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2830,7 +2830,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _FORMAT_WITH_SPEC_r21 }, + { 1, 2, _FORMAT_WITH_SPEC_r21 }, { -1, -1, -1 }, }, }, @@ -2875,7 +2875,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _BINARY_OP_r21 }, + { 1, 2, _BINARY_OP_r21 }, { -1, -1, -1 }, }, }, @@ -3253,6 +3253,8 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_RESUME_CHECK_r22] = _RESUME_CHECK, [_RESUME_CHECK_r33] = _RESUME_CHECK, [_LOAD_FAST_CHECK_r01] = _LOAD_FAST_CHECK, + [_LOAD_FAST_CHECK_r12] = _LOAD_FAST_CHECK, + [_LOAD_FAST_CHECK_r23] = _LOAD_FAST_CHECK, [_LOAD_FAST_0_r01] = _LOAD_FAST_0, [_LOAD_FAST_0_r12] = _LOAD_FAST_0, [_LOAD_FAST_0_r23] = _LOAD_FAST_0, @@ -3526,6 +3528,8 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_STORE_GLOBAL_r10] = _STORE_GLOBAL, [_DELETE_GLOBAL_r00] = _DELETE_GLOBAL, [_LOAD_LOCALS_r01] = _LOAD_LOCALS, + [_LOAD_LOCALS_r12] = _LOAD_LOCALS, + [_LOAD_LOCALS_r23] = _LOAD_LOCALS, [_LOAD_NAME_r01] = _LOAD_NAME, [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, @@ -4647,6 +4651,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_FAST_BORROW_7_r23] = "_LOAD_FAST_BORROW_7_r23", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", [_LOAD_FAST_CHECK_r01] = "_LOAD_FAST_CHECK_r01", + [_LOAD_FAST_CHECK_r12] = "_LOAD_FAST_CHECK_r12", + [_LOAD_FAST_CHECK_r23] = "_LOAD_FAST_CHECK_r23", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", [_LOAD_FROM_DICT_OR_DEREF_r11] = "_LOAD_FROM_DICT_OR_DEREF_r11", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", @@ -4657,6 +4663,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_GLOBAL_MODULE_r01] = "_LOAD_GLOBAL_MODULE_r01", [_LOAD_LOCALS] = "_LOAD_LOCALS", [_LOAD_LOCALS_r01] = "_LOAD_LOCALS_r01", + [_LOAD_LOCALS_r12] = "_LOAD_LOCALS_r12", + [_LOAD_LOCALS_r23] = "_LOAD_LOCALS_r23", [_LOAD_NAME] = "_LOAD_NAME", [_LOAD_NAME_r01] = "_LOAD_NAME_r01", [_LOAD_SMALL_INT] = "_LOAD_SMALL_INT", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 0f6ed3d85f0330..e278a29dc5e17b 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2781,6 +2781,28 @@ def write(id0): pool.submit(write, (1,)) """)) + def test_handling_of_tos_cache_with_side_exits(self): + # https://github.com/python/cpython/issues/142718 + class EvilAttr: + def __init__(self, d): + self.d = d + + def __del__(self): + try: + del self.d['attr'] + except Exception: + pass + + class Obj: + pass + + obj = Obj() + obj.__dict__ = {} + + for _ in range(TIER2_THRESHOLD+1): + obj.attr = EvilAttr(obj.__dict__) + + def global_identity(x): return x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-15-15-07-40.gh-issue-142718.zjiGjS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-15-15-07-40.gh-issue-142718.zjiGjS.rst new file mode 100644 index 00000000000000..022707fe33e8d5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-15-15-07-40.gh-issue-142718.zjiGjS.rst @@ -0,0 +1 @@ +JIT: Fix segfault caused by not flushing the stack to memory at side exits. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2a1156091e3d37..2318a4f733090b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -136,6 +136,7 @@ #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -146,6 +147,7 @@ assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -153,6 +155,7 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -171,6 +174,8 @@ #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -181,6 +186,8 @@ assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -188,6 +195,8 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -208,6 +217,9 @@ #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -218,6 +230,9 @@ assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -225,6 +240,9 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -262,6 +280,65 @@ break; } + case _LOAD_FAST_CHECK_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + value = PyStackRef_DUP(value_s); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_FAST_CHECK_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + value = PyStackRef_DUP(value_s); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + case _LOAD_FAST_0_r01: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); @@ -2195,6 +2272,7 @@ value = _stack_item_0; if (!PyStackRef_BoolCheck(value)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2214,6 +2292,8 @@ value = _stack_item_1; if (!PyStackRef_BoolCheck(value)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -2235,6 +2315,9 @@ value = _stack_item_2; if (!PyStackRef_BoolCheck(value)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -2257,6 +2340,7 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyLong_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2308,6 +2392,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyList_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2330,6 +2415,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyList_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -2351,6 +2438,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyList_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -2390,6 +2480,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyList_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2409,6 +2500,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyList_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -2430,6 +2523,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyList_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -2469,6 +2565,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PySlice_Check(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2488,6 +2585,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PySlice_Check(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -2509,6 +2608,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PySlice_Check(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -2580,6 +2682,7 @@ value = _stack_item_0; if (!PyStackRef_IsNone(value)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2601,6 +2704,8 @@ value = _stack_item_1; if (!PyStackRef_IsNone(value)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -2624,6 +2729,9 @@ value = _stack_item_2; if (!PyStackRef_IsNone(value)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -2666,6 +2774,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2688,6 +2797,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -2709,6 +2820,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -2748,6 +2862,7 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2767,6 +2882,8 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -2788,6 +2905,9 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -2906,6 +3026,7 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!_PyLong_CheckExactAndCompact(left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -2928,6 +3049,8 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!_PyLong_CheckExactAndCompact(left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -2949,6 +3072,9 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!_PyLong_CheckExactAndCompact(left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -2988,6 +3114,7 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -3007,6 +3134,8 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -3028,6 +3157,9 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -3070,6 +3202,7 @@ assert(Py_TYPE(left_o) == &PyLong_Type); if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -3093,6 +3226,8 @@ assert(Py_TYPE(left_o) == &PyLong_Type); if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -3115,6 +3250,9 @@ assert(Py_TYPE(left_o) == &PyLong_Type); if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -3156,6 +3294,7 @@ assert(Py_TYPE(value_o) == &PyLong_Type); if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -3176,6 +3315,8 @@ assert(Py_TYPE(value_o) == &PyLong_Type); if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -3198,6 +3339,9 @@ assert(Py_TYPE(value_o) == &PyLong_Type); if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -3257,6 +3401,7 @@ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -3289,6 +3434,8 @@ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -3320,6 +3467,9 @@ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -3380,6 +3530,7 @@ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -3412,6 +3563,8 @@ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -3443,6 +3596,9 @@ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -3503,6 +3659,7 @@ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -3535,6 +3692,8 @@ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -3566,6 +3725,9 @@ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -3607,6 +3769,7 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -3629,6 +3792,8 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -3650,6 +3815,9 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -3689,6 +3857,7 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyFloat_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -3708,6 +3877,8 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyFloat_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -3729,6 +3900,9 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyFloat_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -4651,7 +4825,9 @@ assert(PyUnicode_CheckExact(left_o)); if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); @@ -4707,7 +4883,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (!res) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_JUMP_TARGET(); } _tos_cache1 = right; @@ -4891,7 +5071,9 @@ assert(PyList_CheckExact(list)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = sub_st; + _tos_cache0 = list_st; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -4905,7 +5087,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = sub_st; + _tos_cache0 = list_st; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); @@ -4913,7 +5099,9 @@ #else if (index >= PyList_GET_SIZE(list)) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = sub_st; + _tos_cache0 = list_st; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); @@ -5008,17 +5196,23 @@ assert(PyUnicode_CheckExact(str)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = str_st; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = str_st; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } if (!PyUnicode_IS_COMPACT_ASCII(str)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = str_st; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5068,6 +5262,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -5090,6 +5285,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5111,6 +5308,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -5150,6 +5350,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -5169,6 +5370,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5190,6 +5393,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -5217,12 +5423,16 @@ assert(PyTuple_CheckExact(tuple)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5279,6 +5489,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -5301,6 +5512,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5322,6 +5535,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -5361,6 +5577,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -5380,6 +5597,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5401,6 +5620,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -5475,6 +5697,8 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5482,6 +5706,8 @@ PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); if (getitem_o == NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5489,6 +5715,8 @@ uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5496,6 +5724,8 @@ assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -5723,12 +5953,18 @@ assert(PyList_CheckExact(list)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = list_st; + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (!LOCK_OBJECT(list)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = list_st; + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -5736,6 +5972,9 @@ UNLOCK_OBJECT(list); if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = list_st; + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -6094,11 +6333,15 @@ PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = v; + _tos_cache0 = receiver; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = v; + _tos_cache0 = receiver; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -6398,6 +6641,7 @@ assert(PyTuple_CheckExact(seq_o)); if (PyTuple_GET_SIZE(seq_o) != 2) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -6434,6 +6678,7 @@ assert(PyTuple_CheckExact(seq_o)); if (PyTuple_GET_SIZE(seq_o) != oparg) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -6468,6 +6713,7 @@ assert(PyList_CheckExact(seq_o)); if (!LOCK_OBJECT(seq_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -6475,6 +6721,7 @@ UNLOCK_OBJECT(seq_o); if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -6677,6 +6924,59 @@ break; } + case _LOAD_LOCALS_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef locals; + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *l = LOCALS(); + if (l == NULL) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + locals = PyStackRef_FromPyObjectNew(l); + _tos_cache1 = locals; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_LOCALS_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef locals; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *l = LOCALS(); + if (l == NULL) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + locals = PyStackRef_FromPyObjectNew(l); + _tos_cache2 = locals; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _LOAD_NAME_r01: { @@ -6771,12 +7071,14 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -6796,12 +7098,16 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -6823,12 +7129,18 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -7744,12 +8056,18 @@ assert(!(oparg & 1)); if (global_super != (PyObject *)&PySuper_Type) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache2 = self_st; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } if (!PyType_Check(class)) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache2 = self_st; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -7810,12 +8128,18 @@ assert(oparg & 1); if (global_super != (PyObject *)&PySuper_Type) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache2 = self_st; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } if (!PyType_Check(class)) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache2 = self_st; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -7971,6 +8295,7 @@ assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -7992,6 +8317,8 @@ assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -8015,6 +8342,9 @@ assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -8067,6 +8397,7 @@ assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8075,6 +8406,7 @@ UNLOCK_OBJECT(owner_o); if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8097,6 +8429,8 @@ assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -8105,6 +8439,8 @@ UNLOCK_OBJECT(owner_o); if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -8129,6 +8465,9 @@ assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -8137,6 +8476,9 @@ UNLOCK_OBJECT(owner_o); if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -8181,6 +8523,7 @@ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8202,6 +8545,8 @@ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -8225,6 +8570,9 @@ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -8249,6 +8597,7 @@ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8257,6 +8606,7 @@ if (!increfed) { if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8293,6 +8643,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8301,6 +8652,7 @@ PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8310,6 +8662,7 @@ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8318,6 +8671,7 @@ if (!increfed) { if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8356,6 +8710,7 @@ PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (dict == NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8364,6 +8719,7 @@ #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8372,6 +8728,7 @@ if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8380,6 +8737,7 @@ if (dk->dk_kind != DICT_KEYS_UNICODE) { if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8388,6 +8746,7 @@ if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8396,6 +8755,7 @@ if (attr_o == NULL) { if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8406,6 +8766,7 @@ if (!increfed) { if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8442,6 +8803,7 @@ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8449,6 +8811,7 @@ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); if (!increfed) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8511,12 +8874,14 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8537,12 +8902,16 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -8565,12 +8934,18 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -8627,21 +9002,25 @@ PyCodeObject *code = (PyCodeObject *)f->func_code; if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (code->co_kwonlyargcount) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (code->co_argcount != 1) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8698,6 +9077,7 @@ UNLOCK_OBJECT(owner_o); if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -8723,6 +9103,8 @@ UNLOCK_OBJECT(owner_o); if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -8750,6 +9132,9 @@ UNLOCK_OBJECT(owner_o); if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -8812,12 +9197,16 @@ PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (dict == NULL) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } if (!LOCK_OBJECT(dict)) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } assert(PyDict_CheckExact((PyObject *)dict)); @@ -8827,7 +9216,9 @@ UNLOCK_OBJECT(dict); if (true) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } } @@ -8836,7 +9227,9 @@ UNLOCK_OBJECT(dict); if (true) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } } @@ -8845,7 +9238,9 @@ UNLOCK_OBJECT(dict); if (true) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } } @@ -8886,6 +9281,8 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!LOCK_OBJECT(owner_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -9235,6 +9632,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyAnySet_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -9254,6 +9652,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyAnySet_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -9275,6 +9675,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyAnySet_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -9964,7 +10367,11 @@ } if (true) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_JUMP_TARGET(); } } @@ -10022,6 +10429,7 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyList_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -10029,6 +10437,7 @@ #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -10054,6 +10463,8 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyList_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -10061,6 +10472,8 @@ #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -10085,6 +10498,9 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyList_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -10092,6 +10508,9 @@ #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -10144,6 +10563,7 @@ assert(Py_TYPE(list_o) == &PyList_Type); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -10171,6 +10591,8 @@ assert(Py_TYPE(list_o) == &PyList_Type); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -10197,6 +10619,9 @@ assert(Py_TYPE(list_o) == &PyList_Type); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -10236,7 +10661,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (result <= 0) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_JUMP_TARGET(); } #else @@ -10289,6 +10718,7 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -10314,6 +10744,8 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -10338,6 +10770,9 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -10387,6 +10822,7 @@ assert(Py_TYPE(tuple_o) == &PyTuple_Type); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -10412,6 +10848,8 @@ assert(Py_TYPE(tuple_o) == &PyTuple_Type); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -10436,6 +10874,9 @@ assert(Py_TYPE(tuple_o) == &PyTuple_Type); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -10556,12 +10997,14 @@ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -10585,12 +11028,16 @@ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -10613,12 +11060,18 @@ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -10664,6 +11117,7 @@ assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -10687,6 +11141,8 @@ assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -10709,6 +11165,9 @@ assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -10833,6 +11292,8 @@ PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -10840,12 +11301,16 @@ if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -11081,6 +11546,7 @@ PyDictValues *ivs = _PyObject_InlineValues(owner_o); if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -11102,6 +11568,8 @@ PyDictValues *ivs = _PyObject_InlineValues(owner_o); if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -11125,6 +11593,9 @@ PyDictValues *ivs = _PyObject_InlineValues(owner_o); if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -11170,6 +11641,7 @@ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -11192,6 +11664,8 @@ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -11216,6 +11690,9 @@ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -11455,6 +11932,7 @@ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -11476,6 +11954,8 @@ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -11499,6 +11979,9 @@ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -11709,6 +12192,7 @@ PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -11729,6 +12213,8 @@ PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -11751,6 +12237,9 @@ PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -11961,6 +12450,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; if (tstate->interp->eval_frame) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -11977,6 +12467,8 @@ _PyStackRef _stack_item_1 = _tos_cache1; if (tstate->interp->eval_frame) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -11995,6 +12487,9 @@ _PyStackRef _stack_item_2 = _tos_cache2; if (tstate->interp->eval_frame) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -12066,6 +12561,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -12082,6 +12578,8 @@ _PyStackRef _stack_item_1 = _tos_cache1; if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -12100,6 +12598,9 @@ _PyStackRef _stack_item_2 = _tos_cache2; if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -12332,6 +12833,7 @@ null = stack_pointer[-1]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -12353,6 +12855,8 @@ null = _stack_item_0; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -12373,6 +12877,9 @@ null = _stack_item_1; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -12413,6 +12920,7 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (o == NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -12435,6 +12943,8 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (o == NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -12456,6 +12966,9 @@ PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (o == NULL) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -12495,6 +13008,7 @@ null = stack_pointer[-2]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -12517,6 +13031,8 @@ null = stack_pointer[-1]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -12540,6 +13056,9 @@ null = _stack_item_0; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -12581,6 +13100,7 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyType_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -12604,6 +13124,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyType_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -12628,6 +13150,9 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyType_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -12705,6 +13230,7 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyUnicode_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -12728,6 +13254,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyUnicode_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -12752,6 +13280,9 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyUnicode_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -12831,6 +13362,7 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -12854,6 +13386,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -12878,6 +13412,9 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -13307,6 +13844,7 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.len) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -13331,6 +13869,8 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.len) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -13356,6 +13896,9 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.len) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -13445,6 +13988,7 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -13469,6 +14013,8 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -13494,6 +14040,9 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -13593,6 +14142,7 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.list_append) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -13617,6 +14167,8 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.list_append) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -13642,6 +14194,9 @@ PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.list_append) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -13670,6 +14225,9 @@ PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = arg; + _tos_cache1 = self; + _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -14112,12 +14670,14 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -14140,22 +14700,26 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (((PyFunctionObject *)func)->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -14209,11 +14773,13 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -15465,11 +16031,13 @@ assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -15488,11 +16056,15 @@ assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -15513,11 +16085,17 @@ assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -15882,6 +16460,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -15898,6 +16477,8 @@ _PyStackRef _stack_item_1 = _tos_cache1; if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -15916,6 +16497,9 @@ _PyStackRef _stack_item_2 = _tos_cache2; if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -16834,6 +17418,7 @@ #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -16842,6 +17427,7 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); if (eval_breaker & _PY_EVAL_EVENTS_MASK) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -16860,6 +17446,8 @@ #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -16868,6 +17456,8 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); if (eval_breaker & _PY_EVAL_EVENTS_MASK) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -16888,6 +17478,9 @@ #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -16896,6 +17489,9 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); if (eval_breaker & _PY_EVAL_EVENTS_MASK) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -16984,6 +17580,7 @@ frame->instr_ptr += OFFSET_OF__PUSH_FRAME; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -17007,6 +17604,8 @@ frame->instr_ptr += OFFSET_OF__PUSH_FRAME; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -17032,6 +17631,9 @@ frame->instr_ptr += OFFSET_OF__PUSH_FRAME; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -17076,6 +17678,7 @@ frame->instr_ptr += OFFSET_OF_YIELD_VALUE; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -17099,6 +17702,8 @@ frame->instr_ptr += OFFSET_OF_YIELD_VALUE; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -17124,6 +17729,9 @@ frame->instr_ptr += OFFSET_OF_YIELD_VALUE; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -17168,6 +17776,7 @@ frame->instr_ptr += OFFSET_OF_RETURN_VALUE; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -17191,6 +17800,8 @@ frame->instr_ptr += OFFSET_OF_RETURN_VALUE; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -17216,6 +17827,9 @@ frame->instr_ptr += OFFSET_OF_RETURN_VALUE; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } @@ -17260,6 +17874,7 @@ frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -17283,6 +17898,8 @@ frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } @@ -17308,6 +17925,9 @@ frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; if (true) { UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index f6575a93e47a31..445a87e49019fc 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1372,11 +1372,6 @@ def get_uop_cache_depths(uop: Uop) -> Iterator[tuple[int, int, int]]: if uop.name == "_ERROR_POP_N": yield 0, 0, 0 return - non_decref_escape = False - for call in uop.properties.escaping_calls.values(): - if "DECREF" in call.call.text or "CLOSE" in call.call.text: - continue - non_decref_escape = True ideal_inputs = 0 has_array = False for item in reversed(uop.stack.inputs): @@ -1394,9 +1389,6 @@ def get_uop_cache_depths(uop: Uop) -> Iterator[tuple[int, int, int]]: ideal_inputs = MAX_CACHED_REGISTER if ideal_outputs > MAX_CACHED_REGISTER: ideal_outputs = MAX_CACHED_REGISTER - if non_decref_escape: - yield ideal_inputs, ideal_outputs, 0 - return at_end = uop.properties.sync_sp or uop.properties.side_exit_at_end exit_depth = ideal_outputs if at_end else ideal_inputs if uop.properties.escapes or uop.properties.sync_sp or has_array or is_large(uop): diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 9c86382cdc5610..478dfcf943276a 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -95,7 +95,9 @@ def exit_if( next(tkn_iter) # Semi colon self.emit(") {\n") self.emit("UOP_STAT_INC(uopcode, miss);\n") - self.emit(f"SET_CURRENT_CACHED_VALUES({self.exit_cache_depth});\n") + storage = storage.copy() + self.cache_items(storage.stack, self.exit_cache_depth, False) + storage.stack.flush(self.out) self.emit("JUMP_TO_JUMP_TARGET();\n") self.emit("}\n") return not always_true(first_tkn) @@ -151,7 +153,7 @@ def tier2_to_tier2( inst: Instruction | None, ) -> bool: assert self.exit_cache_depth == 0, uop.name - cache_items(self, storage.stack, self.exit_cache_depth, False) + self.cache_items(storage.stack, self.exit_cache_depth, False) storage.flush(self.out) self.out.emit(tkn) lparen = next(tkn_iter) @@ -181,31 +183,31 @@ def exit_if_after( self.emit("UOP_STAT_INC(uopcode, miss);\n") storage = storage.copy() storage.clear_inputs("in AT_END_EXIT_IF") - cache_items(self, storage.stack, self.exit_cache_depth, False) + self.cache_items(storage.stack, self.exit_cache_depth, False) storage.flush(self.out) self.emit("JUMP_TO_JUMP_TARGET();\n") self.emit("}\n") return not always_true(first_tkn) - -def cache_items(emitter: Emitter, stack: Stack, cached_items: int, zero_regs: bool) -> None: - emitter.out.start_line() - i = cached_items - while i > 0: - emitter.out.start_line() - item = StackItem(f"_tos_cache{i-1}", "", False, True) - stack.pop(item, emitter.out) - i -= 1 - if zero_regs: - # TO DO -- For compilers that support it, - # replace this with a "clobber" to tell - # the compiler that these values are unused - # without having to emit any code. - for i in range(cached_items, MAX_CACHED_REGISTER): - emitter.out.emit(f"_tos_cache{i} = PyStackRef_ZERO_BITS;\n") - emitter.emit(f"SET_CURRENT_CACHED_VALUES({cached_items});\n") - -def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, tuple[str, str]], cached_items: int = 0) -> tuple[bool, Stack]: + def cache_items(self, stack: Stack, cached_items: int, zero_regs: bool) -> None: + self.out.start_line() + i = cached_items + while i > 0: + self.out.start_line() + item = StackItem(f"_tos_cache{i-1}", "", False, True) + stack.pop(item, self.out) + i -= 1 + if zero_regs: + # TO DO -- For compilers that support it, + # replace this with a "clobber" to tell + # the compiler that these values are unused + # without having to emit any code. + for i in range(cached_items, MAX_CACHED_REGISTER): + self.out.emit(f"_tos_cache{i} = PyStackRef_ZERO_BITS;\n") + self.emit(f"SET_CURRENT_CACHED_VALUES({cached_items});\n") + + +def write_uop(uop: Uop, emitter: Tier2Emitter, stack: Stack, offset_strs: dict[str, tuple[str, str]], cached_items: int = 0) -> tuple[bool, Stack]: locals: dict[str, Local] = {} zero_regs = is_large(uop) or uop.properties.escapes try: @@ -233,7 +235,7 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, t reachable, storage = emitter.emit_tokens(uop, storage, None, False) if reachable: storage.stack._print(emitter.out) - cache_items(emitter, storage.stack, cached_items, zero_regs) + emitter.cache_items(storage.stack, cached_items, zero_regs) storage.flush(emitter.out) if name_offset_pair: emitter.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n")