diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8ed5436eb6838c..f9301c95586453 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -52,6 +52,7 @@ typedef struct _PyExitData { typedef struct _PyExecutorObject { PyObject_VAR_HEAD const _PyUOpInstruction *trace; + PyObject *constant_pool; _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ uint32_t exit_count; uint32_t code_size; @@ -98,7 +99,7 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, - _PyBloomFilter *dependencies); + _PyBloomFilter *dependencies, PyObject **constant_pool_ptr); extern PyTypeObject _PyUOpExecutor_Type; @@ -278,6 +279,7 @@ typedef struct _JitOptContext { bool contradiction; // Has the builtins dict been watched? bool builtins_watched; + PyObject *constant_pool; // The current "executing" frame. _Py_UOpsAbstractFrame *frame; _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; @@ -324,8 +326,9 @@ extern bool _Py_uop_sym_is_compact_int(JitOptRef sym); extern JitOptRef _Py_uop_sym_new_compact_int(JitOptContext *ctx); extern void _Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef sym); -extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); +extern int _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); +extern int _Py_uop_promote_to_constant_pool(JitOptContext *ctx, PyObject *obj); extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( JitOptContext *ctx, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 4e94f62d35eba2..36a0847067c083 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2594,8 +2594,7 @@ def testfunc(n): self.assertIsNotNone(ex) uops = get_opnames(ex) - # For now... until we constant propagate it away. - self.assertIn("_BINARY_OP", uops) + self.assertIn("_LOAD_CONST_INLINE_BORROW", uops) def test_jitted_code_sees_changed_globals(self): "Issue 136154: Check that jitted code spots the change in the globals" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-21-10-54.gh-issue-140928.purSIt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-21-10-54.gh-issue-140928.purSIt.rst new file mode 100644 index 00000000000000..b1f4a784ef57ac --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-21-10-54.gh-issue-140928.purSIt.rst @@ -0,0 +1 @@ +Add a constant pool to all JIT executors and allow promotion of constants to the pool. Patch by Ken Jin. Implementation in CPython inspired by PyPy/RPython. diff --git a/Python/optimizer.c b/Python/optimizer.c index f44f8a9614b846..72fd26d7182f85 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -100,7 +100,7 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, PyObject *constant_pool); static int uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, @@ -388,6 +388,7 @@ static int executor_traverse(PyObject *o, visitproc visit, void *arg) { _PyExecutorObject *executor = _PyExecutorObject_CAST(o); + Py_VISIT(executor->constant_pool); for (uint32_t i = 0; i < executor->exit_count; i++) { Py_VISIT(executor->exits[i].executor); } @@ -1115,13 +1116,15 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) /* Executor side exits */ static _PyExecutorObject * -allocate_executor(int exit_count, int length) +allocate_executor(int exit_count, int length, PyObject *constant_pool) { int size = exit_count*sizeof(_PyExitData) + length*sizeof(_PyUOpInstruction); _PyExecutorObject *res = PyObject_GC_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size); if (res == NULL) { return NULL; } + // Transfer ownership + res->constant_pool = constant_pool; res->trace = (_PyUOpInstruction *)(res->exits + exit_count); res->code_size = length; res->exit_count = exit_count; @@ -1196,10 +1199,10 @@ sanity_check(_PyExecutorObject *executor) * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, PyObject *constant_pool) { int exit_count = count_exits(buffer, length); - _PyExecutorObject *executor = allocate_executor(exit_count, length); + _PyExecutorObject *executor = allocate_executor(exit_count, length, constant_pool); if (executor == NULL) { return NULL; } @@ -1293,6 +1296,7 @@ uop_optimize( _PyBloomFilter dependencies; _Py_BloomFilter_Init(&dependencies); PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *constant_pool = NULL; if (interp->jit_uop_buffer == NULL) { interp->jit_uop_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); if (interp->jit_uop_buffer == NULL) { @@ -1316,7 +1320,7 @@ uop_optimize( if (!is_noopt) { length = _Py_uop_analyze_and_optimize(frame, buffer, length, - curr_stackentries, &dependencies); + curr_stackentries, &dependencies, &constant_pool); if (length <= 0) { return length; } @@ -1339,7 +1343,7 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies, constant_pool); if (executor == NULL) { return -1; } @@ -1512,7 +1516,7 @@ _PyExecutor_GetColdExecutor(void) if (interp->cold_executor != NULL) { return interp->cold_executor; } - _PyExecutorObject *cold = allocate_executor(0, 1); + _PyExecutorObject *cold = allocate_executor(0, 1, NULL); if (cold == NULL) { Py_FatalError("Cannot allocate core JIT code"); } @@ -1575,6 +1579,8 @@ executor_clear(PyObject *op) unlink_executor(executor); executor->vm_data.valid = 0; + Py_CLEAR(executor->constant_pool); + /* It is possible for an executor to form a reference * cycle with itself, so decref'ing a side exit could * free the executor unless we hold a strong reference to it diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a6add301ccb26c..a006ff09971aa8 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -153,6 +153,8 @@ incorrect_keys(PyObject *obj, uint32_t version) (INST)->oparg = ARG; \ (INST)->operand0 = OPERAND; +#define PROMOTE_TO_CONSTANT_POOL _Py_uop_promote_to_constant_pool + /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null #define sym_is_const _Py_uop_sym_is_const @@ -290,7 +292,8 @@ optimize_uops( _PyUOpInstruction *trace, int trace_len, int curr_stacklen, - _PyBloomFilter *dependencies + _PyBloomFilter *dependencies, + PyObject **constant_pool_ptr ) { assert(!PyErr_Occurred()); @@ -310,9 +313,13 @@ optimize_uops( interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; } - _Py_uop_abstractcontext_init(ctx); + if (_Py_uop_abstractcontext_init(ctx)) { + return 0; + } + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, (PyCodeObject *)func->func_code, curr_stacklen, NULL, 0); if (frame == NULL) { + _Py_uop_abstractcontext_fini(ctx); return 0; } frame->func = func; @@ -367,6 +374,7 @@ optimize_uops( /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ + *constant_pool_ptr = Py_NewRef(ctx->constant_pool); _Py_uop_abstractcontext_fini(ctx); if (first_valid_check_stack != NULL) { assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); @@ -522,14 +530,15 @@ _Py_uop_analyze_and_optimize( _PyUOpInstruction *buffer, int length, int curr_stacklen, - _PyBloomFilter *dependencies + _PyBloomFilter *dependencies, + PyObject **constant_pool_ptr ) { OPT_STAT_INC(optimizer_attempts); length = optimize_uops( _PyFrame_GetFunction(frame), buffer, - length, curr_stacklen, dependencies); + length, curr_stacklen, dependencies, constant_pool_ptr); if (length == 0) { return length; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b08099d8e2fc3b..fdce417ff39cad 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -206,10 +206,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-1] = res; break; @@ -248,10 +251,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-1] = res; break; @@ -410,10 +416,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-1] = res; break; @@ -500,10 +509,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -550,10 +562,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -600,10 +615,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -669,10 +687,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -721,10 +742,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -773,10 +797,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -852,10 +879,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1809,10 +1839,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1860,10 +1893,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1910,10 +1946,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1958,10 +1997,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; @@ -2011,10 +2053,13 @@ b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); if (sym_is_const(ctx, b)) { PyObject *result = sym_get_const(ctx, b); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = b; stack_pointer += -1; @@ -3094,10 +3139,13 @@ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (!_Py_IsImmortal(result)) { + if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) { + goto error; + } } + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal/promoted result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } stack_pointer[-2] = res; stack_pointer += -1; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 01cff0b014cc7b..226a64933c89fc 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -870,9 +870,17 @@ _Py_uop_abstractcontext_fini(JitOptContext *ctx) Py_CLEAR(sym->value.value); } } + Py_CLEAR(ctx->constant_pool); } -void +// -1 if err, 0 if success +int +_Py_uop_promote_to_constant_pool(JitOptContext *ctx, PyObject *obj) +{ + return PyList_Append(ctx->constant_pool, obj); +} + +int _Py_uop_abstractcontext_init(JitOptContext *ctx) { static_assert(sizeof(JitOptSymbol) <= 3 * sizeof(uint64_t), "JitOptSymbol has grown"); @@ -898,6 +906,12 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) ctx->out_of_space = false; ctx->contradiction = false; ctx->builtins_watched = false; + + ctx->constant_pool = PyList_New(0); + if (ctx->constant_pool == NULL) { + return -1; + } + return 0; } int @@ -935,7 +949,9 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { JitOptContext context; JitOptContext *ctx = &context; - _Py_uop_abstractcontext_init(ctx); + if (_Py_uop_abstractcontext_init(ctx) < 0) { + return NULL; + } PyObject *val_42 = NULL; PyObject *val_43 = NULL; PyObject *val_big = NULL; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 41df073cf6df23..ffc91a1e967641 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -250,10 +250,13 @@ def replace_opcode_if_evaluates_pure( emitter.emit(f"if (sym_is_const(ctx, {outp.name})) {{\n") emitter.emit(f"PyObject *result = sym_get_const(ctx, {outp.name});\n") - emitter.emit(f"if (_Py_IsImmortal(result)) {{\n") - emitter.emit(f"// Replace with {replacement_uop} since we have {input_desc} and an immortal result\n") - emitter.emit(f"REPLACE_OP(this_instr, {replacement_uop}, 0, (uintptr_t)result);\n") + emitter.emit(f"if (!_Py_IsImmortal(result)) {{\n") + emitter.emit("if (PROMOTE_TO_CONSTANT_POOL(ctx, result) < 0) {\n") + emitter.emit("goto error;\n") + emitter.emit("}\n") emitter.emit("}\n") + emitter.emit(f"// Replace with {replacement_uop} since we have {input_desc} and an immortal/promoted result\n") + emitter.emit(f"REPLACE_OP(this_instr, {replacement_uop}, 0, (uintptr_t)result);\n") emitter.emit("}\n") storage.flush(self.out)