From 57a80904faa2c2931e38fedbb7177f4b2bb3bb47 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sun, 28 Jan 2024 04:35:45 -0800 Subject: [PATCH 01/18] Uninstall executors when invalidating --- Include/cpython/optimizer.h | 5 +- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/libregrtest/refleak.py | 9 ++- Modules/_testinternalcapi.c | 21 ++++++- Modules/clinic/_testinternalcapi.c.h | 62 ++++++++++++++++++- Objects/codeobject.c | 21 ++----- Python/bytecodes.c | 20 ++---- Python/generated_cases.c.h | 22 ++----- Python/optimizer.c | 46 ++++++++++---- 12 files changed, 145 insertions(+), 67 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index ecf3cae4cbc3f10..82b2e2722cb9d0d 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -27,6 +27,8 @@ typedef struct { uint8_t linked; _PyBloomFilter bloom; _PyExecutorLinkListNode links; + PyCodeObject *code; // Weak. + int index; } _PyVMData; typedef struct { @@ -84,7 +86,8 @@ void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); -extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp); +PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp); +void _PyExecutor_Remove(_PyExecutorObject *executor); /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index dd09ff40f39fe6f..8b3484e4034df51 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -882,6 +882,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(defaultaction)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(delete)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dependency)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(depth)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(detect_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 79d6509abcdfd91..d3756fa011a40d0 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -371,6 +371,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(default) STRUCT_FOR_ID(defaultaction) STRUCT_FOR_ID(delete) + STRUCT_FOR_ID(dependency) STRUCT_FOR_ID(depth) STRUCT_FOR_ID(detect_types) STRUCT_FOR_ID(deterministic) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index f3c55acfb3c282f..454d4e96169b7ff 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -880,6 +880,7 @@ extern "C" { INIT_ID(default), \ INIT_ID(defaultaction), \ INIT_ID(delete), \ + INIT_ID(dependency), \ INIT_ID(depth), \ INIT_ID(detect_types), \ INIT_ID(deterministic), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 2e9572382fe0332..a32dfc9d4060885 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -954,6 +954,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(delete); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(dependency); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(depth); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 7da16cf721f0973..6e54b464a497154 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -201,7 +201,14 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear caches clear_caches() - # Clear type cache at the end: previous function calls can modify types + # Clear executors and type cache at the end, since the previous function + # calls can create executors and modify types: + try: + import _testinternalcapi + except ImportError: + pass + else: + _testinternalcapi.invalidate_executors() sys._clear_type_cache() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c4a648a1816392f..ad425e421a8e50b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1029,11 +1029,26 @@ add_executor_dependency(PyObject *self, PyObject *args) Py_RETURN_NONE; } +/*[clinic input] + +_testinternalcapi.invalidate_executors -> object + + dependency: object = NULL + +[clinic start generated code]*/ + static PyObject * -invalidate_executors(PyObject *self, PyObject *obj) +_testinternalcapi_invalidate_executors_impl(PyObject *module, + PyObject *dependency) +/*[clinic end generated code: output=79497cfcf0826eaa input=8c3282b374a51e1f]*/ { PyInterpreterState *interp = PyInterpreterState_Get(); - _Py_Executors_InvalidateDependency(interp, obj); + if (dependency) { + _Py_Executors_InvalidateDependency(interp, dependency); + } + else { + _Py_Executors_InvalidateAll(interp); + } Py_RETURN_NONE; } @@ -1698,7 +1713,7 @@ static PyMethodDef module_functions[] = { {"get_counter_optimizer", get_counter_optimizer, METH_NOARGS, NULL}, {"get_uop_optimizer", get_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, - {"invalidate_executors", invalidate_executors, METH_O, NULL}, + _TESTINTERNALCAPI_INVALIDATE_EXECUTORS_METHODDEF {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index cba2a943d03456a..c5ca4d6f1329944 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -266,6 +266,66 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, return return_value; } +PyDoc_STRVAR(_testinternalcapi_invalidate_executors__doc__, +"invalidate_executors($module, /, dependency=)\n" +"--\n" +"\n"); + +#define _TESTINTERNALCAPI_INVALIDATE_EXECUTORS_METHODDEF \ + {"invalidate_executors", _PyCFunction_CAST(_testinternalcapi_invalidate_executors), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_invalidate_executors__doc__}, + +static PyObject * +_testinternalcapi_invalidate_executors_impl(PyObject *module, + PyObject *dependency); + +static PyObject * +_testinternalcapi_invalidate_executors(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(dependency), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"dependency", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "invalidate_executors", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *dependency = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + dependency = args[0]; +skip_optional_pos: + return_value = _testinternalcapi_invalidate_executors_impl(module, dependency); + +exit: + return return_value; +} + PyDoc_STRVAR(_testinternalcapi_test_long_numbits__doc__, "test_long_numbits($module, /)\n" "--\n" @@ -282,4 +342,4 @@ _testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignore { return _testinternalcapi_test_long_numbits_impl(module); } -/*[clinic end generated code: output=679bf53bbae20085 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=88923251416ab15e input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index dc46b773c26528a..3561f1c65b4b3ad 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1490,26 +1490,17 @@ static void clear_executors(PyCodeObject *co) { for (int i = 0; i < co->co_executors->size; i++) { - Py_CLEAR(co->co_executors->executors[i]); + if (co->co_executors->executors[i]) { + _PyExecutor_Remove(co->co_executors->executors[i]); + } } PyMem_Free(co->co_executors); co->co_executors = NULL; } void -_PyCode_Clear_Executors(PyCodeObject *code) { - int code_len = (int)Py_SIZE(code); - for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - uint8_t opcode = instr->op.code; - uint8_t oparg = instr->op.arg; - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *exec = code->co_executors->executors[oparg]; - assert(exec->vm_data.opcode != ENTER_EXECUTOR); - instr->op.code = exec->vm_data.opcode; - instr->op.arg = exec->vm_data.oparg; - } - } +_PyCode_Clear_Executors(PyCodeObject *code) +{ clear_executors(code); } @@ -2360,10 +2351,10 @@ _PyCode_ConstantKey(PyObject *op) void _PyStaticCode_Fini(PyCodeObject *co) { - deopt_code(co, _PyCode_CODE(co)); if (co->co_executors != NULL) { clear_executors(co); } + deopt_code(co, _PyCode_CODE(co)); PyMem_Free(co->co_extra); if (co->_co_cached != NULL) { Py_CLEAR(co->_co_cached->_co_code); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ebd5b06abb2d4e4..70c1220034194b4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2371,22 +2371,10 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - if (executor->vm_data.valid) { - Py_INCREF(executor); - current_executor = executor; - GOTO_TIER_TWO(); - } - else { - /* ENTER_EXECUTOR will be the first code unit of the instruction */ - assert(oparg < 256); - code->co_executors->executors[oparg] = NULL; - opcode = this_instr->op.code = executor->vm_data.opcode; - this_instr->op.arg = executor->vm_data.oparg; - oparg = executor->vm_data.oparg; - Py_DECREF(executor); - next_instr = this_instr; - DISPATCH_GOTO(); - } + assert(executor->vm_data.valid); + Py_INCREF(executor); + current_executor = executor; + GOTO_TIER_TWO(); } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 16f1db30620d722..a24870b475ae38c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2363,29 +2363,17 @@ } TARGET(ENTER_EXECUTOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); TIER_ONE_ONLY CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - if (executor->vm_data.valid) { - Py_INCREF(executor); - current_executor = executor; - GOTO_TIER_TWO(); - } - else { - /* ENTER_EXECUTOR will be the first code unit of the instruction */ - assert(oparg < 256); - code->co_executors->executors[oparg] = NULL; - opcode = this_instr->op.code = executor->vm_data.opcode; - this_instr->op.arg = executor->vm_data.oparg; - oparg = executor->vm_data.oparg; - Py_DECREF(executor); - next_instr = this_instr; - DISPATCH_GOTO(); - } + assert(executor->vm_data.valid); + Py_INCREF(executor); + current_executor = executor; + GOTO_TIER_TWO(); DISPATCH(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index 0d04b09fef1e846..52f241ead5e101c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -67,6 +67,23 @@ get_index_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) return size; } +void +_PyExecutor_Remove(_PyExecutorObject *executor) +{ + PyCodeObject *code = executor->vm_data.code; + if (code == NULL) { + return; + } + _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; + assert(instruction->op.code == ENTER_EXECUTOR); + int index = instruction->op.arg; + assert(code->co_executors->executors[index] == executor); + instruction->op.code = executor->vm_data.opcode; + instruction->op.arg = executor->vm_data.oparg; + executor->vm_data.code = NULL; + Py_CLEAR(code->co_executors->executors[index]); +} + static void insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorObject *executor) { @@ -74,24 +91,21 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO if (instr->op.code == ENTER_EXECUTOR) { assert(index == instr->op.arg); _PyExecutorObject *old = code->co_executors->executors[index]; - executor->vm_data.opcode = old->vm_data.opcode; - executor->vm_data.oparg = old->vm_data.oparg; - old->vm_data.opcode = 0; - code->co_executors->executors[index] = executor; - Py_DECREF(old); + _PyExecutor_Remove(old); } else { assert(code->co_executors->size == index); assert(code->co_executors->capacity > index); - executor->vm_data.opcode = instr->op.code; - executor->vm_data.oparg = instr->op.arg; - code->co_executors->executors[index] = executor; - assert(index < MAX_EXECUTORS_SIZE); - instr->op.code = ENTER_EXECUTOR; - instr->op.arg = index; code->co_executors->size++; } - return; + executor->vm_data.opcode = instr->op.code; + executor->vm_data.oparg = instr->op.arg; + executor->vm_data.code = code; + executor->vm_data.index = instr - _PyCode_CODE(code); + code->co_executors->executors[index] = executor; + assert(index < MAX_EXECUTORS_SIZE); + instr->op.code = ENTER_EXECUTOR; + instr->op.arg = index; } int @@ -1115,7 +1129,7 @@ _Py_ExecutorClear(_PyExecutorObject *executor) void _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) { - assert(executor->vm_data.valid = true); + assert(executor->vm_data.valid); _Py_BloomFilter_Add(&executor->vm_data.bloom, obj); } @@ -1136,6 +1150,7 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { exec->vm_data.valid = false; unlink_executor(exec); + _PyExecutor_Remove(exec); } exec = next; } @@ -1148,11 +1163,16 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) /* Walk the list of executors */ for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { assert(exec->vm_data.valid); + Py_INCREF(exec); + if (exec->vm_data.code) { + _PyCode_Clear_Executors(exec->vm_data.code); + } _PyExecutorObject *next = exec->vm_data.links.next; exec->vm_data.links.next = NULL; exec->vm_data.links.previous = NULL; exec->vm_data.valid = false; exec->vm_data.linked = false; + Py_DECREF(exec); exec = next; } interp->executor_list_head = NULL; From f80427f300af8cb16f0a2ab053f1d2752222ad53 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sun, 28 Jan 2024 06:34:29 -0800 Subject: [PATCH 02/18] Get rid of vm_data.valid --- Include/cpython/optimizer.h | 1 - Python/bytecodes.c | 9 ++++----- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 7 +++---- Python/optimizer.c | 10 +++------- 5 files changed, 11 insertions(+), 18 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 82b2e2722cb9d0d..31a521d2062193f 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -23,7 +23,6 @@ typedef struct _bloom_filter { typedef struct { uint8_t opcode; uint8_t oparg; - uint8_t valid; uint8_t linked; _PyBloomFilter bloom; _PyExecutorLinkListNode links; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 70c1220034194b4..65baeb117035513 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2370,10 +2370,9 @@ dummy_func( CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.valid); - Py_INCREF(executor); - current_executor = executor; + current_executor = code->co_executors->executors[oparg & 255]; + assert(current_executor->vm_data.code == code); + Py_INCREF(current_executor); GOTO_TIER_TWO(); } @@ -4055,7 +4054,7 @@ dummy_func( op(_CHECK_VALIDITY, (--)) { TIER_TWO_ONLY - DEOPT_IF(!current_executor->vm_data.valid); + DEOPT_IF(current_executor->vm_data.code == NULL); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 241b9056207715d..b1fba17d783a767 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3386,7 +3386,7 @@ case _CHECK_VALIDITY: { TIER_TWO_ONLY - if (!current_executor->vm_data.valid) goto deoptimize; + if (current_executor->vm_data.code == NULL) goto deoptimize; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a24870b475ae38c..de21fe60d85f2ac 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2369,10 +2369,9 @@ TIER_ONE_ONLY CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.valid); - Py_INCREF(executor); - current_executor = executor; + current_executor = code->co_executors->executors[oparg & 255]; + assert(current_executor->vm_data.code == code); + Py_INCREF(current_executor); GOTO_TIER_TWO(); DISPATCH(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index 52f241ead5e101c..3bff6e394dce490 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -229,7 +229,7 @@ PyUnstable_GetExecutor(PyCodeObject *code, int offset) static PyObject * is_valid(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.valid); + return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.code != NULL); } static PyMethodDef executor_methods[] = { @@ -1112,7 +1112,6 @@ unlink_executor(_PyExecutorObject *executor) void _Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) { - executor->vm_data.valid = true; for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { executor->vm_data.bloom.bits[i] = dependency_set->bits[i]; } @@ -1129,7 +1128,7 @@ _Py_ExecutorClear(_PyExecutorObject *executor) void _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) { - assert(executor->vm_data.valid); + assert(executor->vm_data.code != NULL); _Py_BloomFilter_Add(&executor->vm_data.bloom, obj); } @@ -1145,10 +1144,9 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) /* Walk the list of executors */ /* TO DO -- Use a tree to avoid traversing as many objects */ for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { - assert(exec->vm_data.valid); + assert(exec->vm_data.code != NULL); _PyExecutorObject *next = exec->vm_data.links.next; if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { - exec->vm_data.valid = false; unlink_executor(exec); _PyExecutor_Remove(exec); } @@ -1162,7 +1160,6 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) { /* Walk the list of executors */ for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { - assert(exec->vm_data.valid); Py_INCREF(exec); if (exec->vm_data.code) { _PyCode_Clear_Executors(exec->vm_data.code); @@ -1170,7 +1167,6 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) _PyExecutorObject *next = exec->vm_data.links.next; exec->vm_data.links.next = NULL; exec->vm_data.links.previous = NULL; - exec->vm_data.valid = false; exec->vm_data.linked = false; Py_DECREF(exec); exec = next; From d4fcc4741e8f6ddf08542c8e13d0bd4802dba0c4 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sun, 28 Jan 2024 17:52:47 -0800 Subject: [PATCH 03/18] Clean up invalidation --- Include/cpython/optimizer.h | 4 ++-- Python/optimizer.c | 17 ++++++----------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 31a521d2062193f..aec3c38001f357c 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -26,8 +26,8 @@ typedef struct { uint8_t linked; _PyBloomFilter bloom; _PyExecutorLinkListNode links; - PyCodeObject *code; // Weak. - int index; + PyCodeObject *code; // Weak (NULL if invalidated). + int index; // Index of the instruction in the code object. } _PyVMData; typedef struct { diff --git a/Python/optimizer.c b/Python/optimizer.c index 3bff6e394dce490..1285f178f1fe578 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1159,17 +1159,12 @@ void _Py_Executors_InvalidateAll(PyInterpreterState *interp) { /* Walk the list of executors */ - for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { - Py_INCREF(exec); - if (exec->vm_data.code) { - _PyCode_Clear_Executors(exec->vm_data.code); + while (interp->executor_list_head) { + if (interp->executor_list_head->vm_data.code) { + _PyCode_Clear_Executors(interp->executor_list_head->vm_data.code); + } + else { + unlink_executor(interp->executor_list_head); } - _PyExecutorObject *next = exec->vm_data.links.next; - exec->vm_data.links.next = NULL; - exec->vm_data.links.previous = NULL; - exec->vm_data.linked = false; - Py_DECREF(exec); - exec = next; } - interp->executor_list_head = NULL; } From 0c6eb75e088e9e72b385fbf70469fa90c09e8ae6 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 7 Feb 2024 16:10:07 -0800 Subject: [PATCH 04/18] Clean up the diff --- Include/cpython/optimizer.h | 6 +-- .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 3 -- Modules/_testinternalcapi.c | 3 +- Modules/clinic/_testinternalcapi.c.h | 44 ++++--------------- Objects/codeobject.c | 25 +++++------ Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 2 +- Python/optimizer.c | 35 ++++++++------- 11 files changed, 45 insertions(+), 78 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 482909d9ca26b08..82bdaffabea8cc4 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -23,11 +23,11 @@ typedef struct _bloom_filter { typedef struct { uint8_t opcode; uint8_t oparg; - uint8_t linked; + uint8_t valid; _PyBloomFilter bloom; _PyExecutorLinkListNode links; - PyCodeObject *code; // Weak (NULL if invalidated). - int index; // Index of the instruction in the code object. + PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). + int index; // Index of ENTER_EXECUTOR in the above code object. } _PyVMData; typedef struct { diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index da4a440c19abdf5..932738c3049882b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -882,7 +882,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(defaultaction)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(delete)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dependency)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(depth)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(detect_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3611fbc8f8f655a..da62b4f0a951ff8 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -371,7 +371,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(default) STRUCT_FOR_ID(defaultaction) STRUCT_FOR_ID(delete) - STRUCT_FOR_ID(dependency) STRUCT_FOR_ID(depth) STRUCT_FOR_ID(detect_types) STRUCT_FOR_ID(deterministic) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6bf52f01407accd..68fbbcb4378e17f 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -880,7 +880,6 @@ extern "C" { INIT_ID(default), \ INIT_ID(defaultaction), \ INIT_ID(delete), \ - INIT_ID(dependency), \ INIT_ID(depth), \ INIT_ID(detect_types), \ INIT_ID(deterministic), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index cf2f0cdf30586f3..c8458b4e36ccc93 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -954,9 +954,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(delete); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(dependency); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(depth); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 60823869735e86c..3483b951e16e239 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1034,13 +1034,14 @@ add_executor_dependency(PyObject *self, PyObject *args) _testinternalcapi.invalidate_executors -> object dependency: object = NULL + / [clinic start generated code]*/ static PyObject * _testinternalcapi_invalidate_executors_impl(PyObject *module, PyObject *dependency) -/*[clinic end generated code: output=79497cfcf0826eaa input=8c3282b374a51e1f]*/ +/*[clinic end generated code: output=79497cfcf0826eaa input=ffbb09a1f7848473]*/ { PyInterpreterState *interp = PyInterpreterState_Get(); if (dependency) { diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index c5ca4d6f1329944..44ae5f3fe1f886e 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -267,59 +267,31 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, } PyDoc_STRVAR(_testinternalcapi_invalidate_executors__doc__, -"invalidate_executors($module, /, dependency=)\n" +"invalidate_executors($module, dependency=, /)\n" "--\n" "\n"); #define _TESTINTERNALCAPI_INVALIDATE_EXECUTORS_METHODDEF \ - {"invalidate_executors", _PyCFunction_CAST(_testinternalcapi_invalidate_executors), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_invalidate_executors__doc__}, + {"invalidate_executors", _PyCFunction_CAST(_testinternalcapi_invalidate_executors), METH_FASTCALL, _testinternalcapi_invalidate_executors__doc__}, static PyObject * _testinternalcapi_invalidate_executors_impl(PyObject *module, PyObject *dependency); static PyObject * -_testinternalcapi_invalidate_executors(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testinternalcapi_invalidate_executors(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(dependency), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"dependency", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "invalidate_executors", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *dependency = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("invalidate_executors", nargs, 0, 1)) { goto exit; } - if (!noptargs) { - goto skip_optional_pos; + if (nargs < 1) { + goto skip_optional; } dependency = args[0]; -skip_optional_pos: +skip_optional: return_value = _testinternalcapi_invalidate_executors_impl(module, dependency); exit: @@ -342,4 +314,4 @@ _testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignore { return _testinternalcapi_test_long_numbits_impl(module); } -/*[clinic end generated code: output=88923251416ab15e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c4949d6611a6c3a2 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 3561f1c65b4b3ad..57e6b2bc3258bf7 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1486,22 +1486,17 @@ PyCode_GetFreevars(PyCodeObject *code) return _PyCode_GetFreevars(code); } -static void -clear_executors(PyCodeObject *co) -{ - for (int i = 0; i < co->co_executors->size; i++) { - if (co->co_executors->executors[i]) { - _PyExecutor_Remove(co->co_executors->executors[i]); - } - } - PyMem_Free(co->co_executors); - co->co_executors = NULL; -} - void _PyCode_Clear_Executors(PyCodeObject *code) { - clear_executors(code); + assert(code->co_executors); + for (int i = 0; i < code->co_executors->size; i++) { + if (code->co_executors->executors[i]) { + _PyExecutor_Remove(code->co_executors->executors[i]); + } + } + PyMem_Free(code->co_executors); + code->co_executors = NULL; } static void @@ -1724,7 +1719,7 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } if (co->co_executors != NULL) { - clear_executors(co); + _PyCode_Clear_Executors(co); } Py_XDECREF(co->co_consts); @@ -2352,7 +2347,7 @@ void _PyStaticCode_Fini(PyCodeObject *co) { if (co->co_executors != NULL) { - clear_executors(co); + _PyCode_Clear_Executors(co); } deopt_code(co, _PyCode_CODE(co)); PyMem_Free(co->co_extra); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6f285db4da4237d..aac09ca11426da8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4054,7 +4054,7 @@ dummy_func( op(_CHECK_VALIDITY, (--)) { TIER_TWO_ONLY - DEOPT_IF(current_executor->vm_data.code == NULL); + DEOPT_IF(current_executor->vm_data.valid); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7db6c99a4ac8989..addcc329330de6e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3386,7 +3386,7 @@ case _CHECK_VALIDITY: { TIER_TWO_ONLY - if (current_executor->vm_data.code == NULL) goto deoptimize; + if (current_executor->vm_data.valid) goto deoptimize; break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 9c7d940d761f449..9b30e1cd20e74cb 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -67,9 +67,12 @@ get_index_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) return size; } +static void unlink_executor(_PyExecutorObject *executor); + void _PyExecutor_Remove(_PyExecutorObject *executor) { + unlink_executor(executor); PyCodeObject *code = executor->vm_data.code; if (code == NULL) { return; @@ -82,6 +85,7 @@ _PyExecutor_Remove(_PyExecutorObject *executor) instruction->op.arg = executor->vm_data.oparg; executor->vm_data.code = NULL; Py_CLEAR(code->co_executors->executors[index]); + } static void @@ -90,8 +94,7 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO Py_INCREF(executor); if (instr->op.code == ENTER_EXECUTOR) { assert(index == instr->op.arg); - _PyExecutorObject *old = code->co_executors->executors[index]; - _PyExecutor_Remove(old); + _PyExecutor_Remove(code->co_executors->executors[index]); } else { assert(code->co_executors->size == index); @@ -227,7 +230,7 @@ PyUnstable_GetExecutor(PyCodeObject *code, int offset) static PyObject * is_valid(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.code != NULL); + return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.valid); } static PyMethodDef executor_methods[] = { @@ -1085,7 +1088,7 @@ link_executor(_PyExecutorObject *executor) } head->vm_data.links.next = executor; } - executor->vm_data.linked = true; + executor->vm_data.valid = true; /* executor_list_head must be first in list */ assert(interp->executor_list_head->vm_data.links.previous == NULL); } @@ -1093,7 +1096,7 @@ link_executor(_PyExecutorObject *executor) static void unlink_executor(_PyExecutorObject *executor) { - if (!executor->vm_data.linked) { + if (!executor->vm_data.valid) { return; } _PyExecutorLinkListNode *links = &executor->vm_data.links; @@ -1111,13 +1114,14 @@ unlink_executor(_PyExecutorObject *executor) assert(interp->executor_list_head == executor); interp->executor_list_head = next; } - executor->vm_data.linked = false; + executor->vm_data.valid = false; } /* This must be called by optimizers before using the executor */ void _Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) { + executor->vm_data.valid = true; for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { executor->vm_data.bloom.bits[i] = dependency_set->bits[i]; } @@ -1128,13 +1132,13 @@ _Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) void _Py_ExecutorClear(_PyExecutorObject *executor) { - unlink_executor(executor); + _PyExecutor_Remove(executor); } void _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) { - assert(executor->vm_data.code != NULL); + assert(executor->vm_data.valid); _Py_BloomFilter_Add(&executor->vm_data.bloom, obj); } @@ -1150,10 +1154,9 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) /* Walk the list of executors */ /* TO DO -- Use a tree to avoid traversing as many objects */ for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { - assert(exec->vm_data.code != NULL); + assert(exec->vm_data.valid); _PyExecutorObject *next = exec->vm_data.links.next; if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { - unlink_executor(exec); _PyExecutor_Remove(exec); } exec = next; @@ -1164,13 +1167,15 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) void _Py_Executors_InvalidateAll(PyInterpreterState *interp) { - /* Walk the list of executors */ - while (interp->executor_list_head) { - if (interp->executor_list_head->vm_data.code) { - _PyCode_Clear_Executors(interp->executor_list_head->vm_data.code); + _PyExecutorObject *executor; + while ((executor = interp->executor_list_head)) { + PyCodeObject *code = executor->vm_data.code; + if (code) { + // Clear the entire code object so its co_executors array be freed: + _PyCode_Clear_Executors(code); } else { - unlink_executor(interp->executor_list_head); + _PyExecutor_Remove(executor); } } } From afe211ce3eff35afbe493f881ccd3bae2596706a Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 7 Feb 2024 17:08:02 -0800 Subject: [PATCH 05/18] fixup --- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index aac09ca11426da8..bbe6ca9eb8d7e12 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4054,7 +4054,7 @@ dummy_func( op(_CHECK_VALIDITY, (--)) { TIER_TWO_ONLY - DEOPT_IF(current_executor->vm_data.valid); + DEOPT_IF(!current_executor->vm_data.valid); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index addcc329330de6e..2d914b82dbf88f4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3386,7 +3386,7 @@ case _CHECK_VALIDITY: { TIER_TWO_ONLY - if (current_executor->vm_data.valid) goto deoptimize; + if (!current_executor->vm_data.valid) goto deoptimize; break; } From 29a44d9f77f7136beaca41fbef445f2d1f0e3d1f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 7 Feb 2024 17:16:07 -0800 Subject: [PATCH 06/18] _PyExecutor_Remove -> _Py_ExecutorClear --- Include/cpython/optimizer.h | 1 - Objects/codeobject.c | 2 +- Python/optimizer.c | 46 +++++++++++++++---------------------- 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 82bdaffabea8cc4..37240f59b9bb50f 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -89,7 +89,6 @@ void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp); -void _PyExecutor_Remove(_PyExecutorObject *executor); /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 57e6b2bc3258bf7..0c0e5e95532aaf9 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1492,7 +1492,7 @@ _PyCode_Clear_Executors(PyCodeObject *code) assert(code->co_executors); for (int i = 0; i < code->co_executors->size; i++) { if (code->co_executors->executors[i]) { - _PyExecutor_Remove(code->co_executors->executors[i]); + _Py_ExecutorClear(code->co_executors->executors[i]); } } PyMem_Free(code->co_executors); diff --git a/Python/optimizer.c b/Python/optimizer.c index 9b30e1cd20e74cb..7b1da35122abeda 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -67,34 +67,13 @@ get_index_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) return size; } -static void unlink_executor(_PyExecutorObject *executor); - -void -_PyExecutor_Remove(_PyExecutorObject *executor) -{ - unlink_executor(executor); - PyCodeObject *code = executor->vm_data.code; - if (code == NULL) { - return; - } - _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; - assert(instruction->op.code == ENTER_EXECUTOR); - int index = instruction->op.arg; - assert(code->co_executors->executors[index] == executor); - instruction->op.code = executor->vm_data.opcode; - instruction->op.arg = executor->vm_data.oparg; - executor->vm_data.code = NULL; - Py_CLEAR(code->co_executors->executors[index]); - -} - static void insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorObject *executor) { Py_INCREF(executor); if (instr->op.code == ENTER_EXECUTOR) { assert(index == instr->op.arg); - _PyExecutor_Remove(code->co_executors->executors[index]); + _Py_ExecutorClear(code->co_executors->executors[index]); } else { assert(code->co_executors->size == index); @@ -1132,7 +1111,19 @@ _Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) void _Py_ExecutorClear(_PyExecutorObject *executor) { - _PyExecutor_Remove(executor); + unlink_executor(executor); + PyCodeObject *code = executor->vm_data.code; + if (code == NULL) { + return; + } + _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; + assert(instruction->op.code == ENTER_EXECUTOR); + int index = instruction->op.arg; + assert(code->co_executors->executors[index] == executor); + instruction->op.code = executor->vm_data.opcode; + instruction->op.arg = executor->vm_data.oparg; + executor->vm_data.code = NULL; + Py_CLEAR(code->co_executors->executors[index]); } void @@ -1157,7 +1148,7 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) assert(exec->vm_data.valid); _PyExecutorObject *next = exec->vm_data.links.next; if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { - _PyExecutor_Remove(exec); + _Py_ExecutorClear(exec); } exec = next; } @@ -1169,13 +1160,12 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) { _PyExecutorObject *executor; while ((executor = interp->executor_list_head)) { - PyCodeObject *code = executor->vm_data.code; - if (code) { + if (executor->vm_data.code) { // Clear the entire code object so its co_executors array be freed: - _PyCode_Clear_Executors(code); + _PyCode_Clear_Executors(executor->vm_data.code); } else { - _PyExecutor_Remove(executor); + _Py_ExecutorClear(executor); } } } From 2f0cdd8eeb5b0a38bd09c20b6aa90d0d448040fd Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 7 Feb 2024 17:28:58 -0800 Subject: [PATCH 07/18] Add more asserts --- Python/bytecodes.c | 2 ++ Python/generated_cases.c.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bbe6ca9eb8d7e12..197dff4b9888ce7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2371,7 +2371,9 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); current_executor = code->co_executors->executors[oparg & 255]; + assert(current_executor->vm_data.index == INSTR_OFFSET() - 1); assert(current_executor->vm_data.code == code); + assert(current_executor->vm_data.valid); Py_INCREF(current_executor); GOTO_TIER_TWO(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index de21fe60d85f2ac..e5244147d499af8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2370,7 +2370,9 @@ CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); current_executor = code->co_executors->executors[oparg & 255]; + assert(current_executor->vm_data.index == INSTR_OFFSET() - 1); assert(current_executor->vm_data.code == code); + assert(current_executor->vm_data.valid); Py_INCREF(current_executor); GOTO_TIER_TWO(); DISPATCH(); From 877e9ab49cc48c4baee3fafb56e8cfff6bc6cc98 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 7 Feb 2024 17:31:25 -0800 Subject: [PATCH 08/18] Clean up codeobject.c diff --- Objects/codeobject.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0c0e5e95532aaf9..30336fa86111a71 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1486,17 +1486,23 @@ PyCode_GetFreevars(PyCodeObject *code) return _PyCode_GetFreevars(code); } -void -_PyCode_Clear_Executors(PyCodeObject *code) +static void +clear_executors(PyCodeObject *co) { - assert(code->co_executors); - for (int i = 0; i < code->co_executors->size; i++) { - if (code->co_executors->executors[i]) { - _Py_ExecutorClear(code->co_executors->executors[i]); + assert(co->co_executors); + for (int i = 0; i < co->co_executors->size; i++) { + if (co->co_executors->executors[i]) { + _Py_ExecutorClear(co->co_executors->executors[i]); } } - PyMem_Free(code->co_executors); - code->co_executors = NULL; + PyMem_Free(co->co_executors); + co->co_executors = NULL; +} + +void +_PyCode_Clear_Executors(PyCodeObject *code) +{ + clear_executors(code); } static void @@ -1719,7 +1725,7 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } if (co->co_executors != NULL) { - _PyCode_Clear_Executors(co); + clear_executors(co); } Py_XDECREF(co->co_consts); @@ -2347,7 +2353,7 @@ void _PyStaticCode_Fini(PyCodeObject *co) { if (co->co_executors != NULL) { - _PyCode_Clear_Executors(co); + clear_executors(co); } deopt_code(co, _PyCode_CODE(co)); PyMem_Free(co->co_extra); From 6471c0b0339e6b2154943b0374f625bd41da1591 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 27 Jan 2024 16:48:03 -0800 Subject: [PATCH 09/18] Add sys._clear_executors --- Python/clinic/sysmodule.c.h | 20 +++++++++++++++++++- Python/sysmodule.c | 22 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 93b8385a5b4097c..a7262d6d5367ca9 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1131,6 +1131,24 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__clear_type_cache_impl(module); } +PyDoc_STRVAR(sys__clear_executors__doc__, +"_clear_executors($module, /)\n" +"--\n" +"\n" +"Clear and invalidate all tier two executors."); + +#define SYS__CLEAR_EXECUTORS_METHODDEF \ + {"_clear_executors", (PyCFunction)sys__clear_executors, METH_NOARGS, sys__clear_executors__doc__}, + +static PyObject * +sys__clear_executors_impl(PyObject *module); + +static PyObject * +sys__clear_executors(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__clear_executors_impl(module); +} + PyDoc_STRVAR(sys_is_finalizing__doc__, "is_finalizing($module, /)\n" "--\n" @@ -1486,4 +1504,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=3dc3b2cb0ce38ebb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cc7c8baa106132e5 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 437d7f8dfc49580..1a932db6907f5a3 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2127,6 +2127,27 @@ sys__clear_type_cache_impl(PyObject *module) Py_RETURN_NONE; } +/*[clinic input] +sys._clear_executors + +Clear and invalidate all tier two executors. +[clinic start generated code]*/ + +static PyObject * +sys__clear_executors_impl(PyObject *module) +/*[clinic end generated code: output=edab315b685eb29b input=073c8664b9ae8106]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + for (PyCodeObject *code = interp->code_list_head; code; code = code->_co_list_next) { + if (code->co_executors != NULL) { + _PyCode_Clear_Executors(code); + } + } + // Anything left is currently running, and should exit quickly to tier one: + _Py_Executors_InvalidateAll(interp); + Py_RETURN_NONE; +} + /* Note that, for now, we do not have a per-interpreter equivalent for sys.is_finalizing(). */ @@ -2461,6 +2482,7 @@ static PyMethodDef sys_methods[] = { {"audit", _PyCFunction_CAST(sys_audit), METH_FASTCALL, audit_doc }, {"breakpointhook", _PyCFunction_CAST(sys_breakpointhook), METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, + SYS__CLEAR_EXECUTORS_METHODDEF SYS__CLEAR_TYPE_CACHE_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS__CURRENT_EXCEPTIONS_METHODDEF From 8f3552fb6366d8d0182fb50698e1afb88cda78d8 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 27 Jan 2024 16:48:44 -0800 Subject: [PATCH 10/18] Add test for sys._clear_executors --- Lib/test/test_capi/test_opt.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 5c8c0596610303b..d4ca0f31a3d6e0c 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1,5 +1,6 @@ import contextlib import opcode +import sys import textwrap import unittest @@ -181,6 +182,21 @@ def f(): _testinternalcapi.invalidate_executors(f.__code__) self.assertFalse(exe.is_valid()) + def test_sys__clear_executors(self): + def f(): + for _ in range(1000): + pass + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + sys._clear_executors() + self.assertFalse(exe.is_valid()) + exe = get_first_executor(f) + self.assertIsNone(exe) + class TestUops(unittest.TestCase): def test_basic_loop(self): From 0e82bd9ac2005af057e2d9da4ab8d10beedc5fe0 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 27 Jan 2024 16:50:20 -0800 Subject: [PATCH 11/18] Add docs for sys._clear_executors --- Doc/library/sys.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a97a369b77b88a8..a48721f627e6019 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -196,6 +196,16 @@ always available. This function should be used for internal and specialized purposes only. +.. function:: _clear_executors() + + Clear and invalidate all tier two executors, which are used to speed up "hot" + Python code. + + This function should be used for internal and specialized purposes only. + + .. versionadded:: 3.13 + + .. function:: _current_frames() Return a dictionary mapping each thread's identifier to the topmost stack frame @@ -724,7 +734,7 @@ always available. regardless of their size. This function is mainly useful for tracking and debugging memory leaks. Because of the interpreter's internal caches, the result can vary from call to call; you may have to call - :func:`_clear_type_cache()` and :func:`gc.collect()` to get more + :func:`_clear_type_cache()`, :func:`_clear_executors()`, and :func:`gc.collect()` to get more predictable results. If a Python build or implementation cannot reasonably compute this From e3c4bf777c5eb36e718482eb50350328a74d5dcb Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 27 Jan 2024 16:50:51 -0800 Subject: [PATCH 12/18] Use sys._clear_executors during refleak runs --- Lib/test/libregrtest/refleak.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 6e54b464a497154..edb0cccfc08c82d 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -200,15 +200,9 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear caches clear_caches() - - # Clear executors and type cache at the end, since the previous function - # calls can create executors and modify types: - try: - import _testinternalcapi - except ImportError: - pass - else: - _testinternalcapi.invalidate_executors() + # Clear executors and type cache at the end (previous function calls can + # create new executors and modify types): + sys._clear_executors() sys._clear_type_cache() From 438eeacc3bb3b784a7aaee6af91eb486e760bfd5 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 7 Feb 2024 17:49:46 -0800 Subject: [PATCH 13/18] Remove the _testinternalcapi stuff --- Include/cpython/optimizer.h | 2 +- Modules/_testinternalcapi.c | 22 +++--------------- Modules/clinic/_testinternalcapi.c.h | 34 +--------------------------- Python/sysmodule.c | 9 +------- 4 files changed, 6 insertions(+), 61 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 37240f59b9bb50f..e3faffb4491fc82 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -88,7 +88,7 @@ void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); -PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp); +extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp); /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 3483b951e16e239..0bb739b5398b113 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1029,27 +1029,11 @@ add_executor_dependency(PyObject *self, PyObject *args) Py_RETURN_NONE; } -/*[clinic input] - -_testinternalcapi.invalidate_executors -> object - - dependency: object = NULL - / - -[clinic start generated code]*/ - static PyObject * -_testinternalcapi_invalidate_executors_impl(PyObject *module, - PyObject *dependency) -/*[clinic end generated code: output=79497cfcf0826eaa input=ffbb09a1f7848473]*/ +invalidate_executors(PyObject *self, PyObject *obj) { PyInterpreterState *interp = PyInterpreterState_Get(); - if (dependency) { - _Py_Executors_InvalidateDependency(interp, dependency); - } - else { - _Py_Executors_InvalidateAll(interp); - } + _Py_Executors_InvalidateDependency(interp, obj); Py_RETURN_NONE; } @@ -1714,7 +1698,7 @@ static PyMethodDef module_functions[] = { {"get_counter_optimizer", get_counter_optimizer, METH_NOARGS, NULL}, {"get_uop_optimizer", get_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, - _TESTINTERNALCAPI_INVALIDATE_EXECUTORS_METHODDEF + {"invalidate_executors", invalidate_executors, METH_O, NULL}, {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 44ae5f3fe1f886e..cba2a943d03456a 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -266,38 +266,6 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, return return_value; } -PyDoc_STRVAR(_testinternalcapi_invalidate_executors__doc__, -"invalidate_executors($module, dependency=, /)\n" -"--\n" -"\n"); - -#define _TESTINTERNALCAPI_INVALIDATE_EXECUTORS_METHODDEF \ - {"invalidate_executors", _PyCFunction_CAST(_testinternalcapi_invalidate_executors), METH_FASTCALL, _testinternalcapi_invalidate_executors__doc__}, - -static PyObject * -_testinternalcapi_invalidate_executors_impl(PyObject *module, - PyObject *dependency); - -static PyObject * -_testinternalcapi_invalidate_executors(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *dependency = NULL; - - if (!_PyArg_CheckPositional("invalidate_executors", nargs, 0, 1)) { - goto exit; - } - if (nargs < 1) { - goto skip_optional; - } - dependency = args[0]; -skip_optional: - return_value = _testinternalcapi_invalidate_executors_impl(module, dependency); - -exit: - return return_value; -} - PyDoc_STRVAR(_testinternalcapi_test_long_numbits__doc__, "test_long_numbits($module, /)\n" "--\n" @@ -314,4 +282,4 @@ _testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignore { return _testinternalcapi_test_long_numbits_impl(module); } -/*[clinic end generated code: output=c4949d6611a6c3a2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=679bf53bbae20085 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 1a932db6907f5a3..87d6c2a30b0d206 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2137,14 +2137,7 @@ static PyObject * sys__clear_executors_impl(PyObject *module) /*[clinic end generated code: output=edab315b685eb29b input=073c8664b9ae8106]*/ { - PyInterpreterState *interp = _PyInterpreterState_GET(); - for (PyCodeObject *code = interp->code_list_head; code; code = code->_co_list_next) { - if (code->co_executors != NULL) { - _PyCode_Clear_Executors(code); - } - } - // Anything left is currently running, and should exit quickly to tier one: - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(_PyInterpreterState_GET()); Py_RETURN_NONE; } From 28beba771e7e9e94a0210a2a3fae9349674f31c0 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 7 Feb 2024 18:04:45 -0800 Subject: [PATCH 14/18] blurb add --- .../2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst new file mode 100644 index 000000000000000..bc1d09caf23363f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst @@ -0,0 +1,2 @@ +Add :func:`sys._clear_executors`, clears and invalidates all tier two +executors. From a171c00b464b07d66b2cc816efa824cf15a25fc0 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 8 Feb 2024 20:04:12 -0800 Subject: [PATCH 15/18] sys._clear_executors -> sys._clear_internal_caches --- Doc/library/sys.rst | 11 ++++++----- Lib/test/libregrtest/refleak.py | 6 ++---- Lib/test/test_capi/test_opt.py | 4 ++-- ...4-02-07-18-04-36.gh-issue-114695.o9wP5P.rst | 5 +++-- Python/clinic/sysmodule.c.h | 18 +++++++++--------- Python/sysmodule.c | 14 ++++++++------ 6 files changed, 30 insertions(+), 28 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a48721f627e6019..7a1f517f44ebac9 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -195,13 +195,14 @@ always available. This function should be used for internal and specialized purposes only. + .. deprecated:: 3.13 + Use the more general :func:`_clear_internal_caches` function instead. -.. function:: _clear_executors() - Clear and invalidate all tier two executors, which are used to speed up "hot" - Python code. +.. function:: _clear_internal_caches() - This function should be used for internal and specialized purposes only. + Clear all internal performance-related caches. Use this function *only* to + release unnecessary references and memory blocks when hunting for leaks. .. versionadded:: 3.13 @@ -734,7 +735,7 @@ always available. regardless of their size. This function is mainly useful for tracking and debugging memory leaks. Because of the interpreter's internal caches, the result can vary from call to call; you may have to call - :func:`_clear_type_cache()`, :func:`_clear_executors()`, and :func:`gc.collect()` to get more + :func:`_clear_internal_caches()`, and :func:`gc.collect()` to get more predictable results. If a Python build or implementation cannot reasonably compute this diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index edb0cccfc08c82d..745080a48c518d4 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -200,10 +200,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear caches clear_caches() - # Clear executors and type cache at the end (previous function calls can - # create new executors and modify types): - sys._clear_executors() - sys._clear_type_cache() + # Clear other caches last (previous function calls can re-populate them): + sys._clear_internal_caches() def warm_caches(): diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d4ca0f31a3d6e0c..e6b1b554c9af10e 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -182,7 +182,7 @@ def f(): _testinternalcapi.invalidate_executors(f.__code__) self.assertFalse(exe.is_valid()) - def test_sys__clear_executors(self): + def test_sys__clear_internal_caches(self): def f(): for _ in range(1000): pass @@ -192,7 +192,7 @@ def f(): exe = get_first_executor(f) self.assertIsNotNone(exe) self.assertTrue(exe.is_valid()) - sys._clear_executors() + sys._clear_internal_caches() self.assertFalse(exe.is_valid()) exe = get_first_executor(f) self.assertIsNone(exe) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst index bc1d09caf23363f..a1db4de393eecb3 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst @@ -1,2 +1,3 @@ -Add :func:`sys._clear_executors`, clears and invalidates all tier two -executors. +Add :func:`sys._clear_internal_caches`, which clears all internal +performance-related caches (and deprecate the less-general +:func:`sys._clear_type_cache` function). diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index a7262d6d5367ca9..13f4ea81eb89842 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1131,22 +1131,22 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__clear_type_cache_impl(module); } -PyDoc_STRVAR(sys__clear_executors__doc__, -"_clear_executors($module, /)\n" +PyDoc_STRVAR(sys__clear_internal_caches__doc__, +"_clear_internal_caches($module, /)\n" "--\n" "\n" -"Clear and invalidate all tier two executors."); +"Clear all internal performance-related caches."); -#define SYS__CLEAR_EXECUTORS_METHODDEF \ - {"_clear_executors", (PyCFunction)sys__clear_executors, METH_NOARGS, sys__clear_executors__doc__}, +#define SYS__CLEAR_INTERNAL_CACHES_METHODDEF \ + {"_clear_internal_caches", (PyCFunction)sys__clear_internal_caches, METH_NOARGS, sys__clear_internal_caches__doc__}, static PyObject * -sys__clear_executors_impl(PyObject *module); +sys__clear_internal_caches_impl(PyObject *module); static PyObject * -sys__clear_executors(PyObject *module, PyObject *Py_UNUSED(ignored)) +sys__clear_internal_caches(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return sys__clear_executors_impl(module); + return sys__clear_internal_caches_impl(module); } PyDoc_STRVAR(sys_is_finalizing__doc__, @@ -1504,4 +1504,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=cc7c8baa106132e5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b8b1c53e04c3b20c input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 87d6c2a30b0d206..69b6d886ccc3e90 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2128,16 +2128,18 @@ sys__clear_type_cache_impl(PyObject *module) } /*[clinic input] -sys._clear_executors +sys._clear_internal_caches -Clear and invalidate all tier two executors. +Clear all internal performance-related caches. [clinic start generated code]*/ static PyObject * -sys__clear_executors_impl(PyObject *module) -/*[clinic end generated code: output=edab315b685eb29b input=073c8664b9ae8106]*/ +sys__clear_internal_caches_impl(PyObject *module) +/*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/ { - _Py_Executors_InvalidateAll(_PyInterpreterState_GET()); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_Executors_InvalidateAll(interp); + PyType_ClearCache(); Py_RETURN_NONE; } @@ -2475,7 +2477,7 @@ static PyMethodDef sys_methods[] = { {"audit", _PyCFunction_CAST(sys_audit), METH_FASTCALL, audit_doc }, {"breakpointhook", _PyCFunction_CAST(sys_breakpointhook), METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, - SYS__CLEAR_EXECUTORS_METHODDEF + SYS__CLEAR_INTERNAL_CACHES_METHODDEF SYS__CLEAR_TYPE_CACHE_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS__CURRENT_EXCEPTIONS_METHODDEF From 3a5acbc6bf6a85f58bc7bb5c23a126896543b636 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 8 Feb 2024 20:04:28 -0800 Subject: [PATCH 16/18] Feedback from code review --- Include/cpython/optimizer.h | 2 +- Python/optimizer.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index e3faffb4491fc82..3928eca583ba5b7 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -24,10 +24,10 @@ typedef struct { uint8_t opcode; uint8_t oparg; uint8_t valid; + int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). - int index; // Index of ENTER_EXECUTOR in the above code object. } _PyVMData; typedef struct { diff --git a/Python/optimizer.c b/Python/optimizer.c index 7b1da35122abeda..ad9ac382d300ef1 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -83,7 +83,7 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO executor->vm_data.opcode = instr->op.code; executor->vm_data.oparg = instr->op.arg; executor->vm_data.code = code; - executor->vm_data.index = instr - _PyCode_CODE(code); + executor->vm_data.index = (int)(instr - _PyCode_CODE(code)); code->co_executors->executors[index] = executor; assert(index < MAX_EXECUTORS_SIZE); instr->op.code = ENTER_EXECUTOR; @@ -1158,8 +1158,8 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) void _Py_Executors_InvalidateAll(PyInterpreterState *interp) { - _PyExecutorObject *executor; - while ((executor = interp->executor_list_head)) { + while (interp->executor_list_head) { + _PyExecutorObject *executor = interp->executor_list_head; if (executor->vm_data.code) { // Clear the entire code object so its co_executors array be freed: _PyCode_Clear_Executors(executor->vm_data.code); From bf4177afa6d27ae39f36786775ca0c723f637151 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 9 Feb 2024 07:34:35 -0800 Subject: [PATCH 17/18] fixup --- Doc/library/sys.rst | 2 +- Lib/test/libregrtest/refleak.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 7a1f517f44ebac9..ad8857fc2807f70 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -735,7 +735,7 @@ always available. regardless of their size. This function is mainly useful for tracking and debugging memory leaks. Because of the interpreter's internal caches, the result can vary from call to call; you may have to call - :func:`_clear_internal_caches()`, and :func:`gc.collect()` to get more + :func:`_clear_internal_caches()` and :func:`gc.collect()` to get more predictable results. If a Python build or implementation cannot reasonably compute this diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 745080a48c518d4..71a70af6882d16f 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -200,6 +200,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear caches clear_caches() + # Clear other caches last (previous function calls can re-populate them): sys._clear_internal_caches() From c656e229a4ee0bdb7b07e4878cb423b919355460 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 9 Feb 2024 08:16:09 -0800 Subject: [PATCH 18/18] Fix test_mailbox --- Lib/test/test_mailbox.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index c52c014185bec7a..d4628f91daf7e85 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -10,6 +10,7 @@ import tempfile from test import support from test.support import os_helper +from test.support import refleak_helper from test.support import socket_helper import unittest import textwrap @@ -2443,6 +2444,9 @@ def test__all__(self): def tearDownModule(): support.reap_children() + # reap_children may have re-populated caches: + if refleak_helper.hunting_for_refleaks(): + sys._clear_internal_caches() if __name__ == '__main__':