From b8767c330b7d87611dac87978807f9e44806d5db Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Tue, 21 May 2024 13:32:51 -0400 Subject: [PATCH 01/16] Add c-api to have contextvar enter/exit callbacks --- Doc/c-api/contextvars.rst | 44 ++++++++ Include/cpython/context.h | 37 +++++++ Include/internal/pycore_context.h | 1 + Include/internal/pycore_interp.h | 2 + Lib/test/test_capi/test_watchers.py | 85 ++++++++++++++++ Modules/_testcapi/watchers.c | 152 ++++++++++++++++++++++++++++ Python/context.c | 78 ++++++++++++++ Python/pystate.c | 5 + 8 files changed, 404 insertions(+) diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index fe7b8f93f2c6cf..39e4eacf14915b 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -101,6 +101,50 @@ Context object management functions: current context for the current thread. Returns ``0`` on success, and ``-1`` on error. +.. c:function:: int PyContext_AddWatcher(PyCode_WatchCallback callback) + + Register *callback* as a context object watcher for the current interpreter. + Return an ID which may be passed to :c:func:`PyContext_ClearWatcher`. + In case of error (e.g. no more watcher IDs available), + return ``-1`` and set an exception. + + .. versionadded:: 3.14 + +.. c:function:: int PyContext_ClearWatcher(int watcher_id) + + Clear watcher identified by *watcher_id* previously returned from + :c:func:`PyContext_AddWatcher` for the current interpreter. + Return ``0`` on success, or ``-1`` and set an exception on error + (e.g. if the given *watcher_id* was never registered.) + + .. versionadded:: 3.14 + +.. c:type:: PyContextEvent + Enumeration of possible context object watcher events: + - ``PY_CONTEXT_EVENT_ENTER`` + - ``PY_CONTEXT_EVENT_EXIT`` + + .. versionadded:: 3.14 + +.. c:type:: int (*PyContecxt_WatchCallback)(PyContextEvent event, PyContext* ctx) + Type of a context object watcher callback function. + If *event* is ``PY_CONTEXT_EVENT_ENTER``, then the callback is invoked + after `ctx` has been set as the current context for the current thread. + Otherwise, the callback is invoked before the deactivation of *ctx* as the current context + and the restoration of the previous contex object for the current thread. + + If the callback returns with an exception set, it must return ``-1``; this + exception will be printed as an unraisable exception using + :c:func:`PyErr_FormatUnraisable`. Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. + + .. versionadded:: 3.14 + Context variable functions: diff --git a/Include/cpython/context.h b/Include/cpython/context.h index a3249fc29b082e..d59876004bdd2d 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -27,6 +27,43 @@ PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void); PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); +#define PY_FOREACH_CONTEXT_EVENT(V) \ + V(ENTER) \ + V(EXIT) + +typedef enum { + #define PY_DEF_EVENT(op) PY_CONTEXT_EVENT_##op, + PY_FOREACH_CONTEXT_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT +} PyContextEvent; + +/* + * A Callback to clue in non-python contexts impls about a + * change in the active python context. + * + * The callback is invoked with the event and a reference to = + * the context after its entered and before its exited. + * + * if the callback returns with an exception set, it must return -1. Otherwise + * it should return 0 + */ +typedef int (*PyContext_WatchCallback)(PyContextEvent, PyContext *); + +/* + * Register a per-interpreter callback that will be invoked for context object + * enter/exit events. + * + * Returns a handle that may be passed to PyContext_ClearWatcher on success, + * or -1 and sets and error if no more handles are available. + */ +PyAPI_FUNC(int) PyContext_AddWatcher(PyContext_WatchCallback callback); + +/* + * Clear the watcher associated with the watcher_id handle. + * + * Returns 0 on success or -1 if no watcher exists for the provided id. + */ +PyAPI_FUNC(int) PyContext_ClearWatcher(int watcher_id); /* Create a new context variable. diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 2ecb40ba584f7f..c2b98d15da68fa 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -7,6 +7,7 @@ #include "pycore_hamt.h" // PyHamtObject +#define CONTEXT_MAX_WATCHERS 8 extern PyTypeObject _PyContextTokenMissing_Type; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index a1c1dd0c957230..36366429e8db25 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -240,8 +240,10 @@ struct _is { PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; PyCode_WatchCallback code_watchers[CODE_MAX_WATCHERS]; + PyContext_WatchCallback context_watchers[CONTEXT_MAX_WATCHERS]; // One bit is set for each non-NULL entry in code_watchers uint8_t active_code_watchers; + uint8_t active_context_watchers; struct _py_object_state object_state; struct _Py_unicode_state unicode; diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 709b5e1c4b716a..6324bd0b744af0 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -1,4 +1,5 @@ import unittest +import contextvars from contextlib import contextmanager, ExitStack from test.support import ( @@ -571,5 +572,89 @@ def test_allocate_too_many_watchers(self): _testcapi.allocate_too_many_func_watchers() +class TestContextObjectWatchers(unittest.TestCase): + @contextmanager + def context_watcher(self, which_watcher): + wid = _testcapi.add_context_watcher(which_watcher) + try: + yield wid + finally: + _testcapi.clear_context_watcher(wid) + + def assert_event_counts(self, exp_enter_0, exp_exit_0, + exp_enter_1, exp_exit_1): + self.assertEqual( + exp_enter_0, _testcapi.get_context_watcher_num_enter_events(0)) + self.assertEqual( + exp_exit_0, _testcapi.get_context_watcher_num_exit_events(0)) + self.assertEqual( + exp_enter_1, _testcapi.get_context_watcher_num_enter_events(1)) + self.assertEqual( + exp_exit_1, _testcapi.get_context_watcher_num_exit_events(1)) + + def test_context_object_events_dispatched(self): + # verify that all counts are zero before any watchers are registered + self.assert_event_counts(0, 0, 0, 0) + + # verify that all counts remain zero when a context object is + # entered and exited with no watchers registered + ctx = contextvars.copy_context() + ctx.run(self.assert_event_counts, 0, 0, 0, 0) + self.assert_event_counts(0, 0, 0, 0) + + # verify counts are as expected when first watcher is registered + with self.context_watcher(0): + self.assert_event_counts(0, 0, 0, 0) + ctx.run(self.assert_event_counts, 1, 0, 0, 0) + self.assert_event_counts(1, 1, 0, 0) + + # again with second watcher registered + with self.context_watcher(1): + self.assert_event_counts(1, 1, 0, 0) + ctx.run(self.assert_event_counts, 2, 1, 1, 0) + self.assert_event_counts(2, 2, 1, 1) + + # verify counts are reset and don't change after both watchers are cleared + ctx.run(self.assert_event_counts, 0, 0, 0, 0) + self.assert_event_counts(0, 0, 0, 0) + + def test_enter_error(self): + with self.context_watcher(2): + with catch_unraisable_exception() as cm: + ctx = contextvars.copy_context() + ctx.run(int, 0) + self.assertEqual( + cm.unraisable.object, + ctx + # For main branch + # f"PY_CONTEXT_EVENT_ENTER watcher callback for {ctx!r}" + # + ) + self.assertEqual(str(cm.unraisable.exc_value), "boom!") + + def test_exit_error(self): + ctx = contextvars.copy_context() + def _in_context(stack): + stack.enter_context(self.context_watcher(2)) + + with catch_unraisable_exception() as cm: + with ExitStack() as stack: + ctx.run(_in_context, stack) + self.assertEqual(str(cm.unraisable.exc_value), "boom!") + + def test_clear_out_of_range_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"Invalid context watcher ID -1"): + _testcapi.clear_context_watcher(-1) + with self.assertRaisesRegex(ValueError, r"Invalid context watcher ID 8"): + _testcapi.clear_context_watcher(8) # CONTEXT_MAX_WATCHERS = 8 + + def test_clear_unassigned_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"No context watcher set for ID 1"): + _testcapi.clear_context_watcher(1) + + def test_allocate_too_many_watchers(self): + with self.assertRaisesRegex(RuntimeError, r"no more context watcher IDs available"): + _testcapi.allocate_too_many_context_watchers() + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 1eb0db2c2e6576..624aa627c6af4e 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -8,6 +8,7 @@ #define Py_BUILD_CORE #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_code.h" // CODE_MAX_WATCHERS +#include "pycore_context.h" // CONTEXT_MAX_WATCHERS /*[clinic input] module _testcapi @@ -622,6 +623,147 @@ allocate_too_many_func_watchers(PyObject *self, PyObject *args) Py_RETURN_NONE; } +// Test contexct object watchers +#define NUM_CONTEXT_WATCHERS 2 +static context_watcher_ids[NUM_CONTEXT_WATCHERS] = {-1, -1}; +static int num_context_object_enter_events[NUM_CONTEXT_WATCHERS] = {0, 0}; +static int num_context_object_exit_events[NUM_CONTEXT_WATCHERS] = {0, 0}; + +static int +handle_context_watcher_event(int which_watcher, PyContextEvent event, PyContext *ctx) { + if (event == PY_CONTEXT_EVENT_ENTER) { + num_context_object_enter_events[which_watcher]++; + } + else if (event == PY_CONTEXT_EVENT_EXIT) { + num_context_object_exit_events[which_watcher]++; + } + else { + return -1; + } + return 0; +} + +static int +first_context_watcher_callback(PyContextEvent event, PyContext *ctx) { + return handle_context_watcher_event(0, event, ctx); +} + +static int +second_context_watcher_callback(PyContextEvent event, PyContext *ctx) { + return handle_context_watcher_event(1, event, ctx); +} + +static int +noop_context_event_handler(PyContextEvent event, PyContext *ctx) { + return 0; +} + +static int +error_context_event_handler(PyContextEvent event, PyContext *ctx) { + PyErr_SetString(PyExc_RuntimeError, "boom!"); + return -1; +} + +static PyObject * +add_context_watcher(PyObject *self, PyObject *which_watcher) +{ + int watcher_id; + assert(PyLong_Check(which_watcher)); + long which_l = PyLong_AsLong(which_watcher); + if (which_l == 0) { + watcher_id = PyContext_AddWatcher(first_context_watcher_callback); + context_watcher_ids[0] = watcher_id; + num_context_object_enter_events[0] = 0; + num_context_object_exit_events[0] = 0; + } + else if (which_l == 1) { + watcher_id = PyContext_AddWatcher(second_context_watcher_callback); + context_watcher_ids[1] = watcher_id; + num_context_object_enter_events[1] = 0; + num_context_object_exit_events[1] = 0; + } + else if (which_l == 2) { + watcher_id = PyContext_AddWatcher(error_context_event_handler); + } + else { + PyErr_Format(PyExc_ValueError, "invalid watcher %d", which_l); + return NULL; + } + if (watcher_id < 0) { + return NULL; + } + return PyLong_FromLong(watcher_id); +} + +static PyObject * +clear_context_watcher(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + if (PyContext_ClearWatcher(watcher_id_l) < 0) { + return NULL; + } + // reset static events counters + if (watcher_id_l >= 0) { + for (int i = 0; i < NUM_CONTEXT_WATCHERS; i++) { + if (watcher_id_l == context_watcher_ids[i]) { + context_watcher_ids[i] = -1; + num_context_object_enter_events[i] = 0; + num_context_object_exit_events[i] = 0; + } + } + } + Py_RETURN_NONE; +} + +static PyObject * +get_context_watcher_num_enter_events(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + assert(watcher_id_l >= 0 && watcher_id_l < NUM_CONTEXT_WATCHERS); + return PyLong_FromLong(num_context_object_enter_events[watcher_id_l]); +} + +static PyObject * +get_context_watcher_num_exit_events(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + assert(watcher_id_l >= 0 && watcher_id_l < NUM_CONTEXT_WATCHERS); + return PyLong_FromLong(num_context_object_exit_events[watcher_id_l]); +} + +static PyObject * +allocate_too_many_context_watchers(PyObject *self, PyObject *args) +{ + int watcher_ids[CONTEXT_MAX_WATCHERS + 1]; + int num_watchers = 0; + for (unsigned long i = 0; i < sizeof(watcher_ids) / sizeof(int); i++) { + int watcher_id = PyContext_AddWatcher(noop_context_event_handler); + if (watcher_id == -1) { + break; + } + watcher_ids[i] = watcher_id; + num_watchers++; + } + PyObject *exc = PyErr_GetRaisedException(); + for (int i = 0; i < num_watchers; i++) { + if (PyContext_ClearWatcher(watcher_ids[i]) < 0) { + PyErr_WriteUnraisable(Py_None); + break; + } + } + if (exc) { + PyErr_SetRaisedException(exc); + return NULL; + } + else if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] _testcapi.set_func_defaults_via_capi func: object @@ -689,6 +831,16 @@ static PyMethodDef test_methods[] = { _TESTCAPI_SET_FUNC_KWDEFAULTS_VIA_CAPI_METHODDEF {"allocate_too_many_func_watchers", allocate_too_many_func_watchers, METH_NOARGS, NULL}, + + // Code object watchers. + {"add_context_watcher", add_context_watcher, METH_O, NULL}, + {"clear_context_watcher", clear_context_watcher, METH_O, NULL}, + {"get_context_watcher_num_enter_events", + get_context_watcher_num_enter_events, METH_O, NULL}, + {"get_context_watcher_num_exit_events", + get_context_watcher_num_exit_events, METH_O, NULL}, + {"allocate_too_many_context_watchers", + (PyCFunction) allocate_too_many_context_watchers, METH_NOARGS, NULL}, {NULL}, }; diff --git a/Python/context.c b/Python/context.c index 5cafde4dab9336..da396d0d072f17 100644 --- a/Python/context.c +++ b/Python/context.c @@ -99,6 +99,82 @@ PyContext_CopyCurrent(void) return (PyObject *)context_new_from_vars(ctx->ctx_vars); } +static const char * +context_event_name(PyContextEvent event) { + switch (event) { + #define CASE(op) \ + case PY_CONTEXT_EVENT_##op: \ + return "PY_CONTEXT_EVENT_" #op; + PY_FOREACH_CONTEXT_EVENT(CASE) + #undef CASE + } + Py_UNREACHABLE(); +} + +static void notify_context_watchers(PyContextEvent event, PyContext *ctx) +{ + assert(Py_REFCNT(ctx) > 0); + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + uint8_t bits = interp->active_context_watchers; + int i = 0; + while (bits) { + assert(i < CONTEXT_MAX_WATCHERS); + if (bits & 1) { + PyContext_WatchCallback cb = interp->context_watchers[i]; + assert(cb != NULL); + if (cb(event, ctx) < 0) { + PyErr_WriteUnraisable((PyObject *) ctx); + /* For upstream pull req on main + * PyErr_FormatUnraisable( + * "Exception ignored in %s watcher callback for %R", + * context_event_name(event), ctx); + */ + } + } + i++; + bits >>= 1; + } +} + + +int +PyContext_AddWatcher(PyContext_WatchCallback callback) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + + for (int i = 0; i < CONTEXT_MAX_WATCHERS; i++) { + if (!interp->context_watchers[i]) { + interp->context_watchers[i] = callback; + interp->active_context_watchers |= (1 << i); + return i; + } + } + + PyErr_SetString(PyExc_RuntimeError, "no more context watcher IDs available"); + return -1; +} + + +int +PyContext_ClearWatcher(int watcher_id) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + if (watcher_id < 0 || watcher_id >= CONTEXT_MAX_WATCHERS) { + PyErr_Format(PyExc_ValueError, "Invalid context watcher ID %d", watcher_id); + return -1; + } + if (!interp->context_watchers[watcher_id]) { + PyErr_Format(PyExc_ValueError, "No context watcher set for ID %d", watcher_id); + return -1; + } + interp->context_watchers[watcher_id] = NULL; + interp->active_context_watchers &= ~(1 << watcher_id); + return 0; +} + static int _PyContext_Enter(PyThreadState *ts, PyObject *octx) @@ -118,6 +194,7 @@ _PyContext_Enter(PyThreadState *ts, PyObject *octx) ts->context = Py_NewRef(ctx); ts->context_ver++; + notify_context_watchers(PY_CONTEXT_EVENT_ENTER, ctx); return 0; } @@ -151,6 +228,7 @@ _PyContext_Exit(PyThreadState *ts, PyObject *octx) return -1; } + notify_context_watchers(PY_CONTEXT_EVENT_EXIT, ctx); Py_SETREF(ts->context, (PyObject *)ctx->ctx_prev); ts->context_ver++; diff --git a/Python/pystate.c b/Python/pystate.c index 54caf373e91d6c..6bf7ebeb75ff73 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -906,6 +906,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->code_watchers[i] = NULL; } interp->active_code_watchers = 0; + + for (int i=0; i < CONTEXT_MAX_WATCHERS; i++) { + interp->context_watchers[i] = NULL; + } + interp->active_context_watchers = 0; // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. From 55e642af042cba80b80a872aa8e4c7f669561bb5 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 18:28:45 +0000 Subject: [PATCH 02/16] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst diff --git a/Misc/NEWS.d/next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst b/Misc/NEWS.d/next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst new file mode 100644 index 00000000000000..a5c84cf4938338 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyContext_AddWatcher` and :c:func:`PyContext_ClearWatcher` APIs to +register callbacks to receive notification on enter and exit of context objects. From 1e0317a3ef7b5ba652e8905f5efa10e724294e3c Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Tue, 21 May 2024 14:31:15 -0400 Subject: [PATCH 03/16] fix exception codes for 3.13+ --- Lib/test/test_capi/test_watchers.py | 5 +---- Python/context.c | 9 +++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 6324bd0b744af0..7caf31c8fe6249 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -625,10 +625,7 @@ def test_enter_error(self): ctx.run(int, 0) self.assertEqual( cm.unraisable.object, - ctx - # For main branch - # f"PY_CONTEXT_EVENT_ENTER watcher callback for {ctx!r}" - # + f"PY_CONTEXT_EVENT_ENTER watcher callback for {ctx!r}" ) self.assertEqual(str(cm.unraisable.exc_value), "boom!") diff --git a/Python/context.c b/Python/context.c index da396d0d072f17..87425977df24c7 100644 --- a/Python/context.c +++ b/Python/context.c @@ -124,12 +124,9 @@ static void notify_context_watchers(PyContextEvent event, PyContext *ctx) PyContext_WatchCallback cb = interp->context_watchers[i]; assert(cb != NULL); if (cb(event, ctx) < 0) { - PyErr_WriteUnraisable((PyObject *) ctx); - /* For upstream pull req on main - * PyErr_FormatUnraisable( - * "Exception ignored in %s watcher callback for %R", - * context_event_name(event), ctx); - */ + PyErr_FormatUnraisable( + "Exception ignored in %s watcher callback for %R", + context_event_name(event), ctx); } } i++; From 0d54500e330316655fd2cd44811073ff2c306628 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Tue, 21 May 2024 14:38:27 -0400 Subject: [PATCH 04/16] fix docs --- Doc/c-api/contextvars.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index 39e4eacf14915b..9eb4d23819ddb4 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -101,7 +101,7 @@ Context object management functions: current context for the current thread. Returns ``0`` on success, and ``-1`` on error. -.. c:function:: int PyContext_AddWatcher(PyCode_WatchCallback callback) +.. c:function:: int PyContext_AddWatcher(PyContext_WatchCallback callback) Register *callback* as a context object watcher for the current interpreter. Return an ID which may be passed to :c:func:`PyContext_ClearWatcher`. @@ -120,6 +120,7 @@ Context object management functions: .. versionadded:: 3.14 .. c:type:: PyContextEvent + Enumeration of possible context object watcher events: - ``PY_CONTEXT_EVENT_ENTER`` - ``PY_CONTEXT_EVENT_EXIT`` @@ -127,6 +128,7 @@ Context object management functions: .. versionadded:: 3.14 .. c:type:: int (*PyContecxt_WatchCallback)(PyContextEvent event, PyContext* ctx) + Type of a context object watcher callback function. If *event* is ``PY_CONTEXT_EVENT_ENTER``, then the callback is invoked after `ctx` has been set as the current context for the current thread. From 3592633c3454fbf949891331c015014cefdfc9ec Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Tue, 21 May 2024 15:57:42 -0400 Subject: [PATCH 05/16] remove weird line endings from the blurbit tool --- .../next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst b/Misc/NEWS.d/next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst index a5c84cf4938338..6fb6013c4d442d 100644 --- a/Misc/NEWS.d/next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst +++ b/Misc/NEWS.d/next/C API/2024-05-21-18-28-44.gh-issue-119333.OTsYVX.rst @@ -1,2 +1,2 @@ -Add :c:func:`PyContext_AddWatcher` and :c:func:`PyContext_ClearWatcher` APIs to +Add :c:func:`PyContext_AddWatcher` and :c:func:`PyContext_ClearWatcher` APIs to register callbacks to receive notification on enter and exit of context objects. From 80d6f55a410657a06f5d8caa7e8a5436c728e0c3 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Tue, 21 May 2024 16:18:36 -0400 Subject: [PATCH 06/16] fix unraisable error handling in test_capi --- Lib/test/test_capi/test_watchers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 7caf31c8fe6249..6e62dd039efd6f 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -624,8 +624,9 @@ def test_enter_error(self): ctx = contextvars.copy_context() ctx.run(int, 0) self.assertEqual( - cm.unraisable.object, - f"PY_CONTEXT_EVENT_ENTER watcher callback for {ctx!r}" + cm.unraisable.err_msg, + "Exception ignored in " + f"PY_CONTEXT_EVENT_EXIT watcher callback for {ctx!r}" ) self.assertEqual(str(cm.unraisable.exc_value), "boom!") From 9ec2260425e788ecd9a20543d34f7eebe87eaf02 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Wed, 22 May 2024 09:58:05 -0400 Subject: [PATCH 07/16] fix typos and naming for pep7 --- Doc/c-api/contextvars.rst | 4 ++-- Include/cpython/context.h | 8 ++++---- Modules/_testcapi/watchers.c | 2 +- Python/context.c | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index 9eb4d23819ddb4..4fbeeb0a41a729 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -127,11 +127,11 @@ Context object management functions: .. versionadded:: 3.14 -.. c:type:: int (*PyContecxt_WatchCallback)(PyContextEvent event, PyContext* ctx) +.. c:type:: int (*PyContext_WatchCallback)(PyContextEvent event, PyContext* ctx) Type of a context object watcher callback function. If *event* is ``PY_CONTEXT_EVENT_ENTER``, then the callback is invoked - after `ctx` has been set as the current context for the current thread. + after *ctx* has been set as the current context for the current thread. Otherwise, the callback is invoked before the deactivation of *ctx* as the current context and the restoration of the previous contex object for the current thread. diff --git a/Include/cpython/context.h b/Include/cpython/context.h index d59876004bdd2d..186f0a167964ee 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -27,14 +27,14 @@ PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void); PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); -#define PY_FOREACH_CONTEXT_EVENT(V) \ +#define Py_FOREACH_CONTEXT_EVENT(V) \ V(ENTER) \ V(EXIT) typedef enum { - #define PY_DEF_EVENT(op) PY_CONTEXT_EVENT_##op, - PY_FOREACH_CONTEXT_EVENT(PY_DEF_EVENT) - #undef PY_DEF_EVENT + #define Py_DEF_EVENT(op) PY_CONTEXT_EVENT_##op, + Py_FOREACH_CONTEXT_EVENT(Py_DEF_EVENT) + #undef Py_DEF_EVENT } PyContextEvent; /* diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 624aa627c6af4e..90fc30f920d5e6 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -625,7 +625,7 @@ allocate_too_many_func_watchers(PyObject *self, PyObject *args) // Test contexct object watchers #define NUM_CONTEXT_WATCHERS 2 -static context_watcher_ids[NUM_CONTEXT_WATCHERS] = {-1, -1}; +static int context_watcher_ids[NUM_CONTEXT_WATCHERS] = {-1, -1}; static int num_context_object_enter_events[NUM_CONTEXT_WATCHERS] = {0, 0}; static int num_context_object_exit_events[NUM_CONTEXT_WATCHERS] = {0, 0}; diff --git a/Python/context.c b/Python/context.c index 87425977df24c7..33be780e10db69 100644 --- a/Python/context.c +++ b/Python/context.c @@ -105,7 +105,7 @@ context_event_name(PyContextEvent event) { #define CASE(op) \ case PY_CONTEXT_EVENT_##op: \ return "PY_CONTEXT_EVENT_" #op; - PY_FOREACH_CONTEXT_EVENT(CASE) + Py_FOREACH_CONTEXT_EVENT(CASE) #undef CASE } Py_UNREACHABLE(); From 2b4bd53aa9da432c1ccbcc316dc4a39c203ae8c0 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Tue, 28 May 2024 12:49:59 -0700 Subject: [PATCH 08/16] Add ignores to c-analyzer --- Tools/c-analyzer/cpython/ignored.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index b002ada1a8d588..f4dc807198a8ef 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -453,6 +453,9 @@ Modules/_testcapi/watchers.c - num_code_object_destroyed_events - Modules/_testcapi/watchers.c - pyfunc_watchers - Modules/_testcapi/watchers.c - func_watcher_ids - Modules/_testcapi/watchers.c - func_watcher_callbacks - +Modules/_testcapi/watchers.c - context_watcher_ids - +Modules/_testcapi/watchers.c - num_context_object_enter_events - +Modules/_testcapi/watchers.c - num_context_object_exit_events - Modules/_testcapimodule.c - BasicStaticTypes - Modules/_testcapimodule.c - num_basic_static_types_used - Modules/_testcapimodule.c - ContainerNoGC_members - From 1ed9657982f1ed978b29882aca8b58262a03b37b Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Mon, 23 Sep 2024 11:30:25 -0700 Subject: [PATCH 09/16] Update Doc/c-api/contextvars.rst for pep-7 Co-authored-by: Erlend E. Aasland --- Doc/c-api/contextvars.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index 4fbeeb0a41a729..1e610033c7c4b0 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -130,7 +130,7 @@ Context object management functions: .. c:type:: int (*PyContext_WatchCallback)(PyContextEvent event, PyContext* ctx) Type of a context object watcher callback function. - If *event* is ``PY_CONTEXT_EVENT_ENTER``, then the callback is invoked + If *event* is ``Py_CONTEXT_EVENT_ENTER``, then the callback is invoked after *ctx* has been set as the current context for the current thread. Otherwise, the callback is invoked before the deactivation of *ctx* as the current context and the restoration of the previous contex object for the current thread. From 546538a05114b8d5f5b304d987bd6fa78ae6870e Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Mon, 23 Sep 2024 11:30:38 -0700 Subject: [PATCH 10/16] Update Python/context.c - pep7 Co-authored-by: Erlend E. Aasland --- Python/context.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/context.c b/Python/context.c index 33be780e10db69..207162c7a37cd6 100644 --- a/Python/context.c +++ b/Python/context.c @@ -191,7 +191,7 @@ _PyContext_Enter(PyThreadState *ts, PyObject *octx) ts->context = Py_NewRef(ctx); ts->context_ver++; - notify_context_watchers(PY_CONTEXT_EVENT_ENTER, ctx); + notify_context_watchers(Py_CONTEXT_EVENT_ENTER, ctx); return 0; } From a7b60d6f0a945ca6aecf0f1430d200c19b386370 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Mon, 23 Sep 2024 11:33:45 -0700 Subject: [PATCH 11/16] Move macro to internal --- Include/cpython/context.h | 6 +----- Include/internal/pycore_context.h | 3 +++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Include/cpython/context.h b/Include/cpython/context.h index 186f0a167964ee..e1f97165a90037 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -27,13 +27,9 @@ PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void); PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); -#define Py_FOREACH_CONTEXT_EVENT(V) \ - V(ENTER) \ - V(EXIT) - typedef enum { #define Py_DEF_EVENT(op) PY_CONTEXT_EVENT_##op, - Py_FOREACH_CONTEXT_EVENT(Py_DEF_EVENT) + PY_FOREACH_CONTEXT_EVENT(Py_DEF_EVENT) #undef Py_DEF_EVENT } PyContextEvent; diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index c2b98d15da68fa..76d56ef145c14a 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -52,6 +52,9 @@ struct _pycontexttokenobject { int tok_used; }; +#define PY_FOREACH_CONTEXT_EVENT(V) \ + V(ENTER) \ + V(EXIT) // _testinternalcapi.hamt() used by tests. // Export for '_testcapi' shared extension From c7e5b85933c2ddd95ebe9ec59973af230687b32b Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Mon, 23 Sep 2024 11:43:22 -0700 Subject: [PATCH 12/16] Fix the rest of pep-7 issues --- Doc/c-api/contextvars.rst | 4 ++-- Include/cpython/context.h | 2 +- Lib/test/test_capi/test_watchers.py | 2 +- Modules/_testcapi/watchers.c | 4 ++-- Python/context.c | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index 1e610033c7c4b0..0de135b232aaaf 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -122,8 +122,8 @@ Context object management functions: .. c:type:: PyContextEvent Enumeration of possible context object watcher events: - - ``PY_CONTEXT_EVENT_ENTER`` - - ``PY_CONTEXT_EVENT_EXIT`` + - ``Py_CONTEXT_EVENT_ENTER`` + - ``Py_CONTEXT_EVENT_EXIT`` .. versionadded:: 3.14 diff --git a/Include/cpython/context.h b/Include/cpython/context.h index e1f97165a90037..9e5afc33bfe6d2 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -28,7 +28,7 @@ PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); typedef enum { - #define Py_DEF_EVENT(op) PY_CONTEXT_EVENT_##op, + #define Py_DEF_EVENT(op) Py_CONTEXT_EVENT_##op, PY_FOREACH_CONTEXT_EVENT(Py_DEF_EVENT) #undef Py_DEF_EVENT } PyContextEvent; diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 6e62dd039efd6f..f21d2627c6094b 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -626,7 +626,7 @@ def test_enter_error(self): self.assertEqual( cm.unraisable.err_msg, "Exception ignored in " - f"PY_CONTEXT_EVENT_EXIT watcher callback for {ctx!r}" + f"Py_CONTEXT_EVENT_EXIT watcher callback for {ctx!r}" ) self.assertEqual(str(cm.unraisable.exc_value), "boom!") diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 90fc30f920d5e6..689863d098ad8a 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -631,10 +631,10 @@ static int num_context_object_exit_events[NUM_CONTEXT_WATCHERS] = {0, 0}; static int handle_context_watcher_event(int which_watcher, PyContextEvent event, PyContext *ctx) { - if (event == PY_CONTEXT_EVENT_ENTER) { + if (event == Py_CONTEXT_EVENT_ENTER) { num_context_object_enter_events[which_watcher]++; } - else if (event == PY_CONTEXT_EVENT_EXIT) { + else if (event == Py_CONTEXT_EVENT_EXIT) { num_context_object_exit_events[which_watcher]++; } else { diff --git a/Python/context.c b/Python/context.c index 207162c7a37cd6..502b34bb6b73ad 100644 --- a/Python/context.c +++ b/Python/context.c @@ -103,9 +103,9 @@ static const char * context_event_name(PyContextEvent event) { switch (event) { #define CASE(op) \ - case PY_CONTEXT_EVENT_##op: \ - return "PY_CONTEXT_EVENT_" #op; - Py_FOREACH_CONTEXT_EVENT(CASE) + case Py_CONTEXT_EVENT_##op: \ + return "Py_CONTEXT_EVENT_" #op; + P_FOREACH_CONTEXT_EVENT(CASE) #undef CASE } Py_UNREACHABLE(); @@ -225,7 +225,7 @@ _PyContext_Exit(PyThreadState *ts, PyObject *octx) return -1; } - notify_context_watchers(PY_CONTEXT_EVENT_EXIT, ctx); + notify_context_watchers(Py_CONTEXT_EVENT_EXIT, ctx); Py_SETREF(ts->context, (PyObject *)ctx->ctx_prev); ts->context_ver++; From 3480c662049086968739abd0960c534a54081896 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Mon, 23 Sep 2024 11:47:53 -0700 Subject: [PATCH 13/16] Fix Foreach Macro Name --- Include/cpython/context.h | 2 +- Include/internal/pycore_context.h | 2 +- Python/context.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/cpython/context.h b/Include/cpython/context.h index 9e5afc33bfe6d2..f055997404020e 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -29,7 +29,7 @@ PyAPI_FUNC(int) PyContext_Exit(PyObject *); typedef enum { #define Py_DEF_EVENT(op) Py_CONTEXT_EVENT_##op, - PY_FOREACH_CONTEXT_EVENT(Py_DEF_EVENT) + Py_FOREACH_CONTEXT_EVENT(Py_DEF_EVENT) #undef Py_DEF_EVENT } PyContextEvent; diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 76d56ef145c14a..7d45206660afc6 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -52,7 +52,7 @@ struct _pycontexttokenobject { int tok_used; }; -#define PY_FOREACH_CONTEXT_EVENT(V) \ +#define Py_FOREACH_CONTEXT_EVENT(V) \ V(ENTER) \ V(EXIT) diff --git a/Python/context.c b/Python/context.c index 502b34bb6b73ad..58998c61c04599 100644 --- a/Python/context.c +++ b/Python/context.c @@ -105,7 +105,7 @@ context_event_name(PyContextEvent event) { #define CASE(op) \ case Py_CONTEXT_EVENT_##op: \ return "Py_CONTEXT_EVENT_" #op; - P_FOREACH_CONTEXT_EVENT(CASE) + Py_FOREACH_CONTEXT_EVENT(CASE) #undef CASE } Py_UNREACHABLE(); From c21035127c052a4fdaa9437f6a2903c34d11a0b7 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Mon, 23 Sep 2024 11:57:31 -0700 Subject: [PATCH 14/16] Yeah can't internalize foreach macro --- Include/cpython/context.h | 4 ++++ Include/internal/pycore_context.h | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Include/cpython/context.h b/Include/cpython/context.h index f055997404020e..39fb2c5dbb3300 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -27,6 +27,10 @@ PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void); PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); +#define Py_FOREACH_CONTEXT_EVENT(V) \ + V(ENTER) \ + V(EXIT) + typedef enum { #define Py_DEF_EVENT(op) Py_CONTEXT_EVENT_##op, Py_FOREACH_CONTEXT_EVENT(Py_DEF_EVENT) diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 7d45206660afc6..c2b98d15da68fa 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -52,9 +52,6 @@ struct _pycontexttokenobject { int tok_used; }; -#define Py_FOREACH_CONTEXT_EVENT(V) \ - V(ENTER) \ - V(EXIT) // _testinternalcapi.hamt() used by tests. // Export for '_testcapi' shared extension From c80dd9de6e5b6e3cedda697bcebb8cdd344f4804 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Mon, 23 Sep 2024 13:51:15 -0700 Subject: [PATCH 15/16] de-macro-ing as suggested by guido --- Include/cpython/context.h | 9 ++------- Python/context.c | 9 ++++----- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Include/cpython/context.h b/Include/cpython/context.h index 39fb2c5dbb3300..a509f4eaba3d77 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -27,14 +27,9 @@ PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void); PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); -#define Py_FOREACH_CONTEXT_EVENT(V) \ - V(ENTER) \ - V(EXIT) - typedef enum { - #define Py_DEF_EVENT(op) Py_CONTEXT_EVENT_##op, - Py_FOREACH_CONTEXT_EVENT(Py_DEF_EVENT) - #undef Py_DEF_EVENT + Py_CONTEXT_EVENT_ENTER, + Py_CONTEXT_EVENT_EXIT, } PyContextEvent; /* diff --git a/Python/context.c b/Python/context.c index 58998c61c04599..833330349bbabf 100644 --- a/Python/context.c +++ b/Python/context.c @@ -102,11 +102,10 @@ PyContext_CopyCurrent(void) static const char * context_event_name(PyContextEvent event) { switch (event) { - #define CASE(op) \ - case Py_CONTEXT_EVENT_##op: \ - return "Py_CONTEXT_EVENT_" #op; - Py_FOREACH_CONTEXT_EVENT(CASE) - #undef CASE + case Py_CONTEXT_EVENT_ENTER: + return "Py_CONTEXT_EVENT_ENTER"; + case Py_CONTEXT_EVENT_EXIT: + return "Py_CONTEXT_EVENT_EXIT"; } Py_UNREACHABLE(); } From 4b78fd3d808884be2a70920c42875d768d822ee7 Mon Sep 17 00:00:00 2001 From: Jason Fried Date: Mon, 23 Sep 2024 17:43:18 -0700 Subject: [PATCH 16/16] default switch case ? --- Python/context.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/context.c b/Python/context.c index 833330349bbabf..e52efbb6516d5c 100644 --- a/Python/context.c +++ b/Python/context.c @@ -106,6 +106,8 @@ context_event_name(PyContextEvent event) { return "Py_CONTEXT_EVENT_ENTER"; case Py_CONTEXT_EVENT_EXIT: return "Py_CONTEXT_EVENT_EXIT"; + default: + return "?"; } Py_UNREACHABLE(); }