bpo-32436: Implement PEP 567 (#5027)

Include/Python.h

@@ -109,6 +109,7 @@
 #include "pyerrors.h"
 
 #include "pystate.h"
+#include "context.h"
 
 #include "pyarena.h"
 #include "modsupport.h"

Include/context.h

@@ -0,0 +1,86 @@
+#ifndef Py_CONTEXT_H
+#define Py_CONTEXT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_LIMITED_API
+
+
+PyAPI_DATA(PyTypeObject) PyContext_Type;
+typedef struct _pycontextobject PyContext;
+
+PyAPI_DATA(PyTypeObject) PyContextVar_Type;
+typedef struct _pycontextvarobject PyContextVar;
+
+PyAPI_DATA(PyTypeObject) PyContextToken_Type;
+typedef struct _pycontexttokenobject PyContextToken;
+
+
+#define PyContext_CheckExact(o) (Py_TYPE(o) == &PyContext_Type)
+#define PyContextVar_CheckExact(o) (Py_TYPE(o) == &PyContextVar_Type)
+#define PyContextToken_CheckExact(o) (Py_TYPE(o) == &PyContextToken_Type)
+
+
+PyAPI_FUNC(PyContext *) PyContext_New(void);
+PyAPI_FUNC(PyContext *) PyContext_Copy(PyContext *);
+PyAPI_FUNC(PyContext *) PyContext_CopyCurrent(void);
+
+PyAPI_FUNC(int) PyContext_Enter(PyContext *);
+PyAPI_FUNC(int) PyContext_Exit(PyContext *);
+
+
+/* Create a new context variable.
+
+   default_value can be NULL.
+*/
+PyAPI_FUNC(PyContextVar *) PyContextVar_New(
+    const char *name, PyObject *default_value);
+
+
+/* Get a value for the variable.
+
+   Returns -1 if an error occurred during lookup.
+
+   Returns 0 if value either was or was not found.
+
+   If value was found, *value will point to it.
+   If not, it will point to:
+
+   - default_value, if not NULL;
+   - the default value of "var", if not NULL;
+   - NULL.
+
+   '*value' will be a new ref, if not NULL.
+*/
+PyAPI_FUNC(int) PyContextVar_Get(
+    PyContextVar *var, PyObject *default_value, PyObject **value);
+
+
+/* Set a new value for the variable.
+   Returns NULL if an error occurs.
+*/
+PyAPI_FUNC(PyContextToken *) PyContextVar_Set(
+    PyContextVar *var, PyObject *value);
+
+
+/* Reset a variable to its previous value.
+   Returns 0 on sucess, -1 on error.
+*/
+PyAPI_FUNC(int) PyContextVar_Reset(
+    PyContextVar *var, PyContextToken *token);
+
+
+/* This method is exposed only for CPython tests. Don not use it. */
+PyAPI_FUNC(PyObject *) _PyContext_NewHamtForTests(void);
+
+
+PyAPI_FUNC(int) PyContext_ClearFreeList(void);
+
+
+#endif /* !Py_LIMITED_API */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_CONTEXT_H */

Include/internal/context.h

@@ -0,0 +1,41 @@
+#ifndef Py_INTERNAL_CONTEXT_H
+#define Py_INTERNAL_CONTEXT_H
+
+
+#include "internal/hamt.h"
+
+
+struct _pycontextobject {
+    PyObject_HEAD
+    PyContext *ctx_prev;
+    PyHamtObject *ctx_vars;
+    PyObject *ctx_weakreflist;
+    int ctx_entered;
+};
+
+
+struct _pycontextvarobject {
+    PyObject_HEAD
+    PyObject *var_name;
+    PyObject *var_default;
+    PyObject *var_cached;
+    uint64_t var_cached_tsid;
+    uint64_t var_cached_tsver;
+    Py_hash_t var_hash;
+};
+
+
+struct _pycontexttokenobject {
+    PyObject_HEAD
+    PyContext *tok_ctx;
+    PyContextVar *tok_var;
+    PyObject *tok_oldval;
+    int tok_used;
+};
+
+
+int _PyContext_Init(void);
+void _PyContext_Fini(void);
+
+
+#endif /* !Py_INTERNAL_CONTEXT_H */

Include/internal/hamt.h

@@ -0,0 +1,113 @@
+#ifndef Py_INTERNAL_HAMT_H
+#define Py_INTERNAL_HAMT_H
+
+
+#define _Py_HAMT_MAX_TREE_DEPTH 7
+
+
+#define PyHamt_Check(o) (Py_TYPE(o) == &_PyHamt_Type)
+
+
+/* Abstract tree node. */
+typedef struct {
+    PyObject_HEAD
+} PyHamtNode;
+
+
+/* An HAMT immutable mapping collection. */
+typedef struct {
+    PyObject_HEAD
+    PyHamtNode *h_root;
+    PyObject *h_weakreflist;
+    Py_ssize_t h_count;
+} PyHamtObject;
+
+
+/* A struct to hold the state of depth-first traverse of the tree.
+
+   HAMT is an immutable collection.  Iterators will hold a strong reference
+   to it, and every node in the HAMT has strong references to its children.
+
+   So for iterators, we can implement zero allocations and zero reference
+   inc/dec depth-first iteration.
+
+   - i_nodes: an array of seven pointers to tree nodes
+   - i_level: the current node in i_nodes
+   - i_pos: an array of positions within nodes in i_nodes.
+*/
+typedef struct {
+    PyHamtNode *i_nodes[_Py_HAMT_MAX_TREE_DEPTH];
+    Py_ssize_t i_pos[_Py_HAMT_MAX_TREE_DEPTH];
+    int8_t i_level;
+} PyHamtIteratorState;
+
+
+/* Base iterator object.
+
+   Contains the iteration state, a pointer to the HAMT tree,
+   and a pointer to the 'yield function'.  The latter is a simple
+   function that returns a key/value tuple for the 'Items' iterator,
+   just a key for the 'Keys' iterator, and a value for the 'Values'
+   iterator.
+*/
+typedef struct {
+    PyObject_HEAD
+    PyHamtObject *hi_obj;
+    PyHamtIteratorState hi_iter;
+    binaryfunc hi_yield;
+} PyHamtIterator;
+
+
+PyAPI_DATA(PyTypeObject) _PyHamt_Type;
+PyAPI_DATA(PyTypeObject) _PyHamt_ArrayNode_Type;
+PyAPI_DATA(PyTypeObject) _PyHamt_BitmapNode_Type;
+PyAPI_DATA(PyTypeObject) _PyHamt_CollisionNode_Type;
+PyAPI_DATA(PyTypeObject) _PyHamtKeys_Type;
+PyAPI_DATA(PyTypeObject) _PyHamtValues_Type;
+PyAPI_DATA(PyTypeObject) _PyHamtItems_Type;
+
+
+/* Create a new HAMT immutable mapping. */
+PyHamtObject * _PyHamt_New(void);
+
+/* Return a new collection based on "o", but with an additional
+   key/val pair. */
+PyHamtObject * _PyHamt_Assoc(PyHamtObject *o, PyObject *key, PyObject *val);
+
+/* Return a new collection based on "o", but without "key". */
+PyHamtObject * _PyHamt_Without(PyHamtObject *o, PyObject *key);
+
+/* Find "key" in the "o" collection.
+
+   Return:
+   - -1: An error ocurred.
+   - 0: "key" wasn't found in "o".
+   - 1: "key" is in "o"; "*val" is set to its value (a borrowed ref).
+*/
+int _PyHamt_Find(PyHamtObject *o, PyObject *key, PyObject **val);
+
+/* Check if "v" is equal to "w".
+
+   Return:
+   - 0: v != w
+   - 1: v == w
+   - -1: An error occurred.
+*/
+int _PyHamt_Eq(PyHamtObject *v, PyHamtObject *w);
+
+/* Return the size of "o"; equivalent of "len(o)". */
+Py_ssize_t _PyHamt_Len(PyHamtObject *o);
+
+/* Return a Keys iterator over "o". */
+PyObject * _PyHamt_NewIterKeys(PyHamtObject *o);
+
+/* Return a Values iterator over "o". */
+PyObject * _PyHamt_NewIterValues(PyHamtObject *o);
+
+/* Return a Items iterator over "o". */
+PyObject * _PyHamt_NewIterItems(PyHamtObject *o);
+
+int _PyHamt_Init(void);
+void _PyHamt_Fini(void);
+
+#endif /* !Py_INTERNAL_HAMT_H */

Include/pystate.h

@@ -143,6 +143,8 @@ typedef struct _is {
     /* AtExit module */
     void (*pyexitfunc)(PyObject *);
     PyObject *pyexitmodule;
+
+    uint64_t tstate_next_unique_id;
 } PyInterpreterState;
 #endif   /* !Py_LIMITED_API */
 
@@ -270,6 +272,12 @@ typedef struct _ts {
     PyObject *async_gen_firstiter;
     PyObject *async_gen_finalizer;
 
+    PyObject *context;
+    uint64_t context_ver;
+
+    /* Unique thread state id. */
+    uint64_t id;
+
     /* XXX signal handlers should also be here */
 
 } PyThreadState;

Lib/asyncio/base_events.py

@@ -489,7 +489,7 @@ def time(self):
         """
         return time.monotonic()
 
-    def call_later(self, delay, callback, *args):
+    def call_later(self, delay, callback, *args, context=None):
         """Arrange for a callback to be called at a given time.
 
         Return a Handle: an opaque object with a cancel() method that
@@ -505,12 +505,13 @@ def call_later(self, delay, callback, *args):
         Any positional arguments after the callback will be passed to
         the callback when it is called.
         """
-        timer = self.call_at(self.time() + delay, callback, *args)
+        timer = self.call_at(self.time() + delay, callback, *args,
+                             context=context)
         if timer._source_traceback:
             del timer._source_traceback[-1]
         return timer
 
-    def call_at(self, when, callback, *args):
+    def call_at(self, when, callback, *args, context=None):
         """Like call_later(), but uses an absolute time.
 
         Absolute time corresponds to the event loop's time() method.
@@ -519,14 +520,14 @@ def call_at(self, when, callback, *args):
         if self._debug:
             self._check_thread()
             self._check_callback(callback, 'call_at')
-        timer = events.TimerHandle(when, callback, args, self)
+        timer = events.TimerHandle(when, callback, args, self, context)
         if timer._source_traceback:
             del timer._source_traceback[-1]
         heapq.heappush(self._scheduled, timer)
         timer._scheduled = True
         return timer
 
-    def call_soon(self, callback, *args):
+    def call_soon(self, callback, *args, context=None):
         """Arrange for a callback to be called as soon as possible.
 
         This operates as a FIFO queue: callbacks are called in the
@@ -540,7 +541,7 @@ def call_soon(self, callback, *args):
         if self._debug:
             self._check_thread()
             self._check_callback(callback, 'call_soon')
-        handle = self._call_soon(callback, args)
+        handle = self._call_soon(callback, args, context)
         if handle._source_traceback:
             del handle._source_traceback[-1]
         return handle
@@ -555,8 +556,8 @@ def _check_callback(self, callback, method):
                 f'a callable object was expected by {method}(), '
                 f'got {callback!r}')
 
-    def _call_soon(self, callback, args):
-        handle = events.Handle(callback, args, self)
+    def _call_soon(self, callback, args, context):
+        handle = events.Handle(callback, args, self, context)
         if handle._source_traceback:
             del handle._source_traceback[-1]
         self._ready.append(handle)
@@ -579,12 +580,12 @@ def _check_thread(self):
                 "Non-thread-safe operation invoked on an event loop other "
                 "than the current one")
 
-    def call_soon_threadsafe(self, callback, *args):
+    def call_soon_threadsafe(self, callback, *args, context=None):
         """Like call_soon(), but thread-safe."""
         self._check_closed()
         if self._debug:
             self._check_callback(callback, 'call_soon_threadsafe')
-        handle = self._call_soon(callback, args)
+        handle = self._call_soon(callback, args, context)
         if handle._source_traceback:
             del handle._source_traceback[-1]
         self._write_to_self()

Lib/asyncio/base_futures.py

@@ -41,13 +41,13 @@ def format_cb(callback):
         return format_helpers._format_callback_source(callback, ())
 
     if size == 1:
-        cb = format_cb(cb[0])
+        cb = format_cb(cb[0][0])
     elif size == 2:
-        cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1]))
+        cb = '{}, {}'.format(format_cb(cb[0][0]), format_cb(cb[1][0]))
     elif size > 2:
-        cb = '{}, <{} more>, {}'.format(format_cb(cb[0]),
+        cb = '{}, <{} more>, {}'.format(format_cb(cb[0][0]),
                                         size - 2,
-                                        format_cb(cb[-1]))
+                                        format_cb(cb[-1][0]))
     return f'cb=[{cb}]'