diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 963bc1fb82c12f..d481a1921d532b 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -120,9 +120,10 @@ Queue raise :exc:`QueueShutDown`. If *immediate* is true, the queue is terminated immediately. - The queue is drained to be completely empty. All callers of - :meth:`~Queue.join` are unblocked regardless of the number - of unfinished tasks. Blocked callers of :meth:`~Queue.get` + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` are unblocked and will raise :exc:`QueueShutDown` because the queue is empty. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 6dcf06aab00295..1b75582f0cf45b 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -256,9 +256,10 @@ until empty or terminated immediately with a hard shutdown. raise :exc:`ShutDown`. If *immediate* is true, the queue is terminated immediately. - The queue is drained to be completely empty. All callers of - :meth:`~Queue.join` are unblocked regardless of the number - of unfinished tasks. Blocked callers of :meth:`~Queue.get` + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` are unblocked and will raise :exc:`ShutDown` because the queue is empty. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e8e2c1ed6047bf..1128da875a8847 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -230,6 +230,17 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +hashlib +------- + +* Ensure that hash functions guaranteed to be always *available* exist as + attributes of :mod:`hashlib` even if they will not work at runtime due to + missing backend implementations. For instance, ``hashlib.md5`` will no + longer raise :exc:`AttributeError` if OpenSSL is not available and Python + has been built without MD5 support. + (Contributed by Bénédikt Tran in :gh:`136929`.) + + http.client ----------- diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index e5d6f2e4b61e17..084fccaaff2ff7 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -253,9 +253,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', unblock callers of join() regardless of the - number of unfinished tasks. + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. """ self._is_shutdown = True if immediate: diff --git a/Lib/doctest.py b/Lib/doctest.py index e77823f64b67e4..92a2ab4f7e66f8 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -94,6 +94,7 @@ def _test(): import __future__ import difflib +import functools import inspect import linecache import os @@ -1141,7 +1142,9 @@ def _find_lineno(self, obj, source_lines): if inspect.ismethod(obj): obj = obj.__func__ if isinstance(obj, property): obj = obj.fget - if inspect.isfunction(obj) and getattr(obj, '__doc__', None): + if isinstance(obj, functools.cached_property): + obj = obj.func + if inspect.isroutine(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. obj = inspect.unwrap(obj) try: diff --git a/Lib/hashlib.py b/Lib/hashlib.py index a7db778b716537..8e7083ba692348 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -261,16 +261,39 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): return digestobj +__logging = None for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL # version not supporting that algorithm. try: globals()[__func_name] = __get_hash(__func_name) - except ValueError: - import logging - logging.exception('code for hash %s was not found.', __func_name) - + except ValueError as __exc: + import logging as __logging + __logging.error('hash algorithm %s will not be supported at runtime ' + '[reason: %s]', __func_name, __exc) + # The following code can be simplified in Python 3.19 + # once "string" is removed from the signature. + __code = f'''\ +def {__func_name}(data=__UNSET, *, usedforsecurity=True, string=__UNSET): + if data is __UNSET and string is not __UNSET: + import warnings + warnings.warn( + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", + DeprecationWarning, stacklevel=2) + if data is not __UNSET and string is not __UNSET: + raise TypeError("'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version.") + raise ValueError("unsupported hash algorithm {__func_name}") +''' + exec(__code, {"__UNSET": object()}, __locals := {}) + globals()[__func_name] = __locals[__func_name] + del __exc, __code, __locals # Cleanup locals() del __always_supported, __func_name, __get_hash del __py_new, __hash_new, __get_openssl_constructor +del __logging diff --git a/Lib/queue.py b/Lib/queue.py index c90de8edc76c34..c0b359876543f7 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -236,9 +236,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', callers of join() are unblocked regardless of - the number of unfinished tasks. + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. ''' with self.mutex: self.is_shutdown = True diff --git a/Lib/test/test_doctest/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py index 0dbcd9a11eaba2..0bd402e98288d0 100644 --- a/Lib/test/test_doctest/doctest_lineno.py +++ b/Lib/test/test_doctest/doctest_lineno.py @@ -76,3 +76,32 @@ def property_with_doctest(self): @decorator def func_with_docstring_wrapped(): """Some unrelated info.""" + + +# https://github.com/python/cpython/issues/136914 +import functools + + +@functools.cache +def cached_func_with_doctest(value): + """ + >>> cached_func_with_doctest(1) + -1 + """ + return -value + + +@functools.cache +def cached_func_without_docstring(value): + return value + 1 + + +class ClassWithACachedProperty: + + @functools.cached_property + def cached(self): + """ + >>> X().cached + -1 + """ + return 0 diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 72763d4a0132d0..0fa74407e3c436 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -678,6 +678,8 @@ def basics(): r""" >>> for t in tests: ... print('%5s %s' % (t.lineno, t.name)) None test.test_doctest.doctest_lineno + None test.test_doctest.doctest_lineno.ClassWithACachedProperty + 102 test.test_doctest.doctest_lineno.ClassWithACachedProperty.cached 22 test.test_doctest.doctest_lineno.ClassWithDocstring 30 test.test_doctest.doctest_lineno.ClassWithDoctest None test.test_doctest.doctest_lineno.ClassWithoutDocstring @@ -687,6 +689,8 @@ def basics(): r""" 45 test.test_doctest.doctest_lineno.MethodWrapper.method_with_doctest None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest + 86 test.test_doctest.doctest_lineno.cached_func_with_doctest + None test.test_doctest.doctest_lineno.cached_func_without_docstring 4 test.test_doctest.doctest_lineno.func_with_docstring 77 test.test_doctest.doctest_lineno.func_with_docstring_wrapped 12 test.test_doctest.doctest_lineno.func_with_doctest diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 9e519537ca5ed3..7a7a8c40d47d40 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1239,6 +1239,25 @@ def getpass(self): # Make sure the password function isn't called if it isn't needed ctx.load_cert_chain(CERTFILE, password=getpass_exception) + @threading_helper.requires_working_threading() + def test_load_cert_chain_thread_safety(self): + # gh-134698: _ssl detaches the thread state (and as such, + # releases the GIL and critical sections) around expensive + # OpenSSL calls. Unfortunately, OpenSSL structures aren't + # thread-safe, so executing these calls concurrently led + # to crashes. + ctx = ssl.create_default_context() + + def race(): + ctx.load_cert_chain(CERTFILE) + + threads = [threading.Thread(target=race) for _ in range(8)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(threads): + pass + + self.assertIsNone(cm.exc_value) + def test_load_verify_locations(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_verify_locations(CERTFILE) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 00a3037c3e1e01..002a1feeb85c94 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -2042,6 +2042,23 @@ def modify_file(): t.start() t.join() + def test_dummy_thread_on_interpreter_shutdown(self): + # GH-130522: When `threading` held a reference to itself and then a + # _DummyThread() object was created, destruction of the dummy thread + # would emit an unraisable exception at shutdown, due to a lock being + # destroyed. + code = """if True: + import sys + import threading + + threading.x = sys.modules[__name__] + x = threading._DummyThread() + """ + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(out, b"") + self.assertEqual(err, b"") + class ThreadRunFail(threading.Thread): def run(self): diff --git a/Lib/threading.py b/Lib/threading.py index b6c451d1fbaabd..b3e24fe7ddaa0c 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1414,7 +1414,7 @@ def __init__(self, dummy_thread): # the related _DummyThread will be kept forever! _thread_local_info._track_dummy_thread_ref = self - def __del__(self): + def __del__(self, _active_limbo_lock=_active_limbo_lock, _active=_active): with _active_limbo_lock: if _active.get(self._tident) is self._dummy_thread: _active.pop(self._tident, None) diff --git a/Misc/NEWS.d/next/Library/2025-05-26-10-52-27.gh-issue-134698.aJ1mZ1.rst b/Misc/NEWS.d/next/Library/2025-05-26-10-52-27.gh-issue-134698.aJ1mZ1.rst new file mode 100644 index 00000000000000..cf3901aba34ebc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-26-10-52-27.gh-issue-134698.aJ1mZ1.rst @@ -0,0 +1,2 @@ +Fix a crash when calling methods of :class:`ssl.SSLContext` or +:class:`ssl.SSLSocket` across multiple threads. diff --git a/Misc/NEWS.d/next/Library/2025-07-21-15-40-00.gh-issue-136914.-GNG-d.rst b/Misc/NEWS.d/next/Library/2025-07-21-15-40-00.gh-issue-136914.-GNG-d.rst new file mode 100644 index 00000000000000..78ec8025fbc0fd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-21-15-40-00.gh-issue-136914.-GNG-d.rst @@ -0,0 +1,2 @@ +Fix retrieval of :attr:`doctest.DocTest.lineno` for objects decorated with +:func:`functools.cache` or :class:`functools.cached_property`. diff --git a/Misc/NEWS.d/next/Library/2025-07-21-16-13-20.gh-issue-136929.obKZ2S.rst b/Misc/NEWS.d/next/Library/2025-07-21-16-13-20.gh-issue-136929.obKZ2S.rst new file mode 100644 index 00000000000000..31b8563f9d8523 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-21-16-13-20.gh-issue-136929.obKZ2S.rst @@ -0,0 +1,5 @@ +Ensure that hash functions guaranteed to be always *available* exist as +attributes of :mod:`hashlib` even if they will not work at runtime due to +missing backend implementations. For instance, ``hashlib.md5`` will no +longer raise :exc:`AttributeError` if OpenSSL is not available and Python +has been built without MD5 support. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-07-25-09-21-56.gh-issue-130522.Crwq68.rst b/Misc/NEWS.d/next/Library/2025-07-25-09-21-56.gh-issue-130522.Crwq68.rst new file mode 100644 index 00000000000000..6c2246631dda9e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-25-09-21-56.gh-issue-130522.Crwq68.rst @@ -0,0 +1,2 @@ +Fix unraisable :exc:`TypeError` raised during :term:`interpreter shutdown` +in the :mod:`threading` module. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 24c243e330d4bf..5887d380046b00 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -43,14 +43,14 @@ /* Redefined below for Windows debug builds after important #includes */ #define _PySSL_FIX_ERRNO -#define PySSL_BEGIN_ALLOW_THREADS_S(save) \ - do { (save) = PyEval_SaveThread(); } while(0) -#define PySSL_END_ALLOW_THREADS_S(save) \ - do { PyEval_RestoreThread(save); _PySSL_FIX_ERRNO; } while(0) -#define PySSL_BEGIN_ALLOW_THREADS { \ +#define PySSL_BEGIN_ALLOW_THREADS_S(save, mutex) \ + do { (save) = PyEval_SaveThread(); PyMutex_Lock(mutex); } while(0) +#define PySSL_END_ALLOW_THREADS_S(save, mutex) \ + do { PyMutex_Unlock(mutex); PyEval_RestoreThread(save); _PySSL_FIX_ERRNO; } while(0) +#define PySSL_BEGIN_ALLOW_THREADS(self) { \ PyThreadState *_save = NULL; \ - PySSL_BEGIN_ALLOW_THREADS_S(_save); -#define PySSL_END_ALLOW_THREADS PySSL_END_ALLOW_THREADS_S(_save); } + PySSL_BEGIN_ALLOW_THREADS_S(_save, &self->tstate_mutex); +#define PySSL_END_ALLOW_THREADS(self) PySSL_END_ALLOW_THREADS_S(_save, &self->tstate_mutex); } #if defined(HAVE_POLL_H) #include @@ -336,6 +336,9 @@ typedef struct { PyObject *psk_client_callback; PyObject *psk_server_callback; #endif + /* Lock to synchronize calls when the thread state is detached. + See also gh-134698. */ + PyMutex tstate_mutex; } PySSLContext; #define PySSLContext_CAST(op) ((PySSLContext *)(op)) @@ -363,6 +366,9 @@ typedef struct { * and shutdown methods check for chained exceptions. */ PyObject *exc; + /* Lock to synchronize calls when the thread state is detached. + See also gh-134698. */ + PyMutex tstate_mutex; } PySSLSocket; #define PySSLSocket_CAST(op) ((PySSLSocket *)(op)) @@ -912,13 +918,14 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->server_hostname = NULL; self->err = err; self->exc = NULL; + self->tstate_mutex = (PyMutex){0}; /* Make sure the SSL error state is initialized */ ERR_clear_error(); - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(sslctx) self->ssl = SSL_new(ctx); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(sslctx) if (self->ssl == NULL) { Py_DECREF(self); _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); @@ -987,12 +994,12 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, BIO_set_nbio(SSL_get_wbio(self->ssl), 1); } - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) if (socket_type == PY_SSL_CLIENT) SSL_set_connect_state(self->ssl); else SSL_set_accept_state(self->ssl); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) self->socket_type = socket_type; if (sock != NULL) { @@ -1061,10 +1068,10 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) /* Actually negotiate SSL connection */ /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ do { - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) ret = SSL_do_handshake(self->ssl); err = _PySSL_errno(ret < 1, self->ssl, ret); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) self->err = err; if (PyErr_CheckSignals()) @@ -2441,9 +2448,10 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) ms = (int)_PyTime_AsMilliseconds(timeout, _PyTime_ROUND_CEILING); assert(ms <= INT_MAX); - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS rc = poll(&pollfd, 1, (int)ms); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; #else /* Guard against socket too large for select*/ if (!_PyIsSelectable_fd(s->sock_fd)) @@ -2455,13 +2463,14 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) FD_SET(s->sock_fd, &fds); /* Wait until the socket becomes ready */ - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS nfds = Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int); if (writing) rc = select(nfds, NULL, &fds, NULL, &tv); else rc = select(nfds, &fds, NULL, NULL, &tv); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; #endif /* Return SOCKET_TIMED_OUT on timeout, SOCKET_OPERATION_OK otherwise @@ -2579,10 +2588,10 @@ _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, } do { - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) retval = SSL_sendfile(self->ssl, fd, (off_t)offset, size, flags); err = _PySSL_errno(retval < 0, self->ssl, (int)retval); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) self->err = err; if (PyErr_CheckSignals()) { @@ -2710,10 +2719,10 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) } do { - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) retval = SSL_write_ex(self->ssl, b->buf, (size_t)b->len, &count); err = _PySSL_errno(retval == 0, self->ssl, retval); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) self->err = err; if (PyErr_CheckSignals()) @@ -2771,10 +2780,10 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) int count = 0; _PySSLError err; - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) count = SSL_pending(self->ssl); err = _PySSL_errno(count < 0, self->ssl, count); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) self->err = err; if (count < 0) @@ -2865,10 +2874,10 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, deadline = _PyDeadline_Init(timeout); do { - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) retval = SSL_read_ex(self->ssl, mem, (size_t)len, &count); err = _PySSL_errno(retval == 0, self->ssl, retval); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) self->err = err; if (PyErr_CheckSignals()) @@ -2967,7 +2976,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } while (1) { - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) /* Disable read-ahead so that unwrap can work correctly. * Otherwise OpenSSL might read in too much data, * eating clear text data that happens to be @@ -2980,7 +2989,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) SSL_set_read_ahead(self->ssl, 0); ret = SSL_shutdown(self->ssl); err = _PySSL_errno(ret < 0, self->ssl, ret); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) self->err = err; /* If err == 1, a secure shutdown with SSL_shutdown() is complete */ @@ -3375,9 +3384,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) // no other thread can be touching this object yet. // (Technically, we can't even lock if we wanted to, as the // lock hasn't been initialized yet.) - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ctx = SSL_CTX_new(method); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (ctx == NULL) { _setSSLError(get_ssl_state(module), NULL, 0, __FILE__, __LINE__); @@ -3402,6 +3412,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) self->psk_client_callback = NULL; self->psk_server_callback = NULL; #endif + self->tstate_mutex = (PyMutex){0}; /* Don't check host name by default */ if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { @@ -3520,9 +3531,10 @@ context_clear(PyObject *op) Py_CLEAR(self->psk_server_callback); #endif if (self->keylog_bio != NULL) { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS BIO_free_all(self->keylog_bio); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; self->keylog_bio = NULL; } return 0; @@ -4245,7 +4257,8 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) _PySSLPasswordInfo *pw_info = (_PySSLPasswordInfo*) userdata; PyObject *fn_ret = NULL; - PySSL_END_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); + _PySSL_FIX_ERRNO; if (pw_info->error) { /* already failed previously. OpenSSL 3.0.0-alpha14 invokes the @@ -4275,13 +4288,13 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); memcpy(buf, pw_info->password, pw_info->size); return pw_info->size; error: Py_XDECREF(fn_ret); - PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); pw_info->error = 1; return -1; } @@ -4334,10 +4347,10 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, SSL_CTX_set_default_passwd_cb(self->ctx, _password_callback); SSL_CTX_set_default_passwd_cb_userdata(self->ctx, &pw_info); } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_use_certificate_chain_file(self->ctx, PyBytes_AS_STRING(certfile_bytes)); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); if (r != 1) { if (pw_info.error) { ERR_clear_error(); @@ -4352,11 +4365,11 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, } goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_use_PrivateKey_file(self->ctx, PyBytes_AS_STRING(keyfile ? keyfile_bytes : certfile_bytes), SSL_FILETYPE_PEM); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); Py_CLEAR(keyfile_bytes); Py_CLEAR(certfile_bytes); if (r != 1) { @@ -4373,9 +4386,9 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, } goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_check_private_key(self->ctx); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); if (r != 1) { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); goto error; @@ -4592,9 +4605,9 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, cafile_buf = PyBytes_AS_STRING(cafile_bytes); if (capath) capath_buf = PyBytes_AS_STRING(capath_bytes); - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) r = SSL_CTX_load_verify_locations(self->ctx, cafile_buf, capath_buf); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) if (r != 1) { if (errno != 0) { PyErr_SetFromErrno(PyExc_OSError); @@ -4646,10 +4659,11 @@ _ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) return NULL; errno = 0; - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS dh = PEM_read_DHparams(f, NULL, NULL, NULL); fclose(f); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (dh == NULL) { if (errno != 0) { PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, filepath); @@ -4801,6 +4815,7 @@ _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self) Py_BEGIN_ALLOW_THREADS rc = SSL_CTX_set_default_verify_paths(self->ctx); Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (!rc) { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); return NULL; diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index f0a0a1674f32bd..211fe15a439bfc 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -140,13 +140,15 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) * critical debug helper. */ - PySSL_BEGIN_ALLOW_THREADS + assert(PyMutex_IsLocked(&ssl_obj->tstate_mutex)); + Py_BEGIN_ALLOW_THREADS PyThread_acquire_lock(lock, 1); res = BIO_printf(ssl_obj->ctx->keylog_bio, "%s\n", line); e = errno; (void)BIO_flush(ssl_obj->ctx->keylog_bio); PyThread_release_lock(lock); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (res == -1) { errno = e; @@ -187,9 +189,10 @@ _PySSLContext_set_keylog_filename(PyObject *op, PyObject *arg, if (self->keylog_bio != NULL) { BIO *bio = self->keylog_bio; self->keylog_bio = NULL; - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS BIO_free_all(bio); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; } if (arg == Py_None) { @@ -211,13 +214,13 @@ _PySSLContext_set_keylog_filename(PyObject *op, PyObject *arg, self->keylog_filename = Py_NewRef(arg); /* Write a header for seekable, empty files (this excludes pipes). */ - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) if (BIO_tell(self->keylog_bio) == 0) { BIO_puts(self->keylog_bio, "# TLS secrets log file, generated by OpenSSL / Python\n"); (void)BIO_flush(self->keylog_bio); } - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) SSL_CTX_set_keylog_callback(self->ctx, _PySSL_keylog_callback); return 0; } diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 88936bbc699c30..28c004c3c5ac2c 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -379,9 +379,9 @@ static int test_pre_initialization_sys_options(void) /* bpo-20891: Avoid race condition when initialising the GIL */ -static void bpo20891_thread(void *lockp) +static void bpo20891_thread(void *eventp) { - PyThread_type_lock lock = *((PyThread_type_lock*)lockp); + PyEvent *event = (PyEvent *)eventp; PyGILState_STATE state = PyGILState_Ensure(); if (!PyGILState_Check()) { @@ -390,8 +390,7 @@ static void bpo20891_thread(void *lockp) } PyGILState_Release(state); - - PyThread_release_lock(lock); + _PyEvent_Notify(event); } static int test_bpo20891(void) @@ -401,27 +400,16 @@ static int test_bpo20891(void) /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread must not crash. */ - PyThread_type_lock lock = PyThread_allocate_lock(); - if (!lock) { - error("PyThread_allocate_lock failed!"); - return 1; - } - + PyEvent event = {0}; _testembed_initialize(); - unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &lock); + unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &event); if (thrd == PYTHREAD_INVALID_THREAD_ID) { error("PyThread_start_new_thread failed!"); return 1; } - PyThread_acquire_lock(lock, WAIT_LOCK); - Py_BEGIN_ALLOW_THREADS - /* wait until the thread exit */ - PyThread_acquire_lock(lock, WAIT_LOCK); - Py_END_ALLOW_THREADS - - PyThread_free_lock(lock); + PyEvent_Wait(&event); Py_Finalize(); diff --git a/Python/gc.c b/Python/gc.c index 25536270523305..2e697faac032b5 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1396,7 +1396,7 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat assert(_PyObject_GC_IS_TRACKED(op)); if (_Py_IsImmortal(op)) { PyGC_Head *next = GC_NEXT(gc); - gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc_list_move(gc, &gcstate->permanent_generation.head); gc = next; continue; }