Support for free-threaded build of CPython

include/boost/python/detail/pymutex.hpp

@@ -0,0 +1,103 @@
+// Copyright 2025 Boost.Python Contributors
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BOOST_PYTHON_DETAIL_PYMUTEX_HPP
+#define BOOST_PYTHON_DETAIL_PYMUTEX_HPP
+
+#include <boost/python/detail/prefix.hpp>
+#ifdef Py_GIL_DISABLED
+// needed for pymutex wrapper
+#include <atomic>
+#include <cstddef>
+#endif
+
+namespace boost { namespace python { namespace detail {
+
+#ifdef Py_GIL_DISABLED
+
+// Re-entrant wrapper around PyMutex for free-threaded Python
+// Similar to _PyRecursiveMutex or threading.RLock
+class pymutex {
+    PyMutex m_mutex;
+    std::atomic<unsigned long> m_owner;
+    std::size_t m_level;
+
+public:
+    pymutex() : m_mutex({}), m_owner(0), m_level(0) {}
+
+    // Non-copyable, non-movable
+    pymutex(const pymutex&) = delete;
+    pymutex& operator=(const pymutex&) = delete;
+
+    void lock() {
+        unsigned long thread = PyThread_get_thread_ident();
+        if (m_owner.load(std::memory_order_relaxed) == thread) {
+            m_level++;
+            return;
+        }
+        PyMutex_Lock(&m_mutex);
+        m_owner.store(thread, std::memory_order_relaxed);
+        // m_level should be 0 when we acquire the lock
+    }
+
+    void unlock() {
+        unsigned long thread = PyThread_get_thread_ident();
+        // Verify current thread owns the lock
+        if (m_owner.load(std::memory_order_relaxed) != thread) {
+            // This should never happen - programming error
+            return;
+        }
+        if (m_level > 0) {
+            m_level--;
+            return;
+        }
+        m_owner.store(0, std::memory_order_relaxed);
+        PyMutex_Unlock(&m_mutex);
+    }
+
+    bool is_locked_by_current_thread() const {
+        unsigned long thread = PyThread_get_thread_ident();
+        return m_owner.load(std::memory_order_relaxed) == thread;
+    }
+};
+
+
+// RAII lock guard for pymutex
+class pymutex_guard {
+    pymutex& m_mutex;
+
+public:
+    explicit pymutex_guard(pymutex& mutex) : m_mutex(mutex) {
+        m_mutex.lock();
+    }
+
+    ~pymutex_guard() {
+        m_mutex.unlock();
+    }
+
+    // Non-copyable, non-movable
+    pymutex_guard(const pymutex_guard&) = delete;
+    pymutex_guard& operator=(const pymutex_guard&) = delete;
+};
+
+// Global mutex for protecting all Boost.Python internal state
+// Similar to pybind11's internals.mutex
+BOOST_PYTHON_DECL pymutex& get_global_mutex();
+
+// Macro for acquiring the global lock
+// Similar to pybind11's PYBIND11_LOCK_INTERNALS
+#define BOOST_PYTHON_LOCK_STATE() \
+    ::boost::python::detail::pymutex_guard lock(::boost::python::detail::get_global_mutex())
+
+#else
+
+// No-op macro when not in free-threaded mode
+#define BOOST_PYTHON_LOCK_STATE()
+
+#endif // Py_GIL_DISABLED
+
+}}} // namespace boost::python::detail
+
+#endif // BOOST_PYTHON_DETAIL_PYMUTEX_HPP

include/boost/python/module_init.hpp

@@ -11,11 +11,41 @@
 
 # ifndef BOOST_PYTHON_MODULE_INIT
 
-namespace boost { namespace python { namespace detail {
+namespace boost { namespace python {
+
+#ifdef HAS_CXX11
+// Use to activate the Py_MOD_GIL_NOT_USED flag.
+class mod_gil_not_used {
+public:
+    explicit mod_gil_not_used(bool flag = true) : flag_(flag) {}
+    bool flag() const { return flag_; }
+
+private:
+    bool flag_;
+};
+
+namespace detail {
+
+inline bool gil_not_used_option() { return false; }
+template <typename F, typename... O>
+bool gil_not_used_option(F &&, O &&...o);
+template <typename... O>
+inline bool gil_not_used_option(mod_gil_not_used f, O &&...o) {
+    return f.flag() || gil_not_used_option(o...);
+}
+template <typename F, typename... O>
+inline bool gil_not_used_option(F &&, O &&...o) {
+    return gil_not_used_option(o...);
+}
+
+}
+#endif // HAS_CXX11
+
+namespace detail {
 
 #  if PY_VERSION_HEX >= 0x03000000
 
-BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef&, void(*)());
+BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef&, void(*)(), bool gil_not_used = false);
 
 #else
 
@@ -27,7 +57,37 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)());
 
 #  if PY_VERSION_HEX >= 0x03000000
 
-#   define _BOOST_PYTHON_MODULE_INIT(name) \
+#   ifdef HAS_CXX11
+#    define _BOOST_PYTHON_MODULE_INIT(name, ...) \
+  PyObject* BOOST_PP_CAT(PyInit_, name)()  \
+  { \
+    static PyModuleDef_Base initial_m_base = { \
+        PyObject_HEAD_INIT(NULL) \
+        0, /* m_init */ \
+        0, /* m_index */ \
+        0 /* m_copy */ };  \
+    static PyMethodDef initial_methods[] = { { 0, 0, 0, 0 } }; \
+ \
+    static struct PyModuleDef moduledef = { \
+        initial_m_base, \
+        BOOST_PP_STRINGIZE(name), \
+        0, /* m_doc */ \
+        -1, /* m_size */ \
+        initial_methods, \
+        0,  /* m_reload */ \
+        0, /* m_traverse */ \
+        0, /* m_clear */ \
+        0,  /* m_free */ \
+    }; \
+ \
+    return boost::python::detail::init_module( \
+        moduledef, BOOST_PP_CAT(init_module_, name), \
+        boost::python::detail::gil_not_used_option(__VA_ARGS__) ); \
+  } \
+  void BOOST_PP_CAT(init_module_, name)()
+
+#   else // !HAS_CXX11
+#    define _BOOST_PYTHON_MODULE_INIT(name) \
   PyObject* BOOST_PP_CAT(PyInit_, name)()  \
   { \
     static PyModuleDef_Base initial_m_base = { \
@@ -53,6 +113,7 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)());
         moduledef, BOOST_PP_CAT(init_module_, name) ); \
   } \
   void BOOST_PP_CAT(init_module_, name)()
+#   endif // HAS_CXX11
 
 #  else
 
@@ -66,9 +127,15 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)());
 
 #  endif
 
-#  define BOOST_PYTHON_MODULE_INIT(name)                       \
+#  ifdef HAS_CXX11
+#   define BOOST_PYTHON_MODULE_INIT(name, ...)                  \
+  void BOOST_PP_CAT(init_module_,name)();                      \
+extern "C" BOOST_SYMBOL_EXPORT _BOOST_PYTHON_MODULE_INIT(name, __VA_ARGS__)
+#  else
+#   define BOOST_PYTHON_MODULE_INIT(name)                       \
   void BOOST_PP_CAT(init_module_,name)();                      \
 extern "C" BOOST_SYMBOL_EXPORT _BOOST_PYTHON_MODULE_INIT(name)
+#  endif // HAS_CXX11
 
 # endif
 

include/boost/python/object_core.hpp

@@ -419,6 +419,16 @@ inline api::object_base& api::object_base::operator=(api::object_base const& rhs
 
 inline api::object_base::~object_base()
 {
+#ifdef Py_GIL_DISABLED
+    // This is a not very elegant fix for a problem that occurs with the
+    // free-threaded build of Python.  If this is called when the interpreter
+    // has already been finalized, the thread-state can be null.  Unlike the
+    // GIL-enabled build, Py_DECREF() requires a valid thread-state.  This
+    // causes a memory leak, rather than crash, which seems preferable.
+    if (PyThreadState_GetUnchecked() == NULL) {
+        return;
+    }
+#endif
     assert( Py_REFCNT(m_ptr) > 0 );
     Py_DECREF(m_ptr);
 }

src/converter/from_python.cpp

@@ -11,6 +11,7 @@
 
 #include <boost/python/handle.hpp>
 #include <boost/python/detail/raw_pyobject.hpp>
+#include <boost/python/detail/pymutex.hpp>
 #include <boost/python/cast.hpp>
 
 #include <vector>
@@ -145,6 +146,8 @@ namespace
 
   inline bool visit(rvalue_from_python_chain const* chain)
   {
+      BOOST_PYTHON_LOCK_STATE();
+
       visited_t::iterator const p = std::lower_bound(visited.begin(), visited.end(), chain);
       if (p != visited.end() && *p == chain)
           return false;
@@ -157,9 +160,11 @@ namespace
   {
       unvisit(rvalue_from_python_chain const* chain)
           : chain(chain) {}
-      
+
       ~unvisit()
       {
+          BOOST_PYTHON_LOCK_STATE();
+
           visited_t::iterator const p = std::lower_bound(visited.begin(), visited.end(), chain);
           assert(p != visited.end());
           visited.erase(p);

src/converter/registry.cpp

@@ -5,6 +5,7 @@
 #include <boost/python/converter/registry.hpp>
 #include <boost/python/converter/registrations.hpp>
 #include <boost/python/converter/builtin_converters.hpp>
+#include <boost/python/detail/pymutex.hpp>
 
 #include <set>
 #include <stdexcept>
@@ -112,9 +113,9 @@ registration::~registration()
 namespace // <unnamed>
 {
   typedef registration entry;
-  
+
   typedef std::set<entry> registry_t;
-  
+
 #ifndef BOOST_PYTHON_CONVERTER_REGISTRY_APPLE_MACH_WORKAROUND
   registry_t& entries()
   {
@@ -181,6 +182,8 @@ namespace // <unnamed>
 
   entry* get(type_info type, bool is_shared_ptr = false)
   {
+      BOOST_PYTHON_LOCK_STATE();
+
 #  ifdef BOOST_PYTHON_TRACE_REGISTRY
       registry_t::iterator p = entries().find(entry(type));
 
@@ -293,6 +296,8 @@ namespace registry
 
   registration const* query(type_info type)
   {
+      BOOST_PYTHON_LOCK_STATE();
+
       registry_t::iterator p = entries().find(entry(type));
 #  ifdef BOOST_PYTHON_TRACE_REGISTRY
       std::cout << "querying " << type

src/converter/type_id.cpp

@@ -5,6 +5,7 @@
 
 #include <boost/python/type_id.hpp>
 #include <boost/python/detail/decorated_type_id.hpp>
+#include <boost/python/detail/pymutex.hpp>
 #include <utility>
 #include <vector>
 #include <algorithm>
@@ -81,7 +82,7 @@ namespace
   {
       free_mem(char*p)
           : p(p) {}
-    
+
       ~free_mem()
       {
           std::free(p);
@@ -92,6 +93,7 @@ namespace
 
 bool cxxabi_cxa_demangle_is_broken()
 {
+    BOOST_PYTHON_LOCK_STATE();
     static bool was_tested = false;
     static bool is_broken = false;
     if (!was_tested) {
@@ -109,6 +111,8 @@ namespace detail
 {
   BOOST_PYTHON_DECL char const* gcc_demangle(char const* mangled)
   {
+      BOOST_PYTHON_LOCK_STATE();
+
       typedef std::vector<
           std::pair<char const*, char const*>
       > mangling_map;

src/dict.cpp

@@ -68,8 +68,16 @@ object dict_base::get(object_cref k) const
 {
     if (check_exact(this))
     {
+#ifdef Py_GIL_DISABLED
+        PyObject* result;
+        if (PyDict_GetItemRef(this->ptr(),k.ptr(),&result) < 0) {
+            throw_error_already_set();
+        }
+        return object(detail::new_reference(result ? result : Py_None));
+#else
         PyObject* result = PyDict_GetItem(this->ptr(),k.ptr());
         return object(detail::borrowed_reference(result ? result : Py_None));
+#endif
     }
     else
     {

src/errors.cpp

@@ -10,18 +10,35 @@
 #include <boost/python/errors.hpp>
 #include <boost/cast.hpp>
 #include <boost/python/detail/exception_handler.hpp>
+#include <boost/python/detail/pymutex.hpp>
 
 namespace boost { namespace python {
 
+#ifdef Py_GIL_DISABLED
+namespace detail {
+    // Global mutex for protecting all Boost.Python internal state
+    pymutex& get_global_mutex()
+    {
+        static pymutex mutex;
+        return mutex;
+    }
+}
+#endif
+
 error_already_set::~error_already_set() {}
 
 // IMPORTANT: this function may only be called from within a catch block!
 BOOST_PYTHON_DECL bool handle_exception_impl(function0<void> f)
 {
     try
     {
-        if (detail::exception_handler::chain)
-            return detail::exception_handler::chain->handle(f);
+        detail::exception_handler* handler_chain = nullptr;
+        {
+            BOOST_PYTHON_LOCK_STATE();
+            handler_chain = detail::exception_handler::chain;
+        }
+        if (handler_chain)
+            return handler_chain->handle(f);
         f();
         return false;
     }
@@ -80,6 +97,7 @@ exception_handler::exception_handler(handler_function const& impl)
     : m_impl(impl)
     , m_next(0)
 {
+    BOOST_PYTHON_LOCK_STATE();
     if (chain != 0)
         tail->m_next = this;
     else