Introduce cantera_python shim and load only when needed

This eliminates the need to directly link the Cantera shared library
or Cantera applications to libpython.

SConstruct

@@ -2219,6 +2219,9 @@ else:
 env["external_libs"] = []
 env["external_libs"].extend(env["sundials_libs"])
 
+if env["OS"] == "Linux":
+    env["external_libs"].append("dl")
+
 if env["use_hdf5"]:
     env["external_libs"].append("hdf5")
 

include/cantera/base/ExtensionManager.h

@@ -44,6 +44,31 @@ class ExtensionManager
         throw NotImplementedError("ExtensionManager::registerRateBuilders");
     };
 
+    //! Register a user-defined ReactionRate implementation with ReactionRateFactory
+    //! @param extensionName The name of the library/module containing the user-defined
+    //!     rate. For example, the module name for rates implemented in Python.
+    //! @param className The name of the rate in the user's code. For example, the
+    //!     Python class name
+    //! @param rateName The name used to construct a rate of this type using
+    //!     the newReactionRate() function or from a YAML input file
+    virtual void registerRateBuilder(const string& extensionName,
+        const string& className, const string& rateName)
+    {
+        throw NotImplementedError("ExtensionManager::registerRateBuilder");
+    }
+
+    //! Register a user-defined ReactionData implementation
+    //! @param extensionName The name of the library/module containing the user-defined
+    //!     type. For example, the module name for rates implemented in Python.
+    //! @param className The name of the data object in the user's code. For example,
+    //!     the Python class name
+    //! @param rateName The name of the corresponding reaction rate type
+    virtual void registerRateDataBuilder(const string& extensionName,
+        const string& className, const string& rateName)
+    {
+        throw NotImplementedError("ExtensionManager::registerRateDataBuilder");
+    }
+
     //! Create an object in an external language that wraps the specified ReactionData
     //! object
     //!

include/cantera/base/ExtensionManagerFactory.h

@@ -26,11 +26,11 @@ class ExtensionManagerFactory : public Factory<ExtensionManager>
     //! Delete the static instance of this factory
     virtual void deleteFactory();
 
-private:
     //! Static function that returns the static instance of the factory, creating it
     //! if necessary.
     static ExtensionManagerFactory& factory();
 
+private:
     //! static member of the single factory instance
     static ExtensionManagerFactory* s_factory;
 

include/cantera/extensions/PythonExtensionManager.h

@@ -8,6 +8,9 @@
 
 #include "cantera/base/ExtensionManager.h"
 
+#define BOOST_DLL_USE_STD_FS
+#include <boost/dll/alias.hpp>
+
 namespace Cantera
 {
 
@@ -29,18 +32,22 @@ class PythonExtensionManager : public ExtensionManager
     PythonExtensionManager();
     virtual void registerRateBuilders(const std::string& extensionName) override;
 
-    //! Function called from Cython to register an ExtensibleRate implementation
-    static void registerPythonRateBuilder(const std::string& moduleName,
-        const std::string& className, const std::string& rateName);
+    void registerRateBuilder(const string& moduleName,
+        const string& className, const string& rateName) override;
+
+    static ExtensionManager* create() {
+        return new PythonExtensionManager();
+    }
 
-    //! Function called from Cython to register an ExtensibleRateData implementation
-    static void registerPythonRateDataBuilder(const std::string& moduleName,
-        const std::string& className, const std::string& rateName);
+    void registerRateDataBuilder(const string& moduleName,
+        const string& className, const string& rateName) override;
 
 private:
     static bool s_imported;
 };
 
+BOOST_DLL_ALIAS(Cantera::PythonExtensionManager::create, create_manager);
+
 }
 
 #endif

interfaces/cython/SConscript

@@ -43,6 +43,7 @@ for pyxfile in multi_glob(localenv, "cantera", "pyx"):
     obj = localenv.SharedObject(
         f"#build/temp-py/{pyxfile.name.split('.')[0]}", cythonized)
     cython_obj.append(obj)
+cython_obj.extend(env['python_ext_objects'])
 
 module_ext = localenv["py_module_ext"]
 ext = localenv.LoadableModule(f"cantera/_cantera{module_ext}",

interfaces/cython/cantera/delegator.pxd

@@ -62,12 +62,18 @@ cdef extern from "cantera/cython/funcWrapper.h":
     cdef function[int(size_t&, const string&)] pyOverride(
         PyObject*, int(PyFuncInfo&, size_t&, const string&))
 
-cdef extern from "cantera/extensions/PythonExtensionManager.h" namespace "Cantera":
-    cdef cppclass CxxPythonExtensionManager "Cantera::PythonExtensionManager":
-        @staticmethod
-        void registerPythonRateBuilder(string&, string&, string&) except +translate_exception
+cdef extern from "cantera/base/ExtensionManager.h" namespace "Cantera":
+    cdef cppclass CxxExtensionManager "Cantera::ExtensionManager":
+        void registerRateBuilder(string&, string&, string&) except +translate_exception
+        void registerRateDataBuilder(string&, string&, string&) except +translate_exception
+
+        shared_ptr[CxxExtensionManager] build(string&)
+
+cdef extern from "cantera/base/ExtensionManagerFactory.h" namespace "Cantera":
+    cdef cppclass CxxExtensionManagerFactory "Cantera::ExtensionManagerFactory":
         @staticmethod
-        void registerPythonRateDataBuilder(string&, string&, string&) except +translate_exception
+        shared_ptr[CxxExtensionManager] build(string&)
+
 
 ctypedef CxxDelegator* CxxDelegatorPtr
 

interfaces/cython/cantera/delegator.pyx

@@ -392,11 +392,14 @@ def extension(*, name, data=None):
     .. versionadded:: 3.0
     """
     def decorator(cls):
+        cdef shared_ptr[CxxExtensionManager] mgr = (
+            CxxExtensionManagerFactory.build(stringify("python")))
+
         if issubclass(cls, ExtensibleRate):
             cls._reaction_rate_type = name
             # Registering immediately supports the case where the main
             # application is Python
-            CxxPythonExtensionManager.registerPythonRateBuilder(
+            mgr.get().registerRateBuilder(
                 stringify(cls.__module__), stringify(cls.__name__), stringify(name))
 
             # Deferred registration supports the case where the main application
@@ -406,7 +409,7 @@ def extension(*, name, data=None):
             # Register the ReactionData delegator
             if not issubclass(data, ExtensibleRateData):
                 raise ValueError("'data' must inherit from 'ExtensibleRateData'")
-            CxxPythonExtensionManager.registerPythonRateDataBuilder(
+            mgr.get().registerRateDataBuilder(
                 stringify(data.__module__), stringify(data.__name__), stringify(name))
             _rate_data_delegators.append((data.__module__, data.__name__, name))
         else:

src/SConscript

@@ -83,10 +83,9 @@ if env["python_package"] == "full":
          ('''${python_cmd} -c "import Cython.Build; '''
           f'''Cython.Build.cythonize(r'${{SOURCE}}', build_dir='{build_dir}')"''')
     )
-    for pxd in multi_glob(localenv, "#interfaces/cython/cantera", "pxd"):
+    for pxd in multi_glob(pyenv, "#interfaces/cython/cantera", "pxd"):
         localenv.Depends(cythonized, pxd)
 
-    obj = pyenv.SharedObject(cythonized[0])
     if env["OS"] == "Windows":
         escaped_home = '\\"' + pyenv["py_base"].replace("\\", "\\\\") + '\\"'
         pyenv.Append(CPPDEFINES={"CT_PYTHONHOME": escaped_home})
@@ -95,12 +94,23 @@ if env["python_package"] == "full":
 
     env.Command('#src/extensions/pythonExtensions.h', '#build/src/extensions/pythonExtensions.h',
                 Copy('$TARGET', '$SOURCE'))
-    libraryTargets.append(obj)
-    libraryTargets.append(pyenv.SharedObject("extensions/PythonExtensionManager.cpp"))
-    localenv.Append(LIBS=pyenv["py_libs"], LIBPATH=pyenv["py_libpath"])
-    env["cantera_libs"].extend(pyenv["py_libs"])
-    env.Append(LIBPATH=pyenv["py_libpath"])
-    env["extra_lib_dirs"].extend(pyenv["py_libpath"])
+
+    env['python_ext_objects'] = [
+        pyenv.SharedObject(cythonized[0]),
+        pyenv.SharedObject("extensions/PythonExtensionManager.cpp")
+    ]
+    pyenv.Append(LIBS=pyenv["py_libs"], LIBPATH=pyenv["py_libpath"])
+    if pyenv["versioned_shared_library"]:
+        lib = build(pyenv.SharedLibrary("../lib/cantera_python", env['python_ext_objects'],
+                                           SPAWN=get_spawn(pyenv),
+                                           SHLIBVERSION=pyenv["cantera_pure_version"]))
+        install(pyenv.InstallVersionedLib, "$inst_libdir", lib)
+    else:
+        lib = build(pyenv.SharedLibrary("../lib/cantera_python", env['python_ext_objects'],
+                                           SPAWN=get_spawn(pyenv)))
+        install("$inst_libdir", lib)
+
+
 
 # build the Cantera static library
 lib = build(localenv.StaticLibrary('../lib/cantera', libraryTargets,

src/base/ExtensionManagerFactory.cpp

@@ -5,10 +5,6 @@
 
 #include "cantera/base/ExtensionManagerFactory.h"
 
-#ifdef CT_HAS_PYTHON
-#include "cantera/extensions/PythonExtensionManager.h"
-#endif
-
 using namespace std;
 
 namespace Cantera
@@ -19,9 +15,6 @@ mutex ExtensionManagerFactory::s_mutex;
 
 ExtensionManagerFactory::ExtensionManagerFactory()
 {
-    #ifdef CT_HAS_PYTHON
-    reg("python", []() { return new PythonExtensionManager(); });
-    #endif
 }
 
 ExtensionManagerFactory& ExtensionManagerFactory::factory()

src/base/application.cpp

@@ -8,6 +8,9 @@
 #include "cantera/base/stringUtils.h"
 #include "cantera/base/ExtensionManagerFactory.h"
 
+#define BOOST_DLL_USE_STD_FS
+#include <boost/dll/import.hpp>
+
 #include <fstream>
 #include <sstream>
 #include <mutex>
@@ -402,9 +405,22 @@ void Application::loadExtension(const string& extType, const string& name)
     if (m_loaded_extensions.count({extType, name})) {
         return;
     }
-    auto manager = ExtensionManagerFactory::build(extType);
+
+    writelog("Loading cantera_python library\n");
+    typedef ExtensionManager* (creator_t)();
+    auto loader = boost::dll::import_alias<creator_t>( // type of imported symbol must be explicitly specified
+        "cantera_python",                              // path to library
+        "create_manager",                              // symbol to import
+        boost::dll::load_mode::search_system_folders | boost::dll::load_mode::append_decorations | boost::dll::load_mode::rtld_global // append extensions and prefixes, and search normal library path
+    );
+    auto manager = loader();
+    ExtensionManagerFactory::factory().reg("python", [&]() { return loader(); });
+
+    Cantera::writelog("Registered extensions with factory {}\n",
+                      (void*) &ExtensionManagerFactory::factory());
     manager->registerRateBuilders(name);
     m_loaded_extensions.insert({extType, name});
+    writelog("Done loading extension\n");
 }
 
 Application* Application::s_app = 0;

src/extensions/PythonExtensionManager.cpp

@@ -157,7 +157,7 @@ PythonExtensionManager::PythonExtensionManager()
 void PythonExtensionManager::registerRateBuilders(const string& extensionName)
 {
     // Each rate builder class is decorated with @extension, which calls the
-    // registerPythonRateBuilder method to register that class. So all we have
+    // registerRateBuilder method to register that class. So all we have
     // to do here is load the module.
     PyObject* module_name = PyUnicode_FromString(extensionName.c_str());
     PyObject* py_module = PyImport_Import(module_name);
@@ -169,7 +169,7 @@ void PythonExtensionManager::registerRateBuilders(const string& extensionName)
     ct_registerReactionDelegators();
 }
 
-void PythonExtensionManager::registerPythonRateBuilder(
+void PythonExtensionManager::registerRateBuilder(
     const std::string& moduleName, const std::string& className,
     const std::string& rateName)
 {
@@ -198,12 +198,11 @@ void PythonExtensionManager::registerPythonRateBuilder(
     ReactionRateFactory::factory()->reg(rateName, builder);
 }
 
-void PythonExtensionManager::registerPythonRateDataBuilder(
+void PythonExtensionManager::registerRateDataBuilder(
     const string& moduleName, const string& className, const string& rateName)
 {
     // Make sure the helper module has been loaded
     PythonExtensionManager mgr;
-
     // Create a function that links a C++ ReactionDataDelegator
     // object and a Python ExtensibleRateData object of a particular type, and register
     // this function for making that link
@@ -218,7 +217,7 @@ void PythonExtensionManager::registerPythonRateDataBuilder(
         }
         delegator.setWrapper(make_shared<PythonHandle>(extData, false));
     };
-    ExtensionManager::registerReactionDataLinker(rateName, builder);
+    mgr.registerReactionDataLinker(rateName, builder);
 
     // Create a function that will link a Python Solution object to the C++ Solution
     // object that gets passed to the Reaction
@@ -231,7 +230,7 @@ void PythonExtensionManager::registerPythonRateDataBuilder(
         }
         return make_shared<PythonHandle>(pySoln, false);
     };
-    ExtensionManager::registerSolutionLinker("python", solnLinker);
+    mgr.registerSolutionLinker("python", solnLinker);
 }
 
 };

src/extensions/pythonExtensions.pyx

@@ -28,13 +28,15 @@ cdef extern from "cantera/kinetics/ReactionRateDelegator.h" namespace "Cantera":
         CxxReactionRateDelegator()
 
 
-cdef extern from "cantera/extensions/PythonExtensionManager.h" namespace "Cantera":
-    cdef cppclass CxxPythonExtensionManager "Cantera::PythonExtensionManager":
-        @staticmethod
-        void registerPythonRateBuilder(string&, string&, string&)
-        @staticmethod
-        void registerPythonRateDataBuilder(string&, string&, string&)
+cdef extern from "cantera/base/ExtensionManager.h" namespace "Cantera":
+    cdef cppclass CxxExtensionManager "Cantera::ExtensionManager":
+        void registerRateBuilder(string&, string&, string&)
+        void registerRateDataBuilder(string&, string&, string&)
 
+cdef extern from "cantera/base/ExtensionManagerFactory.h" namespace "Cantera":
+    cdef cppclass CxxExtensionManagerFactory "Cantera::ExtensionManagerFactory":
+        @staticmethod
+        shared_ptr[CxxExtensionManager] build(string&)
 
 cdef public char* ct_getExceptionString(object exType, object exValue, object exTraceback):
     import traceback
@@ -67,15 +69,16 @@ cdef public object ct_newPythonExtensibleRateData(CxxReactionDataDelegator* dele
 
 
 cdef public ct_registerReactionDelegators():
+    cdef shared_ptr[CxxExtensionManager] mgr = (
+        CxxExtensionManagerFactory.build(stringify("python")))
+
     for module, cls, name in ct.delegator._rate_delegators:
-        CxxPythonExtensionManager.registerPythonRateBuilder(
-            stringify(module), stringify(cls), stringify(name))
+        mgr.get().registerRateBuilder(stringify(module), stringify(cls), stringify(name))
 
     ct.delegator._rate_delegators.clear()
 
     for module, cls, name in ct.delegator._rate_data_delegators:
-        CxxPythonExtensionManager.registerPythonRateDataBuilder(
-            stringify(module), stringify(cls), stringify(name))
+        mgr.get().registerRateDataBuilder(stringify(module), stringify(cls), stringify(name))
 
     ct.delegator._rate_data_delegators.clear()
 

test/SConscript

@@ -12,7 +12,7 @@ localenv = env.Clone()
 
 # Where possible, link tests against the shared libraries to minimize the sizes
 # of the resulting binaries.
-if localenv['OS'] == 'Linux':
+if localenv['OS'] == 'Linux' or localenv['OS'] == 'Darwin':
     cantera_libs = localenv['cantera_shared_libs']
 else:
     cantera_libs = localenv['cantera_libs']