From 5ae4847239ba2704c1e030e923f3bf64a495e7ff Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Fri, 27 Jan 2023 08:55:31 -0500 Subject: [PATCH] Introduce cantera_python shim and load only when needed This eliminates the need to directly link the Cantera shared library or Cantera applications to libpython. --- SConstruct | 3 +++ include/cantera/base/ExtensionManager.h | 25 ++++++++++++++++++ .../cantera/base/ExtensionManagerFactory.h | 2 +- .../extensions/PythonExtensionManager.h | 19 +++++++++----- interfaces/cython/SConscript | 1 + interfaces/cython/cantera/delegator.pxd | 16 ++++++++---- interfaces/cython/cantera/delegator.pyx | 7 +++-- src/SConscript | 26 +++++++++++++------ src/base/ExtensionManagerFactory.cpp | 7 ----- src/base/application.cpp | 18 ++++++++++++- src/extensions/PythonExtensionManager.cpp | 11 ++++---- src/extensions/pythonExtensions.pyx | 23 +++++++++------- test/SConscript | 2 +- 13 files changed, 113 insertions(+), 47 deletions(-) diff --git a/SConstruct b/SConstruct index bf4c4266cb..a48600be53 100644 --- a/SConstruct +++ b/SConstruct @@ -2219,6 +2219,9 @@ else: env["external_libs"] = [] env["external_libs"].extend(env["sundials_libs"]) +if env["OS"] == "Linux": + env["external_libs"].append("dl") + if env["use_hdf5"]: env["external_libs"].append("hdf5") diff --git a/include/cantera/base/ExtensionManager.h b/include/cantera/base/ExtensionManager.h index b769f5fa1b..4d9b34b2a6 100644 --- a/include/cantera/base/ExtensionManager.h +++ b/include/cantera/base/ExtensionManager.h @@ -44,6 +44,31 @@ class ExtensionManager throw NotImplementedError("ExtensionManager::registerRateBuilders"); }; + //! Register a user-defined ReactionRate implementation with ReactionRateFactory + //! @param extensionName The name of the library/module containing the user-defined + //! rate. For example, the module name for rates implemented in Python. + //! @param className The name of the rate in the user's code. For example, the + //! Python class name + //! @param rateName The name used to construct a rate of this type using + //! the newReactionRate() function or from a YAML input file + virtual void registerRateBuilder(const string& extensionName, + const string& className, const string& rateName) + { + throw NotImplementedError("ExtensionManager::registerRateBuilder"); + } + + //! Register a user-defined ReactionData implementation + //! @param extensionName The name of the library/module containing the user-defined + //! type. For example, the module name for rates implemented in Python. + //! @param className The name of the data object in the user's code. For example, + //! the Python class name + //! @param rateName The name of the corresponding reaction rate type + virtual void registerRateDataBuilder(const string& extensionName, + const string& className, const string& rateName) + { + throw NotImplementedError("ExtensionManager::registerRateDataBuilder"); + } + //! Create an object in an external language that wraps the specified ReactionData //! object //! diff --git a/include/cantera/base/ExtensionManagerFactory.h b/include/cantera/base/ExtensionManagerFactory.h index 77b48fb873..03bd58afae 100644 --- a/include/cantera/base/ExtensionManagerFactory.h +++ b/include/cantera/base/ExtensionManagerFactory.h @@ -26,11 +26,11 @@ class ExtensionManagerFactory : public Factory //! Delete the static instance of this factory virtual void deleteFactory(); -private: //! Static function that returns the static instance of the factory, creating it //! if necessary. static ExtensionManagerFactory& factory(); +private: //! static member of the single factory instance static ExtensionManagerFactory* s_factory; diff --git a/include/cantera/extensions/PythonExtensionManager.h b/include/cantera/extensions/PythonExtensionManager.h index 4a068ced83..49212c1faa 100644 --- a/include/cantera/extensions/PythonExtensionManager.h +++ b/include/cantera/extensions/PythonExtensionManager.h @@ -8,6 +8,9 @@ #include "cantera/base/ExtensionManager.h" +#define BOOST_DLL_USE_STD_FS +#include + namespace Cantera { @@ -29,18 +32,22 @@ class PythonExtensionManager : public ExtensionManager PythonExtensionManager(); virtual void registerRateBuilders(const std::string& extensionName) override; - //! Function called from Cython to register an ExtensibleRate implementation - static void registerPythonRateBuilder(const std::string& moduleName, - const std::string& className, const std::string& rateName); + void registerRateBuilder(const string& moduleName, + const string& className, const string& rateName) override; + + static ExtensionManager* create() { + return new PythonExtensionManager(); + } - //! Function called from Cython to register an ExtensibleRateData implementation - static void registerPythonRateDataBuilder(const std::string& moduleName, - const std::string& className, const std::string& rateName); + void registerRateDataBuilder(const string& moduleName, + const string& className, const string& rateName) override; private: static bool s_imported; }; +BOOST_DLL_ALIAS(Cantera::PythonExtensionManager::create, create_manager); + } #endif diff --git a/interfaces/cython/SConscript b/interfaces/cython/SConscript index ef34af3f78..d39933c9d4 100644 --- a/interfaces/cython/SConscript +++ b/interfaces/cython/SConscript @@ -43,6 +43,7 @@ for pyxfile in multi_glob(localenv, "cantera", "pyx"): obj = localenv.SharedObject( f"#build/temp-py/{pyxfile.name.split('.')[0]}", cythonized) cython_obj.append(obj) +cython_obj.extend(env['python_ext_objects']) module_ext = localenv["py_module_ext"] ext = localenv.LoadableModule(f"cantera/_cantera{module_ext}", diff --git a/interfaces/cython/cantera/delegator.pxd b/interfaces/cython/cantera/delegator.pxd index 67f96a9660..7a8b40ef14 100644 --- a/interfaces/cython/cantera/delegator.pxd +++ b/interfaces/cython/cantera/delegator.pxd @@ -62,12 +62,18 @@ cdef extern from "cantera/cython/funcWrapper.h": cdef function[int(size_t&, const string&)] pyOverride( PyObject*, int(PyFuncInfo&, size_t&, const string&)) -cdef extern from "cantera/extensions/PythonExtensionManager.h" namespace "Cantera": - cdef cppclass CxxPythonExtensionManager "Cantera::PythonExtensionManager": - @staticmethod - void registerPythonRateBuilder(string&, string&, string&) except +translate_exception +cdef extern from "cantera/base/ExtensionManager.h" namespace "Cantera": + cdef cppclass CxxExtensionManager "Cantera::ExtensionManager": + void registerRateBuilder(string&, string&, string&) except +translate_exception + void registerRateDataBuilder(string&, string&, string&) except +translate_exception + + shared_ptr[CxxExtensionManager] build(string&) + +cdef extern from "cantera/base/ExtensionManagerFactory.h" namespace "Cantera": + cdef cppclass CxxExtensionManagerFactory "Cantera::ExtensionManagerFactory": @staticmethod - void registerPythonRateDataBuilder(string&, string&, string&) except +translate_exception + shared_ptr[CxxExtensionManager] build(string&) + ctypedef CxxDelegator* CxxDelegatorPtr diff --git a/interfaces/cython/cantera/delegator.pyx b/interfaces/cython/cantera/delegator.pyx index 0605c9dd11..75d0f1468d 100644 --- a/interfaces/cython/cantera/delegator.pyx +++ b/interfaces/cython/cantera/delegator.pyx @@ -392,11 +392,14 @@ def extension(*, name, data=None): .. versionadded:: 3.0 """ def decorator(cls): + cdef shared_ptr[CxxExtensionManager] mgr = ( + CxxExtensionManagerFactory.build(stringify("python"))) + if issubclass(cls, ExtensibleRate): cls._reaction_rate_type = name # Registering immediately supports the case where the main # application is Python - CxxPythonExtensionManager.registerPythonRateBuilder( + mgr.get().registerRateBuilder( stringify(cls.__module__), stringify(cls.__name__), stringify(name)) # Deferred registration supports the case where the main application @@ -406,7 +409,7 @@ def extension(*, name, data=None): # Register the ReactionData delegator if not issubclass(data, ExtensibleRateData): raise ValueError("'data' must inherit from 'ExtensibleRateData'") - CxxPythonExtensionManager.registerPythonRateDataBuilder( + mgr.get().registerRateDataBuilder( stringify(data.__module__), stringify(data.__name__), stringify(name)) _rate_data_delegators.append((data.__module__, data.__name__, name)) else: diff --git a/src/SConscript b/src/SConscript index 19e337472d..e9e82dc11c 100644 --- a/src/SConscript +++ b/src/SConscript @@ -83,10 +83,9 @@ if env["python_package"] == "full": ('''${python_cmd} -c "import Cython.Build; ''' f'''Cython.Build.cythonize(r'${{SOURCE}}', build_dir='{build_dir}')"''') ) - for pxd in multi_glob(localenv, "#interfaces/cython/cantera", "pxd"): + for pxd in multi_glob(pyenv, "#interfaces/cython/cantera", "pxd"): localenv.Depends(cythonized, pxd) - obj = pyenv.SharedObject(cythonized[0]) if env["OS"] == "Windows": escaped_home = '\\"' + pyenv["py_base"].replace("\\", "\\\\") + '\\"' pyenv.Append(CPPDEFINES={"CT_PYTHONHOME": escaped_home}) @@ -95,12 +94,23 @@ if env["python_package"] == "full": env.Command('#src/extensions/pythonExtensions.h', '#build/src/extensions/pythonExtensions.h', Copy('$TARGET', '$SOURCE')) - libraryTargets.append(obj) - libraryTargets.append(pyenv.SharedObject("extensions/PythonExtensionManager.cpp")) - localenv.Append(LIBS=pyenv["py_libs"], LIBPATH=pyenv["py_libpath"]) - env["cantera_libs"].extend(pyenv["py_libs"]) - env.Append(LIBPATH=pyenv["py_libpath"]) - env["extra_lib_dirs"].extend(pyenv["py_libpath"]) + + env['python_ext_objects'] = [ + pyenv.SharedObject(cythonized[0]), + pyenv.SharedObject("extensions/PythonExtensionManager.cpp") + ] + pyenv.Append(LIBS=pyenv["py_libs"], LIBPATH=pyenv["py_libpath"]) + if pyenv["versioned_shared_library"]: + lib = build(pyenv.SharedLibrary("../lib/cantera_python", env['python_ext_objects'], + SPAWN=get_spawn(pyenv), + SHLIBVERSION=pyenv["cantera_pure_version"])) + install(pyenv.InstallVersionedLib, "$inst_libdir", lib) + else: + lib = build(pyenv.SharedLibrary("../lib/cantera_python", env['python_ext_objects'], + SPAWN=get_spawn(pyenv))) + install("$inst_libdir", lib) + + # build the Cantera static library lib = build(localenv.StaticLibrary('../lib/cantera', libraryTargets, diff --git a/src/base/ExtensionManagerFactory.cpp b/src/base/ExtensionManagerFactory.cpp index 35f2ac1a33..becae86a4f 100644 --- a/src/base/ExtensionManagerFactory.cpp +++ b/src/base/ExtensionManagerFactory.cpp @@ -5,10 +5,6 @@ #include "cantera/base/ExtensionManagerFactory.h" -#ifdef CT_HAS_PYTHON -#include "cantera/extensions/PythonExtensionManager.h" -#endif - using namespace std; namespace Cantera @@ -19,9 +15,6 @@ mutex ExtensionManagerFactory::s_mutex; ExtensionManagerFactory::ExtensionManagerFactory() { - #ifdef CT_HAS_PYTHON - reg("python", []() { return new PythonExtensionManager(); }); - #endif } ExtensionManagerFactory& ExtensionManagerFactory::factory() diff --git a/src/base/application.cpp b/src/base/application.cpp index 221f2d46e7..c3ceb40772 100644 --- a/src/base/application.cpp +++ b/src/base/application.cpp @@ -8,6 +8,9 @@ #include "cantera/base/stringUtils.h" #include "cantera/base/ExtensionManagerFactory.h" +#define BOOST_DLL_USE_STD_FS +#include + #include #include #include @@ -402,9 +405,22 @@ void Application::loadExtension(const string& extType, const string& name) if (m_loaded_extensions.count({extType, name})) { return; } - auto manager = ExtensionManagerFactory::build(extType); + + writelog("Loading cantera_python library\n"); + typedef ExtensionManager* (creator_t)(); + auto loader = boost::dll::import_alias( // type of imported symbol must be explicitly specified + "cantera_python", // path to library + "create_manager", // symbol to import + boost::dll::load_mode::search_system_folders | boost::dll::load_mode::append_decorations | boost::dll::load_mode::rtld_global // append extensions and prefixes, and search normal library path + ); + auto manager = loader(); + ExtensionManagerFactory::factory().reg("python", [&]() { return loader(); }); + + Cantera::writelog("Registered extensions with factory {}\n", + (void*) &ExtensionManagerFactory::factory()); manager->registerRateBuilders(name); m_loaded_extensions.insert({extType, name}); + writelog("Done loading extension\n"); } Application* Application::s_app = 0; diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index e00d78d017..ed3f74b9ea 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -157,7 +157,7 @@ PythonExtensionManager::PythonExtensionManager() void PythonExtensionManager::registerRateBuilders(const string& extensionName) { // Each rate builder class is decorated with @extension, which calls the - // registerPythonRateBuilder method to register that class. So all we have + // registerRateBuilder method to register that class. So all we have // to do here is load the module. PyObject* module_name = PyUnicode_FromString(extensionName.c_str()); PyObject* py_module = PyImport_Import(module_name); @@ -169,7 +169,7 @@ void PythonExtensionManager::registerRateBuilders(const string& extensionName) ct_registerReactionDelegators(); } -void PythonExtensionManager::registerPythonRateBuilder( +void PythonExtensionManager::registerRateBuilder( const std::string& moduleName, const std::string& className, const std::string& rateName) { @@ -198,12 +198,11 @@ void PythonExtensionManager::registerPythonRateBuilder( ReactionRateFactory::factory()->reg(rateName, builder); } -void PythonExtensionManager::registerPythonRateDataBuilder( +void PythonExtensionManager::registerRateDataBuilder( const string& moduleName, const string& className, const string& rateName) { // Make sure the helper module has been loaded PythonExtensionManager mgr; - // Create a function that links a C++ ReactionDataDelegator // object and a Python ExtensibleRateData object of a particular type, and register // this function for making that link @@ -218,7 +217,7 @@ void PythonExtensionManager::registerPythonRateDataBuilder( } delegator.setWrapper(make_shared(extData, false)); }; - ExtensionManager::registerReactionDataLinker(rateName, builder); + mgr.registerReactionDataLinker(rateName, builder); // Create a function that will link a Python Solution object to the C++ Solution // object that gets passed to the Reaction @@ -231,7 +230,7 @@ void PythonExtensionManager::registerPythonRateDataBuilder( } return make_shared(pySoln, false); }; - ExtensionManager::registerSolutionLinker("python", solnLinker); + mgr.registerSolutionLinker("python", solnLinker); } }; diff --git a/src/extensions/pythonExtensions.pyx b/src/extensions/pythonExtensions.pyx index 5a558b304e..48c67524de 100644 --- a/src/extensions/pythonExtensions.pyx +++ b/src/extensions/pythonExtensions.pyx @@ -28,13 +28,15 @@ cdef extern from "cantera/kinetics/ReactionRateDelegator.h" namespace "Cantera": CxxReactionRateDelegator() -cdef extern from "cantera/extensions/PythonExtensionManager.h" namespace "Cantera": - cdef cppclass CxxPythonExtensionManager "Cantera::PythonExtensionManager": - @staticmethod - void registerPythonRateBuilder(string&, string&, string&) - @staticmethod - void registerPythonRateDataBuilder(string&, string&, string&) +cdef extern from "cantera/base/ExtensionManager.h" namespace "Cantera": + cdef cppclass CxxExtensionManager "Cantera::ExtensionManager": + void registerRateBuilder(string&, string&, string&) + void registerRateDataBuilder(string&, string&, string&) +cdef extern from "cantera/base/ExtensionManagerFactory.h" namespace "Cantera": + cdef cppclass CxxExtensionManagerFactory "Cantera::ExtensionManagerFactory": + @staticmethod + shared_ptr[CxxExtensionManager] build(string&) cdef public char* ct_getExceptionString(object exType, object exValue, object exTraceback): import traceback @@ -67,15 +69,16 @@ cdef public object ct_newPythonExtensibleRateData(CxxReactionDataDelegator* dele cdef public ct_registerReactionDelegators(): + cdef shared_ptr[CxxExtensionManager] mgr = ( + CxxExtensionManagerFactory.build(stringify("python"))) + for module, cls, name in ct.delegator._rate_delegators: - CxxPythonExtensionManager.registerPythonRateBuilder( - stringify(module), stringify(cls), stringify(name)) + mgr.get().registerRateBuilder(stringify(module), stringify(cls), stringify(name)) ct.delegator._rate_delegators.clear() for module, cls, name in ct.delegator._rate_data_delegators: - CxxPythonExtensionManager.registerPythonRateDataBuilder( - stringify(module), stringify(cls), stringify(name)) + mgr.get().registerRateDataBuilder(stringify(module), stringify(cls), stringify(name)) ct.delegator._rate_data_delegators.clear() diff --git a/test/SConscript b/test/SConscript index 6b2150d347..2a79f31195 100644 --- a/test/SConscript +++ b/test/SConscript @@ -12,7 +12,7 @@ localenv = env.Clone() # Where possible, link tests against the shared libraries to minimize the sizes # of the resulting binaries. -if localenv['OS'] == 'Linux': +if localenv['OS'] == 'Linux' or localenv['OS'] == 'Darwin': cantera_libs = localenv['cantera_shared_libs'] else: cantera_libs = localenv['cantera_libs']