Refs #4204. Make the old/new python interfaces will blissfully coexist.

The ticket is the best place to look for the reason why this is so convoluted.
Please also read 'PythonAPI/src/why_is_boost_python_here.txt'.

Code/Mantid/Framework/Kernel/inc/MantidKernel/LibraryManager.h

@@ -62,6 +62,8 @@ namespace Mantid
 
       /// Load a given library
       bool loadLibrary(const std::string & filepath);
+      /// Returns true if the library is to be loaded
+      bool skip(const std::string & filename);
       ///Storage for the LibraryWrappers.
       std::map< const std::string, boost::shared_ptr<Mantid::Kernel::LibraryWrapper> > OpenLibs;
 

Code/Mantid/Framework/Kernel/src/LibraryManager.cpp

@@ -2,6 +2,7 @@
 
 #include "MantidKernel/DllOpen.h"
 #include "MantidKernel/LibraryManager.h"
+#include "MantidKernel/ConfigService.h"
 #include "MantidKernel/Logger.h"
 
 #include <Poco/Path.h>
@@ -63,6 +64,7 @@ namespace Mantid
           }
           else
           {
+            if( skip(item.toString()) ) continue;
             if( loadLibrary(item.toString()) ) 
             {
               ++libCount;
@@ -81,6 +83,38 @@ namespace Mantid
     //-------------------------------------------------------------------------
     // Private members
     //-------------------------------------------------------------------------
+    /**
+     * Returns true if the name contains one of the strings given in the
+     * 'plugins.exclude' variable. Each string from the variable is
+     * searched for with the filename so an exact match is not necessary. This
+     * avoids having to specify prefixes and suffixes for different platforms,
+     * i.e. 'plugins.exclude = MantidKernel' will exclude libMantidKernel.so
+     * @param filename :: A string giving the filename/file path
+     * @return True if the library should be skipped
+     */
+    bool LibraryManagerImpl::skip(const std::string & filename)
+    {
+      static std::set<std::string> excludes;
+      static bool initialized(false);
+      if( !initialized )
+      {
+        std::string excludeStr = ConfigService::Instance().getString("plugins.exclude");
+        boost::split(excludes, excludeStr, boost::is_any_of(":;"), boost::token_compress_on);
+        initialized = true;
+      }
+      bool skipme(false);
+      for( std::set<std::string>::const_iterator itr = excludes.begin(); itr != excludes.end();
+          ++itr)
+      {
+        if( filename.find(*itr) != std::string::npos)
+        {
+          skipme = true;
+          break;
+        }
+      }
+      return skipme;
+    }
+
     /** 
     * Load a library
     * @param filepath :: The full path to a library as a string

Code/Mantid/Framework/Properties/Mantid.properties.template

@@ -25,6 +25,9 @@ mantidqt.python_interfaces_directory = @MANTID_ROOT@/scripts
 # Where to find mantid plugin libraries
 plugins.directory = @PLUGINS@
 
+# Libraries to skip. The strings are searched for when loading libraries so they don't need to be exact
+plugins.exclude = MantidPythonAPI
+
 # Where to find Mantid Qt plugin libraries
 mantidqt.plugins.directory = @QTPLUGINS@
 

Code/Mantid/Framework/PythonAPI/CMakeLists.txt

@@ -15,6 +15,8 @@ set ( SRC_FILES
 	src/kernel_exports.cpp
 )
 
+set ( BOOST_SRC src/boostpython.cpp )
+
 set ( SRC_UNITY_IGNORE_FILES )
 
 set ( INC_FILES
@@ -60,11 +62,7 @@ endif(UNITY_BUILD)
 ###########################################################################
 # Add local dependencies
 ###########################################################################
-
-set ( Boost_LIBRARIES ) # Empty out the variable after previous use
-set ( Boost_USE_DEBUG_PYTHON TRUE )
-find_package ( Boost REQUIRED python )
-add_definitions ( -DBOOST_DEBUG_PYTHON -DBOOST_PYTHON_NO_LIB )
+add_definitions ( -DBOOST_DEBUG_PYTHON -DBOOST_PYTHON_NO_LIB -DBOOST_PYTHON_STATIC_LIB -DBOOST_PYTHON_SOURCE )
 
 find_package ( Numpy REQUIRED )
 include_directories ( ${PYTHON_NUMPY_INCLUDE_DIR} )
@@ -90,7 +88,13 @@ endforeach ( PYFILE )
 # Create the target for this directory
 ###########################################################################
 
-add_library ( PythonAPI ${SRC_FILES} ${INC_FILES} ${PYTHON_INSTALL_FILES})
+add_library ( PythonAPI ${SRC_FILES} ${INC_FILES} ${BOOST_SRC} ${PYTHON_INSTALL_FILES} )
+if ( ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" )
+  set ( DLOPEN_SRC src/dlopen.cpp )
+  add_library ( dlopen ${DLOPEN_SRC} )
+  add_dependencies( PythonAPI dlopen )
+endif()
+
 # Set the name of the generated library
 set_target_properties ( PythonAPI PROPERTIES OUTPUT_NAME MantidPythonAPI )
 # Add to the 'Framework' group in VS
@@ -176,9 +180,9 @@ endif ()
 # to ensure that the stdc++ library appears as early in the link list as possible so that it
 # is loaded first, hence the hard coding of it here rather than leaving it to be implicitly defined.
 if ( UNIX ) 
-  set ( PYTHON_DEPS stdc++ ${NEXUS_C_LIBRARIES} ${MANTIDLIBS} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES}  )
+  set ( PYTHON_DEPS stdc++ ${NEXUS_C_LIBRARIES} ${MANTIDLIBS} ${PYTHON_LIBRARIES}  )
 else ()
-  set ( PYTHON_DEPS ${MANTIDLIBS} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES}  )
+  set ( PYTHON_DEPS ${MANTIDLIBS} ${PYTHON_LIBRARIES}  )
 endif ()
 
 ###########################################################################
@@ -242,6 +246,9 @@ endif ()
 ###########################################################################
 
 install ( TARGETS PythonAPI DESTINATION ${BIN_DIR} )
+if ( ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" )
+  install ( TARGETS dlopen DESTINATION ${BIN_DIR} )
+endif ()
 install ( FILES ${PY_FILES} DESTINATION ${BIN_DIR} )
 install ( DIRECTORY PythonAlgorithms/ DESTINATION ${PLUGINS_DIR}/PythonAlgs
                                       PATTERN ".svn" EXCLUDE PATTERN "*.pyc" EXCLUDE )

Code/Mantid/Framework/PythonAPI/MantidFramework.py

@@ -34,31 +34,45 @@
     from MantidPythonAPI import *
     from MantidPythonAPI import _binary_op, _equals_op
 else:
-    # 
-    # To enable symbol sharing across extension modules (i.e. loaded dynamic libraries)
-    # calls to dlopen by Python must also use the RTLD_GLOBAL flag. If the default
-    # dlopen flags are used then the Singleton instance symbols will be multiply 
-    # defined across libraries and multiple intances Singleton instances can be created
-    #
-    saved_dlopenflags = sys.getdlopenflags()
     if platform.system() == "Linux":
-        try:
-            import DLFCN as dynload
-        except:
-            # Try older module
-            try:
-                import dl as dynload
-            except:
-                # If neither is available then this platform is unsupported
-                print "Both the DLFCN and dl modules are unavailable."
-                print "Cannot run Mantid from stand-alone Python on this platform."
-                sys.exit(1)
-
-        sys.setdlopenflags(dynload.RTLD_NOW | dynload.RTLD_GLOBAL)
+        # 
+        # The libMantidPythonAPI module is essentially statically linked
+        # to boost python. However, we need to ensure the Mantid libraries
+        # are loaded with the the RTLD_GLOBAL flag so that the singleton
+        # symbols are shared across the boundaries.
+        # 
+        # We also need to coexist with the new-style interface meaning that
+        # the boost python registry must be kept private in each api
+        # so that multiple converters are not defined. This means that 
+        # we cannot just pass the RTLD_GLOBAL flag here as this will 
+        # cause the registry to be shared with the new api if it is 
+        # loaded on top of this one. The only solution is to cherry
+        # pick the modules that are loaded with the RTLD symbol.
+        #
+        # Another nice issue is that the dl module is broken on 64-bit
+        # systems for Python 2.4 and ctypes doesn't exist there yet!
+        # All in all this meant a small custom module calling dlopen
+        # ourselves was the easiest way 
+        import libdlopen
+        dlloader = libdlopen.loadlibrary
+        import subprocess
+
+        _bin = os.path.abspath(os.path.dirname(__file__))
+        def get_libpath(mainlib, dependency):
+            cmd = 'ldd %s | grep %s' % (mainlib, dependency)
+            subp = subprocess.Popen(cmd,stdout=subprocess.PIPE,
+                                    stderr=subprocess.STDOUT,shell=True)
+            out = subp.communicate()[0]
+            # ldd produces a string that always has 4 columns. The full path
+            # is in the 3rd column
+            return out.split()[2]
+        dlloader(get_libpath(os.path.join('libMantidKernel.so'), 'libNeXus'))
+        dlloader(os.path.join(_bin, 'libMantidKernel.so'))
+        dlloader(os.path.join(_bin, 'libMantidGeometry.so'))
+        dlloader(os.path.join(_bin, 'libMantidAPI.so'))
 
     from libMantidPythonAPI import *
     from libMantidPythonAPI import _binary_op, _equals_op
-    sys.setdlopenflags(saved_dlopenflags)
 # --- End of library load ---
 
 

Code/Mantid/Framework/PythonAPI/inc/boost/python/arg_from_python.hpp

@@ -0,0 +1,76 @@
+// Copyright David Abrahams 2002.
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+#ifndef ARG_FROM_PYTHON_DWA2002128_HPP
+# define ARG_FROM_PYTHON_DWA2002128_HPP
+
+# include <boost/python/detail/prefix.hpp>
+# include <boost/python/converter/arg_from_python.hpp>
+# if BOOST_WORKAROUND(BOOST_MSVC, BOOST_TESTED_AT(1400)) \
+    || BOOST_WORKAROUND(BOOST_INTEL_WIN, BOOST_TESTED_AT(800))
+# include <boost/type_traits/remove_cv.hpp>
+#endif
+
+namespace boost { namespace python { 
+
+template <class T>
+struct arg_from_python
+    : converter::select_arg_from_python<
+# if BOOST_WORKAROUND(BOOST_MSVC, BOOST_TESTED_AT(1400)) \
+    || BOOST_WORKAROUND(BOOST_INTEL_WIN, BOOST_TESTED_AT(800))
+          typename boost::remove_cv<T>::type
+# else
+          T
+# endif 
+      >::type
+{
+    typedef typename converter::select_arg_from_python<
+# if BOOST_WORKAROUND(BOOST_MSVC, BOOST_TESTED_AT(1400)) \
+    || BOOST_WORKAROUND(BOOST_INTEL_WIN, BOOST_TESTED_AT(800))
+          typename boost::remove_cv<T>::type
+# else
+          T
+# endif 
+        >::type base;
+
+    arg_from_python(PyObject*);
+};
+
+// specialization for PyObject*
+template <>
+struct arg_from_python<PyObject*>
+{
+    typedef PyObject* result_type;
+
+    arg_from_python(PyObject* p) : m_source(p) {}
+    bool convertible() const { return true; }
+    PyObject* operator()() const { return m_source; }
+ private:
+    PyObject* m_source;
+};
+
+template <>
+struct arg_from_python<PyObject* const&>
+{
+    typedef PyObject* const& result_type;
+
+    arg_from_python(PyObject* p) : m_source(p) {}
+    bool convertible() const { return true; }
+    PyObject*const& operator()() const { return m_source; }
+ private:
+    PyObject* m_source;
+};
+
+//
+// implementations
+//
+template <class T>
+inline arg_from_python<T>::arg_from_python(PyObject* source)
+    : base(source)
+{
+}
+
+}} // namespace boost::python
+
+#endif // ARG_FROM_PYTHON_DWA2002128_HPP