Ensure Python algorithms are loaded by only 1 API. Refs #4399

Code/Mantid/Framework/PythonAPI/MantidFramework.py

@@ -1132,8 +1132,6 @@ def __init__(self):
         super(MantidPyFramework, self).__init__()
         self._garbage_collector = WorkspaceGarbageCollector()
         self._proxyfactory = WorkspaceProxyFactory(self._garbage_collector, self)
-        self._pyalg_loader = PyAlgLoader(self)
-        self._pyalg_objs = {}
 
     def runtime_mtdsimple(self, is_runtime=True):
         self.sendLogMessage("DEPRECATION WARNING: mtd.runtime_mtdsimple() is no longer used. Simply remove this line from your code.")
@@ -1190,10 +1188,11 @@ def initialise(self, gui=None):
             self.__gui__ = gui
 
         # Run through init steps 
-        self._pyalg_loader.load_modules(refresh=False)
-        # Ensure the fake Python algorithm functions are overwritten by
-        # the real ones
-        self._importSimpleAPIToMain()
+        if 'mantid.kernel' not in sys.modules:
+            import mantidsimple as _mantidsimple
+            pyalg_loader = PyAlgLoader()
+            pyalg_loader.load_modules(refresh=False)
+            _mantidsimple.translate() # Make sure the PythonAlgorithm functions are written
 
         self.__is_initialized = True
 
@@ -1244,14 +1243,6 @@ def keys(self):
             key_list.append(name)
         return key_list
 
-    def registerPyAlgorithm(self, algorithm):
-        """
-        Register an algorithm into the framework
-        """
-        # Keep the original object alive
-        self._pyalg_objs[algorithm.name()] = algorithm
-        super(MantidPyFramework, self).registerPyAlgorithm(algorithm)
-
     def createAlgorithm(self, name, version=-1):
         """Creates an algorithm object. The object
         is then wrapped in an AlgorithmProxy and returned.
@@ -1338,15 +1329,6 @@ def _addToPySearchPath(dirs):
             if not path in sys.path:
                 sys.path.append(path)
 
-
-    def _importSimpleAPIToMain(self):
-        import mantidsimple
-        mantidsimple.translate()
-        for name in dir(mantidsimple):
-            if name.startswith('_'):
-                continue
-            setattr(__main__, name, getattr(mantidsimple, name))
-
     def _refreshPyAlgorithms(self):
         # Reload the modules that have been loaded
         self._pyalg_loader.load_modules(refresh=True)
@@ -1433,21 +1415,17 @@ class PyAlgLoader(object):
 
     __CHECKLINES__ = 100
 
-    def __init__(self, framework):
-        self.framework = framework
-
-
     def load_modules(self, refresh=False):
         """
         Import Python modules containing Python algorithms
         """
         dir_list = mtd.getConfigProperty("pythonalgorithms.directories").split(';')
         if len(dir_list) == 0: 
-            self.framework.sendLogMessage('PyAlgLoader.load_modules: no python algorithm directory found')
+            mtd.sendLogMessage('PyAlgLoader.load_modules: no python algorithm directory found')
             return
 
         # Disable factory updates while everything is (re)imported
-        self.framework._observeAlgFactoryUpdates(False,False)
+        mtd._observeAlgFactoryUpdates(False,False)
 
         # Check defined Python algorithm directories and load any modules
         changes = False
@@ -1458,7 +1436,7 @@ def load_modules(self, refresh=False):
                 changes = True
 
         # Now connect the relevant signals to monitor for algorithm factory updates
-        self.framework._observeAlgFactoryUpdates(True, (refresh and changes))
+        mtd._observeAlgFactoryUpdates(True, (refresh and changes))
 
 #
 # ------- Private methods --------------
@@ -1495,9 +1473,9 @@ def _process_file(file_path, modname):
                     # Cleanup system path
                     del sys.path[0]
             except(StandardError), exp:
-                self.framework.sendLogMessage('Error: Importing module "%s" failed". %s' % (modname,str(exp)))
+                mtd.sendLogMessage('Error: Importing module "%s" failed". %s' % (modname,str(exp)))
             except:
-                self.framework.sendLogMessage('Error: Unknown error on Python algorithm module import. "%s" skipped' % modname)
+                mtd.sendLogMessage('Error: Unknown error on Python algorithm module import. "%s" skipped' % modname)
 
         # Find sub-directories     
         for root, dirs, files in os.walk(path):
@@ -1991,3 +1969,4 @@ def FrameworkSingleton():
     _fullpath = os.path.join(_script_dirs,_file)
     if not 'svn' in _file and os.path.isdir(_fullpath):
         MantidPyFramework._addToPySearchPath(_fullpath)
+

Code/Mantid/Framework/PythonAPI/mantidsimple.py

@@ -265,27 +265,5 @@ def translate():
         _algm_object = mtd.createUnmanagedAlgorithm(algorithm[0], max(algorithm[1]))
         create_algorithm(algorithm[0], max(algorithm[1]), _algm_object)
         create_algorithm_dialog(algorithm[0], max(algorithm[1]), _algm_object)
-
-def fake_python_alg_functions():
-    """
-        Creates fake, no-op functions to fool imports for
-        Python algorithms. The real definitions will
-        be created after all of the algorithms have been registered
-    """
-    def create_fake_function(files):
-        def fake_function():
-            pass
-        for pyfile in files:
-            if pyfile.endswith(".py"):
-                name = pyfile.strip(".py")
-                fake_function.__name__ = name
-                if name not in globals():
-                    globals()[name] = fake_function
-    #
-    directories = mtd.getConfigProperty("pythonalgorithms.directories").split(';')
-    for top in directories:
-        for root, dirs, files in os.walk(top):
-            create_fake_function(files)
 
-translate()
-fake_python_alg_functions()
+translate()

Code/Mantid/Framework/PythonInterface/mantid/__init__.py

@@ -65,3 +65,26 @@ def apiVersion():
 ###############################################################################
 __version__ = version_str()
 
+###############################################################################
+# Load the Python plugins now everything has started
+#
+# Before the plugins are loaded the simpleapi module is called to create
+# fake error-raising functions for all of the plugins. After the plugins have been 
+# loaded the correction translation is applied to create the "real" simple
+# API functions.
+#
+# Although this seems odd it is necessary so that any PythonAlgorithm 
+# can call any other PythonAlgorithm through the simple API mechanism. If left 
+# to the simple import mechanism then plugins that are loaded later cannot
+# be seen by the earlier ones (chicken & the egg essentially). 
+################################################################################
+import kernel.plugins as _plugins
+import sys as _sys
+import simpleapi as _simpleapi
+
+_simpleapi.mockout_api()
+if 'MantidFramework' not in _sys.modules: # Just while the other API is still around
+    loaded = _plugins.load(config['pythonalgorithms.directories'])
+    # Now everything is loaded create the proper definitions
+    _simpleapi.translate()
+

Code/Mantid/Framework/PythonInterface/mantid/api/__init__.py

@@ -30,13 +30,3 @@
 # Attach operators to workspaces 
 ###############################################################################
 import _workspaceops
-
-###############################################################################
-# Starting the FrameworkManager loads the C++ plugin libraries
-# we need to load in the Python plugins as well
-###############################################################################
-import mantid.kernel.plugins as _plugins
-# Algorithms
-from mantid.kernel import config as _config
-# Disabled for the time being as all algorithms are of the old kind
-#_plugins.load(_config['pythonalgorithm.directories']) 

Code/Mantid/Framework/PythonInterface/mantid/api/src/Exports/PythonAlgorithmExport.cpp

@@ -14,6 +14,7 @@
 
 using Mantid::API::Algorithm;
 using Mantid::PythonInterface::AlgorithmWrapper;
+using Mantid::Kernel::Direction;
 using namespace boost::python;
 
 namespace
@@ -41,20 +42,20 @@ void export_leaf_classes()
   // Export the algorithm wrapper that boost.python makes look like a PythonAlgorithm
   class_<AlgorithmWrapper, bases<Algorithm>, boost::noncopyable>("PythonAlgorithm", "Base class for all Python algorithms")
     .def("declareProperty", (declarePropertyType1)&AlgorithmWrapper::declareProperty,
-          declarePropertyType1_Overload(args("prop", "doc")))
+          declarePropertyType1_Overload((arg("prop"), arg("doc") = "")))
 
     .def("declareProperty", (declarePropertyType2)&AlgorithmWrapper::declareProperty,
-          declarePropertyType2_Overload(args("name", "defaultValue", "validator=None","doc=''","direction=Direction.Input"),
+          declarePropertyType2_Overload((arg("name"), arg("defaultValue"), arg("validator")=object(), arg("doc")="",arg("direction")=Direction::Input),
                                         "Declares a named property where the type is taken from "
                                         "the type of the defaultValue and mapped to an appropriate C++ type"))
 
     .def("declareProperty", (declarePropertyType3)&AlgorithmWrapper::declareProperty,
-         declarePropertyType3_Overload(args("name", "defaultValue", "doc","direction=Direction.Input"),
+         declarePropertyType3_Overload((arg("name"), arg("defaultValue"), arg("doc")="",arg("direction")=Direction::Input),
                                        "Declares a named property where the type is taken from the type "
                                        "of the defaultValue and mapped to an appropriate C++ type"))
 
     .def("declareProperty", (declarePropertyType4)&AlgorithmWrapper::declareProperty,
-         args("name", "defaultValue", "direction"),
+        (arg("name"), arg("defaultValue"), arg("direction")=Direction::Input),
          "Declares a named property where the type is taken from the type "
          "of the defaultValue and mapped to an appropriate C++ type")
   ;

Code/Mantid/Framework/PythonInterface/mantid/kernel/plugins.py

@@ -6,7 +6,7 @@
 """
 import os as _os
 import imp as _imp
-from mantid.kernel import Logger, ConfigService
+from mantid.kernel import logger, Logger, ConfigService
 
 
 class PluginLoader(object):
@@ -21,7 +21,7 @@ def __init__(self, filepath):
 
     def run(self):
         """
-            Load the module we are pointing at and return
+            Try and load the module we are pointing at and return
             the module object.
             
             Any ImportErrors raised are not caught and are passed
@@ -48,19 +48,21 @@ def load(path):
         will not included modules that will have attempted to be
         reloaded but had not been changed
     """
-    loaded = []
+    loaded = {}
     if _os.path.isfile(path) and path.endswith('.py'): # Single file
-        loader = PluginLoader(path)
-        module = loader.run()
-        loaded.append(module.__name__)
+        try:
+            name, module = load_plugin(path)
+            loaded[name] = module
+        except Exception, exc:
+            logger.warning("Failed to load plugin %s. Error: %s" % (path, str(exc)))
     elif _os.path.isdir(path): # Directory
-        loaded.extend(load_from_dir(path))
+        loaded.update(load_from_dir(path))
     else: # a list 
         if ';' in path:
             path = split(';')
         if type(path) is list: # Call load again for each one
             for p in path: 
-                loaded.extend(load(p))
+                loaded.update(load(p))
 
     return loaded
 
@@ -72,102 +74,23 @@ def load_from_dir(directory):
     """
     if not _os.path.isdir(directory):
         raise RuntimeError("The path given does not point to an existing directory")
-    loaded = []
+    loaded = {}
     for root, dirs, files in _os.walk(directory):
         for f in files:
             filename = _os.path.join(root, f)
-            loaded.extend(load(filename))
+            loaded.update(load(filename))
 
     return loaded
-
-###############################################################################
-# Backwards compatible loader for PythonAlgorithms written in old style
-# 
-# This will be removed when the old-style API is removed.
-#
-###############################################################################
-class PyAlgLoader(object):
-
-    __CHECKLINES__ = 100
-
-    def __init__(self):
-        self._logger = Logger.get("PyAlgLoader")
-
-    def load_modules(self, refresh=False):
-        """
-        Import Python modules containing Python algorithms
-        """
-        dir_list = ConfigService["pythonalgorithms.directories"].split(';')
-
-        # Check defined Python algorithm directories and load any modules
-        changes = False
-        for path in dir_list:
-            if path == '':
-                continue
-            if self._importAlgorithms(path, refresh):
-                changes = True
 
-#
-# ------- Private methods --------------
-#
-    def _importAlgorithms(self, path, refresh):
-        # Make sure the directory doesn't contain a trailing slash
-        path = path.rstrip("/").rstrip("\\")
-        try:
-            files = _os.listdir(path)
-        except(OSError):
-            return False
-        changes = False
+def load_plugin(plugin_path):
+    """
+        Load a plugin and return the name & module object
         
-        def _process_file(file_path, modname):
-            import sys
-            pyext = '.py'
-            if not modname.endswith(pyext):
-                return
-            original = _os.path.join(file_path, modname)
-            modname = modname[:-len(pyext)]
-            compiled = _os.path.join(file_path, modname + '.pyc')
-            if modname in sys.modules and \
-               _os.path.exists(compiled) and \
-               _os.path.getmtime(compiled) >= _os.path.getmtime(original):
-                return
-            try:               
-                if self._containsPyAlgorithm(original):
-                    # Temporarily insert into path
-                    sys.path.insert(0, file_path)
-                    if modname in sys.modules:
-                        reload(sys.modules[modname])
-                    else:
-                        __import__(modname)
-                    changes = True
-                    # Cleanup system path
-                    del sys.path[0]
-            except(StandardError), exp:
-                self._logger.warning('Error: Importing module "%s" failed". %s' % (modname,str(exp)))
-            except:
-                self._logger.warning('Error: Unknown error on Python algorithm module import. "%s" skipped' % modname)
-
-        # Find sub-directories     
-        for root, dirs, files in _os.walk(path):
-            for f in files:
-                _process_file(root, f)
-
-        return changes
-
-    def _containsPyAlgorithm(self, modfilename):
-        file = open(modfilename,'r')
-        line_count = 0
-        alg_found = False
-        while line_count < self.__CHECKLINES__:
-            line = file.readline()
-            # EOF
-            if line == '':
-                alg_found = False
-                break
-            if line.rfind('PythonAlgorithm') >= 0:
-                alg_found = True
-                break
-            line_count += 1
-        file.close()
-        return alg_found
-
+         @param plugin_path :: A path that must should point
+         to a .py file that will be loaded. An ValueError is raised if
+         path is not a valid plugin path. Any exceptions raised by the 
+         import are passed to the caller
+    """
+    loader = PluginLoader(plugin_path)
+    module = loader.run()
+    return module.__name__, module