Introduce new directory structure for python plugins.

There is now a single directory called plugins that contains sub-
directories for algorithms & functions. These get installed into the
package under ROOT/plugins/python. This requires new keys to find the
directories and deprecates pythonalgorithms.directories key.

Users will receive warnings about the old key being deprecated and are
instructed about what to do.

Refs #970

Code/Mantid/Framework/Kernel/CMakeLists.txt

@@ -381,7 +381,8 @@ set ( PLUGINS "." )
 set ( PV_PLUGINS "./pvplugins" )
 set ( IGNORE_PARAVIEW "0" )
 set ( QTPLUGINS "." )
-set ( PYTHONALG_DIRS "${MANTID_ROOT}/Framework/PythonAPI/PythonAlgorithms;${MANTID_ROOT}/Framework/PythonInterface/PythonAlgorithms" )
+set ( PYTHONALG_DIRS "${MANTID_ROOT}/Framework/PythonAPI/PythonAlgorithms;${MANTID_ROOT}/Framework/PythonInterface/PythonAlgorithms" ) # deprecated
+set ( PYTHONPLUGIN_DIRS "${MANTID_ROOT}/Framework/PythonAPI/PythonAlgorithms;${MANTID_ROOT}/Framework/PythonInterface/plugins" )
 set ( DATADIRS ${MANTID_ROOT}/../../Test/AutoTestData;${MANTID_ROOT}/instrument )
 set ( COLORMAPS_FOLDER ${MANTID_ROOT}/Installers/colormaps/ )
 
@@ -454,7 +455,8 @@ else ()
 endif ()
 
 set ( PLUGINS ${MANTID_ROOT}/plugins )
-set ( PYTHONALG_DIRS ${PLUGINS}/PythonAlgs )
+set ( PYTHONALG_DIRS ${PLUGINS}/PythonAlgs ) # deprecated
+set ( PYTHONPLUGIN_DIRS ${PLUGINS}/python )
 set ( DATADIRS "" )
 
 # Construct script paths by replacing the old MANTID_ROOT with the new one.

Code/Mantid/Framework/Properties/Mantid.properties.template

@@ -34,6 +34,9 @@ pvplugins.directory = @PV_PLUGINS@
 # Where to find Mantid Qt plugin libraries
 mantidqt.plugins.directory = @QTPLUGINS@
 
+# Where to find python plugins
+python.plugins.directories = @PYTHONPLUGIN_DIRS@
+
 # Where to load instrument definition files from
 instrumentDefinition.directory = @MANTID_ROOT@/instrument
 
@@ -63,8 +66,10 @@ requiredpythonscript.directories = @REQUIREDSCRIPT_DIRS@
 # This key is NOT recursive so sub-directories must be added in addition
 pythonscripts.directories = @PYTHONSCRIPT_DIRS@
 
-# The locations of Python algorithms that are to be loaded at startup
-pythonalgorithms.directories = @PYTHONALG_DIRS@
+# The locations of Python algorithms that are to be loaded at startup (deprecated)
+pythonalgorithms.directories = 
+# Require a copy of what they key used to look like so we know if a user had overridden it
+pythonalgorithms.directories.deprecated = @PYTHONALG_DIRS@
 
 # Setting this to 1 will allow python algorithms to be reloaded before execution.
 pythonalgorithms.refresh.allowed = 0

Code/Mantid/Framework/PythonAPI/CMakeLists.txt

@@ -264,6 +264,6 @@ endif ()
 # Pure Python files 
 install ( FILES ${PY_FILES} DESTINATION ${BIN_DIR} )
 # Algorithms
-install ( DIRECTORY PythonAlgorithms/ DESTINATION ${PLUGINS_DIR}/PythonAlgs
+install ( DIRECTORY plugins/ DESTINATION ${PLUGINS_DIR}/python
           PATTERN "*.pyc" EXCLUDE
           PATTERN ".svn" EXCLUDE )

Code/Mantid/Framework/PythonAPI/MantidFramework.py

@@ -1193,7 +1193,9 @@ def initialise(self, gui=None):
 
         # Run through init steps 
         import mantidsimple as _mantidsimple
-        dir_list = mtd.getConfigProperty("pythonalgorithms.directories").split(';')
+        dir_list = mtd.getConfigProperty("python.plugins.directories").split(';')
+        dir_list += mtd.getConfigProperty("user.python.plugins.directories").split(';')
+
         _mantidsimple.mockup(dir_list)
         pyalg_loader = PyAlgLoader()
         plugins = pyalg_loader.load_modules(refresh=False)

Code/Mantid/Framework/PythonAPI/mantidsimple.py

@@ -420,6 +420,8 @@ def fake_function(*args, **kwargs):
         directories = [directories]
     for top_dir in directories:
         for root, dirs, filenames in os.walk(top_dir):
+            if 'functions' in root: # Functions are solely for new API
+                continue
             create_fake_functions(filenames)
 
 #------------------------------------------------------------------------------

Code/Mantid/Framework/PythonInterface/CMakeLists.txt

@@ -67,6 +67,6 @@ add_subdirectory( test )
 ###########################################################################
 
 # Python algorithms
-install ( DIRECTORY PythonAlgorithms/ DESTINATION ${PLUGINS_DIR}/PythonAlgs
+install ( DIRECTORY plugins/ DESTINATION ${PLUGINS_DIR}/python
           PATTERN "*.pyc" EXCLUDE
           PATTERN ".svn" EXCLUDE )

Code/Mantid/Framework/PythonInterface/mantid/__init__.py

@@ -88,22 +88,68 @@ def apiVersion():
 import simpleapi as _simpleapi
 from kernel import plugins as _plugins
 
-plugin_dirs = kernel.config['pythonalgorithms.directories'].split(";")
+def _to_set(key):
+    s = set(kernel.config[key].split(";"))
+    if '' in s:
+        s.remove('')
+    return s
+
+_plugins_key = 'python.plugins.directories'
+_user_key = 'user.%s' % _plugins_key
+plugin_dirs = _to_set(_plugins_key)
+plugin_dirs.update(_to_set(_user_key))
+
+# Check deprecated path too
+# There are 2 situations that require handling:
+#   2) Users defined new algorithms and placed them in other locations and added these
+#      locations to the pythonalgorithms.directories key in their user.props file. Tell them to update the
+#      and use the new keys
+#   2) Users defined new algorithms and placed them in the deprecated location so the key is not in their 
+#      user.properties file. Ask them to move the files to the new location
+_deprecated_key = 'pythonalgorithms.directories'
+_old_locs_key = '%s.deprecated' % (_deprecated_key)
+# 1)
+_user_file = config.getUserFilename()
+_deprecated_locs = _to_set(_deprecated_key)
+_msg="The Python algorithms key '%s' in '%s' has been deprecated. Please add '%s' to the '%s' key instead. " +\
+     "Future release will not check the old key."
+for loc in _deprecated_locs:
+    loc = loc.rstrip("/")
+    if (not loc.endswith('PythonAPI/PythonAlgorithms')) and (not loc.endswith('PythonInterface/PythonAlgorithms')): # Avoid dev warning
+        logger.warning(_msg % (_deprecated_key,_user_file, loc,_user_key))
+        plugin_dirs.add(loc)
+
+# 2)
+_old_locs = _to_set(_old_locs_key)
+_new_loc = _os.path.abspath(_os.path.join(_os.path.dirname(_bindir), '../plugins/python/algorithms'))
+_msg="The packaged Python algorithms have been moved. You have extra algorithms in '%s', please move these files to '%s'. " +\
+     "Future releases will not check this location."
+for loc in _old_locs:
+    loc = loc.rstrip("/")
+    if _plugins.check_for_plugins(loc):
+        if 'PythonAPI/PythonAlgorithms' not in loc: # Avoid a warning for developers until all old algorithms are gone
+            logger.warning(_msg % (loc,_new_loc))
+        plugin_dirs.add(loc)
+
+# Load
 plugin_files = []
+alg_files = []
 for directory in plugin_dirs:
     try:
-        if directory != '':
-            plugin_files += _plugins.find_plugins(directory)
+        all_plugins, algs = _plugins.find_plugins(directory)
+        plugin_files += all_plugins
+        alg_files += algs
     except ValueError, exc:
         logger.warning(str(exc))
         continue
 
 # Mockup the full API first so that any Python algorithm module has something to import
-_simpleapi._mockup(plugin_files)
-# Now actually load the Python plugins
+_simpleapi._mockup(alg_files)
+# Load the plugins
 plugin_modules = _plugins.load(plugin_files)
-# Create the proper definitions in the module
+# Create the proper algorithm definitions in the module
 new_attrs = _simpleapi._translate()
 # Finally, overwrite the mocked function definitions in the loaded modules with the real ones 
 _plugins.sync_attrs(_simpleapi, new_attrs, plugin_modules)
+
 ################################################################################

Code/Mantid/Framework/PythonInterface/mantid/kernel/plugins.py

@@ -37,20 +37,40 @@ def run(self):
 
 #======================================================================================================================
 
+def check_for_plugins(top_dir):
+    """
+        Runs a quick check to see if any plugin files exist in the given directory
+    
+        @returns True if any plugins are found, false otherwise
+    """
+    if not _os.path.isdir(top_dir):
+        return False
+
+    for root, dirs, files in _os.walk(top_dir):
+        for f in files:
+            if f.endswith(PluginLoader.extension):
+                return True
+
+    return False
+
 def find_plugins(top_dir):
     """
        Searches recursively from the given directory to find the list of plugins that should be loaded
        @param top_dir :: A string containing a path to a directory. Throws ValueError if it is not valid
     """
     if not _os.path.isdir(top_dir):
         raise ValueError("Cannot search given path for plugins, path is not a directory: '%s' " % str(top_dir))
-    plugins = []
+    all_plugins = []
+    algs = []
     for root, dirs, files in _os.walk(top_dir):
         for f in files:
             if f.endswith(PluginLoader.extension):
-                plugins.append(_os.path.join(root, f))
-
-    return plugins
+                filename = _os.path.join(root, f)
+                all_plugins.append(filename)
+                if contains_newapi_algorithm(filename):
+                    algs.append(filename)
+
+    return all_plugins, algs
 
 #======================================================================================================================
 
@@ -120,8 +140,8 @@ def load_from_dir(directory):
 
 def load_from_file(filepath):
     """
-        If the algorithm is a new API algorithm then load it
-        
+        Loads the plugin file. Any code present at the top-level will
+        be executed on loading
         @param filepath :: A path that must point to a file
     """
     loaded = []
@@ -189,5 +209,8 @@ def contains_newapi_algorithm(filename):
         if 'registerPyAlgorithm' in line:
             alg_found = False
             break
+        if 'registerAlgorithm' in line:
+            alg_found = True
+            break
     file.close()
     return alg_found

Code/Mantid/Framework/PythonInterface/plugins/algorithms/Examples/README.md

@@ -0,0 +1 @@
+This directory contains example algorithms that can be plugged into the Mantid.

Code/Mantid/Framework/PythonInterface/plugins/functions/Examples/README.md

@@ -0,0 +1 @@
+This directory contains example functions that can be plugged into the Mantid optimisation framework.